├── .gitignore
├── CHANGELOG.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── docs
    ├── Dockerfile
    ├── Makefile
    ├── README.md
    ├── sources
    │   ├── conf.py
    │   ├── examples
    │   │   ├── advanced.rst
    │   │   ├── basics.rst
    │   │   ├── cassandra.rst
    │   │   ├── hadoop.rst
    │   │   ├── index.rst
    │   │   ├── mongodb.rst
    │   │   ├── multiple.rst
    │   │   ├── openmpi.rst
    │   │   ├── pulling.rst
    │   │   └── spark.rst
    │   ├── index.rst
    │   ├── install
    │   │   ├── aws.rst
    │   │   ├── client.rst
    │   │   ├── hpcloud.rst
    │   │   ├── index.rst
    │   │   ├── openstack.rst
    │   │   ├── osclient.rst
    │   │   └── rackspace.rst
    │   ├── references
    │   │   ├── commands.rst
    │   │   └── index.rst
    │   ├── toctree.rst
    │   └── upgrade.rst
    └── theme
    │   └── ferry
    │       ├── layout.html
    │       ├── redirect_build.html
    │       ├── redirect_home.html
    │       ├── static
    │           ├── css
    │           │   ├── bootstrap.css
    │           │   └── main.css
    │           ├── favicon.png
    │           ├── img
    │           │   ├── frontpage_logo.png
    │           │   ├── github.png
    │           │   ├── glyphicons-halflings-white.png
    │           │   ├── glyphicons-halflings.png
    │           │   ├── google.png
    │           │   ├── menu_arrow_down.gif
    │           │   ├── menu_arrow_right.gif
    │           │   ├── opencore.png
    │           │   └── twitter.png
    │           ├── js
    │           │   └── docs.js
    │           └── sitemap
    │           │   └── sitemap.txt
    │       └── theme.conf
├── ferry-dockerfile
    ├── 50-ferryheader
    ├── 51-ferryhelp
    ├── Dockerfile
    └── make.sh
├── ferry
    ├── __init__.py
    ├── cli
    │   ├── __init__.py
    │   └── cli.py
    ├── config
    │   ├── __init__.py
    │   ├── cassandra
    │   │   ├── __init__.py
    │   │   ├── cassandraclientconfig.py
    │   │   └── cassandraconfig.py
    │   ├── gluster
    │   │   ├── __init__.py
    │   │   └── glusterconfig.py
    │   ├── hadoop
    │   │   ├── __init__.py
    │   │   ├── hadoopclientconfig.py
    │   │   ├── hadoopconfig.py
    │   │   ├── hiveconfig.py
    │   │   └── metastore.py
    │   ├── mongo
    │   │   ├── __init__.py
    │   │   ├── mongoclientconfig.py
    │   │   └── mongoconfig.py
    │   ├── openmpi
    │   │   ├── __init__.py
    │   │   ├── mpiclientconfig.py
    │   │   └── mpiconfig.py
    │   ├── spark
    │   │   ├── __init__.py
    │   │   ├── sparkclientconfig.py
    │   │   └── sparkconfig.py
    │   ├── system
    │   │   ├── __init__.py
    │   │   ├── aws.py
    │   │   └── info.py
    │   └── titan
    │   │   ├── __init__.py
    │   │   └── titanconfig.py
    ├── data
    │   ├── conf
    │   │   ├── ferry-aws-example.yaml
    │   │   ├── ferry-default.yaml
    │   │   └── ferry-hp-example.yaml
    │   ├── dockerfiles
    │   │   ├── cassandra-client
    │   │   │   ├── Dockerfile
    │   │   │   ├── gods.json
    │   │   │   ├── loadgods.groovy
    │   │   │   ├── naturalgraph.groovy
    │   │   │   ├── restart01.sh
    │   │   │   ├── rexexample.sh
    │   │   │   ├── start01.sh
    │   │   │   ├── startnode
    │   │   │   ├── stop10.sh
    │   │   │   ├── test01.sh
    │   │   │   └── testusers.db
    │   │   ├── cassandra
    │   │   │   ├── Dockerfile
    │   │   │   └── startnode
    │   │   ├── ferry-base
    │   │   │   ├── Dockerfile
    │   │   │   ├── halt01.sh
    │   │   │   ├── init01.sh
    │   │   │   └── setup
    │   │   ├── gluster
    │   │   │   ├── Dockerfile
    │   │   │   └── startnode
    │   │   ├── hadoop-base
    │   │   │   └── Dockerfile
    │   │   ├── hadoop-client
    │   │   │   ├── Dockerfile
    │   │   │   ├── count.pig
    │   │   │   ├── createtable.sql
    │   │   │   ├── mounthelper.py
    │   │   │   ├── restart01.sh
    │   │   │   ├── start01.sh
    │   │   │   ├── startnode
    │   │   │   ├── stop10.sh
    │   │   │   ├── test01.sh
    │   │   │   └── test02.sh
    │   │   ├── hadoop
    │   │   │   ├── Dockerfile
    │   │   │   ├── mounthelper.py
    │   │   │   └── startnode
    │   │   ├── hive-metastore
    │   │   │   ├── Dockerfile
    │   │   │   ├── createstore.sql
    │   │   │   ├── hive-schema.sql
    │   │   │   └── startnode
    │   │   ├── mongodb
    │   │   │   ├── Dockerfile
    │   │   │   ├── createadmin.js
    │   │   │   └── startnode
    │   │   ├── openmpi-client
    │   │   │   ├── Dockerfile
    │   │   │   ├── helloworld.cpp
    │   │   │   └── test01.sh
    │   │   ├── openmpi
    │   │   │   ├── Dockerfile
    │   │   │   ├── mounthelper.py
    │   │   │   ├── restart01.sh
    │   │   │   ├── start01.sh
    │   │   │   ├── startnode
    │   │   │   ├── stop10.sh
    │   │   │   └── utils
    │   │   ├── spark
    │   │   │   ├── Dockerfile
    │   │   │   ├── als_test.data
    │   │   │   ├── classification.py
    │   │   │   ├── clustering.py
    │   │   │   ├── filtering.py
    │   │   │   ├── kmeans_test.data
    │   │   │   ├── lpsa_test.data
    │   │   │   ├── mounthelper.py
    │   │   │   ├── regression.py
    │   │   │   ├── restart01.sh
    │   │   │   ├── simpleapp.py
    │   │   │   ├── start01.sh
    │   │   │   ├── startnode
    │   │   │   ├── stop10.sh
    │   │   │   ├── svm_test.data
    │   │   │   ├── test01.sh
    │   │   │   └── tree_test.data
    │   │   └── titan
    │   │   │   ├── Dockerfile
    │   │   │   └── startnode
    │   ├── key
    │   │   ├── insecure_ferry_key
    │   │   └── insecure_ferry_key.pem
    │   ├── plans
    │   │   ├── cassandra.yaml
    │   │   ├── hadoop.yaml
    │   │   ├── mongo.yaml
    │   │   ├── openmpi.yaml
    │   │   ├── spark.yaml
    │   │   └── yarn.yaml
    │   └── templates
    │   │   ├── cassandra
    │   │       ├── cassandra.yaml.template
    │   │       ├── log4j-server.properties
    │   │       └── titan.properties
    │   │   ├── gluster
    │   │       └── configure.template
    │   │   ├── hadoop
    │   │       ├── core-site.xml.template
    │   │       ├── hadoop-env.sh
    │   │       ├── hdfs-site.xml.template
    │   │       ├── httpfs-site.xml.template
    │   │       ├── log4j.properties
    │   │       ├── mapred-env.sh
    │   │       ├── mapred-site.xml.template
    │   │       ├── yarn-env.sh.template
    │   │       └── yarn-site.xml.template
    │   │   ├── hive-metastore
    │   │       ├── hive-site.xml.template
    │   │       ├── pg_hba.conf
    │   │       └── postgresql.conf
    │   │   ├── linux
    │   │       └── configure.template
    │   │   ├── mongo
    │   │       ├── mongodb.conf.template
    │   │       └── trusted.conf.template
    │   │   ├── openmpi
    │   │       └── openmpi-mca-params.conf
    │   │   ├── spark
    │   │       └── spark_env.sh.template
    │   │   └── titan
    │   │       ├── rexster.xml.template
    │   │       └── titan.properties
    ├── docker
    │   ├── __init__.py
    │   ├── configfactory.py
    │   ├── deploy.py
    │   ├── docker.py
    │   ├── manager.py
    │   └── resolve.py
    ├── fabric
    │   ├── __init__.py
    │   ├── aws
    │   │   ├── __init__.py
    │   │   └── awslauncher.py
    │   ├── cloud.py
    │   ├── com.py
    │   ├── local.py
    │   └── openstack
    │   │   ├── __init__.py
    │   │   └── singlelauncher.py
    ├── http
    │   ├── __init__.py
    │   └── httpapi.py
    ├── install.py
    ├── ip
    │   ├── __init__.py
    │   ├── client.py
    │   ├── dhcp.py
    │   └── nat.py
    ├── logging.conf
    ├── options.py
    └── table
    │   ├── __init__.py
    │   └── prettytable.py
├── install
    ├── README.md
    ├── ferry-dust
    └── ferry-install
├── requirements.txt
└── setup.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | *pyc
3 | *pyo


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | ## 0.3 series
 4 | 
 5 | ### 0.3.3 (2014-10-06)
 6 | 
 7 | - Upgrade Spark to 1.1.0
 8 | - Upgrade Hadoop to 2.5.1
 9 | - Upgrade Cassandra to 2.1.0
10 | - Upgrade GlusterFS to 3.5.0
11 | - Remove Docker fork
12 | 
13 | ### 0.3.2 (2014-09-29)
14 | 
15 | - AWS backend
16 | - Various bug fixes
17 | 
18 | ### 0.3.1 (2014-09-01)
19 | 
20 | - Fix local mode refactor
21 | 	
22 | ### 0.3.0 (2014-08-27)
23 | 
24 | - Asynchronous stack operations
25 | - Better key management
26 | - More flexible way to acquire user parameters
27 | - Inspect command shows individual services
28 | - Initial OpenStack backend
29 | 
30 | ## 0.2 series
31 | 
32 | ### 0.2.0 (2014-05-15)
33 | 
34 | - Spark support
35 | - Pig support in Hadoop client
36 | - Various bug fixes
37 | 
38 | ### 0.2.1 (2014-06-04)
39 | 
40 | - Support for building images
41 | - Support for pushing and pulling applications
42 | - Enhanced  'ls' and 'inspect' commands
43 | - Fix Cassandra nodetool RMI port
44 | - Fix Hadoop jobhistoryserver
45 | 
46 | ### 0.2.2 (2014-06-11)
47 | 
48 | - MongoDB storage service
49 | - Support for multiple storage backends
50 | - Support for multiple compute backends
51 | 
52 | ### 0.2.3 (2014-07-07)
53 | 
54 | - Various bugfixes
55 | - Ability to use OpenMPI client without compute
56 | - Print out status messages when starting service
57 | - Better custom key support
58 | 
59 | ### 0.2.3.1 (2014-07-11)
60 | 
61 | - Fix PostGreSQL 9.3 upgrade
62 | 
63 | ## 0.1 series
64 | 
65 | ### 0.1.23 (2014-03-24)
66 | 
67 | - Hadoop and Cassandra clients make symlinks to `hadoop`, `cql`, etc commands
68 | - Specify Python logging configuration file using `-l` when starting server
69 | - Remove ghosted Mongo instances when starting
70 | - Update documentation to reflect upgrades
71 | - Fix specifying `-u` and `-k` at same time when installing
72 | 
73 | ### 0.1.24 (2014-03-28)
74 | 
75 | - Images now mount the key directory instead of being hard-coded
76 | - Dockerfiles no longer need privileged to build
77 | - Images are now tagged with version
78 | - Images are now hosted on Docker public index
79 | - YAML support
80 | 
81 | ### 0.1.26 (2014-04-05)
82 | 
83 | - Skipped version 0.1.25 due to packaging error
84 | - Stop/restart storage services without losing data
85 | - Use custom Dockerfile images for connectors
86 | - Connectors get random names unless supplied with user-defined name
87 | 
88 | ### 0.1.27 (2014-04-18)
89 | 
90 | - Port forwarding
91 | - Fix issues restarting GlusterFS/Yarn stacks
92 | 
93 | ### 0.1.28 (2014-04-25)
94 | 
95 | - Vagrant support
96 | - Check LXC version during install
97 | 
98 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
 1 | include LICENSE
 2 | include requirements.txt
 3 | include *.md
 4 | include docker/*
 5 | include ferry/logging.conf
 6 | include ferry/data/conf/*
 7 | recursive-include ferry/data/dockerfiles *
 8 | include ferry/data/key/*
 9 | include ferry/data/plans/*
10 | recursive-include ferry/data/templates *
11 | global-exclude *.pyc
12 | global-exclude *.pyo
13 | global-exclude *~
14 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Ferry: Big Data Development Environment using Docker
 2 | ====================================================
 3 | 
 4 | Ferry lets you launch, run, and manage big data clusters on AWS, OpenStack, and your local machine. It does this by leveraging awesome technologies such as 
 5 | [Docker](https://www.docker.io). Ferry currently supports:
 6 | 
 7 | * Hadoop/YARN (version 2.5.1)
 8 | * Cassandra (version 2.1.0) + Titan (0.3.1)
 9 | * Spark (version 1.1.0)
10 | * GlusterFS (version 3.5)
11 | * Open MPI (version 1.8.1)
12 | 
13 | All you have to do to start is specify your stack using YAML or download a pre-existing application. 
14 | 
15 | Why?
16 | ====
17 | 
18 | Ferry is made for developers and data scientists that want to develop big data applications without the fuss of setting up the infrastructure. It will help you: 
19 | 
20 | * Experiment with big data technologies, such as Hadoop or Cassandra without having to learn the intricacies of configuring each software
21 | * Share and evaluate other people's big data application quickly and safely via Dockerfiles
22 | * Develop and test applications locally before being deployed onto an operational cluster
23 | 
24 | Because Ferry uses Docker underneath, each virtual cluster is completely isolated. That means you can create multiple clusters for different applications.
25 | 
26 | Getting started
27 | ===============
28 | 
29 | Ferry is a Python application and runs on your local machine. All you have to do to get started is have
30 | `docker` installed and type the following `pip install -U ferry`. More detailed installation instructions and examples can be found [here](http://ferry.opencore.io). 
31 | 
32 | 
33 | Once installed, you can create your big data application using YAML files. 
34 | 
35 | ```yaml
36 |    backend:
37 |       - storage:
38 |            personality: "gluster"
39 |            instances: 2
40 |         compute:
41 |            - personality: "yarn"
42 |              instances: 2
43 |    connectors:
44 |       - personality: "hadoop-client"
45 |         name: "control-0"
46 | ```
47 | 
48 | This stack consists of two GlusterFS data nodes, and two Hadoop/YARN compute nodes. There's also an Ubuntu-based
49 | client that automatically connects to those backend components. Of course you can substitute your own
50 | customized client. 
51 | 
52 | To create this stack, just type `ferry start yarn`. Once you create the stack, 
53 | you can log in by typing `ferry ssh control-0`. 
54 | 
55 | Contributing
56 | ============
57 | 
58 | Contributions are totally welcome. Here are some suggestions on how to get started:
59 | 
60 | * Use Ferry, report bugs, and file new features! By filing issues and sharing your experience, you will help improve the software for others.
61 | * Create Dockerfiles for your favorite backend, especially if you think the installation process is harder than it should be. The Dockerfile can be basic and we'll work together to get it ready for other users. 
62 | * Create a new configuration module for your backend. This one is more complicated since it will involve actually hacking Ferry, but it's not so hard if we work together. 
63 | 
64 | I strongly recommend using GitHub issues + pull requests for contributions. Tweets sent to @open_core_io are also welcome. Happy hacking!
65 | 
66 | Under the hood
67 | ==============
68 | 
69 | Ferry leverages some awesome open source projects:
70 | 
71 | * [Docker](https://www.docker.io) simplifies the management of Linux containers
72 | * [Python](http://www.python.org) programming language
73 | * [Hadoop](http://hadoop.apache.org) is a general-purpose big data storage and processing framework
74 | * [GlusterFS](http://www.gluster.org) is a parallel filesystem actively developed by Redhat
75 | * [OpenMPI](http://www.open-mpi.org) is a scalable MPI implementation focused on modeling & simulation
76 | * [Cassandra](http://cassandra.apache.org) is a highly scalable column store
77 | * [PostgreSQL](http://postgresql.org) is a popular relational database
78 | 


--------------------------------------------------------------------------------
/docs/Dockerfile:
--------------------------------------------------------------------------------
1 | # Based on Dockerfile from Docker.io documentation. 
2 | FROM base
3 | 
4 | RUN apt-get update;apt-get --yes install make python-pip python-setuptools
5 | RUN pip install Sphinx==1.2.1
6 | RUN pip install sphinxcontrib-httpdomain==1.2.0
7 | 
8 | CMD ["make", "-C", "/docs", "clean", "server"]
9 | EXPOSE 8000


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
 1 | # Makefile for Sphinx documentation
 2 | #
 3 | 
 4 | SPHINXOPTS   = 
 5 | SPHINXBUILD  = sphinx-build
 6 | BUILDDIR     = _build
 7 | PYTHON       = python
 8 | DOCKER       = docker
 9 | 
10 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) sources
11 | 
12 | clean:
13 | 	-rm -rf $(BUILDDIR)/*
14 | image:
15 | 	$(DOCKER) build -t $(USER)/ferry-doc .
16 | run:
17 | 	$(DOCKER) run -d -v ./:/docs $(USER)/ferry-doc
18 | docs:
19 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/html
20 | 	@echo
21 | 	@echo "Build finished. The documentation pages are now in $(BUILDDIR)/html."
22 | 
23 | server: docs
24 | 	@cd $(BUILDDIR)/html; $(PYTHON) -m SimpleHTTPServer 8000


--------------------------------------------------------------------------------
/docs/README.md:
--------------------------------------------------------------------------------
 1 | Ferry documentation
 2 | =====================
 3 | 
 4 | Overview
 5 | --------
 6 | 
 7 | The source for all documentation is under ``sources/`` in the form of .rst files. 
 8 | These files use [reStructuredText](http://docutils.sourceforge.net/rst.html) formatting
 9 | with [Sphinx](http://sphinx-doc.org/) extensions for structure, cross-linking, and
10 | indexing. 
11 | 
12 | Getting Started
13 | ---------------
14 | 
15 | To edit and compile the docs you'll need to have Docker installed. Then
16 | it's just a matter of creating the documentation image and starting the server. 
17 | 
18 | ### Installation
19 | 
20 | The preferred way of generating the documentation is using ``docker``. First, 
21 | you'll want to create the image:
22 | 
23 | ```bash
24 | make image
25 | 
26 | This will create an image called $USER/ferry-doc. Next you'll want to generate
27 | the documentation and start the HTML server. 
28 | 
29 | ```bash
30 | docker run -d -v ./:/docs $USER/ferry-doc
31 | # Or equivalently
32 | make run
33 | 
34 | You should be able to access the documentation under ``_build/html``, or by
35 | visiting ``http://localhost:8000``. 
36 | 


--------------------------------------------------------------------------------
/docs/sources/examples/advanced.rst:
--------------------------------------------------------------------------------
  1 | :title: Ferry Advanced
  2 | :description: Basic examples
  3 | :keywords: ferry, examples
  4 | 
  5 | .. _advanced:
  6 | 
  7 | Advanced Topics
  8 | ===============
  9 | 
 10 | Creating a Dockerfile
 11 | ---------------------
 12 | 
 13 | All connectors in Ferry can be customized and saved during runtime. However, sometimes it's nice to specify all these runtime dependencies *outside* the connector. This would let us share applications easily with others and help us organize our own applications. Fortunately for us, we can use special files called Dockerfiles to help us do this. Here's an example Dockerfile. 
 14 | 
 15 | .. code-block:: bash
 16 | 
 17 |    FROM ferry/cassandra-client
 18 |    NAME james/cassandra-examples
 19 |    RUN apt-get --yes install python-pip python-dev
 20 |    RUN pip install cassandra-driver
 21 |    RUN git clone https://github.com/opencore/cassandra-examples.git /home/ferry/cassandra-examples
 22 | 
 23 | Let's take a look line by line. The first states that 
 24 | 
 25 | .. code-block:: bash
 26 | 
 27 |    FROM ferry/cassandra-client
 28 | 
 29 | That just means that our client is based on the official Ferry ``cassandra`` client. This ensures that all the right drivers, etc. will be automatically installed. The ``cassandra`` client, in turn, is based on an Ubuntu 12.04 image, so you can use convenient tools like ``apt-get`` in the Dockerfile. The next line:
 30 | 
 31 | .. code-block:: bash
 32 | 
 33 |    NAME james/cassandra-examples
 34 | 
 35 | specifies the official name of this Dockerfile (for those that have created Dockerfiles before, you may recognize that the ``NAME`` command is not an officially supported Docker command). Now the next few lines tells Ferry what to install:
 36 | 
 37 | .. code-block:: bash
 38 | 
 39 |    RUN apt-get --yes install python-pip python-dev
 40 |    RUN pip install cassandra-driver
 41 | 
 42 | In our case, we just want to use the Python Cassandra driver. Finally, we're going to download a copy of some convenient Cassandra examples from GitHub. 
 43 | 
 44 | .. code-block:: bash
 45 | 
 46 |    RUN git clone https://github.com/opencore/cassandra-examples.git /home/ferry/cassandra-examples
 47 | 
 48 | Once you have the Dockerfile specified, you can use it in a Ferry application by specifying it's name in the ``connector`` section. Here's an example:
 49 | 
 50 | .. code-block:: yaml
 51 | 
 52 |    backend:
 53 |       - storage:
 54 |            personality: "cassandra"
 55 |            instances: 2
 56 |    connectors:
 57 |       - personality: "james/cassandra-examples"
 58 | 
 59 | Let's assume that your ``yaml`` file is called ``cassandra_examples.yml`` and stored in the directory ``~/my_app``. In order to start your new custom application, just type the following:
 60 | 
 61 | .. code-block:: bash
 62 | 
 63 |    $ ferry start ~/my_app/cassandra_examples.yml -b ~/my_app
 64 | 
 65 | The ``-b`` flag tells Ferry where to find your Dockerfile. Without that flag, it won't be able to compile the ``james/cassandra-examples`` image. After you type that command the first time, you can omit the the ``-b`` flag (since the compiled image will reside on your local Ferry repository). Once you log into your connector, you find the Cassandra example applications like this:
 66 | 
 67 | .. code-block:: bash
 68 | 
 69 |    $ su ferry
 70 |    $ ls /home/ferry/cassandra-examples
 71 |    sensorapp
 72 |    twissandra
 73 |    kairosdb
 74 | 
 75 | Creating a Dockerfile for your application is a convenient way to store and share your application. By providing the Dockerfile (along with any files that are included in the Dockerfile), any user can run the same application in Ferry. 
 76 | 
 77 | Port forwarding
 78 | ---------------
 79 | 
 80 | If your connector exposes a web service, you can find the IP address of your connector using the ``inspect`` command. This IP can then be used to access your connector so long as you're on the same host (this IP is not exposed to the outside world). However, if you wanted to expose this web service to the outside world, you can use *port forwarding*. This concept is very similar to the native Docker *port* feature. Simply add the ``ports`` argument to your YAML stack file like below:
 81 | 
 82 | .. code-block:: yaml
 83 | 
 84 |    backend:
 85 |       - storage:
 86 |            personality: "cassandra"
 87 |            instances: 2
 88 |    connectors:
 89 |       - personality: "james/cassandra-examples"
 90 | 	ports: ["7888:8000"]
 91 | 
 92 | Here we're specifying both the exposed port on the host (7888) and the internal port used by your web service (8000). If you use a single value ("8000"`), Ferry will simply choose a random port to expose on the host. You can find the exposed port value via the ``inspect`` command. For example, here's a shortened version of what you should see:
 93 | 
 94 | .. code-block:: bash
 95 | 
 96 |    {
 97 |      "connectors":[
 98 |        {
 99 |          "containers":[
100 |            {
101 |              "internal_ip":"10.1.0.5",
102 |              "ports":{
103 |                "8000":[
104 |                  {
105 |                    "HostIp":"0.0.0.0",
106 |                    "HostPort":"7888"
107 |                  }
108 |                ]
109 |              }
110 | 


--------------------------------------------------------------------------------
/docs/sources/examples/basics.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Basics
 2 | :description: Basic examples
 3 | :keywords: ferry, examples
 4 | 
 5 | .. _basics:
 6 | 
 7 | Quick start
 8 | ===========
 9 | 
10 | Let's get a basic Hadoop cluster up and running using Ferry. First you'll need to 
11 | install Docker. If you're running the latest version of Ubuntu, it's fairly straightforward. 
12 | 
13 | .. code-block:: bash
14 | 
15 |     $ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
16 |     $ sudo sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
17 |     $ sudo apt-get update
18 |     $ sudo apt-get install lxc-docker-0.8.1
19 | 
20 | *Please note that you'll need to install Docker version 0.8.1. This will install additional libraries that Ferry needs.*
21 | 
22 | You'll also want to create a new group called ``docker`` to simplify interacting with Docker and
23 | Ferry as a non-root user. There are more detailed instructions on the Docker_ homepage. 
24 | 
25 | .. _Docker: http://docs.docker.io/en/latest/installation/
26 | 
27 | Next you'll want to install Ferry. 
28 | 
29 | .. code-block:: bash
30 | 
31 |     $ sudo pip install -U ferry
32 | 
33 | Now you'll want to build the various Ferry images. Just type:
34 | 
35 | .. code-block:: bash
36 | 
37 |     $ sudo ferry install
38 | 
39 | This will take at least tens of minutes, so sit back and relax. After all the images are built, check by typing:
40 | 
41 | .. code-block:: bash
42 | 
43 |     $ sudo ferry server
44 |     $ ferry info
45 | 
46 | Congratulations! 
47 | 
48 | 
49 | You can examine the pre-installed Ferry applications by typing: 
50 | 
51 | .. code-block:: bash
52 | 
53 |     $ ferry ls
54 |        App           Author       Version           Description
55 |        ---           ------       -------           -----------
56 |     cassandra     James Horey      0.2.0         Cassandra stack...
57 |       hadoop      James Horey      0.2.0          Hadoop stack...
58 |      openmpi      James Horey      0.2.0      Open MPI over Gluster...
59 |       spark       James Horey      0.2.0        Spark over Hadoop...
60 |        yarn       James Horey      0.2.0      Hadoop YARN over Glus...
61 | 
62 | Say you're interested in the Hadoop application, just type the following to create a new Spark cluster: 
63 | 
64 | .. code-block:: bash
65 | 
66 |     $ ferry start hadoop
67 | 
68 | At this point, you probably want to head over to the :ref:`Hadoop <hadoop>` tutorial to understand how to interact with your new cluster. 
69 | 
70 | When you're all done writing your application, you can stop the Ferry servers by typing:
71 | 
72 | .. code-block:: bash
73 | 
74 |     $ sudo ferry quit
75 | 


--------------------------------------------------------------------------------
/docs/sources/examples/index.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Examples
 2 | :description: Examples running Ferry
 3 | :keywords: ferry, examples
 4 | 
 5 | .. _examples:
 6 | 
 7 | Getting Started
 8 | ===============
 9 | .. toctree::
10 |    :maxdepth: 1
11 | 
12 |    basics
13 |    advanced
14 |    pulling
15 |    cassandra
16 |    hadoop
17 |    mongodb
18 |    openmpi
19 |    spark
20 |    multiple
21 | 


--------------------------------------------------------------------------------
/docs/sources/examples/mongodb.rst:
--------------------------------------------------------------------------------
 1 | :title: MongoDB example
 2 | :description: A simple example using MongoDB
 3 | :keywords: ferry, example, mongodb
 4 | 
 5 | .. _mongodb:
 6 | 
 7 | Getting started with MongoDB
 8 | ============================
 9 | 
10 | MongoDB is a popular document store that makes it super simple store, retrieve, and analyze 
11 | JSON-based datasets. It is often used as a transactional store for web applications. Although
12 | it's not strictly a "big data" tool, it is an important element in many applications. 
13 | 
14 | To get started, let's define our stack in a file (let's call it ``mongo.yaml``). 
15 | The file should look something like this:
16 | 
17 | .. code-block:: yaml
18 | 
19 |    backend:
20 |       - storage:
21 |            personality: "mongodb"
22 |            instances: 1
23 |    connectors:
24 |       - personality: "ferry/mongodb-client"
25 | 
26 | Although MongoDB does support sharding for scalability, Ferry currently does not support this (as of version 0.2.2). 
27 | Consequently, you probably want just a single instance of MongoDB. 
28 | 
29 | Running an example
30 | ------------------
31 | 
32 | Once you have the application defined, go ahead and start it. Once it's started login to your client to see what
33 | environment variables have been populated by MongoDB. 
34 | 
35 | .. code-block:: bash
36 | 
37 |    $ env | grep BACKEND
38 |    BACKEND_STORAGE_TYPE=mongodb
39 |    BACKEND_STORAGE_MONGO=10.1.0.1
40 |    BACKEND_STORAGE_MONGO_PASS=eec68b55-9819-4461-bb55-f1494cfe364e
41 |    BACKEND_STORAGE_MONGO_USER=9b845bdf-657c-4402-ac45-c08a3c91e3d8
42 | 
43 | Note that MongoDB by default generates a random username and password for authentication. You'll need to pass those values
44 | into your client before accessing MongoDB. 
45 | 
46 | More resources
47 | --------------
48 | 
49 | As mentioned, MongoDB is often used in combination with another data store that's more focused on analytics, such as 
50 | Hadoop or GlusterFS. :ref:`Here's <multiple>` where to learn how to create a multi-storage application. Also, here
51 | are a few links for additional MongoDB documentation.  
52 | 
53 | - `MongoDB <http://www.mongodb.org/>`_
54 | 
55 | 


--------------------------------------------------------------------------------
/docs/sources/examples/multiple.rst:
--------------------------------------------------------------------------------
 1 | :title: Multiple storage and compute
 2 | :description: How to push and pull new applications
 3 | :keywords: ferry, example 
 4 | 
 5 | .. _multiple:
 6 | 
 7 | Multiple Storage and Compute
 8 | ============================
 9 | 
10 | Although many application consist of a single storage layer, you can actually combine
11 | multiple storage layers. This lets you construct complex applications that uses the
12 | storage in different ways. For example, here we're using both Hadoop (HDFS) and
13 | MongoDB, a popular document store.  
14 | 
15 | .. code-block:: yaml
16 | 
17 |     backend:
18 |        - storage:
19 |             personality: "hadoop"
20 |             instances: ">=1"
21 |             layers:
22 |                - "hive"
23 |        - storage:
24 |             personality: "mongodb"
25 |             instances: 1
26 |     connectors:
27 |        - personality: "ferry/hadoop-client"
28 | 
29 | Hadoop is great at storing the archival data for analytic purposes, while MongoDB
30 | may be used for transactional data. Once you're logged into the client, you'll find
31 | environment variables that refer to both sets of storage. 
32 | 
33 | *One word of warning: the bult-in client for Hadoop doesn't have MongoDB drivers installed, so you'll want to install that before anything else*. 
34 | 
35 | You can also combine multiple compute layers. For example, we might create a *data science* application that uses both Hadoop/YARN and OpenMPI. 
36 | 
37 | .. code-block:: yaml
38 | 
39 |     backend:
40 |        - storage:
41 |             personality: "gluster"
42 |             instances: 1
43 |          compute:
44 |             - personality: "yarn"
45 |               instances: 1
46 |               layers: 
47 |                  - "hive"
48 |             - personality: "openmpi"
49 |               instances: 1
50 |     connectors:
51 |        - personality: "ferry-user/data-science"
52 | 
53 | In this example, we're supplying our own custom client that has both YARN and Open MPI drivers installed. 
54 | 


--------------------------------------------------------------------------------
/docs/sources/examples/pulling.rst:
--------------------------------------------------------------------------------
 1 | :title: Pulling new applications
 2 | :description: How to push and pull new applications
 3 | :keywords: ferry, example 
 4 | 
 5 | .. _pulling:
 6 | 
 7 | Pushing and Pulling Applications
 8 | ================================
 9 | 
10 | Ferry lets you easily share your application with others. 
11 | 
12 | Once you're done developing your app, you can upload your application to the Ferry servers to share it with others. 
13 | For now, you'll need an account on Docker.io to upload your images and another on Ferry to store your application stack. Once
14 | you have these accounts, you'll need to create an *authorization* file. 
15 | 
16 | .. code-block:: bash
17 | 
18 |     $ cat ~/.ferry.docker
19 |     docker:
20 |          user: <DOCKER LOGIN>
21 |          password: <DOCKER PASSWORD>
22 |          email: <DOCKER EMAIL>
23 |          server: https://index.docker.io/v1/
24 |     ferry:
25 |          user: <FERRY LOGIN>
26 |          key: <FERRY KEY>
27 |          server: http://apps.opencore.io
28 |          registry: docker
29 | 
30 | Note that currently all Ferry applications are *public*. It doesn't necessarily mean that all your customizations are publicly accessible, 
31 | just that anybody can download your application. Private applications are something that we plan on supporting in the near future. 
32 | 
33 | To upload the application, just perform a *push* command and point it to your application stack. 
34 | 
35 | .. code-block:: bash
36 | 
37 |     $ ferry push app:///home/ferry/mortar.yml
38 |     opencore/mortar
39 | 
40 | The final name of the application is your Ferry user name appended with the name of the application file. 
41 | 
42 | Pulling Applications
43 | --------------------
44 | 
45 | To download an application, you'll want to perform a *pull* command. Ferry lets you download Docker images manually or an entire Ferry application. Here
46 | we're going to pull the *Mortar Recommendation System* application. 
47 | 
48 | .. code-block:: bash
49 | 
50 |     $ sudo ferry pull app://opencore/mortar
51 |     $ ferry inspect opencore/mortar
52 |     backend:
53 |        - storage:
54 |             personality: "hadoop"
55 |             instances: ">=1"
56 |             layers:
57 |                - "hive"
58 |     connectors:
59 |        - personality: "ferry/mortar-recsys"
60 | 
61 | This will pull all the Docker images the application needs and will register the
62 | application on your local machine. If you just want to download an *image*, just type:
63 | 
64 | .. code-block:: bash
65 | 
66 |     $ sudo ferry pull image://opencore/mortar-recsys
67 | 
68 | Once you downloaded the application, you can start it like
69 | any other application:
70 | 
71 | .. code-block:: bash
72 | 
73 |     $ ferry start opencore/mortar
74 | 


--------------------------------------------------------------------------------
/docs/sources/examples/spark.rst:
--------------------------------------------------------------------------------
  1 | :title: Spark example
  2 | :description: A simple example using Spark
  3 | :keywords: ferry, example, spark
  4 | 
  5 | .. _spark:
  6 | 
  7 | Getting started with Spark
  8 | ==========================
  9 | 
 10 | Apache Spark is a new parallel, in-memory processing framework from U.C. Berkeley's AMPLab. Spark has two main advantages relative to other similar frameworks. First, because of its in-memory design, Spark is able to run certain computations much faster, for example machine learning algorithms. Second, Spark has a very clean API. Spark is written in Scala, a functional language for the JVM but also has strong support for Python. Having said this, Spark is still compatible with Hadoop and can easily read/write data from HDFS. 
 11 | 		
 12 | The first thing to do is to define our application stack. Let's call our new application stack ``spark.yaml``. 
 13 | The file should look something like this:
 14 | 
 15 | .. code-block:: yaml
 16 | 
 17 |    backend:
 18 |       - storage:
 19 |            personality: "hadoop"
 20 |            instances: 2
 21 |         compute:
 22 |            - personality: "spark"
 23 |              instances: 2
 24 |    connectors:
 25 |       - personality: "ferry/spark-client"
 26 | 
 27 | Note that Spark relies on Hadoop for the actual data storage. Ferry runs Spark in "stand-alone" mode (deeper YARN integration will come in the future). So to create the actual Spark cluster, we'll need to specify the ``compute`` section. Finally, we'll need at least one ``spark-client`` so that we can launch jobs and interact with the rest of the cluster. 
 28 | 
 29 | Running an example
 30 | ------------------
 31 | 
 32 | Now that we've defined our stack, let's start it up. Don't forget that you need the Ferry server to be up and running (via ``sudo ferry server``). Afterwards type ``ferry start spark`` into your terminal. ``spark`` should be replaced with the path to your specific file. Otherwise it will use a default Spark
 33 | stack. The entire process should take less than a minute. 
 34 | 
 35 | Before we continue, let's take a step back to examine what just happened. After typing ``start``, ``ferry`` created the following Docker
 36 | containers:
 37 | 
 38 | - Two Hadoop data nodes
 39 | - Hadoop namenode
 40 | - Hadoop YARN resource manager
 41 | - Two Spark compute nodes
 42 | - A Linux client
 43 | 
 44 | Now that the environment is created, let's interact with it by connecting to the Linux client. 
 45 | Just type ``ferry ssh sa-0`` (where ``sa-0`` is replaced with your application ID). Once you're logged in, you should be able to run all the examples. Remember, the connector is just a Docker container. That means you can completely customize the environment including installing packages and even modify configuration files. 
 46 | 
 47 | Python Examples
 48 | ---------------
 49 | 
 50 | Now we should be able to run Spark jobs. If you're really impatient, you can run some Python examples by typing:
 51 | 
 52 | .. code-block:: bash
 53 | 
 54 |    $ /service/runscripts/test/test01.sh load
 55 |    $ /service/runscripts/test/test01.sh python regression.py
 56 | 
 57 | This downloads some data and runs a *linear regression* example over that data. You can check out more examples in the directory */service/examples/python*. Here's a Python example showing how to perform *collaborative filtering* (a popular method for recommendations). 
 58 | 
 59 | .. code-block:: python
 60 | 
 61 |    import sys
 62 |    from pyspark import SparkContext
 63 |    from pyspark.mllib.recommendation import ALS
 64 |    from numpy import array
 65 |    
 66 |    if __name__ == "__main__":
 67 |        data_file = '/spark/data/als.data'
 68 |    
 69 |        sc = SparkContext(sys.argv[1], "Collaborative Filtering")
 70 |        data = sc.textFile(data_file)
 71 |        ratings = data.map(lambda line: array([float(x) for x in line.split(',')]))
 72 |    
 73 |        # Build the recommendation model using Alternating Least Squares
 74 |        model = ALS.train(ratings, 1, 20)
 75 |    
 76 |        # Evaluate the model on training data
 77 |        testdata = ratings.map(lambda p: (int(p[0]), int(p[1])))
 78 |        predictions = model.predictAll(testdata).map(lambda r: ((r[0], r[1]), r[2]))
 79 |        ratesAndPreds = ratings.map(lambda r: ((r[0], r[1]), r[2])).join(predictions)
 80 |        MSE = ratesAndPreds.map(lambda r: (r[1][0] - r[1][1])**2).reduce(lambda x, y: x + y)/ratesAndPreds.count()
 81 |        print("Mean Squared Error = " + str(MSE))
 82 | 
 83 | As you can see the source is fairly short for what it does. Spark includes the *MLLib* machine-learning library which simplifies creating advanced data mining algorithms. If you specified more than a single node for your Spark cluster, this example will run (virtually) in parallel. 
 84 | 
 85 | If you want to run your own Python application, just type the following (as the ``ferry`` user):
 86 | 
 87 | .. code-block:: bash
 88 | 
 89 |    $ $SPARK_HOME/bin/pyspark my_spark_app.py spark://$BACKEND_COMPUTE_MASTER:7077
 90 | 
 91 | 
 92 | More resources
 93 | --------------
 94 | 
 95 | Once you're done running the built-in examples, check out these additional resources to learn more. 
 96 | 
 97 | - `Apache Spark <http://spark.apache.org/>`_
 98 | - `U.C. Berkeley Amp Lab <https://amplab.cs.berkeley.edu/>`_
 99 | 
100 | 


--------------------------------------------------------------------------------
/docs/sources/index.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Introduction
 2 | :description: Introduction
 3 | :keywords: ferry, introduction
 4 | 
 5 | .. _intro:
 6 | 
 7 | Big Data Development Environment using Docker
 8 | =============================================
 9 | 
10 | Ferry helps you create big data clusters on your local machine. Define your big data stack using YAML and
11 | share your application with :ref:`Dockerfiles <advanced>`. Ferry supports Hadoop, Cassandra, Spark, GlusterFS, and Open MPI. 
12 | 
13 | Here's an example Hadoop cluster:
14 | 
15 | .. code-block:: yaml
16 | 
17 |    backend:
18 |       - storage:
19 |            personality: "hadoop"
20 |            instances: 2
21 |            layers:
22 |               - "hive"
23 |    connectors:
24 |       - personality: "hadoop-client"
25 | 
26 | Then get started by typing ``ferry start hadoop``. This will automatically create a two node
27 | Hadoop cluster and a single Linux client. You can customize the Linux client during runtime or define your own using a Dockerfile. In addition to Hadoop, Ferry also supports Spark, Cassandra, GlusterFS, and Open MPI. 
28 | 
29 | Ferry is useful for:
30 | 
31 | - Data scientists that want to experiment and learn about big data technologies
32 | - Developers that need a locally accessible big data development environment
33 | - Users that want to share big data application quickly and safely
34 | 
35 | Ferry provides several useful commands for your applications: 
36 | 
37 | - Start and stop services
38 | - View status and create snapshots
39 | - SSH into clients
40 | - Copy over log files to a host directory
41 | 
42 | For example, let's inspect all the running services
43 | 
44 | .. code-block:: bash
45 | 
46 |    $ ferry ps
47 |    UUID Storage  Compute  Connectors  Status   Base  Time
48 |    ---- ------- --------- ---------- ------- ------- ----
49 |    sa-2    se-6 [u'se-7']       se-8 removed hadoop    --
50 |    sa-1    se-3 [u'se-4']       se-5 stopped openmpi   --
51 |    sa-0    se-0 [u'se-1']       se-2 running cassandra --
52 | 
53 | Ferry is under active development, so follow us on `Twitter <https://twitter.com/open_core_io/>`_ to keep up to date. 
54 | 
55 | If you're interested in collaborating or have any questions, feel free to send an email to info@opencore.io.
56 | 


--------------------------------------------------------------------------------
/docs/sources/install/index.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Installation
 2 | :description: Ferry installation instructions
 3 | :keywords: ferry, installation
 4 | 
 5 | .. _install:
 6 | 
 7 | Installation
 8 | ============
 9 | .. toctree::
10 |    :maxdepth: 1
11 | 
12 |    client
13 |    openstack
14 |    osclient
15 |    hpcloud
16 |    rackspace
17 |    aws
18 | 


--------------------------------------------------------------------------------
/docs/sources/install/openstack.rst:
--------------------------------------------------------------------------------
  1 | :title: Ferry OpenStack Installation
  2 | :description: Installing Ferry on your OpenStack cluster
  3 | :keywords: ferry, installation, openstack
  4 | 
  5 | .. _openstack:
  6 | 
  7 | OpenStack Server Installation
  8 | =============================
  9 | 
 10 | This documentation is meant for system adminstrators and data engineers that are interested 
 11 | in installing Ferry in their own OpenStack private cloud. If you're an end-user, you probably
 12 | want to read :ref:`this <hpcloud>`. 
 13 | 
 14 | Installing Ferry on your OpenStack cluster is relatively straightfoward, and simply requires
 15 | creating a "Ferry Server" image. 
 16 | 
 17 | Quick Installation 
 18 | -------------------
 19 | 
 20 | 1. Instantiate an Ubuntu 14.04 image 
 21 | 2. Create a `/ferry/master` directory and export "FERRY_DIR=/ferry/master"
 22 | 3. Install Ferry and all Ferry images
 23 | 4. Save the image as "Ferry Server (small)"
 24 | 
 25 | Ubuntu 14.04
 26 | ------------
 27 | 
 28 | The Ferry server must be based on Ubuntu 14.04. If your OpenStack cluster 
 29 | does not have an Ubuntu 14.04 image already, you can download an image from
 30 | the Ubuntu website:
 31 | 
 32 | - `Ubuntu 14.04 Cloud Image <https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img/>`_ 
 33 | 
 34 | Afterwards you can import the image into Glance by typing: 
 35 | Once you have an Ubuntu 14.04 instance running, 
 36 | 
 37 | .. code-block:: bash
 38 | 
 39 |     $ glance image-create --name "Ubuntu 14.04 (amd64)" --disk-format=raw --container-format=bare --file=./trusty-server-cloudimg-amd64-disk1.img
 40 | 
 41 | After importing the image, boot up a new Ubuntu instance. You'll want to use an instance type with at least a 10GB root directory, since
 42 | we'll be installing all the Ferry images. 
 43 | 
 44 | Installing Ferry
 45 | ----------------
 46 | 
 47 | After instantiating the Ubuntu instance, you'll need to install Ferry. The easiest way is to use our automated installation script:
 48 | 
 49 | .. code-block:: bash
 50 | 
 51 |     $ git clone https://github.com/opencore/ferry.git
 52 |     $ sudo ferry/install/ferry-install
 53 | 
 54 | However you can also install Ferry manually.
 55 | 
 56 | - Detailed Ferry :ref:`instructions <client>`
 57 | 
 58 | One point of note: you'll need to first create and set an alternative Ferry data directory. 
 59 | 
 60 | .. code-block:: bash
 61 | 
 62 |     $ mkdir -p /ferry/master && export FERRY_DIR=/ferry/master
 63 | 
 64 | The installation process can take quite a while (it will download approximately
 65 | 5GB of Docker images). 
 66 | 
 67 | To verify if Ferry has been installed, you can just type:
 68 | 
 69 | .. code-block:: bash
 70 | 
 71 |     $ sudo ferry server
 72 |     ...
 73 |     using backend local
 74 |     $ sudo ferry ps
 75 |     UUID        Storage     Compute     Connectors      Status         Base       Time
 76 |     ----        -------     -------     ----------      ------         ----       ----
 77 | 
 78 | OpenStack Heat
 79 | --------------
 80 | 
 81 | Ferry uses the Heat orchestration engine to launch clusters on OpenStack. If your
 82 | OpenStack cluster already has Heat installed, then you can skip this step. Otherwise, Ferry
 83 | will use its own "stand-alone" Heat server. To use the stand-alone Heat server, you'll need 
 84 | to download the Ferry Heat image. 
 85 | 
 86 | .. code-block:: bash
 87 | 
 88 |     $ sudo ferry server
 89 |     $ sudo ferry pull image://ferry/heatserver
 90 | 
 91 | Save Image
 92 | ----------
 93 | 
 94 | Now that you have Ferry installed, go ahead and stop Ferry.  
 95 | 
 96 | .. code-block:: bash
 97 | 
 98 |     $ sudo ferry quit
 99 | 
100 | Afterwards, create a *snapshot* of the instance. You can name the
101 | snapshot whatever you want, but users will need this name later when configuring the client. 
102 | Something like "Ferry Server" should do. 
103 | 
104 | Next Steps
105 | ----------
106 | 
107 | Once the Ferry image is created, users should be able to start using Ferry to create 
108 | big data clusters. The "Ferry Server" image can be used either as a client or server. 
109 | 
110 | - Configuring the client :ref:`instructions <osclient>`
111 | - HP Cloud client :ref:`instructions <hpcloud>`
112 | 
113 | 


--------------------------------------------------------------------------------
/docs/sources/install/rackspace.rst:
--------------------------------------------------------------------------------
1 | :title: Ferry on Rackspace
2 | :description: Running Ferry on Rackspace
3 | :keywords: ferry, installation, hpcloud
4 | 
5 | Rackspace
6 | =========
7 | 
8 | Coming soon!
9 | 


--------------------------------------------------------------------------------
/docs/sources/references/index.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Reference
 2 | :description: Reference Ferry commands
 3 | :keywords: ferry, reference
 4 | 
 5 | .. _reference:
 6 | 
 7 | Reference
 8 | =========
 9 | 
10 | .. toctree::
11 |    :maxdepth: 1
12 | 
13 |    commands
14 | 
15 | 


--------------------------------------------------------------------------------
/docs/sources/toctree.rst:
--------------------------------------------------------------------------------
 1 | :title: Documentation
 2 | :description: Ferry documentation
 3 | :keywords: ferry, documentation
 4 | 
 5 | Documentation
 6 | =============
 7 | 
 8 | This documentation has the following resources:
 9 | 
10 | .. toctree::
11 |    :maxdepth: 1
12 |    
13 |    Home <index>
14 |    install/index
15 |    examples/index
16 |    references/index
17 | 


--------------------------------------------------------------------------------
/docs/sources/upgrade.rst:
--------------------------------------------------------------------------------
 1 | :title: Ferry Upgrade Installation
 2 | :description: Upgrading Ferry on your local machine
 3 | :keywords: ferry, installation, upgrade
 4 | 
 5 | .. _upgrade:
 6 | 
 7 | Updating Ferry
 8 | ==============
 9 | 
10 | We are trying to make upgrading Ferry a breeze. Assuming that you already have a 
11 | working Ferry installation, the first thing you'll want to do is stop any
12 | running Ferry instances:
13 | 
14 | .. code-block:: bash
15 | 
16 |     $ sudo ferry quit
17 | 
18 | Now let's upgrade the Python packages.
19 | 
20 | .. code-block:: bash
21 | 
22 |     $ sudo pip install -U ferry
23 | 
24 | Finally, we'll need to update the Ferry images. 
25 | 
26 | .. code-block:: bash
27 | 
28 |     $ sudo ferry install -u
29 | 
30 | We're constantly trying to improve the process, so if anything goes wrong, don't
31 | hesitate to file an issue on `GitHub <https://github.com/opencore/ferry/>`_ or 
32 | leave a message on the `Ferry Google Group <https://groups.google.com/d/forum/ferry-user/>`_. 
33 | 


--------------------------------------------------------------------------------
/docs/theme/ferry/layout.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <head>
  3 |     <meta charset="utf-8">
  4 |     <meta http-equiv="X-UA-Compatible" content="IE=edge" />
  5 |     <title>Ferry | Big Data Development Environment Using Docker</title>
  6 |     <meta name="description" content="Ferry helps developers deploy big data applications
  7 | 				      using Docker. Ferry supports Spark, Hadoop, Cassandra, GlusterFS, and Open MPI" />
  8 |     <meta name="keywords" content="ferry, docker, spark, hadoop, cassandra, glusterfs, openmpi, big data" />
  9 |     <meta name="google-site-verification" content="oWhdaCFYAlFrFuw5oaBQTTuPSRQtXwMO1WSCtoY6T3A" />
 10 | 
 11 |     {%- set url_root = pathto('', 1) %}
 12 |     {%- if url_root == '#' %}{% set url_root = '' %}{% endif %}
 13 | 
 14 |     <script type="text/javascript">
 15 |         // This is included here for Javascript that doesn't have access to the templates.
 16 |         var doc_version = "{{ current_version }}";
 17 |         var doc_slug = "{{ slug }}";
 18 |     </script>
 19 | 
 20 |     {%- set css_files = css_files + ['_static/css/bootstrap.css'] %}
 21 |     {%- set css_files = css_files + ['_static/pygments.css'] %}
 22 |     {%- set css_files = css_files + ['_static/css/main.css'] %}
 23 | 
 24 |     {%- set script_files =
 25 |     ['//code.jquery.com/jquery-1.10.1.min.js']
 26 |     + ['//fonts.googleapis.com/css?family=Cabin:400,700,400italic']
 27 |     %}
 28 | 
 29 |     {%- for cssfile in css_files %}
 30 |     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
 31 |     {%- endfor %}
 32 | 
 33 |     {%- for scriptfile in script_files if scriptfile != '_static/jquery.js' %}
 34 |     <script type="text/javascript" src="{{ pathto(scriptfile, 1) }}"></script>
 35 |     {%- endfor %}
 36 |     <link rel="shortcut icon" href="{{ pathto('_static/favicon.png', 1) }}"/>
 37 |     {%- block extrahead %}{% endblock %}
 38 | 
 39 |     <!-- Google analytics -->
 40 |     <script type="text/javascript" >
 41 |       (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
 42 |       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
 43 |       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
 44 |       })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
 45 |       ga('create', 'UA-48786933-1', 'opencore.io');
 46 |       ga('require', 'displayfeatures');
 47 |       ga('send', 'pageview');
 48 |     </script>
 49 | </head>
 50 | 
 51 | <body>
 52 | 
 53 | <div class='container-fluid navbar-fixed-top header'>
 54 |   <div class='row-fluid'>
 55 |     <div class='span1 toplevel'></div>
 56 |     <div class='span10 toplevel'>
 57 |       <div class='row-fluid'>
 58 | 	<div class="span4">
 59 |           <a href="http://www.opencore.io">
 60 |             <img src="{{ pathto('_static/img/frontpage_logo.png', 1) }}"/>
 61 | 	  </a>
 62 | 	</div>
 63 | 	<div class="span2"></div>
 64 | 	<div class='span6 toplevel-content'>
 65 |           <div class='pull-right'>
 66 |             <ul class="nav nav-pills nav-colors">
 67 |               <li class='link'> 
 68 | 		<a href='http://ferry.opencore.io'>Ferry</a>
 69 | 	      </li>
 70 |               <li class='link'>
 71 | 		<a href='http://blog.opencore.io'>Blog</a>
 72 | 	      </li>
 73 |               <li class='link'> 
 74 | 		<a href='http://www.opencore.io/signup'>Sign Up</a>
 75 | 	      </li>
 76 | 	    </ul>
 77 | 	  </div>
 78 | 	</div>
 79 |       </div>
 80 |     </div>
 81 |     <div class='span1 toplevel'></div>
 82 |   </div>
 83 | </div>
 84 | 
 85 | <div id="wrap">
 86 | 
 87 | <div class="container-fluid">
 88 |     <div class="box">
 89 |       <div class="row-fluid">
 90 |         <div class="column span3 nav-column">
 91 |           <div class="container-fluid">
 92 | 	    <div class="row-fluid nav-title">
 93 | 	       Ferry
 94 | 	    </div>
 95 | 
 96 | 	    <div class="row-fluid">
 97 |               <div class="sidebar bs-docs-sidebar">
 98 |                 {{ toctree(collapse=False, maxdepth=2) }}
 99 |               </div>
100 |               <div class="footer-sidebar">
101 | 		<ul>
102 | 		  <li class="github">
103 | 		    <a href="https://github.com/opencore/ferry">Follow on GitHub</a>
104 | 		  </li>
105 | 		  <li class="twitter">
106 | 		    <a href="https://twitter.com/open_core_io">Follow on Twitter</a>
107 | 		  </li>
108 | 		  <li class="google">
109 | 		    <a href="https://groups.google.com/d/forum/ferry-user">Discuss in Google Groups</a>
110 | 		  </li>
111 | 		</ul>
112 | 
113 | 		<div class="badges">
114 | 		  <iframe src="http://ghbtns.com/github-btn.html?user=opencore&repo=ferry&type=watch&count=true"
115 | 			  allowtransparency="true" frameborder="0" scrolling="0" width="110" height="20">
116 | 		  </iframe>
117 | 		</div>
118 | 
119 |               </div>
120 | 	    </div>
121 |           </div>
122 | 	</div>
123 | 
124 | 	<div class="column span9">
125 |           <div class="main-content">
126 |             <section id="global" class="containerblock">
127 |               {% block body %}{% endblock %}
128 |             </section>
129 |           </div>
130 | 	</div>
131 |       </div>
132 |     </div>
133 | </div>
134 | 
135 | <script type="text/javascript" src="{{ pathto('_static/js/docs.js', 1) }}"></script>
136 | 
137 | </body>
138 | </html>
139 | 


--------------------------------------------------------------------------------
/docs/theme/ferry/redirect_build.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |     <title>Page Moved</title>
 5 |     <meta http-equiv="refresh" content="0; url=http://docs.docker.io/en/latest/use/builder/">
 6 | </head>
 7 | <body>
 8 | 
 9 | This page has moved. Perhaps you should visit the <a href="http://docs.docker.io/en/latest/use/builder/" title="builder page">Builder page</a>
10 | 
11 | </body>
12 | </html>
13 | 


--------------------------------------------------------------------------------
/docs/theme/ferry/redirect_home.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |     <title>Page Moved</title>
 5 |     <meta http-equiv="refresh" content="0; url=http://docs.docker.io/en/latest/">
 6 | </head>
 7 | <body>
 8 | 
 9 | This page has moved. Perhaps you should visit the <a href="http://docs.docker.io/" title="documentation homepage">Documentation Homepage</a>
10 | 
11 | </body>
12 | </html>
13 | 


--------------------------------------------------------------------------------
/docs/theme/ferry/static/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/favicon.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/frontpage_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/frontpage_logo.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/github.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/github.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/glyphicons-halflings-white.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/glyphicons-halflings-white.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/glyphicons-halflings.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/glyphicons-halflings.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/google.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/google.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/menu_arrow_down.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/menu_arrow_down.gif


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/menu_arrow_right.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/menu_arrow_right.gif


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/opencore.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/opencore.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/img/twitter.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/docs/theme/ferry/static/img/twitter.png


--------------------------------------------------------------------------------
/docs/theme/ferry/static/js/docs.js:
--------------------------------------------------------------------------------
 1 | 
 2 | // This script should be included at the END of the document. 
 3 | // For the fastest loading it does not inlude $(document).ready()
 4 | 
 5 | 
 6 | $(function(){
 7 | 
 8 |     // sidebar accordian-ing
 9 |     // don't apply on last object (it should be the FAQ) or the first (it should be introduction)
10 | 
11 |     // define an array to which all opened items should be added
12 |     var openmenus = [];
13 | 
14 |     var elements = $('.toctree-l1');
15 |     for (var i = 0; i < elements.length; i += 1) {
16 |         var current = $(elements[i]);
17 | 
18 |         if (current.hasClass('current')) {
19 |             current.addClass('open');
20 |             currentlink = current.children('a')[0].href;
21 |             openmenus.push(currentlink);
22 | 
23 |             // do nothing
24 |         } else {
25 |             // collapse children
26 |             current.children('ul').hide();
27 |         }
28 |     }
29 | 
30 |     // attached handler on click
31 |     // We're removing the first two and last elements. A better way
32 |     // would be to filter out elements that don't have any children.
33 |     // However that requires someone that actually understands Javascript. 
34 |     $('.sidebar > ul > li > a').not(':last').not(':first').not(':first').click(function(){
35 |         var index = $.inArray(this.href, openmenus)
36 |         if (index > -1) {
37 |             console.log(index);
38 |             openmenus.splice(index, 1);
39 | 
40 |             $(this).parent().children('ul').slideUp(200, function() {
41 |                 $(this).parent().removeClass('open'); // toggle after effect
42 |             });
43 |         }
44 |         else {
45 |             openmenus.push(this.href);
46 | 
47 |             var current = $(this);
48 | 
49 |             setTimeout(function() {
50 |                 // $('.sidebar > ul > li').removeClass('current');
51 |                 current.parent().addClass('current').addClass('open'); // toggle before effect
52 |                 current.parent().children('ul').hide();
53 |                 current.parent().children('ul').slideDown(200);
54 |             }, 100);
55 |         }
56 |         return false;
57 |     });
58 | 
59 |     // add class to all those which have children
60 |     $('.sidebar > ul > li').not(':last').not(':first').not(':first').addClass('has-children');
61 | });


--------------------------------------------------------------------------------
/docs/theme/ferry/static/sitemap/sitemap.txt:
--------------------------------------------------------------------------------
 1 | http://ferry.opencore.io/en/latest/index.html
 2 | http://ferry.opencore.io/en/latest/install.html
 3 | http://ferry.opencore.io/en/latest/examples/basics.html
 4 | http://ferry.opencore.io/en/latest/examples/advanced.html
 5 | http://ferry.opencore.io/en/latest/examples/cassandra.html
 6 | http://ferry.opencore.io/en/latest/examples/hadoop.html
 7 | http://ferry.opencore.io/en/latest/examples/openmpi.html
 8 | http://ferry.opencore.io/en/latest/references/index.html
 9 | http://ferry.opencore.io/en/latest/references/commands.html
10 | 


--------------------------------------------------------------------------------
/docs/theme/ferry/theme.conf:
--------------------------------------------------------------------------------
 1 | [theme]
 2 | inherit = basic
 3 | pygments_style = monokai
 4 | 
 5 | [options]
 6 | full_logo         = true
 7 | textcolor         = #444444
 8 | headingcolor      = #0c3762
 9 | linkcolor         = #8C7B65
10 | visitedlinkcolor  = #AFA599
11 | hoverlinkcolor    = #4e4334
12 | 


--------------------------------------------------------------------------------
/ferry-dockerfile/50-ferryheader:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | #
 3 | #    50-ferryheader - create the Ferry MOTD header
 4 | #    Copyright (C) 2014 OpenCore LLC
 5 | #
 6 | #    Authors: James Horey <jlh@opencore.io>
 7 | #    Licensed under the Apache License, Version 2.0 (the "sLicense");
 8 | #    you may not use this file except in compliance with the License.
 9 | #    You may obtain a copy of the License at
10 | #
11 | #        http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | #    Unless required by applicable law or agreed to in writing, software
14 | #    distributed under the License is distributed on an "AS IS" BASIS,
15 | #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | #    See the License for the specific language governing permissions and
17 | #    limitations under the License.
18 | 
19 | echo ''
20 | echo '   _       __     __                             __      '
21 | echo '  | |     / /__  / /________  ____ ___  ___     / /_____ '
22 | echo '  | | /| / / _ \/ / ___/ __ \/ __ `__ \/ _ \   / __/ __ \'
23 | echo '  | |/ |/ /  __/ / /__/ /_/ / / / / / /  __/  / /_/ /_/ /'
24 | echo '  |__/|__/\___/_/\___/\____/_/ /_/ /_/\___/   \__/\____/ '
25 | echo '    ______                     '
26 | echo '   / ____/__  ____________  __ '
27 | echo '  / /_  / _ \/ ___/ ___/ / / / '
28 | echo ' / __/ /  __/ /  / /  / /_/ /  '
29 | echo '/_/    \___/_/  /_/   \__, /   '
30 | echo '                     /____/    '
31 | echo ''
32 | 


--------------------------------------------------------------------------------
/ferry-dockerfile/51-ferryhelp:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | #
 3 | #    50-ferryheader - create the Ferry MOTD header
 4 | #    Copyright (C) 2014 OpenCore LLC
 5 | #
 6 | #    Authors: James Horey <jlh@opencore.io>
 7 | #    Licensed under the Apache License, Version 2.0 (the "License");
 8 | #    you may not use this file except in compliance with the License.
 9 | #    You may obtain a copy of the License at
10 | #
11 | #        http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | #    Unless required by applicable law or agreed to in writing, software
14 | #    distributed under the License is distributed on an "AS IS" BASIS,
15 | #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | #    See the License for the specific language governing permissions and
17 | #    limitations under the License.
18 | 
19 | echo 'Start the Ferry server by typing:'
20 | echo 'sudo ferry server'
21 | echo 'Then start using the Ferry client. Try typing:'
22 | echo 'ferry help'
23 | echo 'For more documentation, visit http://ferry.opencore.io'
24 | 


--------------------------------------------------------------------------------
/ferry-dockerfile/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM ubuntu:14.04
 2 | 
 3 | # Install Docker
 4 | RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
 5 | RUN sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
 6 | 
 7 | RUN apt-get update
 8 | RUN DEBIAN_FRONTEND=noninteractive apt-get --yes upgrade
 9 | RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install lxc lxc-docker iptables ca-certificates cgroup-lite aufs-tools build-essential python-dev python-pip python-babel openssh-server
10 | 
11 | # Create docker group & ferry user
12 | # RUN groupadd docker
13 | RUN adduser --disabled-password --gecos "" ferry;usermod -a -G docker ferry
14 | 
15 | RUN mkdir -p /service/sbin /var/run/sshd
16 | 
17 | # Install Ferry from GitHub master branch
18 | WORKDIR /home/ferry
19 | RUN pip install ferry
20 | 
21 | # Add the dind/start script
22 | ADD ./make.sh /service/sbin/
23 | RUN chmod +x /service/sbin/*
24 | 
25 | # Add customized ssh banner 
26 | RUN rm /etc/update-motd.d/*
27 | RUN rm /var/run/motd.dynamic
28 | ADD ./50-ferryheader /etc/update-motd.d/
29 | ADD ./51-ferryhelp /etc/update-motd.d/
30 | RUN chmod +x /etc/update-motd.d/50-ferryheader
31 | RUN chmod +x /etc/update-motd.d/51-ferryhelp
32 | RUN /etc/update-motd.d/50-ferryheader >> /var/run/motd.dynamic
33 | RUN /etc/update-motd.d/51-ferryhelp >> /var/run/motd.dynamic


--------------------------------------------------------------------------------
/ferry-dockerfile/make.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # Bash colors
  4 | RED='\e[0;31m'
  5 | GREEN='\e[0;32m'
  6 | ORANGE='\e[0;33m'
  7 | BLUE='\e[0;34m'
  8 | NC='\e[0m'
  9 | 
 10 | VERSION='0.3.3.4'
 11 | 
 12 | # Prepare the docker environment for docker-in-docker. This code is from
 13 | # jpetazzo/dind
 14 | function prepare_dind {
 15 |     # First, make sure that cgroups are mounted correctly.
 16 |     CGROUP=/sys/fs/cgroup
 17 | 
 18 |     [ -d $CGROUP ] || 
 19 |             mkdir $CGROUP
 20 | 
 21 |     mountpoint -q $CGROUP || 
 22 |             mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
 23 | 		    echo -e "${RED}Could not make a tmpfs mount. Did you use -privileged?${NC}"
 24 | 		    exit 1
 25 | 	    }
 26 | 
 27 |     if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
 28 |     then
 29 | 	mount -t securityfs none /sys/kernel/security || {
 30 |             echo -e "${RED}Could not mount /sys/kernel/security.${NC}"
 31 |             echo -e "${RED}AppArmor detection and -privileged mode might break.${NC}"
 32 | 	}
 33 |     fi
 34 | 
 35 |     # Mount the cgroup hierarchies exactly as they are in the parent system.
 36 |     for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
 37 |     do
 38 | 	    [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
 39 |             mountpoint -q $CGROUP/$SUBSYS || 
 40 |                     mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
 41 | 
 42 |             # The two following sections address a bug which manifests itself
 43 |             # by a cryptic "lxc-start: no ns_cgroup option specified" when
 44 |             # trying to start containers withina container.
 45 |             # The bug seems to appear when the cgroup hierarchies are not
 46 |             # mounted on the exact same directories in the host, and in the
 47 |             # container.
 48 | 
 49 |             # Named, control-less cgroups are mounted with "-o name=foo"
 50 |             # (and appear as such under /proc/<pid>/cgroup) but are usually
 51 |             # mounted on a directory named "foo" (without the "name=" prefix).
 52 |             # Systemd and OpenRC (and possibly others) both create such a
 53 |             # cgroup. To avoid the aforementioned bug, we symlink "foo" to
 54 |             # "name=foo". This shouldn't have any adverse effect.
 55 |             echo $SUBSYS | grep -q ^name= && {
 56 |                     NAME=$(echo $SUBSYS | sed s/^name=//)
 57 |                     ln -s $SUBSYS $CGROUP/$NAME
 58 |             }
 59 | 
 60 |             # Likewise, on at least one system, it has been reported that
 61 |             # systemd would mount the CPU and CPU accounting controllers
 62 |             # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
 63 |             # but on a directory called "cpu,cpuacct" (note the inversion
 64 |             # in the order of the groups). This tries to work around it.
 65 |             [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
 66 |     done
 67 | 
 68 |     # Note: as I write those lines, the LXC userland tools cannot setup
 69 |     # a "sub-container" properly if the "devices" cgroup is not in its
 70 |     # own hierarchy. Let's detect this and issue a warning.
 71 |     grep -q :devices: /proc/1/cgroup ||
 72 |     	    echo -e "${ORANGE}WARNING: the 'devices' cgroup should be in its own hierarchy.${NC}"
 73 |     grep -qw devices /proc/1/cgroup ||
 74 | 	    echo -e "${ORANGE}WARNING: it looks like the 'devices' cgroup is not mounted.${NC}"
 75 | 
 76 |     # Now, close extraneous file descriptors.
 77 |     pushd /proc/self/fd >/dev/null
 78 |     for FD in *
 79 |     do
 80 | 	    case "$FD" in
 81 | 	    # Keep stdin/stdout/stderr
 82 | 	    [012])
 83 | 		    ;;
 84 | 	    # Nuke everything else
 85 | 	    *)
 86 | 		    eval exec "$FD>&-"
 87 | 		    ;;
 88 | 	    esac
 89 |     done
 90 |     popd >/dev/null
 91 | 
 92 |     # If a pidfile is still around (for example after a container restart),
 93 |     # delete it so that docker can start.
 94 |     rm -rf /var/run/ferry.pid
 95 | }
 96 | 
 97 | # Build all the images. Assumed that it is being
 98 | # called from internally. 
 99 | function make_images_internal {
100 |     OUTPUT=$(ferry install $1)
101 |     echo -e "${GREEN}${OUTPUT}${NC}"
102 | }
103 | 
104 | # Parse any options. 
105 | ARG=""
106 | if [ "$#" == 2 ]; then
107 |     ARG=$2
108 | fi
109 | 
110 | # Ok parse the user arguments. 
111 | if [[ $1 == "internal" ]]; then
112 |     prepare_dind
113 |     make_images_internal $ARG
114 | elif [[ $1 == "console" ]]; then
115 |     prepare_dind
116 |     exec bash
117 | fi
118 | 


--------------------------------------------------------------------------------
/ferry/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.3.3.4'
2 | 


--------------------------------------------------------------------------------
/ferry/cli/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/cli/__init__.py


--------------------------------------------------------------------------------
/ferry/config/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/__init__.py


--------------------------------------------------------------------------------
/ferry/config/cassandra/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/cassandra/__init__.py


--------------------------------------------------------------------------------
/ferry/config/gluster/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/gluster/__init__.py


--------------------------------------------------------------------------------
/ferry/config/hadoop/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/hadoop/__init__.py


--------------------------------------------------------------------------------
/ferry/config/hadoop/hiveconfig.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2014 OpenCore LLC
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | 
 16 | import os
 17 | import sh
 18 | import sys
 19 | from string import Template
 20 | 
 21 | class HiveClientInitializer(object):
 22 |     """
 23 |     Create a new initializer
 24 |     Param user The user login for the git repo
 25 |     """
 26 |     def __init__(self, system):
 27 |         self.template_dir = None
 28 |         self.template_repo = None
 29 | 
 30 |         self.container_data_dir = HiveClientConfig.data_directory
 31 |         self.container_log_dir = HiveClientConfig.log_directory
 32 | 
 33 |     """
 34 |     Generate a new hostname
 35 |     """
 36 |     def new_host_name(self, instance_id):
 37 |         return 'hive-client' + str(instance_id)
 38 | 
 39 |     """
 40 |     Start the service on the containers. 
 41 |     """
 42 |     def _execute_service(self, containers, entry_point, fabric, cmd):
 43 |         output = fabric.cmd(containers, '/service/sbin/startnode %s client' % cmd)
 44 | 
 45 |     """
 46 |     Generate a new configuration.
 47 |     """
 48 |     def _generate_config_dir(self, uuid):
 49 |         return 'hive_client_' + str(uuid)
 50 | 
 51 |     def get_public_ports(self, num_instances):
 52 |         """
 53 |         Ports to expose to the outside world. 
 54 |         """
 55 |         return []
 56 | 
 57 |     def get_internal_ports(self, num_instances):
 58 |         """
 59 |         Ports needed for communication within the network. 
 60 |         This is usually used for internal IPC.
 61 |         """
 62 |         return []
 63 | 
 64 |     def get_working_ports(self, num_instances):
 65 |         """
 66 |         Ports necessary to get things working. 
 67 |         """
 68 |         ports = []
 69 | 
 70 |     """
 71 |     Generate a new configuration
 72 |     """
 73 |     def generate(self, num):
 74 |         return HiveClientConfig(num)
 75 | 
 76 |     """
 77 |     Generate the hive site configuration. 
 78 |     """
 79 |     def _generate_hive_site(self, config, new_config_dir):
 80 |         in_file = open(self.template_dir + '/hive-site.xml.template', 'r')
 81 |         out_file = open(new_config_dir + '/hive-site.xml', 'w+')
 82 |         
 83 |         changes = { "DB":config.metastore,
 84 |                     "USER": os.environ['USER'] }
 85 |         for line in in_file:
 86 |             s = Template(line).substitute(changes)
 87 |             out_file.write(s)
 88 | 
 89 |         in_file.close()
 90 |         out_file.close()
 91 | 
 92 |     """
 93 |     Apply the configuration to the instances
 94 |     """
 95 |     def apply(self, config, containers):
 96 |         # The "entry point" is the way to contact the storage service.
 97 |         # For gluster this is the IP address of the "master" and the volume name. 
 98 |         entry_point = { 'type' : 'hive' }
 99 | 
100 |         # Create a new configuration directory, and place
101 |         # into the template directory. 
102 |         config_dirs = []
103 |         new_config_dir = "/tmp/" + self._generate_config_dir(config.uuid)
104 |         try:
105 |             sh.mkdir('-p', new_config_dir)
106 |         except:
107 |             sys.stderr.write('could not create config dir ' + new_config_dir)
108 | 
109 |         self._generate_hive_site(config, new_config_dir)
110 | 
111 |         # Each container needs to point to a new config dir. 
112 |         for c in containers:
113 |             config_files = new_config_dir + '/*'
114 |             config_dirs.append([c['container'],
115 |                                 config_files, 
116 |                                 config.config_directory])
117 | 
118 |         return config_dirs, entry_point
119 | 
120 | class HiveClientConfig(object):
121 |     data_directory = '/service/data/main'
122 |     log_directory = '/service/data/logs'
123 |     config_directory = '/service/conf/hive'
124 |     def __init__(self, num):
125 |         self.num = num
126 |         self.config_directory = HiveClientConfig.config_directory
127 |         self.hadoop_config_dir = None
128 |         self.metastore = None
129 | 


--------------------------------------------------------------------------------
/ferry/config/mongo/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/mongo/__init__.py


--------------------------------------------------------------------------------
/ferry/config/mongo/mongoclientconfig.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2014 OpenCore LLC
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | 
 16 | import sys
 17 | import sh
 18 | from string import Template
 19 | from ferry.config.mongo.mongoconfig import *
 20 | 
 21 | class MongoClientInitializer(object):
 22 |     def __init__(self, system):
 23 |         """
 24 |         Create a new initializer
 25 |         Param user The user login for the git repo
 26 |         """
 27 |         self.mongo = MongoInitializer(system)
 28 |         self.container_data_dir = self.mongo.container_data_dir
 29 |         self.container_log_dir = self.mongo.container_log_dir
 30 | 
 31 |     @property
 32 |     def fabric(self):
 33 |         self.mongo.fabric
 34 | 
 35 |     @fabric.setter
 36 |     def fabric(self, value):
 37 |         self.mongo.fabric = value
 38 | 
 39 |     @property
 40 |     def template_dir(self):
 41 |         self.mongo.template_dir
 42 | 
 43 |     @template_dir.setter
 44 |     def template_dir(self, value):
 45 |         self.mongo.template_dir = value
 46 | 
 47 |     @property
 48 |     def template_repo(self):
 49 |         self.mongo.template_repo
 50 | 
 51 |     @template_repo.setter
 52 |     def template_repo(self, value):
 53 |         self.mongo.template_repo = value
 54 | 
 55 |     """
 56 |     Generate a new hostname
 57 |     """
 58 |     def new_host_name(self, instance_id):
 59 |         return 'mongo_client' + str(instance_id)
 60 | 
 61 |     """
 62 |     Start the service on the containers. 
 63 |     """
 64 |     def start_service(self, containers, entry_point, fabric):
 65 |         return self.mongo.start_service(containers, entry_point, fabric)
 66 |     def restart_service(self, containers, entry_point, fabric):
 67 |         return self.mongo.restart_service(containers, entry_point, fabric)
 68 |     def stop_service(self, containers, entry_point, fabric):        
 69 |         return self.mongo.stop_service(containers, entry_point, fabric)
 70 | 
 71 |     def get_public_ports(self, num_instances):
 72 |         """
 73 |         Ports to expose to the outside world. 
 74 |         """
 75 |         return []
 76 | 
 77 |     def get_internal_ports(self, num_instances):
 78 |         """
 79 |         Ports needed for communication within the network. 
 80 |         This is usually used for internal IPC.
 81 |         """
 82 |         return self.mongo.get_internal_ports(num_instances)
 83 | 
 84 |     def get_working_ports(self, num_instances):
 85 |         """
 86 |         Ports necessary to get things working. 
 87 |         """
 88 |         return self.mongo.get_working_ports(num_instances)
 89 | 
 90 |     """
 91 |     Generate a new configuration
 92 |     Param num Number of instances that need to be configured
 93 |     Param image Image type of the instances
 94 |     """
 95 |     def generate(self, num):
 96 |         return self.mongo.generate(num)
 97 | 
 98 |     """
 99 |     Apply the configuration to the instances
100 |     """
101 |     def apply(self, config, containers):
102 |         config_dirs, entry_point = self.mongo.apply(config, containers)
103 |         entry_point['type'] = 'mongo-client'
104 |         entry_point['ip'] = entry_point['mongo']
105 |         return config_dirs, entry_point
106 | 


--------------------------------------------------------------------------------
/ferry/config/openmpi/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/openmpi/__init__.py


--------------------------------------------------------------------------------
/ferry/config/openmpi/mpiclientconfig.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import logging
17 | import sh
18 | import sys
19 | from string import Template
20 | from ferry.config.openmpi.mpiconfig import *
21 | 
22 | class OpenMPIClientInitializer(object):
23 |     def __init__(self, system):
24 |         self.mpi = OpenMPIInitializer(system)
25 |         self.container_data_dir = self.mpi.container_data_dir
26 |         self.container_log_dir = self.mpi.container_log_dir
27 | 
28 |     @property
29 |     def template_dir(self):
30 |         self.mpi.template_dir
31 | 
32 |     @template_dir.setter
33 |     def template_dir(self, value):
34 |         self.mpi.template_dir = value
35 | 
36 |     @property
37 |     def template_repo(self):
38 |         self.mpi.template_repo
39 | 
40 |     @template_repo.setter
41 |     def template_repo(self, value):
42 |         self.mpi.template_repo = value
43 | 
44 |     def new_host_name(self, instance_id):
45 |         return 'openmpi_client' + str(instance_id)
46 | 
47 |     def start_service(self, containers, entry_point, fabric):
48 |         return self.mpi.start_service(containers, entry_point, fabric)
49 |     def restart_service(self, containers, entry_point, fabric):
50 |         return self.mpi.restart_service(containers, entry_point, fabric)
51 |     def stop_service(self, containers, entry_point, fabric):        
52 |         return self.mpi.stop_service(containers, entry_point, fabric)
53 | 
54 |     def get_public_ports(self, num_instances):
55 |         """
56 |         Ports to expose to the outside world. 
57 |         """
58 |         return []
59 | 
60 |     def get_internal_ports(self, num_instances):
61 |         """
62 |         Ports needed for communication within the network. 
63 |         This is usually used for internal IPC.
64 |         """
65 |         return self.mpi.get_internal_ports(num_instances)
66 | 
67 |     def get_working_ports(self, num_instances):
68 |         """
69 |         Ports necessary to get things working. 
70 |         """
71 |         return self.mpi.get_working_ports(num_instances)
72 | 
73 |     def generate(self, num):
74 |         return self.mpi.generate(num)
75 | 
76 |     def apply(self, config, containers):
77 |         config_dirs, entry_point = self.mpi.apply(config, containers)
78 |         entry_point['type'] = 'mpi-client'
79 |         return config_dirs, entry_point
80 | 


--------------------------------------------------------------------------------
/ferry/config/spark/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/spark/__init__.py


--------------------------------------------------------------------------------
/ferry/config/spark/sparkclientconfig.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import logging
17 | import sh
18 | import sys
19 | from string import Template
20 | from ferry.config.spark.sparkconfig import *
21 | 
22 | class SparkClientInitializer(object):
23 |     """
24 |     Create a new initializer
25 |     Param user The user login for the git repo
26 |     """
27 |     def __init__(self, system):
28 |         self.spark = SparkInitializer(system)
29 |         self.container_data_dir = self.spark.container_data_dir
30 |         self.container_log_dir = self.spark.container_log_dir
31 | 
32 |     @property
33 |     def template_dir(self):
34 |         self.spark.template_dir
35 | 
36 |     @template_dir.setter
37 |     def template_dir(self, value):
38 |         self.spark.template_dir = value
39 | 
40 |     @property
41 |     def template_repo(self):
42 |         self.spark.template_repo
43 | 
44 |     @template_repo.setter
45 |     def template_repo(self, value):
46 |         self.spark.template_repo = value
47 | 
48 |     """
49 |     Generate a new hostname
50 |     """
51 |     def new_host_name(self, instance_id):
52 |         return 'spark_client' + str(instance_id)
53 | 
54 |     """
55 |     Start the service on the containers. 
56 |     """
57 |     def start_service(self, containers, entry_point, fabric):
58 |         return self.spark.start_service(containers, entry_point, fabric)
59 |     def restart_service(self, containers, entry_point, fabric):
60 |         return self.spark.restart_service(containers, entry_point, fabric)
61 |     def stop_service(self, containers, entry_point, fabric):        
62 |         return self.spark.stop_service(containers, entry_point, fabric)
63 | 
64 |     def get_public_ports(self, num_instances):
65 |         """
66 |         Ports to expose to the outside world. 
67 |         """
68 |         return []
69 | 
70 |     def get_internal_ports(self, num_instances):
71 |         """
72 |         Ports needed for communication within the network. 
73 |         This is usually used for internal IPC.
74 |         """
75 |         return self.spark.get_internal_ports(num_instances)
76 | 
77 |     def get_working_ports(self, num_instances):
78 |         """
79 |         Ports necessary to get things working. 
80 |         """
81 |         return self.spark.get_working_ports(num_instances)
82 | 
83 |     """
84 |     Generate a new configuration
85 |     Param num Number of instances that need to be configured
86 |     Param image Image type of the instances
87 |     """
88 |     def generate(self, num):
89 |         return self.spark.generate(num)
90 | 
91 |     """
92 |     Apply the configuration to the instances
93 |     """
94 |     def apply(self, config, containers):
95 |         config_dirs, entry_point = self.spark.apply(config, containers)
96 |         entry_point['type'] = 'spark-client'
97 |         return config_dirs, entry_point
98 | 


--------------------------------------------------------------------------------
/ferry/config/system/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/system/__init__.py


--------------------------------------------------------------------------------
/ferry/config/system/aws.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import logging
17 | 
18 | AWS_INSTANCE_INFO = {
19 |     "t2.micro" : { "mem" : 1, "cores" : 1 },
20 |     "t2.small" : { "mem" : 2, "cores" : 1 },
21 |     "t2.medium" : { "mem" : 4, "cores" : 2 },
22 |     "m3.medium" : { "mem" : 3.75, "cores" : 1 },
23 |     "m3.large" : { "mem" : 7.5, "cores" : 2 },
24 |     "m3.xlarge" : { "mem" : 15, "cores" : 4 },
25 |     "m3.2xlarge" : { "mem" : 30, "cores" : 8 },
26 |     "c3.large" : { "mem" : 3.75, "cores" : 2 },
27 |     "c3.xlarge" : { "mem" : 7.5, "cores" : 4 },
28 |     "c3.2xlarge" : { "mem" : 15, "cores" : 8 },
29 |     "c3.4xlarge" : { "mem" : 30, "cores" : 16 },
30 |     "c3.8xlarge" : { "mem" : 60, "cores" : 32 },
31 |     "r3.large" : { "mem" : 15.25, "cores" : 2 },
32 |     "r3.xlarge" : { "mem" : 30.5, "cores" : 4 },
33 |     "r3.2xlarge" : { "mem" : 61, "cores" : 8 },
34 |     "r3.4xlarge" : { "mem" : 122, "cores" : 16 },
35 |     "r3.8xlarge" : { "mem" : 244, "cores" : 32 },
36 |     "i2.xlarge" : { "mem" : 30.5, "cores" : 4 },
37 |     "i2.2xlarge" : { "mem" : 61, "cores" : 8 },
38 |     "i2.4xlarge" : { "mem" : 122, "cores" : 16 },
39 |     "i2.8xlarge" : { "mem" : 244, "cores" : 32 },
40 |     "g2.2xlarge" : { "mem" : 15, "cores" : 8 },
41 |     "hs1.8xlarge" : { "mem" : 117, "cores" : 16 }
42 | }
43 | 
44 | class System(object):
45 |     def __init__(self):
46 |         self.instance_type = "t2.small"
47 | 
48 |     def get_total_memory(self):
49 |         """
50 |         Get total memory of current system. 
51 |         """
52 |         if self.instance_type in AWS_INSTANCE_INFO:
53 |             return AWS_INSTANCE_INFO[self.instance_type]["mem"] * 1024
54 |         else:
55 |             logging.warning("AWS system could not find " + self.instance_type)
56 |             return 1024
57 | 
58 |     def get_free_memory(self):
59 |         """
60 |         Get free memory of current system. 
61 |         """
62 |         if self.instance_type in AWS_INSTANCE_INFO:
63 |             return AWS_INSTANCE_INFO[self.instance_type]["mem"] * 1024
64 |         else:
65 |             return 1024
66 | 
67 |     def get_num_cores(self):
68 |         """
69 |         Get total number of cores. 
70 |         """
71 |         if self.instance_type in AWS_INSTANCE_INFO:
72 |             return AWS_INSTANCE_INFO[self.instance_type]["cores"]
73 |         else:
74 |             return 1
75 | 


--------------------------------------------------------------------------------
/ferry/config/system/info.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import os
17 | import sys
18 | from subprocess import Popen, PIPE
19 | 
20 | class System(object):
21 |     def get_total_memory(self):
22 |         """
23 |         Get total memory of current system. 
24 |         """
25 |         cmd = "cat /proc/meminfo | grep  MemTotal | awk '{print $2}'"
26 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
27 |         return int(output.strip()) / 1000
28 | 
29 |     def get_free_memory(self):
30 |         """
31 |         Get free memory of current system. 
32 |         """
33 |         cmd = "cat /proc/meminfo | grep  MemFree | awk '{print $2}'"
34 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
35 |         return int(output.strip()) / 1000
36 | 
37 |     def get_num_cores(self):
38 |         """
39 |         Get total number of cores. 
40 |         """
41 |         cmd = "nproc"
42 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
43 |         return int(output.strip())
44 | 


--------------------------------------------------------------------------------
/ferry/config/titan/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/config/titan/__init__.py


--------------------------------------------------------------------------------
/ferry/data/conf/ferry-aws-example.yaml:
--------------------------------------------------------------------------------
 1 | system:
 2 |   provider: aws
 3 |   backend: ferry.fabric.cloud/CloudFabric
 4 |   mode: ferry.fabric.aws.awslauncher/AWSLauncher
 5 |   proxy: false
 6 | web:
 7 |   workers: 1
 8 |   bind: 0.0.0.0
 9 |   port: 4000
10 | aws:
11 |   params:
12 |     dc: us-east-1
13 |     zone: us-east-1b
14 |     volume: ebs:8
15 |   deploy:
16 |      image: ami-xxxyyy
17 |      personality: t2.micro
18 |      personalities: 
19 |         hive: t2.micro
20 |         hadoop: t2.small
21 |      default-user: ubuntu
22 |      ssh: ferry-aws
23 |      ssh-user: ferry
24 |      public: false
25 |      user: APIUSER
26 |      access: APIACCESSKEY
27 |      secret: APISECRETKEY


--------------------------------------------------------------------------------
/ferry/data/conf/ferry-default.yaml:
--------------------------------------------------------------------------------
1 | system:
2 |   backend: ferry.fabric.local/LocalFabric
3 | web:
4 |   workers: 1
5 |   bind: 127.0.0.1
6 |   port: 4000
7 | 


--------------------------------------------------------------------------------
/ferry/data/conf/ferry-hp-example.yaml:
--------------------------------------------------------------------------------
 1 | system:
 2 |   provider: hp
 3 |   network: eth0
 4 |   backend: ferry.fabric.cloud/CloudFabric
 5 |   mode: ferry.fabric.openstack.singlelauncher/SingleLauncher
 6 |   proxy: false
 7 | web:
 8 |   workers: 1
 9 |   bind: 0.0.0.0
10 |   port: 4000
11 | hp:
12 |   params:
13 |     dc: uswest
14 |     zone: az2
15 |   uswest:
16 |      region: region-a.geo-1
17 |      keystone: https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/
18 |      neutron: https://region-a.geo-1.network.hpcloudsvc.com
19 |      nova: https://region-a.geo-1.compute.hpcloudsvc.com/v2/10089763026941
20 |      swift: https://region-a.geo-1.images.hpcloudsvc.com:443/v1.0
21 |      cinder: https://region-a.geo-1.block.hpcloudsvc.com/v1/10089763026941
22 |      extnet: 122c72de-0924-4b9f-8cf3-b18d5d3d292c
23 |      network: 123-123-123
24 |      router: 123-123-123
25 |   useast:
26 |      region: region-b.geo-1
27 |      keystone: https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/
28 |      neutron: https://region-b.geo-1.network.hpcloudsvc.com
29 |      swift: https://region-b.geo-1.images.hpcloudsvc.com:443/v1.0 
30 |      cinder: https://region-b.geo-1.block.hpcloudsvc.com/v1/10089763026941
31 |      nova: https://region-b.geo-1.compute.hpcloudsvc.com/v2/10089763026941
32 |      extnet: 122c72de-0924-4b9f-8cf3-b18d5d3d292c
33 |      network: 123-123-123
34 |      router: 123-123-123
35 |   deploy:
36 |      image: Ferry Server (small)
37 |      personality: standard.small
38 |      default-user: ubuntu
39 |      ssh: ferry-keys
40 |      ssh-user: ferry
41 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/titan
 2 | NAME cassandra-client
 3 | 
 4 | # Download the various packages needed
 5 | RUN apt-get --yes install git maven
 6 | 
 7 | # Create necessary directories
 8 | RUN mkdir -p /service/runscripts/start /service/runscripts/stop /service/runscripts/restart /service/runscripts/test
 9 | 
10 | # Add the control script to the image. 
11 | ADD ./startnode /service/sbin/
12 | ADD ./testusers.db /service/scripts/
13 | ADD ./gods.json /service/scripts/
14 | ADD ./naturalgraph.groovy /service/scripts/
15 | ADD ./loadgods.groovy /service/scripts/
16 | ADD ./rexexample.sh /service/scripts/
17 | 
18 | # Add our runscripts
19 | ADD ./start01.sh /service/runscripts/start/
20 | ADD ./restart01.sh /service/runscripts/restart/
21 | ADD ./stop10.sh /service/runscripts/stop/
22 | ADD ./test01.sh /service/runscripts/test/
23 | RUN chmod a+x /service/sbin/startnode;chmod a+x /service/runscripts/start/*;chmod a+x /service/runscripts/stop/*;chmod a+x /service/runscripts/test/*;chmod a+x /service/runscripts/restart/*;chmod a+x /service/scripts/rexexample.sh
24 | 
25 | # Download the latest package
26 | RUN wget $DOWNLOAD_URL/rexster-console-2.3.0.zip -P /service/packages
27 | RUN cd /service/packages/;unzip rexster-console-2.3.0.zip
28 | 
29 | # Get rid of the tarballs.
30 | RUN rm /service/packages/*zip
31 | RUN rm -rf /var/cache/apt/archives/*
32 | RUN rm -rf /var/lib/apt/lists/*


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/gods.json:
--------------------------------------------------------------------------------
  1 | {
  2 |     "graph": {
  3 |         "mode":"NORMAL",
  4 | 	"vertices": [
  5 | 	    {
  6 | 	       "name":"saturn",
  7 | 	       "type":"titan",
  8 | 	       "_id":0,
  9 | 	       "_type":"vertex"
 10 | 	    },
 11 | 	    {
 12 | 		"name":"jupiter",
 13 | 		"type":"god",
 14 | 		"_id":1,
 15 | 	       	"_type":"vertex"
 16 | 	    },
 17 | 	    {
 18 | 		"name":"neptune",
 19 | 		"type":"god",
 20 | 		"_id":2,
 21 | 	       "_type":"vertex"
 22 | 	    },
 23 | 	    {
 24 | 		"name":"pluto",
 25 | 		"type":"god",
 26 | 		"_id":3,
 27 | 	       "_type":"vertex"
 28 | 	    },
 29 | 	    {
 30 | 		"name":"sky",
 31 | 		"type":"location",
 32 | 		"_id":4,
 33 | 	       "_type":"vertex"
 34 | 	    },
 35 | 	    {
 36 | 		"name":"sea",
 37 | 		"type":"location",
 38 | 		"_id":5,
 39 | 	       "_type":"vertex"
 40 | 	    },
 41 | 	    {
 42 | 		"name":"tartarus",
 43 | 		"type":"location",
 44 | 		"_id":6,
 45 | 	       "_type":"vertex"
 46 | 	   },
 47 | 	   {
 48 | 		"name":"hercules",
 49 | 		"type":"demigod",
 50 | 		"_id":7,
 51 | 	       "_type":"vertex"
 52 | 	   },
 53 | 	   {
 54 | 		"name":"alcmene",
 55 | 		"type":"human",
 56 | 		"_id":8,
 57 | 	       "_type":"vertex"
 58 | 	   },
 59 | 	   {
 60 | 		"name":"nemean",
 61 | 		"type":"monster",
 62 | 		"_id":9,
 63 | 	       "_type":"vertex"
 64 | 	   },
 65 | 	   {
 66 | 		"name":"hydra",
 67 | 		"type":"monster",
 68 | 		"_id":10,
 69 | 	       "_type":"vertex"
 70 | 	   },
 71 | 	   {
 72 | 		"name":"cerberus",
 73 | 		"type":"monster",
 74 | 		"_id":11,
 75 | 	       	"_type":"vertex"
 76 | 	   }],
 77 |        "edges":[
 78 | 	   {
 79 | 		"_id": 12,
 80 |                 "_type": "edge",
 81 |                 "_outV": 1,
 82 |                 "_inV": 0,
 83 |                 "_label": "father"
 84 | 	   },
 85 | 	   {
 86 | 		"_id": 13,
 87 |                 "_type": "edge",
 88 |                 "_outV": 4,
 89 |                 "_inV": 1,
 90 |                 "_label": "lives"
 91 | 	   },
 92 | 	   {
 93 | 		"_id": 14,
 94 |                 "_type": "edge",
 95 |                 "_outV": 3,
 96 |                 "_inV": 1,
 97 |                 "_label": "brother"
 98 | 	   },
 99 | 	   {
100 | 		"_id": 15,
101 |                 "_type": "edge",
102 |                 "_outV": 1,
103 |                 "_inV": 3,
104 |                 "_label": "brother"
105 | 	   },
106 | 	   {
107 | 		"_id": 16,
108 |                 "_type": "edge",
109 |                 "_outV": 2,
110 |                 "_inV": 1,
111 |                 "_label": "brother"
112 | 	   },
113 | 	   {
114 | 		"_id": 17,
115 |                 "_type": "edge",
116 |                 "_outV": 1,
117 |                 "_inV": 2,
118 |                 "_label": "brother"
119 | 	   },
120 | 	   {
121 | 		"_id": 18,
122 |                 "_type": "edge",
123 |                 "_outV": 0,
124 |                 "_inV": 1,
125 |                 "_label": "father"
126 | 	   },
127 | 	   {
128 | 		"_id": 19,
129 |                 "_type": "edge",
130 |                 "_outV": 5,
131 |                 "_inV": 2,
132 |                 "_label": "lives"
133 | 	   },
134 | 	   {
135 | 		"_id": 20,
136 |                 "_type": "edge",
137 |                 "_outV": 3,
138 |                 "_inV": 2,
139 |                 "_label": "brother"
140 | 	   },
141 | 	   {
142 | 		"_id": 21,
143 |                 "_type": "edge",
144 |                 "_outV": 2,
145 |                 "_inV": 3,
146 |                 "_label": "brother"
147 | 	   },
148 | 	   {
149 | 		"_id": 22,
150 |                 "_type": "edge",
151 |                 "_outV": 2,
152 |                 "_inV": 1,
153 |                 "_label": "brother"
154 | 	   },
155 | 	   {
156 | 		"_id": 23,
157 |                 "_type": "edge",
158 |                 "_outV": 1,
159 |                 "_inV": 2,
160 |                 "_label": "brother"
161 | 	   }
162 |        ]
163 |    }
164 | }


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/loadgods.groovy:
--------------------------------------------------------------------------------
1 | g = TitanFactory.open('/service/conf/titan.properties');
2 | g.loadGraphSON('/service/scripts/gods.json');
3 | println g.V('name','pluto').out('brother').name;


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/naturalgraph.groovy:
--------------------------------------------------------------------------------
 1 | g = TitanFactory.open('/service/conf/titan.properties');
 2 | 
 3 | size = 100; ids = [g.addVertex().id]; rand = new Random();
 4 | (1..size).each{
 5 |     v = g.addVertex();
 6 |     u = g.v(ids.get(rand.nextInt(ids.size())))
 7 | 	g.addEdge(v,u,'linked');
 8 |     ids.add(u.id);
 9 |     ids.add(v.id);
10 |     if(it % 1000 == 0) 
11 | 	g.commit()
12 | }
13 | 
14 | indegree = [:].withDefault{0}
15 |     ids.unique().each{ 
16 | 	count = g.v(it).inE.count();
17 | 	indegree[count] = indegree[count] + 1;
18 |     }
19 | indegree.sort{a,b -> b.value <=> a.value}
20 | println indegree


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/restart01.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | cat /service/conf/servers >> /etc/hosts
4 | echo export CQLSH_HOST=$1 >> /etc/profile
5 | source /etc/profile


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/rexexample.sh:
--------------------------------------------------------------------------------
1 | wget --http-user=rexster --http-password=rexster http://rexserver:8182/graphs/users/vertices?key=name\&value=pluto -O pluto.json
2 | wget --http-user=rexster --http-password=rexster http://rexserver:8182/graphs/users/vertices/40016/outIds -O relationships.json
3 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/start01.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | cat /service/conf/cassandra/servers >> /etc/hosts
4 | echo export CQLSH_HOST=$1 >> /etc/profile
5 | source /etc/profile
6 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | else
12 |     for f in /service/runscripts/$1/*
13 |     do
14 | 	echo "start script ${1}/${f}" >> /tmp/startnode.log
15 | 	$f $2
16 |     done
17 | fi
18 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/stop10.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | # There's nothing to actually stop for the Cassandra client.


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/test01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /etc/profile
 4 | 
 5 | # Bash colors
 6 | GREEN='\e[0;32m'
 7 | NC='\e[0m'
 8 | 
 9 | function run_as_ferry {
10 |     echo -e "${GREEN} ${2} ${NC}"
11 |     if [ $USER == "root" ]; then
12 | 	su ferry -c "$1"
13 |     else
14 | 	$1
15 |     fi
16 | }
17 | 
18 | if [ $1 == "cql" ]; then
19 |     CMD='/service/bin/cqlsh -f /service/scripts/testusers.db'
20 |     run_as_ferry "$CMD" "Running CQL example"
21 | elif [ $1 == "natural" ]; then
22 |     CMD='/service/packages/titan/bin/gremlin.sh -e /service/scripts/naturalgraph.groovy'
23 |     run_as_ferry "$CMD" "Running Titan example"
24 | elif [ $1 == "gods" ]; then
25 |     CMD='/service/packages/titan/bin/gremlin.sh -e /service/scripts/loadgods.groovy'
26 |     run_as_ferry "$CMD" "Running Titan example"
27 | fi


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra-client/testusers.db:
--------------------------------------------------------------------------------
 1 | CREATE KEYSPACE mykeyspace WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };
 2 | 
 3 | USE mykeyspace;
 4 | 
 5 | CREATE TABLE users (
 6 |   user_id int PRIMARY KEY,
 7 |   fname text,
 8 |   lname text
 9 | );
10 | 
11 | INSERT INTO users (user_id,  fname, lname) VALUES (1745, 'john', 'smith');
12 | INSERT INTO users (user_id,  fname, lname) VALUES (1744, 'john', 'doe');
13 | INSERT INTO users (user_id,  fname, lname) VALUES (1746, 'john', 'smith');
14 | 
15 | CREATE INDEX ON users (lname);
16 | 
17 | SELECT * FROM users WHERE lname = 'smith';
18 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/ferry-base
 2 | NAME cassandra
 3 | 
 4 | # Download the various packages needed
 5 | RUN apt-get --yes install openjdk-7-jdk
 6 | 
 7 | # Create necessary directories
 8 | RUN mkdir -p /service/data/main /service/logs /service/conf /service/data/cache /service/data/commits /service/sbin /service/package;chown -R ferry:docker /service/data;chown -R ferry:docker /service/logs;chown -R ferry:docker /service/conf
 9 | 
10 | # Download the latest package
11 | RUN wget $DOWNLOAD_URL/apache-cassandra-2.1.0-bin.tar.gz -P /service/package/
12 | RUN cd /service/package;tar -xzf apache-cassandra-2.1.0-bin.tar.gz
13 | 
14 | # Get rid of the tarballs
15 | RUN rm /service/package/*tar.gz
16 | 
17 | # Make soft links to the bin and configuration for easy access
18 | RUN ln -s /service/package/apache-cassandra-2.1.0/bin /service;ln -s /service/package/apache-cassandra-2.1.0/conf /service/conf/cassandra
19 | RUN ln -s /service/package/apache-cassandra-2.1.0/bin/* /usr/local/bin/
20 | 
21 | # Environment variables.
22 | ENV CASSANDRA_HOME /service/package/apache-cassandra-2.1.0
23 | ENV CASSANDRA_CONF /service/conf/cassandra
24 | ENV CASSANDRA_INCLUDE /service/bin/cassandra.in.sh
25 | RUN echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /etc/profile
26 | RUN echo export CASSANDRA_HOME=/service/package/apache-cassandra-2.1.0 >> /etc/profile
27 | RUN echo export CASSANDRA_CONF=/service/conf/cassandra >> /etc/profile
28 | RUN echo export CASSANDRA_INCLUDE=/service/bin/cassandra.in.sh >> /etc/profile
29 | 
30 | # Add the control script to the image. 
31 | ADD ./startnode /service/sbin/
32 | RUN chmod a+x /service/sbin/startnode
33 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/cassandra/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | elif [ $1 == "start" ]; then 
12 |     export JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=7200"
13 |     su ferry -c 'nohup /service/bin/cassandra &'
14 |     sleep 2
15 | elif [ $1 == "restart" ]; then 
16 |     export JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=7200"
17 |     su ferry -c 'nohup /service/bin/cassandra &'
18 |     sleep 2
19 | elif [ $1 == "stop" ]; then 
20 |     # Stop the Cassandra daemon.
21 |     pkill -f CassandraDaemon
22 | fi
23 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/ferry-base/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM ubuntu:14.04
 2 | NAME ferry-base
 3 | 
 4 | # Get rid of Upstart and FUSE errors
 5 | # RUN dpkg-divert --local --rename --add /sbin/initctl && ln -s /bin/true /sbin/initctl
 6 | RUN dpkg-divert --local --add --rename --divert /bin/mknod.real /bin/mknod && ln -s /bin/true /bin/mknod
 7 | 
 8 | # Fake a fuse install
 9 | RUN apt-get update;apt-get install libfuse2
10 | RUN cd /tmp ; apt-get download fuse
11 | RUN cd /tmp ; dpkg-deb -x fuse_* .
12 | RUN cd /tmp ; dpkg-deb -e fuse_*
13 | RUN cd /tmp ; rm fuse_*.deb
14 | RUN cd /tmp ; echo -en '#!/bin/bash\nexit 0\n' > DEBIAN/postinst
15 | RUN cd /tmp ; dpkg-deb -b . /fuse.deb
16 | RUN cd /tmp ; dpkg -i /fuse.deb
17 | 
18 | # Get the "mknod" command back. 
19 | RUN rm /bin/mknod
20 | RUN dpkg-divert --rename --remove /bin/mknod
21 | 
22 | # The previous set of commands messes up the /tmp permission for some
23 | # reason. This just fixes it.
24 | RUN chmod 1777 /tmp
25 | 
26 | # Make the various directories needed
27 | RUN mkdir -p /service/keys /service/sbin /service/sconf /var/run/sshd
28 | 
29 | # Download some dependencies
30 | RUN apt-get --yes install openssh-server
31 | 
32 | # Create ferry user
33 | RUN addgroup supergroup;addgroup --gid $DOCKER docker;adduser --disabled-password --gecos "" ferry;usermod -a -G sudo ferry;usermod -a -G docker ferry;usermod -a -G supergroup ferry;chown -R ferry:docker /service;chown -R ferry:ferry /home/ferry
34 | 
35 | # Generate an ssh key for this image.
36 | RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' > /dev/null
37 | RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config;touch /etc/mtab
38 | 
39 | # Disable UsePAM in the sshd config. 
40 | RUN sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
41 | 
42 | ADD ./setup /service/sbin/
43 | ADD ./halt01.sh /service/sbin/
44 | ADD ./init01.sh /service/sbin/
45 | RUN chmod a+x /service/sbin/*
46 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/ferry-base/halt01.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | 
 3 | # 
 4 | # This the halt script used by all the Ferry images. The halt script is the
 5 | # last thing executed by the container before it is stopped. This script should
 6 | # not be modified by the container. 
 7 | # 
 8 | # Currently this script just stops the SSH daemon. 
 9 | # 
10 |     
11 | pkill -f sshd


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/ferry-base/init01.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | 
 3 | # 
 4 | # This the initialization script used by all the Ferry images. The init script is the
 5 | # first script run after the container is started. This script should not be modified 
 6 | # by the container. 
 7 | # 
 8 | #
 9 | 
10 | #
11 | # Contains basic functions. 
12 | # 
13 | source /service/sbin/setup
14 | 
15 | #
16 | # Create the FUSE device. This won't be necessary
17 | # once we upgrade Docker, but keep it as a fail-safe
18 | # 
19 | create_fuse_dev    
20 | 
21 | #
22 | # Authorize the ssh key so that we can do password-less entry. 
23 | #
24 | cat /service/keys/$KEY >> /root/.ssh/authorized_keys
25 | 
26 | #
27 | # Start the ssh daemon in the foreground so that the container won't quit. 
28 | #
29 | /usr/sbin/sshd -D
30 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/ferry-base/setup:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # 
 4 | # Helper functions needed by all the Ferry images. 
 5 | # This script should not be modified by the container. 
 6 | #
 7 | 
 8 | function create_fuse_dev {
 9 |     # We'll need to create /dev/fuse since we removed 
10 |     # it during the installation (since build doesn't support
11 |     # privileged operations)
12 |     if [ ! -e /dev/fuse ]; then
13 | 	mknod /dev/fuse c 10 229
14 |     fi
15 | }
16 | 
17 | function pophosts {
18 |     # Populate the /etc/hosts file with the contents of
19 |     # 'instances' file. 
20 |     input=/service/sconf/instances
21 |     while read line
22 |     do
23 | 	split=( $line )
24 | 	name=${split[1]}
25 | 	echo $line >> /etc/hosts
26 |     done < "$input"
27 | }


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/gluster/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/ferry-base
 2 | NAME gluster
 3 | 
 4 | # Create necessary directories
 5 | RUN mkdir -p /service/data /service/logs /service/conf /service/sbin;chown -R ferry:docker /service/data;chown -R ferry:docker /service/logs;chown -R ferry:docker /service/conf
 6 | 
 7 | # Add the latest stable GlusterFS repo
 8 | RUN apt-get update; apt-get --yes install software-properties-common;add-apt-repository --yes ppa:semiosis/ubuntu-glusterfs-3.5
 9 | 
10 | # Tell apt to use the latest repo packages. 
11 | RUN echo "Package: *" >> /etc/apt/preferences
12 | RUN echo "Pin: release o=LP-PPA-semiosis-ubuntu-glusterfs-3.5,a=trusty" >> /etc/apt/preferences
13 | RUN echo "Pin-Priority: 900" >> /etc/apt/preferences
14 | 
15 | # Download the various packages needed
16 | RUN apt-get update; apt-get --yes install git glusterfs-common glusterfs-client glusterfs-server attr
17 | 
18 | # Add the control script to the image. 
19 | ADD ./startnode /service/sbin/
20 | RUN chmod a+x /service/sbin/startnode
21 | 
22 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/gluster/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | elif [ $1 == "start" ]; then 
12 |     nohup /usr/sbin/glusterd &
13 |     sleep 2
14 |     if [ $2 == "master" ]; then
15 | 	/service/conf/gluster/configure start
16 |     fi
17 | elif [ $1 == "restart" ]; then 
18 |     nohup /usr/sbin/glusterd &
19 |     sleep 2
20 |     if [ $2 == "master" ]; then
21 | 	/service/conf/gluster/configure restart
22 |     fi
23 | elif [ $1 == "stop" ]; then 
24 |     # Stop the gluster volume
25 |     if [ $2 == "master" ]; then
26 | 	/service/conf/gluster/configure stop
27 |     fi
28 |     # Stop the gluster daemon
29 |     pkill -f glusterd
30 | elif [ $1 == "clean" ]; then 
31 |     /service/conf/gluster/configure clean    
32 | fi
33 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-base/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/gluster
 2 | NAME hadoop-base
 3 | 
 4 | # Make the various directories needed
 5 | RUN mkdir -p /service/packages /service/conf /service/data/main /service/data/hive /service/data/logs /service/data/tmp /service/data/staging /service/scripts
 6 | 
 7 | # Download some dependencies
 8 | RUN apt-get --yes install openjdk-7-jdk zlib1g lib32z1 libpostgresql-jdbc-java software-properties-common
 9 | RUN wget $DOWNLOAD_URL/hadoop-2.5.1.tar.gz -P /service/packages
10 | RUN wget $DOWNLOAD_URL/hive-0.13.1.tar.gz -P /service/packages
11 | RUN cd /service/packages;tar -xzf hadoop-2.5.1.tar.gz
12 | RUN cd /service/packages;tar -xzf hive-0.13.1.tar.gz
13 | 
14 | # Get rid of the tarballs. 
15 | RUN rm /service/packages/*.tar.gz
16 | 
17 | # Make symlinks
18 | RUN ln -s /service/packages/hadoop-2.5.1 /service/packages/hadoop
19 | RUN ln -s /service/packages/hadoop/etc/hadoop /service/conf/hadoop
20 | RUN ln -s /service/packages/hadoop/logs /service/logs/hadoop
21 | 
22 | RUN ln -s /service/packages/apache-hive-0.13.1-bin /service/packages/hive
23 | RUN ln -s /service/packages/hive/conf /service/conf/hive
24 | RUN ln -s /usr/share/java/postgresql-jdbc4.jar /service/packages/hive/lib/postgresql-jdbc4.jar
25 | 
26 | # Soft links to make life easier
27 | RUN ln -s /service/packages/hadoop/bin/* /usr/local/bin/
28 | RUN ln -s /service/packages/hive/bin/* /usr/local/bin/
29 | 
30 | # Make ferry owner of everything. 
31 | RUN chown -R ferry:docker /service
32 | 
33 | ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64
34 | ENV HADOOP_HOME /service/packages/hadoop
35 | ENV HADOOP_CONF_DIR /service/conf/hadoop
36 | ENV HIVE_HOME /service/packages/hive
37 | ENV HADOOP_LIBEXEC_DIR /service/packages/hadoop/libexec
38 | ENV JAVA_LIBRARY_PATH /service/packages/hadoop/lib/native/
39 | RUN echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /etc/profile;echo export HADOOP_HOME=/service/packages/hadoop >> /etc/profile;echo export HIVE_HOME=/service/packages/hive >> /etc/profile;echo export HADOOP_LIBEXEC_DIR=/service/packages/hadoop/libexec >> /etc/profile;echo export JAVA_LIBRARY_PATH=/service/packages/hadoop/lib/native/ >> /etc/profile;echo export HADOOP_CONF_DIR=/service/conf/hadoop >> /etc/profile
40 | 
41 | # Set new ulimit.
42 | RUN echo "*               soft    nofile            2048" >> /etc/security/limits.conf


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/hadoop-base
 2 | NAME hadoop-client
 3 | 
 4 | # Make the various directories needed
 5 | RUN mkdir -p /service/runscripts/start /service/runscripts/restart /service/runscripts/test /service/runscripts/stop
 6 | 
 7 | # The Pig package should just reside on the client.
 8 | RUN wget $DOWNLOAD_URL/pig-0.13.0.tar.gz -P /service/packages
 9 | RUN cd /service/packages;tar -xzf pig-0.13.0.tar.gz
10 | 
11 | # Create a symlink to the package dir. 
12 | RUN ln -s /service/packages/pig-0.13.0 /service/packages/pig;
13 | RUN ln -s /service/packages/pig/bin/pig /usr/local/bin/;
14 | 
15 | # Make ferry owner of everything. 
16 | RUN chown -R ferry:docker /service
17 | 
18 | # Environment variables. 
19 | ENV PIG_HOME /service/packages/pig
20 | ENV PIG_CLASSPATH /service/conf/hadoop
21 | RUN echo export PIG_HOME=/service/packages/pig >> /etc/profile;echo export PIG_CLASSPATH=/service/conf/hadoop >> /etc/profile;
22 | 
23 | # Add the control script to the image. 
24 | ADD ./startnode /service/sbin/
25 | ADD ./createtable.sql /service/scripts/
26 | ADD ./count.pig /service/scripts/
27 | ADD ./mounthelper.py /service/scripts/
28 | ADD ./start01.sh /service/runscripts/start/
29 | ADD ./restart01.sh /service/runscripts/restart/
30 | ADD ./stop10.sh /service/runscripts/stop/
31 | ADD ./test01.sh /service/runscripts/test/
32 | ADD ./test02.sh /service/runscripts/test/
33 | RUN chmod a+x /service/sbin/startnode;chmod a+x /service/runscripts/start/*;chmod a+x /service/runscripts/stop/*;chmod a+x /service/runscripts/test/*;chmod a+x /service/runscripts/restart/*
34 | 
35 | # Get rid of the apt-cache 
36 | RUN rm -rf /var/cache/apt/archives/*
37 | RUN rm -rf /var/lib/apt/lists/*
38 | RUN rm /service/packages/*.tar.gz


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/count.pig:
--------------------------------------------------------------------------------
1 | /**
2 |  * Simple Pig example
3 |  * Counts the number of movies rated grouped by user id. 
4 |  **/
5 | 
6 | movies = LOAD '/service/data/movielens/u.data' USING PigStorage() AS (userid:int, movieid:int, rating:int, unixtime:chararray);
7 | g = GROUP movies BY userid;
8 | c = FOREACH g GENERATE COUNT($1);


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/createtable.sql:
--------------------------------------------------------------------------------
 1 | CREATE TABLE  movielens_users_text (
 2 |        userid INT,
 3 |        movieid INT,
 4 |        rating INT,
 5 |        unixtime STRING
 6 | ) 
 7 | ROW FORMAT DELIMITED
 8 | FIELDS TERMINATED BY '\t'
 9 | STORED AS TEXTFILE;
10 | 
11 | CREATE TABLE movielens_users (
12 |        userid INT,
13 |        movieid INT,
14 |        rating INT,
15 |        unixtime STRING
16 | ) STORED AS RCFILE;
17 | 
18 | LOAD DATA INPATH '/service/data/movielens/u.data'
19 | OVERWRITE INTO TABLE movielens_users_text;
20 | 
21 | INSERT INTO TABLE movielens_users SELECT * FROM movielens_users_text;
22 | DROP TABLE movielens_users_text;


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/mounthelper.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import os
 3 | import sys
 4 | import logging
 5 | from subprocess import Popen, PIPE
 6 | 
 7 | def mkdir(directory):
 8 |     if not os.path.isdir(directory):
 9 |         cmd = 'mkdir -p %s' % directory
10 |         Popen(cmd, shell=True)
11 | 
12 | def mount(entry_point, mount_point):
13 |     # Check if the mount point exists. If not
14 |     # go ahead and create it. 
15 |     # mount -t glusterfs entry_point mount_point
16 |     cmd = 'mount -t glusterfs %s %s' % (entry_point,
17 |                                         mount_point)
18 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
19 |     logging.info(cmd)
20 |     logging.info(output)
21 | 
22 | def umount(mount_point):
23 |     cmd = 'cat /etc/mtab | grep /service/data | awk \'{print $2}\''
24 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
25 |     if output.strip() != "":
26 |         cmd = 'umount %s' % mount_point
27 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
28 |         logging.info(cmd)
29 |         logging.info(output)
30 | 
31 | cmd = sys.argv[1]
32 | if cmd == "mount":
33 |     entry = sys.argv[2]
34 |     mkdir('/service/data')
35 |     mount(entry, '/service/data')
36 | elif cmd == "umount":
37 |     umount('/service/data')
38 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/restart01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Make sure we get the env variables in place. 
 4 | source /etc/profile
 5 | 
 6 | if [ $1 == "gluster" ]; then 
 7 |     sleep 3
 8 |     python /service/scripts/mounthelper.py umount
 9 |     python /service/scripts/mounthelper.py mount $2
10 |     chown -R ferry:docker /service/data
11 | fi
12 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/start01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Make sure we get the env variables in place. 
 4 | source /etc/profile
 5 | 
 6 | if [ $1 == "hadoop" ]; then
 7 |     #
 8 |     # This Hadoop cluster is using the HDFS backend. That means
 9 |     # we need to create various directories in HDFS for Hive.
10 |     #
11 |     su ferry -c '/service/packages/hadoop/bin/hdfs dfs -mkdir /tmp' >> /tmp/hadoop.log 2>> /tmp/hadoop.err
12 |     su ferry -c '/service/packages/hadoop/bin/hdfs dfs -chmod g+w /tmp' >> /tmp/hadoop.log 2>> /tmp/hadoop.err
13 |     su ferry -c '/service/packages/hadoop/bin/hdfs dfs -mkdir -p /service/data/hive' >> /tmp/hadoop.log 2>> /tmp/hadoop.err
14 |     su ferry -c '/service/packages/hadoop/bin/hdfs dfs -chmod g+w /service/data/hive' >> /tmp/hadoop.log 2>> /tmp/hadoop.err
15 | elif [ $1 == "gluster" ]; then 
16 |     #
17 |     # This Hadoop cluster is using the GlusterFS backend. That means
18 |     # we need to mount the volume.
19 |     #
20 |     python /service/scripts/mounthelper.py umount >> /tmp/gluster.log 2>> /tmp/gluster.err
21 |     python /service/scripts/mounthelper.py mount $2  >> /tmp/gluster.log 2>> /tmp/gluster.err
22 |     chown -R ferry:docker /service/data
23 | fi
24 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | else
12 |     for f in /service/runscripts/$1/*
13 |     do
14 | 	$f $2 $3
15 |     done
16 | fi
17 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/stop10.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | source /service/sbin/setup
4 | 
5 | # Unmount gluster if necessary
6 | if [ $1 == "gluster" ]; then 
7 | 	python /service/scripts/mounthelper.py umount
8 | fi
9 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/test01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Bash colors
 4 | GREEN='\e[0;32m'
 5 | NC='\e[0m'
 6 | 
 7 | function run_as_ferry {
 8 |     echo -e "${GREEN} ${2} ${NC}"
 9 |     if [ $USER == "root" ]; then
10 | 	su ferry -c "$1"
11 |     else
12 | 	$1
13 |     fi
14 | }
15 | 
16 | if [ $1 == "tera" ]; then
17 |     TERAGEN='hadoop jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar teragen 256 /service/data/teragen-output'
18 |     TERASORT='hadoop jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar terasort /service/data/teragen-output /service/data/terasort-output'
19 |     run_as_ferry "$TERAGEN" "Running Teragen"
20 |     run_as_ferry "$TERASORT" "Running Terasort"
21 | elif [ $1 == "text" ]; then
22 |     WGET1='wget http://www.gutenberg.org/cache/epub/20417/pg20417.txt -P /tmp/example-text/'
23 |     WGET2='wget http://www.gutenberg.org/cache/epub/5000/pg5000.txt -P /tmp/example-text/'
24 |     COPY='hdfs dfs -copyFromLocal /tmp/example-text /service/data/example-text'
25 |     WORDCOUNT='hadoop jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordcount /service/data/example-text /service/data/wordcount-output'
26 |     WORDMEAN='hadoop jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmean /service/data/example-text /service/data/wordmean-output'
27 |     WORDMEDIAN='hadoop jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmedian /service/data/example-text /service/data/wordmedian-output'
28 | 
29 |     run_as_ferry "$WGET1" "Downloading first text"
30 |     run_as_ferry "$WGET2" "Downloading second text"
31 |     run_as_ferry "$COPY" "Copy text to HDFS"
32 |     run_as_ferry "$WORDCOUNT" "Running word count"
33 |     run_as_ferry "$WORDMEAN" "Running word mean"
34 |     run_as_ferry "$WORDMEDIAN" "Running word median"
35 | elif [ $1 == "hive" ]; then
36 |     WGET='wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /tmp/movielens/'
37 |     MKDIR='hdfs dfs -mkdir -p /service/data/movielens'
38 |     COPY='hdfs dfs -copyFromLocal /tmp/movielens/u.data /service/data/movielens'
39 |     HIVE='hive -f /service/scripts/createtable.sql'
40 | 
41 |     run_as_ferry "$WGET" "Downloading movielens dataset"
42 |     run_as_ferry "$MKDIR" "Making movielens directory"
43 |     run_as_ferry "$COPY" "Copy data to HDFS"
44 |     run_as_ferry "$HIVE" "Running Hive"
45 | elif [ $1 == "pig" ]; then
46 |     WGET='wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /tmp/movielens/'
47 |     MKDIR='hdfs dfs -mkdir -p /service/data/movielens'
48 |     COPY='hdfs dfs -copyFromLocal /tmp/movielens/u.data /service/data/movielens'
49 |     PIG='pig -f /service/scripts/count.pig'
50 | 
51 |     run_as_ferry "$WGET" "Downloading movielens dataset"
52 |     run_as_ferry "$MKDIR" "Making movielens directory"
53 |     run_as_ferry "$COPY" "Copy data to HDFS"
54 |     run_as_ferry "$PIG" "Running Pig command"
55 | elif [ $1 == "gluster" ]; then
56 |     WGET='wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /service/data/movielens/'
57 |     HIVE='hive -f /service/scripts/createtable.sql'
58 | 
59 |     run_as_ferry "$WGET" "Downloading movielens dataset"
60 |     run_as_ferry "$HIVE" "Running Hive"
61 | elif [ $1 == "dfs" ]; then
62 |     DFS='hadoop dfsadmin -report'
63 |     run_as_ferry "$DFS" "Running HDFS report"
64 | fi
65 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop-client/test02.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /etc/profile
 4 | 
 5 | # Prerequisites
 6 | apt-get --yes install unzip
 7 | su ferry -c 'mkdir /tmp/build'
 8 | su ferry -c 'mkdir /tmp/src'
 9 | su ferry -c 'mkdir /tmp/text'
10 | su ferry -c 'export PATH=$PATH:$HADOOP_HOME/bin'
11 | 
12 | # Compile the example Wordcount application
13 | su ferry -c 'cp /service/packages/hadoop/share/hadoop/mapreduce/sources/hadoop-mapreduce-examples-2.3.0-sources.jar /tmp/src'
14 | su ferry -c 'cd /tmp/src; unzip hadoop-mapreduce-examples-2.3.0-sources.jar'
15 | su ferry -c 'javac -cp `hadoop classpath`:. -d /tmp/build org/apache/hadoop/examples/WordCount.java'
16 | su ferry -c 'cd /tmp/build; jar cf WordCount.jar org/apache/hadoop/examples/WordCount.class'
17 | 
18 | # Execute the example Wordcount application
19 | su ferry -c 'wget http://www.gutenberg.org/cache/epub/20417/pg20417.txt -P /tmp/text/'
20 | su ferry -c '/service/packages/hadoop/bin/hadoop --config /service/conf dfs -copyFromLocal /tmp/text /service/data/example-text'
21 | su ferry -c 'hadoop jar WordCount.jar org.apache.hadoop.examples.WordCount /service/data/example-text /service/data/wordcount-output'


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/hadoop-base
 2 | NAME hadoop
 3 | NAME yarn
 4 | 
 5 | ## Add the control script to the image. 
 6 | ADD ./startnode /service/sbin/
 7 | ADD ./mounthelper.py /service/scripts/
 8 | RUN chmod a+x /service/sbin/startnode
 9 | 
10 | # Run the ssh daemon in the foreground (Docker does
11 | # like daemons running in background)
12 | CMD ["/service/sbin/startnode", "init"]


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop/mounthelper.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import os
 3 | import sys
 4 | import logging
 5 | from subprocess import Popen, PIPE
 6 | 
 7 | def mkdir(directory):
 8 |     if not os.path.isdir(directory):
 9 |         cmd = 'mkdir -p %s' % directory
10 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
11 |         logging.info(cmd)
12 |         logging.info(output)
13 | 
14 | def mount(entry_point, mount_point):
15 |     # Check if the mount point exists. If not
16 |     # go ahead and create it. 
17 |     # mount -t glusterfs entry_point mount_point
18 |     cmd = 'mount -t glusterfs %s %s' % (entry_point,
19 |                                         mount_point)
20 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
21 |     logging.info(cmd)
22 |     logging.info(output)
23 | 
24 | def umount(mount_point):
25 |     cmd = 'cat /etc/mtab | grep /service/data | awk \'{print $2}\''
26 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
27 |     if output.strip() != "":
28 |         cmd = 'umount %s' % mount_point
29 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
30 |         logging.info(cmd)
31 |         logging.info(output)
32 | 
33 | cmd = sys.argv[1]
34 | if cmd == "mount":
35 |     entry = sys.argv[2]
36 |     mkdir('/service/data')
37 |     mount(entry, '/service/data')
38 | elif cmd == "umount":
39 |     umount('/service/data')
40 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hadoop/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /etc/profile
 4 | source /service/sbin/setup
 5 | 
 6 | if [ $1 == "init" ]; then 
 7 |     /service/sbin/init01.sh
 8 | elif [ $1 == "halt" ]; then 
 9 |     /service/sbin/halt01.sh
10 | elif [ $1 == "hosts" ]; then 
11 |     pophosts
12 | elif [ $1 == "start" ]; then 
13 |     # Record the services running on this node.
14 |     echo $2 >> /tmp/hadoop_services.log
15 | 
16 |     if [ $2 == "namenode" ]; then
17 | 	su ferry -c '/service/packages/hadoop/bin/hdfs --config /service/conf/hadoop namenode -format HADOOP_CLUSTER'
18 | 	su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs start namenode'
19 |     elif [ $2 == "datanode" ]; then
20 | 	su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs start datanode'
21 |     elif [ $2 == "yarnmaster" ]; then	
22 | 	su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop start resourcemanager'
23 | 	su ferry -c '/service/packages/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver --config /service/conf/hadoop'
24 |     elif [ $2 == "yarnslave" ]; then	
25 | 	su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop start nodemanager'
26 |     elif [ $2 == "gluster" ]; then 
27 | 	python /service/scripts/mounthelper.py umount
28 | 	python /service/scripts/mounthelper.py mount $3	
29 | 	chown -R ferry:docker /service/data
30 |     fi
31 | elif [ $1 == "restart" ]; then 
32 |     pophosts
33 |     if [ $2 == "namenode" ]; then
34 | 	su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs start namenode'
35 |     elif [ $2 == "datanode" ]; then
36 | 	su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs start datanode'
37 |     elif [ $2 == "yarnmaster" ]; then	
38 | 	su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop start resourcemanager'
39 | 	su ferry -c '/service/packages/hadoop/sbin/mr-jobhistory-daemon.sh stop historyserver --config /service/conf/hadoop'
40 |     elif [ $2 == "yarnslave" ]; then	
41 | 	su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop start nodemanager'
42 |     elif [ $2 == "gluster" ]; then 
43 | 	python /service/scripts/mounthelper.py umount
44 | 	python /service/scripts/mounthelper.py mount $3	
45 | 	chown -R ferry:docker /service/data
46 |     fi
47 | elif [ $1 == "stop" ]; then 
48 |     input=/tmp/hadoop_services.log
49 |     while read line
50 |     do
51 | 	if [ "$line" == "namenode" ]; then
52 | 	    su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs stop namenode'
53 | 	elif [ "$line" == "datanode" ]; then
54 | 	    su ferry -c '/service/packages/hadoop/sbin/hadoop-daemon.sh --config /service/conf/hadoop --script hdfs stop datanode'
55 | 	elif [ "$line" == "yarnmaster" ]; then	
56 | 	    su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop stop resourcemanager'
57 | 	elif [ "$line" == "yarnslave" ]; then	
58 | 	    su ferry -c '/service/packages/hadoop/sbin/yarn-daemon.sh --config /service/conf/hadoop stop nodemanager'
59 | 	elif [ "$line" == "gluster" ]; then 
60 | 	    python /service/scripts/mounthelper.py umount
61 | 	fi
62 |     done < "$input"
63 | fi
64 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hive-metastore/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/hadoop-base
 2 | NAME hive-metastore
 3 | NAME hive
 4 | 
 5 | # Add the control script to the image. 
 6 | ADD ./startnode /service/sbin/
 7 | ADD ./createstore.sql /service/sbin/
 8 | ADD ./hive-schema.sql /service/sbin/
 9 | RUN chmod a+x /service/sbin/startnode
10 | 
11 | # Download some dependencies
12 | RUN apt-get --yes install language-pack-en-base postgresql
13 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hive-metastore/createstore.sql:
--------------------------------------------------------------------------------
1 | CREATE USER hiveuser WITH PASSWORD '1hiveuser2';
2 | CREATE DATABASE metastore;
3 | \c metastore
4 | \i /service/sbin/hive-schema.sql
5 | CREATE DATABASE hive_stats;
6 | GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO hiveuser;


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/hive-metastore/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /etc/profile
 4 | source /service/sbin/setup
 5 | 
 6 | if [ $1 == "init" ]; then 
 7 |     /service/sbin/init01.sh
 8 | elif [ $1 == "halt" ]; then 
 9 |     /service/sbin/halt01.sh    
10 | elif [ $1 == "hosts" ]; then 
11 |     pophosts
12 | elif [ $1 == "start" ]; then 
13 |     if [ $2 == "metastore" ]; then
14 |         # Start the PostGres service
15 | 	service postgresql start
16 | 
17 |         # Now create the Hive metastore
18 | 	su postgres -c 'psql -f /service/sbin/createstore.sql'
19 | 
20 |         # Sstart the metastore service and Hiveserver2.
21 | 	su ferry -c 'nohup /service/packages/hive/bin/hive --service metastore > /service/data/logs/metastore.out 2> /service/data/logs/metastore.err &'
22 | 	su ferry -c 'nohup /service/packages/hive/bin/hive --service hiveserver2 > /service/data/logs/hive.out 2> /service/data/logs/hive.err &'
23 | 
24 | 	# Give everything a couple seconds to start
25 | 	sleep 3
26 |     fi
27 | elif [ $1 == "restart" ]; then 
28 |     if [ $2 == "metastore" ]; then
29 |         # Start the PostGres service
30 | 	service postgresql start
31 | 
32 |         # Start the metastore service. Give it a couple seconds to start 
33 | 	# before stopping the script otherwise sometimes it doesn't start. 
34 | 	su ferry -c 'nohup /service/packages/hive/bin/hive --service metastore > /service/data/logs/metastore.out 2> /service/data/logs/metastore.err &'
35 | 	su ferry -c 'nohup /service/packages/hive/bin/hive --service hiveserver2 > /service/data/logs/hive.out 2> /service/data/logs/hive.err &'
36 | 	sleep 3
37 |     fi
38 | elif [ $1 == "stop" ]; then 
39 |     # Stop Hive and PostGres
40 |     pkill -f hive 
41 |     service postgresql stop
42 | fi
43 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/mongodb/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/ferry-base
 2 | NAME mongodb
 3 | NAME mongodb-client
 4 | 
 5 | # Add the 10Gen key and repo
 6 | RUN apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10;echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" >> /etc/apt/sources.list
 7 | 
 8 | # Update the repository for latest packages
 9 | RUN apt-get update;apt-get --yes install mongodb-org python-pip python-dev build-essential
10 | 
11 | # To download pymongo
12 | RUN pip install --upgrade virtualenv;pip install pymongo
13 | 
14 | # Create default data directory
15 | RUN mkdir -p /service/com /service/sbin /service/data /service/logs /service/conf/mongodb
16 | RUN chown -R ferry:docker /service
17 | 
18 | # Export data directory path
19 | ENV MONGO_DATA /service/data
20 | ENV MONGO_LOG /service/logs/mongodb.log
21 | RUN echo export MONGO_DATA=/service/data >> /etc/profile;echo export MONGO_LOG=/service/logs/mongodb.log >> /etc/profile
22 | 
23 | # Used to address Mongo bug
24 | ENV LC_ALL C
25 | RUN echo export LC_ALL=C >> /etc/profile
26 | 
27 | # Add the start script
28 | ADD ./createadmin.js /service/sbin/
29 | ADD ./startnode /service/sbin/
30 | RUN chmod a+x /service/sbin/startnode
31 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/mongodb/createadmin.js:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Create a new MongoDB adminstrator account. This script should be run
 3 |  * using the localhost exception. Once the adminstrator account is created
 4 |  * all future connections will be authenticated. 
 5 |  **/
 6 | 
 7 | db.createUser(
 8 |     {
 9 | 	user: "LOGIN",
10 | 	pwd: "PASS",
11 | 	roles: [ "root" ]
12 |     }
13 | )


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/mongodb/startnode:
--------------------------------------------------------------------------------
  1 | #! /bin/bash
  2 | 
  3 | source /service/sbin/setup
  4 | 
  5 | #
  6 | # Read in the login information from the 
  7 | # stored files. Stores these in global vars. 
  8 | #
  9 | LOGIN=''
 10 | PASS=''
 11 | function read_login {
 12 |     linput=/service/conf/mongodb/login
 13 |     while read line
 14 |     do
 15 | 	LOGIN=$line
 16 |     done < "$linput"
 17 |     pinput=/service/conf/mongodb/pass
 18 |     while read line
 19 |     do
 20 | 	PASS=$line
 21 |     done < "$pinput"
 22 | }
 23 | 
 24 | #
 25 | # Save the login information in stored files. 
 26 | #
 27 | function save_login {
 28 |     echo $LOGIN > /service/conf/mongodb/login
 29 |     echo $PASS > /service/conf/mongodb/pass
 30 | }
 31 | 
 32 | #
 33 | # Generate a new random login and password. 
 34 | #
 35 | function generate_login {
 36 |     LOGIN=$(cat /proc/sys/kernel/random/uuid)
 37 |     PASS=$(cat /proc/sys/kernel/random/uuid)
 38 | }
 39 | 
 40 | #
 41 | # Record that we performed a login event. 
 42 | #
 43 | function record_login {
 44 |     touch /tmp/login
 45 | }
 46 | 
 47 | #
 48 | # Start the Mongo server. 
 49 | #
 50 | function start_mongo {
 51 |     # Change the file ownership back to ferry user. This is 
 52 |     # necessary since the host may not have the ferry user. 
 53 |     chown -R ferry:ferry /service/data
 54 |     chown -R ferry:ferry /service/logs
 55 | 
 56 |     # Check if there was an unclean shutdown. If so, remove
 57 |     # the Mongo lock, and run repair. 
 58 |     if [[ -e /service/data/mongod.lock ]]; then
 59 | 	rm /service/data/mongod.lock
 60 |     fi
 61 | 
 62 |     # Start the mongo server. MongoDB can be started in either
 63 |     # trusted or non-trusted modes. Trusted means that authentication
 64 |     # is turned off (should only be used by the installer). 
 65 |     if [[ $1  == "trust" ]]; then
 66 | 	su ferry -c 'source /etc/profile;mongod --config /service/conf/mongodb/trusted.conf' >> /service/logs/start.log 2>> /service/logs/err.log
 67 |     else
 68 | 	su ferry -c 'source /etc/profile;mongod --config /service/conf/mongodb/mongodb.conf' >> /service/logs/start.log 2>> /service/logs/err.log
 69 |     fi
 70 |     sleep 2
 71 | }
 72 | 
 73 | if [[ $1 == "init" ]]; then 
 74 |     # Generate a unique login/password
 75 |     # for the admin account and save it.
 76 |     generate_login
 77 |     save_login
 78 |     record_login
 79 | 
 80 |     # Now finish the init process
 81 |     /service/sbin/init01.sh
 82 | elif [[ $1 == "halt" ]]; then 
 83 |     /service/sbin/halt01.sh
 84 | elif [[ $1 == "login" ]]; then 
 85 |     # Print out login info. 
 86 |     read_login
 87 |     echo -e "{\"user\":\"${LOGIN}\",\"pass\":\"${PASS}\"}"
 88 | elif [[ $1 == "start" ]]; then 
 89 |     start_mongo $2
 90 | 
 91 |     # Check if the adminstrator account has already
 92 |     # been created. If not, we need to create one. 
 93 |     if [[ $2 == "notrust" ]] && [[ -e /tmp/login ]]; then
 94 | 	read_login
 95 | 	sed -i "s/LOGIN/$LOGIN/g" /service/sbin/createadmin.js
 96 | 	sed -i "s/PASS/$PASS/g" /service/sbin/createadmin.js
 97 | 	su ferry -c 'mongo localhost:27017/admin /service/sbin/createadmin.js'
 98 | 	rm /tmp/login
 99 |     fi
100 | elif [[ $1 == "restart" ]]; then 
101 |     start_mongo $2
102 | elif [[ $1 == "stop" ]]; then 
103 |     # Shutdown MongoDB. We have to use the right
104 |     # configuration file so that we know which to shutdown. 
105 |     if [[ -e /service/conf/mongodb/trusted.conf ]]; then
106 | 	su ferry -c 'mongod --config /service/conf/mongodb/trusted.conf --shutdown'
107 |     else
108 | 	su ferry -c 'mongod --config /service/conf/mongodb/mongodb.conf --shutdown'
109 |     fi
110 | fi
111 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi-client/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/openmpi
 2 | NAME openmpi-client
 3 | 
 4 | # Add the control script to the image. 
 5 | ADD ./helloworld.cpp /service/examples/
 6 | ADD ./test01.sh /service/runscripts/test/
 7 | RUN chmod a+x /service/runscripts/test/*
 8 | 
 9 | # Clear the apt cache. 
10 | RUN rm -rf /var/cache/apt/archives/*
11 | RUN rm -rf /var/lib/apt/lists/*
12 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi-client/helloworld.cpp:
--------------------------------------------------------------------------------
 1 | #include <mpi.h>
 2 | 
 3 | int main(int argc, char **argv)
 4 | {
 5 |   int numprocs, rank, namelen;
 6 |   char proc_name[MPI_MAX_PROCESSOR_NAME];
 7 | 
 8 |   MPI_Init(&argc, &argv);
 9 |   MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
10 |   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
11 |   MPI_Get_processor_name(proc_name, &namelen);
12 | 
13 |   if(rank == 0) {
14 |     std::cout << "Processor name: " << proc_name << "\n";
15 |     std::cout << "master (" << rank << "/" << numprocs << ")\n";
16 |   }
17 |   else {
18 |     std::cout << "slave (" << rank << "/" << numprocs << ")\n";
19 |   }
20 | 
21 |   MPI_Finalize();
22 | }
23 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi-client/test01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /etc/profile
 4 | 
 5 | # Bash colors
 6 | GREEN='\e[0;32m'
 7 | NC='\e[0m'
 8 | 
 9 | # Commands
10 | MKBINS='mkdir /service/data/binaries'
11 | MKOUT='mkdir /service/data/outputs'
12 | COMPILE='mpic++ -W -Wall /service/examples/helloworld.cpp -o /service/data/binaries/helloworld.o'
13 | RUN='mpirun -np 4 --hostfile $MPI_CONF/hosts /service/data/binaries/helloworld.o >> /service/data/outputs/mpitest.out'
14 | 
15 | function run_as_ferry {
16 |     echo -e "${GREEN} ${2} ${NC}"
17 |     if [ $USER == "root" ]; then
18 | 	su ferry -c "$1"
19 |     else
20 | 	$1
21 |     fi
22 | }
23 | 
24 | run_as_ferry "$MKBINS" "Creating binary directory"
25 | run_as_ferry "$MKOUT" "Creating output directory"
26 | run_as_ferry "$COMPILE" "Compiling MPI application"
27 | run_as_ferry "$RUN" "Running MPI application"
28 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/gluster
 2 | NAME openmpi
 3 | 
 4 | RUN mkdir -p /service/examples /service/packages /service/runscripts/start /service/runscripts/restart /service/runscripts/stop /service/runscripts/test
 5 | 
 6 | # Download the various packages needed
 7 | RUN apt-get --yes install autotools-dev autoconf automake g++ gfortran build-essential libevent-dev 
 8 | 
 9 | # Add the package
10 | RUN wget $DOWNLOAD_URL/openmpi-1.8.1-bin.tar.gz -P /service/packages
11 | RUN cd /service/packages;tar -xzf openmpi-1.8.1-bin.tar.gz
12 | RUN cd /service/packages/openmpi-1.8.1;make install
13 | RUN ldconfig
14 | 
15 | # Get rid of the tarballs.
16 | RUN rm /service/packages/*.tar.gz
17 | 
18 | # Make symlinks
19 | RUN ln -s /service/packages/openmpi-1.8.1 /service/packages/openmpi
20 | RUN ln -s /usr/local/etc /service/conf/openmpi
21 | 
22 | # Add the control script to the image. 
23 | ADD ./startnode /service/sbin/
24 | ADD ./utils /service/sbin/
25 | ADD ./mounthelper.py /service/sbin/
26 | ADD ./start01.sh /service/runscripts/start/
27 | ADD ./stop10.sh /service/runscripts/stop/
28 | ADD ./restart01.sh /service/runscripts/restart/
29 | RUN chmod a+x /service/sbin/startnode;chmod a+x /service/sbin/utils;chmod a+x /service/runscripts/start/*;chmod a+x /service/runscripts/stop/*;chmod a+x /service/runscripts/restart/*
30 | 
31 | # Make environment variables
32 | RUN echo export MPI_HOME=/service/packages/openmpi >> /etc/profile
33 | RUN echo export MPI_CONF=/service/conf/openmpi >> /etc/profile
34 | 
35 | # Generate an ssh key for this image.
36 | RUN mkdir /home/ferry/.ssh;ssh-keygen -f /home/ferry/.ssh/id_rsa -t rsa -N '' > /dev/null
37 | RUN cat /home/ferry/.ssh/id_rsa.pub >> /home/ferry/.ssh/authorized_keys;echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config;touch /etc/mtab
38 | RUN chown -R ferry:ferry /home/ferry/.ssh
39 | 
40 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/mounthelper.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import os
 3 | import sys
 4 | import logging
 5 | from subprocess import Popen, PIPE
 6 | 
 7 | def mount(entry_point, mount_point):
 8 |     # Check if the mount point exists. If not
 9 |     # go ahead and create it. 
10 |     cmd = 'mount -t glusterfs %s %s' % (entry_point,
11 |                                         mount_point)
12 |     proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
13 |     output = proc.stdout.read()
14 |     err = proc.stderr.read()
15 |     print output
16 |     print err
17 | 
18 | def umount(mount_point):
19 |     cmd = 'cat /etc/mtab | grep /service/data | awk \'{print $2}\''
20 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
21 |     if output.strip() != "":
22 |         cmd = 'umount %s' % mount_point
23 |         Popen(cmd, shell=True)
24 | 
25 | cmd = sys.argv[1]
26 | if cmd == "mount":
27 |     entry = sys.argv[2]
28 |     mount(entry, '/service/data')
29 | elif cmd == "umount":
30 |     umount('/service/data')
31 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/restart01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # 
 4 | # Basic restart script for Open MPI clients. 
 5 | # This script is executed whenever the container restarts.
 6 | # 
 7 | 
 8 | source /etc/profile
 9 | source /service/sbin/utils
10 | 
11 | #
12 | # Remount the data directory. 
13 | #
14 | remount $1 $2
15 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/start01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # 
 4 | # Basic start script for Open MPI clients. 
 5 | # This script is executed whenever the container starts.
 6 | # 
 7 | 
 8 | source /etc/profile
 9 | source /service/sbin/utils
10 | 
11 | #
12 | # Remount the data directory. 
13 | #
14 | remount $1 $2
15 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | else
12 |     for f in /service/runscripts/$1/*
13 |     do
14 | 	$f $2 $3
15 |     done
16 | fi
17 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/stop10.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | # Unmount & stop the ssh daemon. 
4 | python /service/sbin/mounthelper.py umount
5 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/openmpi/utils:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | function remount {
 4 |     # We want to make sure that this is a valid Gluster
 5 |     # command. Otherwise we may start unmounting for no good reason. 
 6 |     if [[ $2 == "glustermaster" ]] || [[ $2 == "glusterslave" ]]; then
 7 | 	# First we should unmount previous filesystems. This might happen
 8 | 	# if we started a snapshotted image.
 9 | 	python /service/sbin/mounthelper.py umount >> /tmp/gluster.out 2>> /tmp/gluster.err
10 | 
11 | 	# Mount GlusterFS and set the owner as 'ferry' user. 
12 | 	python /service/sbin/mounthelper.py mount $1 >> /tmp/gluster.out 2>> /tmp/gluster.err
13 | 	chown -R ferry:docker /service/data
14 | 
15 | 	# Make a copy of the communication keys to a
16 | 	# shared directory.
17 | 	if [[ $2 == "glustermaster" ]]; then
18 | 	    if [[ ! -e /service/data/.comkeys ]]; then
19 | 		mkdir /service/data/.comkeys
20 | 	    fi
21 | 	    cp /home/ferry/.ssh/id_rsa* /service/data/.comkeys/
22 | 	fi    
23 |     fi
24 | }


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/hadoop-base
 2 | NAME spark
 3 | NAME spark-client
 4 | 
 5 | # Make the various directories needed
 6 | RUN mkdir -p /service/runscripts/start /service/runscripts/restart /service/runscripts/test /service/runscripts/stop
 7 | 
 8 | # Install everything. Spark needs a newer version of Numpy, so we'll need to
 9 | # use `pip` to install it. 
10 | RUN apt-get update
11 | RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install build-essential git scala libgfortran3 jblas python-pip python-dev
12 | RUN pip install numpy
13 | 
14 | # Add the binaries
15 | RUN wget $DOWNLOAD_URL/spark-1.1.0-bin-hadoop2.4.tgz -P /service/packages/ 
16 | # RUN wget $DOWNLOAD_URL/spark-assembly_2.10-0.9.1-hadoop2.3.0.jar -P /service/packages/
17 | RUN cd /service/packages; tar -xzf spark-1.1.0-bin-hadoop2.4.tgz
18 | 
19 | # Make some symlinks
20 | RUN ln -s /service/packages/spark-1.1.0-bin-hadoop2.4 /service/packages/spark
21 | RUN ln -s /service/packages/spark/conf /service/conf/spark
22 | 
23 | # Add all the scripts. 
24 | ADD ./startnode /service/sbin/
25 | ADD ./start01.sh /service/runscripts/start/
26 | ADD ./stop10.sh /service/runscripts/stop/
27 | ADD ./restart01.sh /service/runscripts/restart/
28 | ADD ./test01.sh /service/runscripts/test/
29 | RUN chmod a+x /service/sbin/startnode;chmod a+x /service/runscripts/start/*;chmod a+x /service/runscripts/stop/*;chmod a+x /service/runscripts/restart/*;chmod a+x /service/runscripts/test/*
30 | 
31 | # Example scripts
32 | RUN mkdir -p /service/examples/python
33 | RUN mkdir -p /service/examples/data
34 | ADD ./classification.py /service/examples/python/
35 | ADD ./clustering.py /service/examples/python/
36 | ADD ./filtering.py /service/examples/python/
37 | ADD ./regression.py /service/examples/python/
38 | ADD ./kmeans_test.data /service/examples/data/
39 | ADD ./lpsa_test.data /service/examples/data/
40 | ADD ./svm_test.data /service/examples/data/
41 | ADD ./als_test.data /service/examples/data/
42 | ADD ./tree_test.data /service/examples/data/
43 | 
44 | # Environment variables
45 | ENV SPARK_HOME /service/packages/spark
46 | RUN echo export SPARK_HOME=/service/packages/spark >> /etc/profile
47 | # ENV SPARK_JAR /service/packages/spark-assembly_2.10-0.9.1-hadoop2.3.0.jar
48 | ENV SPARK_JAR /service/packages/spark/lib/spark-assembly-1.1.0-hadoop2.4.0.jar
49 | # RUN echo export SPARK_JAR=/service/packages/spark-assembly_2.10-0.9.1-hadoop2.3.0.jar >> /etc/profile
50 | RUN echo export SPARK_JAR=/service/packages/spark/lib/spark-assembly-1.1.0-hadoop2.4.0.jar >> /etc/profile
51 | RUN echo export PATH=$$PATH:$$SPARK_HOME/sbin:$$SPARK_HOME/bin >> /etc/profile
52 | ENV SPARK_YARN_APP_JAR /service/packages/spark/examples/target/spark-examples_2.10-0.9.1-sources.jar
53 | RUN echo export SPARK_YARN_APP_JAR=/service/packages/spark/examples/target/spark-examples_2.10-0.9.1-sources.jar >> /etc/profile
54 | 
55 | # Generate an ssh key for this image.
56 | RUN mkdir /home/ferry/.ssh;ssh-keygen -f /home/ferry/.ssh/id_rsa -t rsa -N '' > /dev/null
57 | RUN cat /home/ferry/.ssh/id_rsa.pub >> /home/ferry/.ssh/authorized_keys;echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config;touch /etc/mtab
58 | RUN chown -R ferry:ferry /home/ferry/.ssh
59 | 
60 | # Remove any intermediate packages
61 | RUN rm /service/packages/*.tgz
62 | RUN rm -rf /var/cache/apt/archives/*
63 | RUN rm -rf /var/lib/apt/lists/*


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/als_test.data:
--------------------------------------------------------------------------------
 1 | 1,1,5.0
 2 | 1,2,1.0
 3 | 1,3,5.0
 4 | 1,4,1.0
 5 | 2,1,5.0
 6 | 2,2,1.0
 7 | 2,3,5.0
 8 | 2,4,1.0
 9 | 3,1,1.0
10 | 3,2,5.0
11 | 3,3,1.0
12 | 3,4,5.0
13 | 4,1,1.0
14 | 4,2,5.0
15 | 4,3,1.0
16 | 4,4,5.0
17 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/classification.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import os.path
 3 | import sys
 4 | from pyspark import SparkContext
 5 | from pyspark.mllib.classification import LogisticRegressionWithSGD
 6 | from pyspark.mllib.regression import LabeledPoint
 7 | from numpy import array
 8 | 
 9 | if __name__ == "__main__":
10 |     data_file = '/spark/data/svm.data'
11 | 
12 |     if len(sys.argv) == 1:
13 |         print >> sys.stderr, "Usage: classification.py <master>"
14 |         exit(-1)
15 |     else:
16 |         # Load and parse the data
17 |         sc = SparkContext(sys.argv[1], "Binary Classification")
18 |         data = sc.textFile(data_file)
19 |         parsedData = data.map(lambda line: array([float(x) for x in line.split(' ')]))
20 |         model = LogisticRegressionWithSGD.train(parsedData)
21 | 
22 |         # Build the model and evaluate on training data. 
23 |         labelsAndPreds = parsedData.map(lambda p: (p.label, model.predict(p.features)))
24 |         trainErr = labelsAndPreds.filter(lambda (v, p): v != p).count() / float(parsedData.count())
25 |         print("Training Error = " + str(trainErr))
26 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/clustering.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import os.path
 3 | import sys
 4 | from pyspark import SparkContext
 5 | from pyspark.mllib.clustering import KMeans
 6 | from numpy import array
 7 | from math import sqrt
 8 | 
 9 | # Evaluate clustering by computing Within Set Sum of Squared Errors
10 | def error(point):
11 |     center = clusters.centers[clusters.predict(point)]
12 |     return sqrt(sum([x**2 for x in (point - center)]))
13 | 
14 | if __name__ == "__main__":
15 |     data_file = '/spark/data/kmeans.data'
16 | 
17 |     if len(sys.argv) == 1:
18 |         print >> sys.stderr, "Usage: filtering.py <master>"
19 |         exit(-1)
20 |     else:
21 |         # Load and parse the data
22 |         sc = SparkContext(sys.argv[1], "KMeans Clustering")
23 |         data = sc.textFile(data_file)
24 |         parsedData = data.map(lambda line: array([float(x) for x in line.split(' ')]))
25 | 
26 |         # Build the model (cluster the data)
27 |         clusters = KMeans.train(parsedData, 2, maxIterations=10,
28 |                                 runs=30, initialization_mode="random")
29 | 
30 | 
31 |         WSSSE = parsedData.map(lambda point: error(point)).reduce(lambda x, y: x + y)
32 |         print("Within Set Sum of Squared Error = " + str(WSSSE))
33 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/filtering.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import os.path
 3 | import sys
 4 | from pyspark import SparkContext
 5 | from pyspark.mllib.recommendation import ALS
 6 | from numpy import array
 7 | 
 8 | if __name__ == "__main__":
 9 |     data_file = '/spark/data/als.data'
10 | 
11 |     if len(sys.argv) == 1:
12 |         print >> sys.stderr, "Usage: filtering.py <master>"
13 |         exit(-1)
14 |     else:
15 |         sc = SparkContext(sys.argv[1], "Collaborative Filtering")
16 |         data = sc.textFile(data_file)
17 |         ratings = data.map(lambda line: array([float(x) for x in line.split(',')]))
18 | 
19 |         # Build the recommendation model using Alternating Least Squares
20 |         model = ALS.train(ratings, 1, 20)
21 | 
22 |         # Evaluate the model on training data
23 |         testdata = ratings.map(lambda p: (int(p[0]), int(p[1])))
24 |         predictions = model.predictAll(testdata).map(lambda r: ((r[0], r[1]), r[2]))
25 |         ratesAndPreds = ratings.map(lambda r: ((r[0], r[1]), r[2])).join(predictions)
26 |         MSE = ratesAndPreds.map(lambda r: (r[1][0] - r[1][1])**2).reduce(lambda x, y: x + y)/ratesAndPreds.count()
27 |         print("Mean Squared Error = " + str(MSE))
28 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/kmeans_test.data:
--------------------------------------------------------------------------------
1 | 0.0 0.0 0.0
2 | 0.1 0.1 0.1
3 | 0.2 0.2 0.2
4 | 9.0 9.0 9.0
5 | 9.1 9.1 9.1
6 | 9.2 9.2 9.2
7 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/mounthelper.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import os
 3 | import sys
 4 | import logging
 5 | from subprocess import Popen, PIPE
 6 | 
 7 | def mkdir(directory):
 8 |     if not os.path.isdir(directory):
 9 |         cmd = 'mkdir -p %s' % directory
10 |         output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
11 |         logging.info(cmd)
12 |         logging.info(output)
13 | 
14 | def mount(entry_point, mount_point):
15 |     # Check if the mount point exists. If not
16 |     # go ahead and create it. 
17 |     # mount -t glusterfs entry_point mount_point
18 |     cmd = 'mount -t glusterfs %s %s' % (entry_point,
19 |                                         mount_point)
20 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
21 |     logging.info(cmd)
22 |     logging.info(output)
23 |     print output
24 | 
25 | def umount(mount_point):
26 |     cmd = 'umount %s' % mount_point
27 |     output = Popen(cmd, stdout=PIPE, shell=True).stdout.read()
28 |     logging.info(cmd)
29 |     logging.info(output)
30 | 
31 | cmd = sys.argv[1]
32 | if cmd == "mount":
33 |     entry = sys.argv[2]
34 |     mkdir('/service/data')
35 |     mount(entry, '/service/data')
36 | elif cmd == "umount":
37 |     umount('/service/data')
38 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/regression.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import os.path
 3 | import sys
 4 | from pyspark import SparkContext
 5 | from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD
 6 | from numpy import array
 7 | 
 8 | # Load and parse the data
 9 | def parsePoint(line):
10 |     values = [float(x) for x in line.replace(',', ' ').split(' ')]
11 |     return LabeledPoint(values[0], values[1:])
12 | 
13 | if __name__ == "__main__":
14 |     data_file = '/spark/data/lpsa.data'
15 | 
16 |     if len(sys.argv) == 1:
17 |         print >> sys.stderr, "Usage: regression.py <master>"
18 |         exit(-1)
19 |     else:
20 |         # Load and parse the data
21 |         sc = SparkContext(sys.argv[1], "Logistic Regression")
22 |         data = sc.textFile(data_file)
23 |         parsedData = data.map(parsePoint)
24 | 
25 |         # Build the model
26 |         model = LinearRegressionWithSGD.train(parsedData)
27 | 
28 |         # Evaluate the model on training data
29 |         valuesAndPreds = parsedData.map(lambda p: (p.label, 
30 |                                                    model.predict(p.features)))
31 |         MSE = valuesAndPreds.map(lambda (v, p): (v - p)**2).reduce(lambda x, y: x + y) / valuesAndPreds.count()
32 |         print("Mean Squared Error = " + str(MSE))
33 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/restart01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Make sure we get the env variables in place. 
 4 | source /etc/profile
 5 | 
 6 | if [ $1 == "master" ]; then
 7 |     su ferry -c '$SPARK_HOME/sbin/start-master.sh' >> /tmp/spark.log 2>> /tmp/spark.err
 8 |     su ferry -c '$SPARK_HOME/sbin/start-slaves.sh' >> /tmp/spark.log 2>> /tmp/spark.err
 9 | fi
10 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/simpleapp.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import os.path
 3 | from pyspark import SparkContext
 4 | 
 5 | if 'BACKEND_COMPUTE_MASTER' in os.environ:
 6 |     master = os.environ['BACKEND_COMPUTE_MASTER']
 7 | else:
 8 |     master = 'localhost'
 9 | 
10 | logFile = os.path.join("/tmp/data/README.md")
11 | 
12 | from pyspark import SparkConf, SparkContext
13 | conf = SparkConf()
14 | conf.setMaster('spark://' + master + ':7077')
15 | conf.setAppName("simpleapp")
16 | sc = SparkContext(conf = conf)
17 | logData = sc.textFile(logFile).cache()
18 | 
19 | numAs = logData.filter(lambda s: 'a' in s).count()
20 | numBs = logData.filter(lambda s: 'b' in s).count()
21 | 
22 | print "Lines with a: %i, lines with b: %i" % (numAs, numBs)
23 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/start01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Make sure we get the env variables in place. 
 4 | source /etc/profile
 5 | 
 6 | if [ $1 == "master" ]; then
 7 |     if [[ ! -f /tmp/spark_services.log ]]; then
 8 | 	su ferry -c '$SPARK_HOME/sbin/start-master.sh' >> /tmp/spark.log 2>> /tmp/spark.err
 9 | 	su ferry -c '$SPARK_HOME/sbin/start-slaves.sh' >> /tmp/spark.log 2>> /tmp/spark.err
10 |     fi
11 | 
12 |     # Record the Spark service. 
13 |     echo $1 > /tmp/spark_services.log
14 | fi
15 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | else
12 |     for f in /service/runscripts/$1/*
13 |     do
14 | 	$f $2 $3
15 |     done
16 | fi
17 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/stop10.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | # Make sure we get the env variables in place. 
4 | source /etc/profile
5 | 
6 | # Stop the Spark daemons
7 | su ferry -c '$SPARK_HOME/sbin/stop-all.sh'
8 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/spark/test01.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | #
 4 | # Run some simple Spark examples.
 5 | # Examples can be found in the following directories:
 6 | # - /service/examples/python
 7 | # - $SPARK_HOME/examples
 8 | # 
 9 | 
10 | # Bash colors
11 | GREEN='\e[0;32m'
12 | NC='\e[0m'
13 | 
14 | function run_as_ferry {
15 |     echo -e "${GREEN} ${2} ${NC}"
16 |     if [ $USER == "root" ]; then
17 | 	su ferry -c "$1"
18 |     else
19 | 	$1
20 |     fi
21 | }
22 | 
23 | source /etc/profile
24 | if [ $1 == "python" ]; then
25 |     if [ $# == 2 ]; then
26 | 	run_as_ferry "$SPARK_HOME/bin/spark-submit /service/examples/python/$2 spark://$BACKEND_COMPUTE_MASTER:7077" "Running $2"
27 |     fi
28 | elif [ $1 == "load" ]; then
29 |     MKDIR='hdfs dfs -mkdir -p /spark/data'
30 |     COPY1='hdfs dfs -copyFromLocal /service/examples/data/als_test.data /spark/data/als.data'
31 |     COPY2='hdfs dfs -copyFromLocal /service/examples/data/svm_test.data /spark/data/svm.data'
32 |     COPY3='hdfs dfs -copyFromLocal /service/examples/data/lpsa_test.data /spark/data/lpsa.data'
33 |     COPY4='hdfs dfs -copyFromLocal /service/examples/data/kmeans_test.data /spark/data/kmeans.data'
34 | 
35 |     run_as_ferry "$MKDIR" "Making Spark data directories"
36 |     run_as_ferry "$COPY1" "Upload dataset 1"
37 |     run_as_ferry "$COPY2" "Upload dataset 2"
38 |     run_as_ferry "$COPY3" "Upload dataset 3"
39 |     run_as_ferry "$COPY4" "Upload dataset 4"
40 | elif [ $1 == "example" ]; then
41 |     run_as_ferry '$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://$BACKEND_COMPUTE_MASTER:7077 $SPARK_HOME/lib/spark-examples-1.1.0-hadoop2.4.0.jar'
42 | fi
43 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/titan/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM $USER/cassandra
 2 | NAME titan
 3 | 
 4 | # Download the various packages needed
 5 | RUN apt-get --yes install unzip
 6 | 
 7 | # Create necessary directories
 8 | RUN mkdir -p /service/packages /service/conf/titan /service/keys /service/sbin /service/bin /service/scripts /service/ext
 9 | 
10 | # Download the latest package
11 | RUN wget $DOWNLOAD_URL/titan-all-0.3.1.zip -P /service/packages
12 | RUN wget $DOWNLOAD_URL/rexster-server-2.3.0.zip -P /service/packages
13 | RUN cd /service/packages/;unzip titan-all-0.3.1.zip;unzip rexster-server-2.3.0.zip
14 | 
15 | # Get rid of the tarballs. 
16 | RUN rm /service/packages/*zip
17 | 
18 | # Make soft links
19 | RUN ln -s /service/packages/titan-all-0.3.1 /service/packages/titan;ln -s /service/packages/rexster-server-2.3.0 /service/packages/rexster-server;ln -s /service/packages/titan-all-0.3.1/lib /service/packages/rexster-server-2.3.0/ext/titan
20 | 
21 | # Soft links to make life easier
22 | RUN ln -s /service/packages/rexster-server-2.3.0/bin/* /usr/local/bin/
23 | RUN ln -s /service/packages/titan-all-0.3.1/bin/* /usr/local/bin/
24 | 
25 | # Remember the various home directories
26 | ENV REXSTER_HOME /service/packages/rexster-server-2.3.0
27 | ENV TITAN_HOME /service/packages/titan-all-0.3.1
28 | RUN echo export REXSTER_HOME=/service/packages/rexster-server-2.3.0 >> /etc/profile;echo export TITAN_HOME=/service/packages/titan-all-0.3.1 >> /etc/profile;echo export CLASSPATH=/service/packages/titan/lib/*: >> /etc/profile
29 | 
30 | # Add the control script to the image. 
31 | ADD ./startnode /service/sbin/
32 | RUN chmod a+x /service/sbin/startnode
33 | 
34 | # Run the ssh daemon in the foreground (Docker does
35 | # like daemons running in background)
36 | CMD ["/service/sbin/startnode", "init"]
37 | 


--------------------------------------------------------------------------------
/ferry/data/dockerfiles/titan/startnode:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | source /service/sbin/setup
 4 | 
 5 | if [ $1 == "init" ]; then 
 6 |     /service/sbin/init01.sh
 7 | elif [ $1 == "halt" ]; then 
 8 |     /service/sbin/halt01.sh    
 9 | elif [ $1 == "hosts" ]; then 
10 |     pophosts
11 | elif [ $1 == "start" ]; then 
12 |     su ferry -c 'nohup /service/packages/rexster-server/bin/rexster.sh -s -c /service/conf/titan/rexster.xml > /service/data/logs/rexster.log 2> /service/data/logs/rexster.err &'
13 | elif [ $1 == "restart" ]; then 
14 |     su ferry -c 'nohup /service/packages/rexster-server/bin/rexster.sh -s -c /service/conf/titan/rexster.xml > /service/data/logs/rexster.log 2> /service/data/logs/rexster.err &'
15 | elif [ $1 == "stop" ]; then 
16 |     su ferry -c 'pkill -f rexster'
17 | fi
18 | 


--------------------------------------------------------------------------------
/ferry/data/key/insecure_ferry_key:
--------------------------------------------------------------------------------
1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCcrMS25hDjZaEh2+QxE82yYIBj1h27/WJu+2ofI+JPpmWmiNgGFQQYrar/Iqs+xDJP9eF2CmSRDYUCjzsIbz0dA0h36CTRpTTIrZdCgcSQAhg3kCgtOtqAnb0TMKzFRg9TBolY4zcNJTr4ZUmoW8T8MSiix9ILng+JbJ7TG4R/mNWjE6WE7qwLmWrj/TONNHzE7TNmfU8xjK+6MHHlQzs7lS5vMKDuwIs6G+bzY6dsUROT7qXDwwTs3FmOclNRob+GfqKzhoN91NtiAujxvtDTt1e2G9UnOqbGF9Xo/kpYXQL8zLmmc4EBuKheO0u3RSbZ5Q1X27G2yHX78k2xUYup jhorey@peel
2 | 


--------------------------------------------------------------------------------
/ferry/data/key/insecure_ferry_key.pem:
--------------------------------------------------------------------------------
 1 | -----BEGIN RSA PRIVATE KEY-----
 2 | MIIEowIBAAKCAQEAnKzEtuYQ42WhIdvkMRPNsmCAY9Ydu/1ibvtqHyPiT6ZlpojY
 3 | BhUEGK2q/yKrPsQyT/XhdgpkkQ2FAo87CG89HQNId+gk0aU0yK2XQoHEkAIYN5Ao
 4 | LTragJ29EzCsxUYPUwaJWOM3DSU6+GVJqFvE/DEoosfSC54PiWye0xuEf5jVoxOl
 5 | hO6sC5lq4/0zjTR8xO0zZn1PMYyvujBx5UM7O5UubzCg7sCLOhvm82OnbFETk+6l
 6 | w8ME7NxZjnJTUaG/hn6is4aDfdTbYgLo8b7Q07dXthvVJzqmxhfV6P5KWF0C/My5
 7 | pnOBAbioXjtLt0Um2eUNV9uxtsh1+/JNsVGLqQIDAQABAoIBAH7/YHktJjVdVO6A
 8 | MolBlzPnG0tR7UpLyvVRICVbzf9oTu9d71WGxYpHg9tJu+ul126PTZGEPR6DCfUY
 9 | Ku7cVHpnBeAEMYwqUQ32AC0tySo+OGkZlieLlEA7HF8oznMeJh/TpZOCokYqjpF4
10 | cOtib9tI8bp5JJ68IbXwcRGINa/C61SKVzBFyTCfHsp8l8psMnQbJumJakzqvvNM
11 | SZAEIJ8QwLqONDqu8HKp/VZG6NkGfPSXEPXO2bzmFZKysqmWqkegkRV3Bkqc2SgB
12 | ODSP5GIFSHTZJ9EDLhB9u91jwnaBrNSXXzy7qjxP4cY8XALeKZNtPxPKqjYa6fxV
13 | ldUSEQECgYEAzyPl4UDYhnSHE0p0sEq+A1nO35MCk5B4ET+2JvWB4JEuLhy9U/OR
14 | hFwJcTXc8BNhosw1URg6tWv/UboiYastxQ7fB/w78UquqAfY1X559Wv11iB7jwwN
15 | UzrvpWgby205+gjjrawI0PUTB7euvxZksySQfGhxwSu8g3hAAEVe+GECgYEAwaGI
16 | +PL+DykXQDIbHXtu4VCRUYYr3uXd5l95xd9qYLA++cCVkdcqZ57b7tEJhc2z+CQN
17 | GMtlie9dMc8J+tpmfF3xl2ts2FPkyKGYO4uiJf/Ep9AhcurSu8aCQoB43Shd0Mk2
18 | hdODbbm9gVr7R3v3mHSI7BYlObTWI0vInRYXuEkCgYEArlbBaUfBtdCny1W0Rphx
19 | urJAyq2Z0ocadzIPZlTl3hNzMb5gnyL2S9eG3vD9plY5q+KB0WI6L23ZM3kc7OU9
20 | HYj5sGfcL6ayv5vzJSVREn2uKG2ocOos4OtOvgGUuy+QR2Bp+Ec+GFjcjndLrWr0
21 | kE5rCTqC25mrDWANRe0dFKECgYB5Mp/mm0oUuTaZcwNv37vWgjSDX8UD/sN3GBnD
22 | lgUiwg6ZC0dTgudC9rdY8zTPsKLOVdpww5cp0E54B8yKXLu7ap32lkIJ4fEY97Su
23 | yMbUnBQ+qdLV+E+xQx5ze7CrVUHrlpZvC6Z/Prx0RDt2Ugz3OzIc+/uO9Db43s2S
24 | LarpKQKBgCwd6BCawRxidwVs/XpC+Wpc0AN/DAtAI9x+bwGzr0pu/jizsjgPgKsw
25 | zApmZLubn9fRURrJIS7vOZdg6yOPm7R1wAGxVpf0tEYI0XXLDPRlX4v13iYgQfk8
26 | j7VQsHb8q10W5g7EDWSvvR95k8US3qaDaeRWA+ZZUrdf69pofGRP
27 | -----END RSA PRIVATE KEY-----
28 | 


--------------------------------------------------------------------------------
/ferry/data/plans/cassandra.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    description: The Apache Cassandra database is the right choice when you need scalability and high availability without compromising performance. Linear scalability and proven fault-tolerance on commodity hardware or cloud infrastructure make it the perfect platform for mission-critical data. Cassandra's data model offers the convenience of column indexes with the performance of log-structured updates, strong support for denormalization and materialized views, and powerful built-in caching.
 5 |    icon: cassandra.png
 6 |    tags: [infrastructure,big data,nosql,cassandra]
 7 |    url: http://cassandra.apache.org
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | questions:
11 |    - question: Number of instances for cassandra
12 |      param: "{instances}"
13 | backend:
14 |    - storage:
15 |         personality: "cassandra"
16 |         instances: "{instances}"
17 |         layers: 
18 |            - "titan"
19 |         args: 
20 |            db: "users"
21 | connectors:
22 |    - personality: "ferry/cassandra-client"
23 |      args: 
24 |         db: "users"


--------------------------------------------------------------------------------
/ferry/data/plans/hadoop.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    description: The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models. It is designed to scale up from single servers to thousands of machines, each offering local computation and storage. Rather than rely on hardware to deliver high-availability, the library itself is designed to detect and handle failures at the application layer, so delivering a highly-available service on top of a cluster of computers, each of which may be prone to failures. 
 5 |    icon: hadoop.png
 6 |    tags: [infrastructure,big data,hadoop]
 7 |    url: http://hadoop.apache.org/
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | questions:
11 |    - question: Number of instances for hadoop
12 |      param: "{instances}"
13 | backend:
14 |    - storage:
15 |         personality: "hadoop"
16 |         instances: "{instances}"
17 |         layers:
18 |            - "hive"
19 | connectors:
20 |    - personality: "ferry/hadoop-client"
21 |      ports: ["19888:19888","8088:8088","50070:50070"]


--------------------------------------------------------------------------------
/ferry/data/plans/mongo.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    description: MongoDB is open-source document database. MongoDB stores JSON-like documents, making it perfect for modern, high-performance web application development.
 5 |    icon: mongodb.png
 6 |    tags: [infrastructure,nosql,mongodb]
 7 |    url: http://www.mongodb.org
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | backend:
11 |    - storage:
12 |         personality: "mongodb"
13 |         instances: 1
14 | connectors:
15 |    - personality: "ferry/mongodb-client"
16 | 


--------------------------------------------------------------------------------
/ferry/data/plans/openmpi.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    icon: openmpi.png
 5 |    description: The Open MPI Project is an open source Message Passing Interface implementation that is developed and maintained by a consortium of academic, research, and industry partners. This stack combines OpenMPI with GlusterFS, an distributed file system capable of scaling to several petabytes and handling thousands of clients.
 6 |    tags: [infrastructure,openmpi,glusterfs]
 7 |    url: http://www.open-mpi.org
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | questions:
11 |    - question: Number of instances for gluster
12 |      param: "{numgluster}"
13 |    - question: Number of instances for openmpi
14 |      param: "{nummpi}"
15 | backend:
16 |    - storage:
17 |         personality: "gluster"
18 |         instances: "{numgluster}"
19 |      compute:
20 |         - personality: "openmpi"
21 |           instances: "{nummpi}"
22 | connectors:
23 |    - personality: "ferry/openmpi-client"
24 |      name: "control-0"


--------------------------------------------------------------------------------
/ferry/data/plans/spark.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    icon: spark.png
 5 |    description: Spark is a fast and powerful engine for processing Hadoop data. It can process data in HDFS, HBase, Cassandra, Hive, and any Hadoop InputFormat. It is designed to perform both general data processing (similar to MapReduce) and new workloads like streaming, interactive queries, and machine learning.
 6 |    tags: [infrastructure,big data,hadoop,spark]
 7 |    url: https://spark.apache.org
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | questions:
11 |    - question: Number of instances for hadoop
12 |      param: "{numhadoop}"
13 |    - question: Number of instances for spark
14 |      param: "{numspark}"
15 | backend:
16 |    - storage:
17 |         personality: "hadoop"
18 |         instances: "{numhadoop}"
19 |      compute:
20 |         - personality: "spark"
21 |           instances: "{numspark}"
22 | connectors:
23 |    - personality: "ferry/spark-client"
24 |      ports: [8080]


--------------------------------------------------------------------------------
/ferry/data/plans/yarn.yaml:
--------------------------------------------------------------------------------
 1 | metadata:
 2 |    author: James Horey
 3 |    version: 0.2.0
 4 |    icon: hadoop.png
 5 |    description: This stack provides complete Hadoop functionality over a GlusterFS storage layer. GlusterFS is an open source, distributed file system capable of scaling to several petabytes and handling thousands of clients. GlusterFS is based on a stackable user space design and can deliver exceptional performance for diverse workloads. By combining GlusterFS with Hadoop, users are able to access data via traditional POSIX mounts but still execute traditional Hadoop jobs over the data.  
 6 |    tags: [infrastructure,big data,hadoop,glusterfs]
 7 |    url: http://hadoop.apache.org
 8 | params:
 9 |    resolver: ferry.docker.resolve.DefaultResolver
10 | questions:
11 |    - question: Number of instances for gluster
12 |      param: "{numgluster}"
13 |    - question: Number of instances for yarn
14 |      param: "{numyarn}"
15 | backend:
16 |    - storage:
17 |         personality: "gluster"
18 |         instances: "{numgluster}"
19 |      compute:
20 |         - personality: "yarn"
21 |           instances: "{numyarn}"
22 |           layers: 
23 |              - "hive"
24 | connectors:
25 |    - personality: "ferry/hadoop-client"


--------------------------------------------------------------------------------
/ferry/data/templates/cassandra/log4j-server.properties:
--------------------------------------------------------------------------------
 1 | # Licensed to the Apache Software Foundation (ASF) under one
 2 | # or more contributor license agreements.  See the NOTICE file
 3 | # distributed with this work for additional information
 4 | # regarding copyright ownership.  The ASF licenses this file
 5 | # to you under the Apache License, Version 2.0 (the
 6 | # "License"); you may not use this file except in compliance
 7 | # with the License.  You may obtain a copy of the License at
 8 | #
 9 | #     http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | 
17 | # for production, you should probably set pattern to %c instead of %l.  
18 | # (%l is slower.)
19 | 
20 | # output messages into a rolling log file as well as stdout
21 | log4j.rootLogger=INFO,stdout,R
22 | 
23 | # stdout
24 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
25 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
26 | log4j.appender.stdout.layout.ConversionPattern=%5p %d{HH:mm:ss,SSS} %m%n
27 | 
28 | # rolling log file
29 | log4j.appender.R=org.apache.log4j.RollingFileAppender
30 | log4j.appender.R.maxFileSize=20MB
31 | log4j.appender.R.maxBackupIndex=50
32 | log4j.appender.R.layout=org.apache.log4j.PatternLayout
33 | log4j.appender.R.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
34 | # Edit the next line to point to your logs directory
35 | log4j.appender.R.File=$LOG_DIR/system.log
36 | 
37 | # Application logging options
38 | #log4j.logger.org.apache.cassandra=DEBUG
39 | #log4j.logger.org.apache.cassandra.db=DEBUG
40 | #log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG
41 | 
42 | # Adding this to avoid thrift logging disconnect errors.
43 | log4j.logger.org.apache.thrift.server.TNonblockingServer=ERROR
44 | 
45 | 


--------------------------------------------------------------------------------
/ferry/data/templates/cassandra/titan.properties:
--------------------------------------------------------------------------------
1 | storage.backend=$BACKEND
2 | storage.hostname=$IP
3 | storage.tablename=$DB
4 | storage.keyspace=$DB


--------------------------------------------------------------------------------
/ferry/data/templates/gluster/configure.template:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | 
 3 | # Run these commands on the head node
 4 | if [ $$1 == "start" ]; then
 5 |    $PEER_PROBE
 6 |    echo "peer probe success";sleep 2
 7 |    $VOLUME_LIST force >> /tmp/gluster_deploy.log 2>> /tmp/gluster_deploy.err
 8 |    echo "volume create success";sleep 2
 9 |    gluster volume set $VOLUME_ID nfs.disable on >> /tmp/gluster_deploy.log 2>> /tmp/gluster_deploy.err
10 |    gluster volume start $VOLUME_ID >> /tmp/gluster_deploy.log 2>> /tmp/gluster_deploy.err
11 |    echo "volume start success"
12 | elif [ $$1 == "restart" ]; then
13 |    gluster volume start $VOLUME_ID >> /tmp/gluster_deploy.log 2>> /tmp/gluster_deploy.err
14 | elif [ $$1 == "stop" ]; then
15 |    yes | gluster volume stop $VOLUME_ID >> /tmp/gluster_deploy.log 2>> /tmp/gluster_deploy.err
16 | elif [ $$1 == "clean" ]; then 
17 |     setfattr -x trusted.glusterfs.volume-id $BRICK_DIR
18 |     setfattr -x trusted.gfid $BRICK_DIR
19 |     rm -rf $BRICK_DIR/.glusterfs
20 | fi


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/core-site.xml.template:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 3 | <!--
 4 |   Licensed under the Apache License, Version 2.0 (the "License");
 5 |   you may not use this file except in compliance with the License.
 6 |   You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 |   Unless required by applicable law or agreed to in writing, software
11 |   distributed under the License is distributed on an "AS IS" BASIS,
12 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |   See the License for the specific language governing permissions and
14 |   limitations under the License. See accompanying LICENSE file.
15 | -->
16 | 
17 | <!-- Put site-specific property overrides in this file. -->
18 | 
19 | <configuration>
20 | 
21 |   <property>
22 |     <name>hadoop.proxyuser.ferry.groups</name>
23 |     <value>*</value>
24 |   </property>
25 | 
26 |   <property>
27 |     <name>hadoop.proxyuser.ferry.hosts</name>
28 |     <value>*</value>
29 |   </property>
30 | 
31 |   <property>
32 |     <name>hadoop.proxyuser.oozie.groups</name>
33 |     <value>*</value>
34 |   </property>
35 | 
36 |   <property>
37 |     <name>hadoop.proxyuser.oozie.hosts</name>
38 |     <value>*</value>
39 |   </property>
40 |            
41 |   <property>
42 |     <name>hadoop.proxyuser.hue.hosts</name>
43 |     <value>*</value>
44 |   </property>
45 | 
46 |   <property>
47 |     <name>hadoop.proxyuser.hue.groups</name>
48 |     <value>*</value>
49 |   </property>
50 | 
51 |   <property>
52 |     <name>hadoop.proxyuser.hcat.groups</name>
53 |     <value>*</value>
54 |   </property>
55 | 
56 |   <property>
57 |     <name>hadoop.proxyuser.hcat.hosts</name>
58 |     <value>*</value>
59 |   </property>
60 | 
61 |   <property>
62 |     <name>fs.defaultFS</name>
63 |     <value>$DEFAULT_NAME</value>
64 |   </property>
65 | 
66 |   <!-- <property> -->
67 |   <!--   <name>fs.AbstractFileSystem.file.impl</name> -->
68 |   <!--   <value>org.apache.hadoop.fs.local.RawLocalFs</value> -->
69 |   <!--   <description>The AbstractFileSystem for file: uris.</description> -->
70 |   <!-- </property> -->
71 | 
72 |   <property>
73 |     <name>hadoop.tmp.dir</name>
74 |     <value>$DATA_TMP</value>
75 |   </property>
76 | 
77 |   <!-- <property> -->
78 |   <!--   <name>io.file.buffer.size</name> -->
79 |   <!--   <value>131072</value> -->
80 |   <!-- </property> -->
81 | 
82 |   <property>
83 |     <name>hadoop.security.authentication</name>
84 |     <value>simple</value>
85 |     <description>A value of "simple" would disable security. </description>
86 |   </property>
87 | 
88 |   <property>
89 |     <name>hadoop.security.authorization</name>
90 |     <value>false</value>
91 |   </property>
92 | 
93 | </configuration>
94 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/hadoop-env.sh:
--------------------------------------------------------------------------------
 1 | # Copyright 2011 The Apache Software Foundation
 2 | # 
 3 | # Licensed to the Apache Software Foundation (ASF) under one
 4 | # or more contributor license agreements.  See the NOTICE file
 5 | # distributed with this work for additional information
 6 | # regarding copyright ownership.  The ASF licenses this file
 7 | # to you under the Apache License, Version 2.0 (the
 8 | # "License"); you may not use this file except in compliance
 9 | # with the License.  You may obtain a copy of the License at
10 | #
11 | #     http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing, software
14 | # distributed under the License is distributed on an "AS IS" BASIS,
15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | # See the License for the specific language governing permissions and
17 | # limitations under the License.
18 | 
19 | # Set Hadoop-specific environment variables here.
20 | 
21 | # The only required environment variable is JAVA_HOME.  All others are
22 | # optional.  When running a distributed configuration it is best to
23 | # set JAVA_HOME in this file, so that it is correctly defined on
24 | # remote nodes.
25 | 
26 | # The java implementation to use.
27 | export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/jre/bin/java::")
28 | 
29 | # The jsvc implementation to use. Jsvc is required to run secure datanodes.
30 | #export JSVC_HOME=${JSVC_HOME}
31 | 
32 | export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
33 | 
34 | # Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
35 | for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
36 |   if [ "$HADOOP_CLASSPATH" ]; then
37 |     export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
38 |   else
39 |     export HADOOP_CLASSPATH=$f
40 |   fi
41 | done
42 | 
43 | # The maximum amount of heap to use, in MB. Default is 1000.
44 | #export HADOOP_HEAPSIZE=
45 | #export HADOOP_NAMENODE_INIT_HEAPSIZE=""
46 | 
47 | # Extra Java runtime options.  Empty by default.
48 | export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -XX:+UseParNewGC $HADOOP_CLIENT_OPTS"
49 | 
50 | # Command specific options appended to HADOOP_OPTS when specified
51 | export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
52 | # export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
53 | export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=DEBUG,ERROR,RFAS $HADOOP_DATANODE_OPTS"
54 | 
55 | export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
56 | 
57 | # The following applies to multiple commands (fs, dfs, fsck, distcp etc)
58 | export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS"
59 | #HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
60 | 
61 | # On secure datanodes, user to run the datanode as after dropping privileges
62 | export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
63 | 
64 | # Where log files are stored.  $HADOOP_HOME/logs by default.
65 | export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}
66 | 
67 | # Where log files are stored in the secure data environment.
68 | export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
69 | 
70 | # The directory where pid files are stored. /tmp by default.
71 | # NOTE: this should be set to a directory that can only be written to by 
72 | #       the user that will run the hadoop daemons.  Otherwise there is the
73 | #       potential for a symlink attack.
74 | export HADOOP_PID_DIR=${HADOOP_PID_DIR}
75 | export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
76 | 
77 | # A string representing this instance of hadoop. $USER by default.
78 | export HADOOP_IDENT_STRING=$USER
79 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/hdfs-site.xml.template:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 3 | <!--
 4 |   Licensed under the Apache License, Version 2.0 (the "License");
 5 |   you may not use this file except in compliance with the License.
 6 |   You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 |   Unless required by applicable law or agreed to in writing, software
11 |   distributed under the License is distributed on an "AS IS" BASIS,
12 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |   See the License for the specific language governing permissions and
14 |   limitations under the License. See accompanying LICENSE file.
15 | -->
16 | 
17 | <!-- Put site-specific property overrides in this file. -->
18 | 
19 | <configuration>
20 |   <property>
21 |     <name>dfs.webhdfs.enabled</name>
22 |     <value>true</value>
23 |   </property>
24 | 
25 |   <property>
26 |     <name>dfs.support.append</name>
27 |     <value>true</value>
28 |   </property>
29 | 
30 |   <property>
31 |     <name>dfs.replication</name>
32 |     <value>1</value>
33 |   </property>
34 | 
35 |   <property>
36 |     <name>dfs.permissions</name>
37 |     <value>true</value>
38 |   </property>
39 | 
40 |   <property>
41 |     <name>dfs.blocksize</name>
42 |     <value>268435456</value>
43 |   </property>
44 | 
45 |   <property>
46 |     <name>dfs.namenode.handler.count</name>
47 |     <value>100</value>
48 |   </property>
49 | 
50 |   <property>
51 |     <name>dfs.namenode.name.dir</name>
52 |     <value>$DATA_DIR/dfs/name</value>
53 |   </property>
54 | 
55 |   <property>
56 |     <name>dfs.datanode.data.dir</name>
57 |     <value>$DATA_DIR</value>
58 |   </property>
59 | </configuration>
60 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/httpfs-site.xml.template:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 3 | <!--
 4 |   Licensed under the Apache License, Version 2.0 (the "License");
 5 |   you may not use this file except in compliance with the License.
 6 |   You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 |   Unless required by applicable law or agreed to in writing, software
11 |   distributed under the License is distributed on an "AS IS" BASIS,
12 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |   See the License for the specific language governing permissions and
14 |   limitations under the License. See accompanying LICENSE file.
15 | -->
16 | 
17 | <!-- Put site-specific property overrides in this file. -->
18 | 
19 | <configuration>
20 |   <property>
21 |     <name>httpfs.proxyuser.hue.hosts</name>
22 |     <value>*</value>
23 |   </property>
24 | 
25 |   <property>
26 |     <name>httpfs.proxyuser.hue.groups</name>
27 |     <value>*</value>
28 |   </property>
29 | 
30 |   <property>
31 |     <name>httpfs.hadoop.config.dir</name>
32 |     <value>/service/conf/hadoop</value>
33 |   </property>
34 | 
35 | </configuration>
36 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/mapred-env.sh:
--------------------------------------------------------------------------------
 1 | # Licensed to the Apache Software Foundation (ASF) under one or more
 2 | # contributor license agreements.  See the NOTICE file distributed with
 3 | # this work for additional information regarding copyright ownership.
 4 | # The ASF licenses this file to You under the Apache License, Version 2.0
 5 | # (the "License"); you may not use this file except in compliance with
 6 | # the License.  You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
17 | 
18 | export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
19 | 
20 | # export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
21 | export HADOOP_MAPRED_ROOT_LOGGER=DEBUG,RFA
22 | 
23 | #export HADOOP_JOB_HISTORYSERVER_OPTS=
24 | #export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored.  $HADOOP_MAPRED_HOME/logs by default.
25 | #export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
26 | #export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
27 | #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
28 | #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
29 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/mapred-site.xml.template:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  3 | <!--
  4 |   Licensed under the Apache License, Version 2.0 (the "License");
  5 |   you may not use this file except in compliance with the License.
  6 |   You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 |   Unless required by applicable law or agreed to in writing, software
 11 |   distributed under the License is distributed on an "AS IS" BASIS,
 12 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |   See the License for the specific language governing permissions and
 14 |   limitations under the License. See accompanying LICENSE file.
 15 | -->
 16 | 
 17 | <!-- Put site-specific property overrides in this file. -->
 18 | 
 19 | <configuration>
 20 |   <!-- <property> -->
 21 |   <!--   <name>mapreduce.task.tmp.dir</name> -->
 22 |   <!--   <value>/service/data/tmp/hadoop_tasks</value> -->
 23 |   <!-- </property> -->
 24 | 
 25 |   <property>
 26 |     <name>mapreduce.jobtracker.jobhistory.location</name>
 27 |     <value>/service/data/logs/mapred/jobhistory</value>
 28 |   </property>
 29 | 
 30 |   <property>
 31 |     <name>mapreduce.jobtracker.jobhistory.completed.location</name>
 32 |     <value>/service/data/logs/mapred/completed</value>
 33 |   </property>
 34 | 
 35 |   <!-- <property> -->
 36 |   <!--   <name>mapreduce.jobtracker.system.dir</name> -->
 37 |   <!--   <value>/service/data/tmp/mapred/system</value> -->
 38 |   <!-- </property> -->
 39 | 
 40 |   <property>
 41 |     <name>mapreduce.job.ubertask.enable</name>
 42 |     <value>false</value>
 43 |   </property>
 44 | 
 45 |   <property>
 46 |     <name>mapreduce.framework.name</name>
 47 |     <value>yarn</value>
 48 |   </property>
 49 | 
 50 |   <property>
 51 |     <name>mapreduce.job.maps</name>
 52 |     <value>$JOB_MAPS</value>
 53 |   </property>
 54 | 
 55 |   <property>
 56 |     <name>mapreduce.job.reduces</name>
 57 |     <value>$JOB_REDUCES</value>
 58 |   </property>
 59 | 
 60 |   <property>
 61 |     <name>mapreduce.tasktracker.map.tasks.maximum</name>
 62 |     <value>$NODE_MAPS</value>
 63 |   </property>
 64 | 
 65 |   <property>
 66 |     <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
 67 |     <value>$NODE_REDUCES</value>
 68 |   </property>
 69 | 
 70 |   <property>
 71 |     <name>mapreduce.map.memory.mb</name>
 72 |     <value>$MMEM</value>
 73 |     <description>memory for maps (set to mem-per-container)</description>
 74 |   </property>
 75 | 
 76 |   <property>
 77 |     <name>mapreduce.reduce.memory.mb</name>
 78 |     <value>$RMEM</value>
 79 |     <description>memory for reduce (set to 2x map)</description>
 80 |   </property>
 81 | 
 82 |   <property>
 83 |     <name>mapreduce.map.java.opts</name>
 84 |     <value>$MOPTS</value>
 85 |     <description>set to 0.8x map</description>
 86 |   </property>
 87 | 
 88 |   <property>
 89 |     <name>mapreduce.reduce.java.opts</name>
 90 |     <value>$ROPTS</value>
 91 |     <description>set to 0.8x reduce</description>
 92 |   </property>
 93 | 
 94 |   <!-- <property> -->
 95 |   <!--   <name>mapreduce.task.io.sort.factor</name> -->
 96 |   <!--   <value>50</value> -->
 97 |   <!-- </property> -->
 98 | 
 99 |   <!-- <property> -->
100 |   <!--   <name>mapreduce.task.io.sort.mb</name> -->
101 |   <!--   <value>500</value> -->
102 |   <!-- </property> -->
103 | 
104 |   <!-- <property> -->
105 |   <!--   <name>mapreduce.reduce.shuffle.parallelcopies</name> -->
106 |   <!--   <value>8</value> -->
107 |   <!-- </property> -->
108 | 
109 |   <property>
110 |     <name>mapreduce.jobhistory.address</name>
111 |     <value>$HISTORY_SERVER:10020</value>
112 |   </property>
113 | 
114 |   <property>
115 |     <name>mapreduce.jobhistory.webapp.address</name>
116 |     <value>$HISTORY_SERVER:19888</value>
117 |   </property>
118 | 
119 |   <property>
120 |     <name>yarn.app.mapreduce.am.job.client.port-range</name>
121 |     <value>50100-50200</value>
122 |   </property>
123 | </configuration>
124 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/yarn-env.sh.template:
--------------------------------------------------------------------------------
  1 | # Licensed to the Apache Software Foundation (ASF) under one or more
  2 | # contributor license agreements.  See the NOTICE file distributed with
  3 | # this work for additional information regarding copyright ownership.
  4 | # The ASF licenses this file to You under the Apache License, Version 2.0
  5 | # (the "License"); you may not use this file except in compliance with
  6 | # the License.  You may obtain a copy of the License at
  7 | #
  8 | #     http://www.apache.org/licenses/LICENSE-2.0
  9 | #
 10 | # Unless required by applicable law or agreed to in writing, software
 11 | # distributed under the License is distributed on an "AS IS" BASIS,
 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | # See the License for the specific language governing permissions and
 14 | # limitations under the License.
 15 | 
 16 | # User for YARN daemons
 17 | export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
 18 | 
 19 | # resolve links - $0 may be a softlink
 20 | export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
 21 | 
 22 | # some Java parameters
 23 | # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
 24 | if [ "$JAVA_HOME" != "" ]; then
 25 |   #echo "run java in $JAVA_HOME"
 26 |   JAVA_HOME=$JAVA_HOME
 27 | fi
 28 |   
 29 | if [ "$JAVA_HOME" = "" ]; then
 30 |   echo "Error: JAVA_HOME is not set."
 31 |   exit 1
 32 | fi
 33 | 
 34 | JAVA=$JAVA_HOME/bin/java
 35 | JAVA_HEAP_MAX=-Xmx1000m 
 36 | 
 37 | # For setting YARN specific HEAP sizes please use this
 38 | # Parameter and set appropriately
 39 | # YARN_HEAPSIZE=1000
 40 | 
 41 | # check envvars which might override default args
 42 | if [ "$YARN_HEAPSIZE" != "" ]; then
 43 |   JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
 44 | fi
 45 | 
 46 | # Resource Manager specific parameters
 47 | 
 48 | # Specify the max Heapsize for the ResourceManager using a numerical value
 49 | # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
 50 | # the value to 1000.
 51 | # This value will be overridden by an Xmx setting specified in either YARN_OPTS
 52 | # and/or YARN_RESOURCEMANAGER_OPTS.
 53 | # If not specified, the default value will be picked from either YARN_HEAPMAX
 54 | # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
 55 | #export YARN_RESOURCEMANAGER_HEAPSIZE=1000
 56 | 
 57 | # Specify the JVM options to be used when starting the ResourceManager.
 58 | # These options will be appended to the options specified as YARN_OPTS
 59 | # and therefore may override any similar flags set in YARN_OPTS
 60 | #export YARN_RESOURCEMANAGER_OPTS=
 61 | 
 62 | # Node Manager specific parameters
 63 | 
 64 | # Specify the max Heapsize for the NodeManager using a numerical value
 65 | # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
 66 | # the value to 1000.
 67 | # This value will be overridden by an Xmx setting specified in either YARN_OPTS
 68 | # and/or YARN_NODEMANAGER_OPTS.
 69 | # If not specified, the default value will be picked from either YARN_HEAPMAX
 70 | # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
 71 | #export YARN_NODEMANAGER_HEAPSIZE=1000
 72 | 
 73 | # Specify the JVM options to be used when starting the NodeManager.
 74 | # These options will be appended to the options specified as YARN_OPTS
 75 | # and therefore may override any similar flags set in YARN_OPTS
 76 | #export YARN_NODEMANAGER_OPTS=
 77 | 
 78 | # so that filenames w/ spaces are handled correctly in loops below
 79 | IFS=
 80 | 
 81 | 
 82 | # default log directory & file
 83 | if [ "$YARN_LOG_DIR" = "" ]; then
 84 |   YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
 85 | fi
 86 | if [ "$YARN_LOGFILE" = "" ]; then
 87 |   YARN_LOGFILE='yarn.log'
 88 | fi
 89 | 
 90 | # default policy file for service-level authorization
 91 | if [ "$YARN_POLICYFILE" = "" ]; then
 92 |   YARN_POLICYFILE="hadoop-policy.xml"
 93 | fi
 94 | 
 95 | # restore ordinary behaviour
 96 | unset IFS
 97 | 
 98 | 
 99 | YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
100 | YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
101 | YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
102 | YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
103 | YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
104 | YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
105 | YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
106 | YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
107 | # YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-DEBUG,console}"
108 | # YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-DEBUG,console}"
109 | YARN_OPTS="$YARN_OPTS -Djava.library.path=/service/packages/hadoop/lib/native"
110 | YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
111 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hadoop/yarn-site.xml.template:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | <!--
  3 |   Licensed under the Apache License, Version 2.0 (the "License");
  4 |   you may not use this file except in compliance with the License.
  5 |   You may obtain a copy of the License at
  6 | 
  7 |     http://www.apache.org/licenses/LICENSE-2.0
  8 | 
  9 |   Unless required by applicable law or agreed to in writing, software
 10 |   distributed under the License is distributed on an "AS IS" BASIS,
 11 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 |   See the License for the specific language governing permissions and
 13 |   limitations under the License. See accompanying LICENSE file.
 14 | -->
 15 | <configuration>
 16 | 
 17 |   <property>
 18 |     <name>yarn.log-aggregation-enable</name>
 19 |     <value>true</value>
 20 |   </property>
 21 | 
 22 |   <property>
 23 |     <name>yarn.nodemanager.remote-app-log-dir</name>
 24 |     <value>/service/data/logs/yarn/apps</value>
 25 |   </property>
 26 | 
 27 |   <property>
 28 |     <name>yarn.nodemanager.resource.cpu-vcores</name>
 29 |     <value>$CORES</value>
 30 |     <description>number of cores to use</description>
 31 |   </property>
 32 | 
 33 | 
 34 |   <property>
 35 |     <name>yarn.nodemanager.resource.memory-mb</name>
 36 |     <value>$MEM</value>
 37 |     <description>total available memory</description>
 38 |   </property>
 39 | 
 40 |   <property>
 41 |     <name>yarn.scheduler.maximum-allocation-mb</name>
 42 |     <value>$MEM</value>
 43 |     <description>max that yarn can use</description>
 44 |   </property>
 45 | 
 46 |   <property>
 47 |     <name>yarn.scheduler.minimum-allocation-mb</name>
 48 |     <value>$CMEM</value>
 49 |     <description>memory per container</description>
 50 |   </property>
 51 | 
 52 |   <property>
 53 |     <name>yarn.app.mapreduce.am.resource.mb</name>
 54 |     <value>$RMEM</value>
 55 |     <description>set to 2x mem-per-container</description>
 56 |   </property>
 57 | 
 58 |   <property>
 59 |     <name>yarn.app.mapreduce.am.command-opts</name>
 60 |     <value>$ROPTS</value>
 61 |     <description>set to 0.8x to above option</description>
 62 |   </property>
 63 | 
 64 |   <property>
 65 |     <name>yarn.app.mapreduce.am.staging-dir</name>
 66 |     <value>$DATA_STAGING</value>
 67 |   </property>
 68 | 
 69 |   <property>
 70 |     <name>yarn.nodemanager.aux-services</name>
 71 |     <value>mapreduce_shuffle</value>
 72 |   </property>
 73 | 
 74 |   <property>
 75 |     <description>The hostname of the RM.</description>
 76 |     <name>yarn.resourcemanager.hostname</name>
 77 |     <value>$YARN_MASTER</value>
 78 |   </property>    
 79 | 
 80 |   <property>
 81 |     <description>The http address of the RM web application.</description>
 82 |     <name>yarn.resourcemanager.webapp.address</name>
 83 |     <value>$YARN_MASTER:8088</value>
 84 |   </property>
 85 | 
 86 |   <property>
 87 |     <description>The https adddress of the RM web application.</description>
 88 |     <name>yarn.resourcemanager.webapp.https.address</name>
 89 |     <value>$YARN_MASTER:8090</value>
 90 |   </property>
 91 | 
 92 |   <property>
 93 |     <description>The address of the RM admin interface.</description>
 94 |     <name>yarn.resourcemanager.admin.address</name>
 95 |     <value>$YARN_MASTER:8033</value>
 96 |   </property>
 97 | 
 98 |   <property>
 99 |     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
100 |     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
101 |   </property>
102 | 
103 |   <property>
104 |     <name>yarn.resourcemanager.resource-tracker.address</name>
105 |     <value>$YARN_MASTER:8025</value>
106 |   </property>
107 |   <property>
108 |     <name>yarn.resourcemanager.scheduler.address</name>
109 |     <value>$YARN_MASTER:8030</value>
110 |   </property>
111 |   <property>
112 |     <name>yarn.resourcemanager.address</name>
113 |     <value>$YARN_MASTER:8041</value>
114 |   </property>
115 |   <property>
116 |     <name>yarn.nodemanager.vmem-check-enabled</name>
117 |     <value>false</value>
118 |     <description>Prevent YARN from killing containers that use too much VMEM </description>
119 |   </property>
120 | 
121 | </configuration>
122 | 


--------------------------------------------------------------------------------
/ferry/data/templates/hive-metastore/hive-site.xml.template:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  3 | <!--
  4 |    Licensed to the Apache Software Foundation (ASF) under one or more
  5 |    contributor license agreements.  See the NOTICE file distributed with
  6 |    this work for additional information regarding copyright ownership.
  7 |    The ASF licenses this file to You under the Apache License, Version 2.0
  8 |    (the "License"); you may not use this file except in compliance with
  9 |    the License.  You may obtain a copy of the License at
 10 | 
 11 |        http://www.apache.org/licenses/LICENSE-2.0
 12 | 
 13 |    Unless required by applicable law or agreed to in writing, software
 14 |    distributed under the License is distributed on an "AS IS" BASIS,
 15 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 16 |    See the License for the specific language governing permissions and
 17 |    limitations under the License.
 18 | -->
 19 | 
 20 | <configuration>
 21 | 
 22 | <!-- Hive Execution Parameters -->
 23 | 
 24 | <property>
 25 |   <name>mapred.reduce.tasks</name>
 26 |   <value>1</value>
 27 |   <description>reduce tasks per job</description>
 28 | </property>
 29 | 
 30 | <property>
 31 |   <name>hive.exec.reducers.bytes.per.reducer</name>
 32 |   <value>100000000</value>
 33 |   <description>input size per reducer (100MB)</description>
 34 | </property>
 35 | 
 36 | <property>
 37 |   <name>hive.exec.reducers.max</name>
 38 |   <value>9</value>
 39 |   <description>max number of reducers</description>
 40 | </property>
 41 | 
 42 | <property>
 43 |   <name>hive.cli.print.header</name>
 44 |   <value>true</value>
 45 | </property>
 46 | 
 47 | <property>
 48 |   <name>datanucleus.cache.level2</name>
 49 |   <value>true</value>
 50 |   <description>Use a level 2 cache. Turn this off if metadata is changed independently of Hive metastore server</description>
 51 | </property>
 52 | 
 53 | <property>
 54 |   <name>hive.exec.scratchdir</name>
 55 |   <value>/service/data/tmp/hive/ferry-$USER</value>
 56 |   <description>Scratch space for Hive jobs</description>
 57 | </property>
 58 | 
 59 | <property>
 60 |   <name>hive.exec.local.scratchdir</name>
 61 |   <value>/tmp/ferry/hive/ferry-$USER</value>
 62 |   <description>Local scratch space for Hive jobs</description>
 63 | </property>
 64 | 
 65 | <property>
 66 |   <name>hive.metastore.uris</name>
 67 |   <value>thrift://$DB:9083</value>
 68 |   <description>Thrift uri for the remote metastore. Used by metastore client to connect to remote metastore.</description>
 69 | </property>
 70 | 
 71 | <property>
 72 |   <name>hive.metastore.warehouse.dir</name>
 73 |   <value>/service/data/hive</value>
 74 | </property>
 75 | 
 76 | <property>
 77 |   <name>javax.jdo.option.ConnectionURL</name>
 78 |   <value>jdbc:postgresql://$DB/metastore</value>
 79 |   <description>JDBC connect string for a JDBC metastore</description>
 80 | </property>
 81 | 
 82 | <property>
 83 |   <name>javax.jdo.option.ConnectionDriverName</name>
 84 |   <value>org.postgresql.Driver</value>
 85 |   <description>Driver class name for a JDBC metastore</description>
 86 | </property>
 87 | 
 88 | <property>
 89 |   <name>javax.jdo.option.ConnectionUserName</name>
 90 |   <value>hiveuser</value>
 91 |   <description>username to use against metastore database</description>
 92 | </property>
 93 | 
 94 | <property>
 95 |   <name>javax.jdo.option.ConnectionPassword</name>
 96 |   <value>1hiveuser2</value>
 97 |   <description>password to use against metastore database</description>
 98 | </property>
 99 | 
100 | <property>
101 |   <name>datanucleus.autoCreateSchema</name>
102 |   <value>true</value>
103 |   <description>creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once</description>
104 | </property>
105 | 
106 | <property>
107 |   <name>datanucleus.fixedDatastore</name>
108 |   <value>true</value>
109 | </property>
110 | 
111 | </configuration>
112 | 


--------------------------------------------------------------------------------
/ferry/data/templates/linux/configure.template:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | 
 3 | # Run these commands on the head node
 4 | if [ $$1 = "start" ]; then
 5 |    python /connector/bin/mounthelper.py mount $VOLUME_IP:/$VOLUME_ID
 6 |    echo "volume mounted successfully"
 7 | elif [ $$1 == "stop" ];
 8 |    echo "not implemented yet"
 9 |    python /connector/bin/mounthelper.py umount $VOLUME_IP:/$VOLUME_ID
10 | fi


--------------------------------------------------------------------------------
/ferry/data/templates/mongo/mongodb.conf.template:
--------------------------------------------------------------------------------
 1 | security:
 2 |    authorization: enabled
 3 | systemLog:
 4 |    destination: file
 5 |    path: "$MONGO_LOG/mongodb.log"
 6 |    quiet: false
 7 |    logAppend: true
 8 | storage:
 9 |    journal:
10 |       enabled: true
11 |    dbPath: "$MONGO_DATA"
12 |    smallFiles: true
13 | processManagement:
14 |    fork: true
15 | net:
16 |    bindIp: 0.0.0.0
17 |    port: 27017


--------------------------------------------------------------------------------
/ferry/data/templates/mongo/trusted.conf.template:
--------------------------------------------------------------------------------
 1 | systemLog:
 2 |    destination: file
 3 |    path: "$MONGO_LOG/mongodb.log"
 4 |    quiet: false
 5 |    logAppend: true
 6 | storage:
 7 |    journal:
 8 |       enabled: true
 9 |    dbPath: "$MONGO_DATA"
10 |    smallFiles: true
11 | processManagement:
12 |    fork: true
13 | net:
14 |    bindIp: 0.0.0.0
15 |    port: 27017


--------------------------------------------------------------------------------
/ferry/data/templates/openmpi/openmpi-mca-params.conf:
--------------------------------------------------------------------------------
1 | btl_tcp_port_min_v4=$BTL_PORT_MIN
2 | btl_tcp_port_range_v4=$BTL_PORT_RANGE
3 | oob_tcp_port_min_v4=$OOB_PORT_MIN 
4 | oob_tcp_port_range_v4=$OOB_PORT_RANGE 


--------------------------------------------------------------------------------
/ferry/data/templates/spark/spark_env.sh.template:
--------------------------------------------------------------------------------
1 | export SPARK_MASTER_IP=$MASTER
2 | 


--------------------------------------------------------------------------------
/ferry/data/templates/titan/rexster.xml.template:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <rexster>
  3 |   <http>
  4 |     <server-port>8182</server-port>
  5 |     <server-host>0.0.0.0</server-host>
  6 |     <base-uri>http://$IP</base-uri>
  7 |     <web-root>public</web-root>
  8 |     <character-set>UTF-8</character-set>
  9 |     <enable-jmx>false</enable-jmx>
 10 |     <enable-doghouse>true</enable-doghouse>
 11 |     <max-post-size>2097152</max-post-size>
 12 |     <max-header-size>8192</max-header-size>
 13 |     <upload-timeout-millis>30000</upload-timeout-millis>
 14 |     <thread-pool>
 15 |       <worker>
 16 |         <core-size>8</core-size>
 17 |         <max-size>8</max-size>
 18 |       </worker>
 19 |       <kernal>
 20 |         <core-size>4</core-size>
 21 |         <max-size>4</max-size>
 22 |       </kernal>
 23 |     </thread-pool>
 24 |     <io-strategy>leader-follower</io-strategy>
 25 |   </http>
 26 |   <rexpro>
 27 |     <server-port>8184</server-port>
 28 |     <server-host>0.0.0.0</server-host>
 29 |     <session-max-idle>1790000</session-max-idle>
 30 |     <session-check-interval>3000000</session-check-interval>
 31 |     <enable-jmx>false</enable-jmx>
 32 |     <thread-pool>
 33 |       <worker>
 34 |           <core-size>8</core-size>
 35 |         <max-size>8</max-size>
 36 |       </worker>
 37 |       <kernal>
 38 |         <core-size>4</core-size>
 39 |         <max-size>4</max-size>
 40 |       </kernal>
 41 |     </thread-pool>
 42 |     <io-strategy>leader-follower</io-strategy>
 43 |   </rexpro>
 44 |   <shutdown-port>8183</shutdown-port>
 45 |   <shutdown-host>127.0.0.1</shutdown-host>
 46 |   <script-engines>
 47 |     <script-engine>
 48 |       <name>gremlin-groovy</name>
 49 |       <reset-threshold>-1</reset-threshold>
 50 |       <init-scripts>config/init.groovy</init-scripts>
 51 |       <imports>com.tinkerpop.rexster.client.*</imports>
 52 |       <static-imports>java.lang.Math.PI</static-imports>
 53 |     </script-engine>
 54 |   </script-engines>
 55 |   <security>
 56 |     <authentication>
 57 |         <type>default</type>
 58 |         <configuration>
 59 |           <users>
 60 |             <user>
 61 |               <username>rexster</username>
 62 |               <password>rexster</password>
 63 |             </user>
 64 |           </users>
 65 |         </configuration>
 66 |     </authentication>
 67 |   </security>
 68 |     <metrics>
 69 |         <reporter>
 70 |             <type>jmx</type>
 71 |         </reporter>
 72 |         <reporter>
 73 |             <type>http</type>
 74 |         </reporter>
 75 |         <reporter>
 76 |             <type>console</type>
 77 |             <properties>
 78 |                 <rates-time-unit>SECONDS</rates-time-unit>
 79 |                 <duration-time-unit>SECONDS</duration-time-unit>
 80 |                 <report-period>10</report-period>
 81 |                 <report-time-unit>MINUTES</report-time-unit>
 82 |                 <includes>http.rest.*</includes>
 83 |                 <excludes>http.rest.*.delete</excludes>
 84 |             </properties>
 85 |         </reporter>
 86 |     </metrics>
 87 |   <graphs>
 88 |     <graph>
 89 |       <graph-name>$GRAPH_NAME</graph-name>
 90 |       <graph-type>com.thinkaurelius.titan.tinkerpop.rexster.TitanGraphConfiguration</graph-type>
 91 |       <graph-location></graph-location>
 92 |       <graph-read-only>false</graph-read-only>
 93 |       <properties>
 94 | 	<storage.backend>$GRAPH_BACKEND</storage.backend>
 95 | 	<storage.hostname>$GRAPH_HOST</storage.hostname>
 96 | 	<storage.tablename>$GRAPH_NAME</storage.tablename>
 97 | 	<storage.keyspace>$GRAPH_NAME</storage.keyspace>
 98 |       </properties>
 99 |       <extensions>
100 |         <allows>
101 |           <allow>tp:frames</allow>
102 |           <allow>tp:gremlin</allow>
103 |         </allows>
104 |       </extensions>
105 |     </graph>
106 |   </graphs>
107 | </rexster>


--------------------------------------------------------------------------------
/ferry/data/templates/titan/titan.properties:
--------------------------------------------------------------------------------
1 | storage.backend=$BACKEND
2 | storage.hostname=$IP
3 | storage.tablename=$DB
4 | storage.keyspace=$DB


--------------------------------------------------------------------------------
/ferry/docker/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/docker/__init__.py


--------------------------------------------------------------------------------
/ferry/docker/deploy.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import os
17 | import json
18 | import uuid
19 | import datetime
20 | import logging
21 | import importlib
22 | import inspect
23 | from pymongo import MongoClient
24 | from ferry.docker.docker import DockerInstance
25 | 
26 | class DeployEngine(object):
27 |     def __init__(self, docker):
28 |         self.docker = docker
29 |         self.engines = {}
30 |         self._load_engines()
31 | 
32 |     """
33 |     Deploy the containers. This is a local version that
34 |     uses either the local registry or a private registry. 
35 |     """
36 |     def deploy(self, cluster_uuid, containers, conf=None):
37 |         if conf['_mode'] in self.engines:
38 |             engine = self.engines[conf['_mode']]
39 |             engine.deploy(cluster_uuid, containers, conf)
40 | 
41 |     """
42 |     Dynamically load all the deployment engines
43 |     """
44 |     def _load_engines(self):
45 |         engine_dir = os.environ['FERRY_HOME'] + '/deploy'
46 |         files = os.listdir(engine_dir)
47 |         for f in files:
48 |             p = engine_dir + os.sep + f
49 |             c = self._load_class(p)
50 |             if c:
51 |                 self.engines[c.type] = c
52 | 
53 |     """
54 |     Dynamically load a class from a string
55 |     """
56 |     def _load_class(self, full_name):
57 |         class_info = full_name.split("/")[-1].split(".")
58 |         module_name = class_info[0]
59 |         file_extension = class_info[1]
60 | 
61 |         # Ignore the init and compiled files. 
62 |         if module_name == "__init__" or file_extension == "pyc":
63 |             return None
64 | 
65 |         # Construct the full module path. This lets us filter out only
66 |         # the deployment engines and ignore all the other imported packages. 
67 |         module_path = "ferry.deploy.%s" % module_name
68 |         module = importlib.import_module(module_path)
69 |         for n, o in inspect.getmembers(module):
70 |             if inspect.isclass(o):
71 |                 if o.__module__ == module_path:
72 |                     return o(self.docker)
73 |         return None
74 | 
75 |     """
76 |     Find the deployed application. 
77 |     """
78 |     def find(self, one=False, spec=None, conf=None):
79 |         all_engines = []
80 |         if conf and conf['_mode'] in self.engines:
81 |             all_engines.append(self.engines[conf['_mode']])
82 |         else:
83 |             all_engines = self.engines.values()
84 | 
85 |         all_v = []
86 |         for e in all_engines:
87 |             v = e.find(one, spec, conf)
88 |             if v and one:
89 |                 return v
90 |             elif v:
91 |                 all_v.append(v)
92 |                 
93 |         if one:
94 |             return None
95 |         else:
96 |             return all_v
97 | 


--------------------------------------------------------------------------------
/ferry/docker/resolve.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | import json
16 | import logging
17 | import yaml
18 | 
19 | class DefaultResolver(object):
20 | 
21 |     def resolve(self, questions):
22 |         """
23 |         For each question, map the answer to a parameter value. 
24 |         """
25 |         values = {}
26 |         for q in questions:
27 |             values[q['param']] = q['_answer']
28 |         return values
29 | 
30 |     def replace(self, payload, values):
31 |         for b in payload['backend']:
32 |             # Check the storage.             
33 |             if b['storage']['instances'] in values.keys():
34 |                 b['storage']['instances'] = values[b['storage']['instances']]
35 |                 
36 |             # Check the compute. 
37 |             if 'compute' in b:
38 |                 for c in b['compute']:
39 |                     if c['instances'] in values.keys():
40 |                         c['instances'] = values[c['instances']]
41 | 
42 |         return payload
43 | 


--------------------------------------------------------------------------------
/ferry/fabric/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.2.3.1'
2 | 


--------------------------------------------------------------------------------
/ferry/fabric/aws/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.2.3.1'
2 | 


--------------------------------------------------------------------------------
/ferry/fabric/com.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import logging
17 | import re
18 | from subprocess import Popen, PIPE
19 | import time
20 | 
21 | # Maximum number of tries to contact. 
22 | MAX_COM_RETRIES = 10
23 | 
24 | def robust_com(cmd):
25 |     # All the possible errors that might happen when
26 |     # we try to connect via ssh. 
27 |     route_closed = re.compile('.*No route to host.*', re.DOTALL)
28 |     conn_closed = re.compile('.*Connection closed.*', re.DOTALL)
29 |     refused_closed = re.compile('.*Connection refused.*', re.DOTALL)
30 |     timed_out = re.compile('.*timed out*', re.DOTALL)
31 |     permission = re.compile('.*Permission denied.*', re.DOTALL)
32 | 
33 |     num_tries = 0
34 |     while(True):
35 |         proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
36 |         output = proc.stdout.read()
37 |         err = proc.stderr.read()
38 |         if route_closed.match(err) or conn_closed.match(err) or refused_closed.match(err) or timed_out.match(err) or permission.match(err):            
39 |             if num_tries < MAX_COM_RETRIES:
40 |                 logging.warning("com error, trying again...")
41 |                 num_tries += 1
42 |                 time.sleep(10 * num_tries)
43 |             else: 
44 |                 logging.error("could not communicate")
45 |                 return None, None, False
46 |         else:
47 |             logging.warning("com msg: " + err)
48 |             break
49 |             
50 |     return output, err, True
51 | 


--------------------------------------------------------------------------------
/ferry/fabric/openstack/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.2.3.1'
2 | 


--------------------------------------------------------------------------------
/ferry/http/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jhorey/ferry/bbaa047df08386e17130a939e20fde5e840d1ffa/ferry/http/__init__.py


--------------------------------------------------------------------------------
/ferry/ip/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/ferry/ip/client.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2014 OpenCore LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | 
16 | import json
17 | import logging
18 | import requests
19 | 
20 | DHCP_SERVER = 'http://localhost:5000'
21 | class DHCPClient(object):
22 |     def __init__(self, cidr_block=None):
23 |         if cidr_block:
24 |             payload = { 'cidr' : cidr_block }
25 |             res = requests.post(DHCP_SERVER + '/cidr', data=payload)
26 | 
27 |     def assign_ip(self, container):
28 |         payload = { 'container' : json.dumps(container) }
29 |         res = requests.get(DHCP_SERVER + '/ip', params=payload)
30 |         j = json.loads(res.text)
31 |         return j['ip']
32 | 
33 |     def reserve_ip(self, ip):
34 |         payload = { 'ip' : ip }
35 |         res = requests.put(DHCP_SERVER + '/ip', data=payload)
36 | 
37 |     def set_owner(self, ip, container):
38 |         payload = { 'args' : json.dumps({ 'ip' : ip,
39 |                                           'container' : container}) }
40 |         requests.post(DHCP_SERVER + '/node', data=payload)
41 | 
42 |     def random_port(self):
43 |         res = requests.get(DHCP_SERVER + '/port')
44 |         return res.text
45 | 
46 |     def forward_rule(self, source_ip, source_port, dest_ip, dest_port):
47 |         payload = { 'src_ip' : source_ip,
48 |                     'src_port' : source_port,
49 |                     'dest_ip' : dest_ip,
50 |                     'dest_port' : dest_port }
51 |         requests.post(DHCP_SERVER + '/port', data={'args': json.dumps(payload)})
52 | 
53 |     def delete_rule(self, dest_ip, dest_port):
54 |         payload = { 'dest_ip' : dest_ip,
55 |                     'dest_port' : dest_port }
56 |         requests.delete(DHCP_SERVER + '/port', data={'args': json.dumps(payload)})
57 | 
58 |     def clean_rules(self):
59 |         requests.delete(DHCP_SERVER + '/ports')
60 | 
61 |     def stop_ip(self, ip):
62 |         payload = { 'ip' : ip }
63 |         requests.post(DHCP_SERVER + '/ip', data=payload)
64 | 
65 |     def free_ip(self, ip):
66 |         payload = { 'ip' : ip }
67 |         requests.delete(DHCP_SERVER + '/ip', data=payload)
68 | 


--------------------------------------------------------------------------------
/ferry/logging.conf:
--------------------------------------------------------------------------------
 1 | [loggers]
 2 | keys=root
 3 | 
 4 | [logger_root]
 5 | level=INFO
 6 | handlers=consoleHandler
 7 | 
 8 | [handlers]
 9 | keys=consoleHandler
10 | 
11 | [formatters]
12 | keys=simpleFormatter
13 | 
14 | [handler_consoleHandler]
15 | class=StreamHandler
16 | level=INFO
17 | formatter=simpleFormatter
18 | args=(sys.stdout,)
19 | 
20 | [formatter_simpleFormatter]
21 | format=[%(asctime)s] [%(levelname)s] %(message)s
22 | datefmt=%m/%d/%Y %I:%M:%S
23 | 
24 | 


--------------------------------------------------------------------------------
/ferry/options.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2014 OpenCore LLC
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | 
 16 | class CmdHelp(object):
 17 |     def __init__(self):
 18 |         self.options = {}
 19 |         self.cmds = {}
 20 |         self.usage = ''
 21 |         self.description = ''
 22 | 
 23 |     def add_option(self, short_flag, long_flag, help):
 24 |         self.options[short_flag] = { 'short' : short_flag,
 25 |                                      'long' : long_flag,
 26 |                                      'help' : help,
 27 |                                      'args' : [] }
 28 | 
 29 |     def add_cmd(self, cmd, help):
 30 |         self.cmds[cmd] = { 'cmd' : cmd,
 31 |                            'help' : help,
 32 |                            'args' : [] }
 33 | 
 34 |     def _parse_values(self, i, args):
 35 |         values = []
 36 |         if i == len(args):
 37 |             return i - 1, values
 38 |         elif args[i] in self.options or args[i] in self.cmds:
 39 |             return i - 1, values
 40 |         elif i < len(args):
 41 |             values.append(args[i])
 42 |             j, v = self._parse_values(i + 1, args)
 43 | 
 44 |             if i + 1 == j:
 45 |                 i = j
 46 |                 values += v
 47 | 
 48 |         return i, values
 49 | 
 50 |     def _is_option(self, flag):
 51 |         if flag in self.options:
 52 |             return True
 53 |         else:
 54 |             for f in self.options.keys():
 55 |                 if self.options[f]['long'] == flag:
 56 |                     return True
 57 |         return False
 58 | 
 59 |     def _get_canonical_option(self, flag):
 60 |         if flag in self.options:
 61 |             return flag
 62 |         else:
 63 |             for f in self.options.keys():
 64 |                 if self.options[f]['long'] == flag:
 65 |                     return f
 66 | 
 67 |     def parse_args(self, args):
 68 |         i = 0
 69 |         while i < len(args):
 70 |             s = args[i].strip()
 71 |             if self._is_option(s):
 72 |                 j, values = self._parse_values(i + 1, args)
 73 |                 s = self._get_canonical_option(s)
 74 |                 if len(values) > 0:
 75 |                     i = j
 76 |                     self.options[s]['args'] += values
 77 |                 else:
 78 |                     i += 1
 79 |                     self.options[s]['args'].append(True)
 80 |             elif s in self.cmds:
 81 |                 j, values = self._parse_values(i + 1, args)
 82 |                 if len(values) > 0:
 83 |                     i = j
 84 |                     self.cmds[s]['args'] += values
 85 |                 else:
 86 |                     i += 1
 87 |                     self.cmds[s]['args'].append(True)
 88 |             else:
 89 |                 i += 1
 90 | 
 91 |     def get_cmds(self):
 92 |         ac = {}
 93 |         for c in self.cmds:
 94 |             a = self.cmds[c]['args']
 95 |             if len(a) > 0:
 96 |                 ac[c] = a
 97 |         return ac
 98 | 
 99 |     def get_options(self):
100 |         ac = {}
101 |         for c in self.options:
102 |             a = self.options[c]['args']
103 |             if len(a) > 0:
104 |                 ac[c] = a
105 |         return ac
106 | 
107 |     def print_help(self):
108 |         help_string = 'Usage: ' + self.usage + '\n'
109 |         help_string += '\n'
110 |         help_string += self.description + '\n'
111 |         help_string += '\n'
112 | 
113 |         help_string += 'Options:\n'
114 | 
115 |         for k in sorted(self.options.iterkeys()):
116 |             cmd_string = '    {:10s} {:13s} {:10s}'.format(k, self.options[k]['long'], self.options[k]['help'])
117 |             help_string += cmd_string + '\n'
118 |         help_string += '\n'
119 | 
120 |         help_string += 'Commands:\n'
121 |         for k in sorted(self.cmds.iterkeys()):
122 |             cmd_string = '    {:10s} {:10s}'.format(k, self.cmds[k]['help'])
123 |             help_string += cmd_string + '\n'
124 | 
125 |         return help_string
126 | 


--------------------------------------------------------------------------------
/ferry/table/__init__.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.1.10'
2 | 


--------------------------------------------------------------------------------
/install/README.md:
--------------------------------------------------------------------------------
 1 | Installation Scripts
 2 | ====================
 3 | 
 4 | Base Image
 5 | ==========
 6 | 
 7 | 
 8 | Ferry Volume
 9 | ============
10 | 


--------------------------------------------------------------------------------
/install/ferry-dust:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Bash colors
 4 | RED='\e[0;31m'
 5 | GREEN='\e[0;32m'
 6 | ORANGE='\e[0;33m'
 7 | BLUE='\e[0;34m'
 8 | NC='\e[0m'
 9 | 
10 | VERSION='0.3.3.3'
11 | 
12 | # Helper to build the Ferry image. 
13 | function make_ferry_image {
14 |     OUTPUT=$(docker inspect ferry/ferry-server | grep "\[")
15 |     if [[ $OUTPUT == "[]" ]]; then
16 |     	echo -e "${GREEN}pulling the ferry image${NC}"
17 |     	docker pull ferry/ferry-server
18 |     else
19 |     	echo -e "${BLUE}found ferry image, proceeding${NC}"
20 |     fi
21 | }
22 | 
23 | # Build all the images. Assumed that it is being 
24 | # called from externally. 
25 | function make_images_external {
26 |     # Check if the 'ferry' image is actually built. If not
27 |     # go ahead and build the image. 
28 |     make_ferry_image
29 | 
30 |     # Now go ahead and run the ferry server. 
31 |     if [ -z "$FERRY_DIR" ]; then
32 |     	echo -e "${RED}you must set the FERRY_DIR environment variable${NC}"
33 |     else
34 |     	echo -e "${BLUE}starting ferry, using $FERRY_DIR to save state${NC}"
35 | 	docker run --privileged -v $FERRY_DIR:/var/lib/ferry -e FERRY_SCRATCH=/var/lib/ferry/scratch -e USER=$USER -e HOME=/var/lib/ferry -e FERRY_HOME=/usr/local/lib/python2.7/dist-packages/ferry -i -t ferry/ferry-server /service/sbin/make.sh internal $1
36 |     fi
37 | }
38 | 
39 | # Function to connect to the Ferry image. 
40 | function run_ferry_server {
41 |     # Check if the 'ferry' image is actually built. If not
42 |     # go ahead and build the image. 
43 |     make_ferry_image
44 | 
45 |     if [ -z "$FERRY_DIR" ]; then
46 |     	echo -e "${RED}you must set the FERRY_DIR environment variable${NC}"
47 |     else
48 |     	echo -e "${BLUE}starting ferry client${NC}"
49 | 	docker run --privileged -v $FERRY_DIR:/var/lib/ferry -e FERRY_SCRATCH=/var/lib/ferry/scratch -e USER=$USER -e HOME=/var/lib/ferry -e FERRY_HOME=/usr/local/lib/python2.7/dist-packages/ferry -i -t ferry/ferry-server /service/sbin/make.sh console
50 |     fi
51 | }
52 | 
53 | #
54 | # Authorize the public key for password-less ssh. 
55 | #
56 | function authorize_keys {
57 |     cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
58 | }
59 | 
60 | #
61 | # Start an SSH server. 
62 | #
63 | function start_ssh {
64 |     /usr/sbin/sshd -D
65 | }
66 | 
67 | # Display the usage information. 
68 | if [ "$#" == 0 ]; then
69 |     echo "Usage: ferry-dust CMD"
70 |     echo ""
71 |     echo "Commands:"
72 |     echo "    install  Install all the Ferry images"
73 |     echo "    start    Log into the Ferry client"
74 |     echo ""
75 |     exit 1
76 | fi 
77 | 
78 | # Parse any options. 
79 | ARG=""
80 | if [ "$#" == 2 ]; then
81 |     ARG=$2
82 | fi
83 | 
84 | # Ok parse the user arguments. 
85 | if [[ $1 == "start" ]]; then
86 |     run_ferry_server
87 | elif [[ $1 == "install" ]]; then
88 |     make_images_external $ARG
89 | fi
90 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | boto>=2.32.1
 2 | Flask>=0.10.1
 3 | PyYAML>=3.10
 4 | pymongo>=2.6.3
 5 | python-novaclient==2.18.1
 6 | python-neutronclient==2.3.5
 7 | python-heatclient==0.2.10
 8 | requests>=1.2.3
 9 | sh>=1.09
10 | six>=1.7.0
11 | Sphinx>=1.2.1
12 | sphinxcontrib-httpdomain>=1.2.0
13 | termcolor==1.1.0
14 | threading2==0.3.1
15 | tornado>=4.0.2
16 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from __future__ import unicode_literals
 4 | from __future__ import absolute_import
 5 | from setuptools import setup, find_packages
 6 | import re
 7 | import os
 8 | import codecs
 9 | 
10 | def read(*parts):
11 |     path = os.path.join(os.path.dirname(__file__), *parts)
12 |     with codecs.open(path, encoding='utf-8') as fobj:
13 |         return fobj.read()
14 | 
15 | 
16 | def find_version(*file_paths):
17 |     version_file = read(*file_paths)
18 |     version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
19 |                               version_file, re.M)
20 |     if version_match:
21 |         return version_match.group(1)
22 |     raise RuntimeError("Unable to find version string.")
23 | 
24 | with open('requirements.txt') as f:
25 |     install_requires = f.read().splitlines()
26 | 
27 | setup(
28 |     name='ferry',
29 |     version=find_version("ferry", "__init__.py"),
30 |     description=('Big data development environments using Docker'),
31 |     url='http://ferry.opencore.io',
32 |     author='OpenCore LLC',
33 |     author_email='jlh@opencore.io',
34 |     license='Apache License 2.0',
35 |     packages=find_packages(),
36 |     include_package_data=True,
37 |     install_requires=install_requires,
38 |     scripts=['install/ferry-install', 'install/ferry-dust'], 
39 |     entry_points="""
40 |     [console_scripts]
41 |     ferry=ferry.cli.cli:main
42 |     """
43 | )
44 | 


--------------------------------------------------------------------------------