├── Rakefile ├── lib ├── ActiveOozie │ ├── version.rb │ ├── configuration.rb │ ├── Actions │ │ └── shell.rb │ ├── bundle.rb │ ├── coordinator.rb │ └── workflow.rb └── ActiveOozie.rb ├── Gemfile ├── .gitignore ├── examples ├── list_bundles.rb ├── run_workflow.rb ├── workflow_with_coordinator.rb └── workflow_with_coordinator_with_bundle.rb ├── README.md ├── ActiveOozie.gemspec ├── LICENSE.txt └── LICENSE /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | -------------------------------------------------------------------------------- /lib/ActiveOozie/version.rb: -------------------------------------------------------------------------------- 1 | module ActiveOozie 2 | VERSION = "0.0.1" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in ActiveOozie.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | *.swp 3 | /.yardoc 4 | /Gemfile.lock 5 | /_yardoc/ 6 | /coverage/ 7 | /doc/ 8 | /pkg/ 9 | /spec/reports/ 10 | /tmp/ 11 | *.bundle 12 | *.so 13 | *.o 14 | *.a 15 | mkmf.log 16 | -------------------------------------------------------------------------------- /examples/list_bundles.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'ActiveOozie' 4 | require 'uri' 5 | 6 | # create the client that manages everything 7 | client = ActiveOozie::Client.new(URI(ENV['WEBHDFS_URL']), URI(ENV['OOZIE_URL'])) 8 | 9 | puts client.bundles 10 | -------------------------------------------------------------------------------- /lib/ActiveOozie/configuration.rb: -------------------------------------------------------------------------------- 1 | require 'nokogiri' 2 | 3 | module ActiveOozie 4 | class Configuration 5 | def initialize(properties) 6 | @properties = properties 7 | end 8 | 9 | def to_xml 10 | builder = Nokogiri::XML::Builder.new do |xml| 11 | xml.configuration do 12 | @properties.each do |(name, value)| 13 | xml.property do 14 | xml.name do 15 | xml.text(name) 16 | end 17 | 18 | xml.value do 19 | xml.text(value) 20 | end 21 | end 22 | end 23 | end 24 | 25 | end 26 | 27 | builder.to_xml 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /examples/run_workflow.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'ActiveOozie' 4 | require 'uri' 5 | 6 | # create the client that manages everything 7 | client = ActiveOozie::Client.new(URI(ENV['WEBHDFS_URL']), URI(ENV['OOZIE_URL'])) 8 | 9 | # make a basic workflow that does stuff 10 | wf = ActiveOozie::Workflow.new(client, "example", "/user/orenmazor/", "oren.mazor@gmail.com") 11 | wf.add(ActiveOozie::Actions::Shell.new("foo", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 12 | wf.add(ActiveOozie::Actions::Shell.new("foo2", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 13 | wf.add(ActiveOozie::Actions::Shell.new("foo3", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 14 | 15 | wf.save! 16 | 17 | wf.submit! 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ActiveOozie 2 | 3 | This is my very, very rudimentary wrapper around the Oozie API. It's good enough for me. 4 | 5 | # Usage 6 | 7 | ``` 8 | #!/usr/bin/env ruby 9 | 10 | require 'ActiveOozie' 11 | require 'ActiveOozie/Workflow' 12 | require 'ActiveOozie/Actions/Shell' 13 | require 'uri' 14 | 15 | # create the client that manages everything 16 | client = ActiveOozie::Client.new(URI(ENV['OOZIE_URL']), URI(ENV['WEBHDFS_URL'])) 17 | 18 | # make a basic workflow that does stuff 19 | wf = ActiveOozie::Workflow.new(client, "example", "/user/orenmazor/", "oren.mazor@gmail.com") 20 | wf.add(ActiveOozie::Actions::Shell.new("foo", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 21 | wf.add(ActiveOozie::Actions::Shell.new("foo2", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 22 | wf.add(ActiveOozie::Actions::Shell.new("foo3", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 23 | 24 | wf.save! 25 | wf.submit! 26 | ``` 27 | -------------------------------------------------------------------------------- /lib/ActiveOozie/Actions/shell.rb: -------------------------------------------------------------------------------- 1 | module ActiveOozie 2 | module Actions 3 | class Shell 4 | attr_reader :name 5 | 6 | def initialize(name, command, job_tracker, name_node, args= []) 7 | @name = name 8 | @command = command 9 | @arguments = args 10 | @job_tracker = job_tracker 11 | @name_node = name_node 12 | end 13 | 14 | def to_xml(nokogiri) 15 | nokogiri.shell(xmlns: "uri:oozie:shell-action:0.2") do 16 | nokogiri.send(:"job-tracker") do 17 | nokogiri.text(@job_tracker) 18 | end 19 | nokogiri.send(:"name-node") do 20 | nokogiri.text(@name_node) 21 | end 22 | nokogiri.exec do 23 | nokogiri.text(@command) 24 | end 25 | 26 | @arguments.each do |arg| 27 | nokogiri.argument do 28 | nokogiri.text(arg) 29 | end 30 | end 31 | end 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /examples/workflow_with_coordinator.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'time' 4 | require 'ActiveOozie' 5 | require 'uri' 6 | 7 | # create the client that manages everything 8 | client = ActiveOozie::Client.new(URI(ENV['WEBHDFS_URL']), URI(ENV['OOZIE_URL'])) 9 | 10 | # make a basic workflow that does stuff 11 | wf = ActiveOozie::Workflow.new(client, "example-coordinator", "/user/orenmazor/", "oren.mazor@gmail.com") 12 | wf.add(ActiveOozie::Actions::Shell.new("foo", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 13 | wf.add(ActiveOozie::Actions::Shell.new("foo2", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 14 | wf.add(ActiveOozie::Actions::Shell.new("foo3", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 15 | 16 | wf.save! 17 | 18 | # create a coordinator to run the above once an hour from now until 20 days from now 19 | coord = ActiveOozie::Coordinator.new(client, "/user/orenmazor/", "example-coordinator", Time.now, Time.now + (60 * 60 * 24 * 20), 60) 20 | coord.add(wf) 21 | 22 | coord.save! 23 | coord.submit! 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /ActiveOozie.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'ActiveOozie/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "ActiveOozie" 8 | spec.version = ActiveOozie::VERSION 9 | spec.authors = ["oren mazor"] 10 | spec.email = ["oren.mazor@gmail.com"] 11 | spec.summary = %q{Oozie API Client} 12 | spec.description = %q{Oozie API Client} 13 | spec.homepage = "" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files -z`.split("\x0") 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler", "~> 1.7" 22 | spec.add_development_dependency "rake", "~> 10.0" 23 | spec.add_dependency 'webhdfs' 24 | spec.add_dependency 'rest-client' 25 | spec.add_dependency 'nokogiri' 26 | spec.add_dependency 'byebug' 27 | end 28 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 oren mazor 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /lib/ActiveOozie/bundle.rb: -------------------------------------------------------------------------------- 1 | module ActiveOozie 2 | class Bundle 3 | def initialize(client, path, name) 4 | @client = client 5 | @path = path + name 6 | @name = name 7 | @coords = [] 8 | end 9 | 10 | def add(coord) 11 | @coords << coord 12 | end 13 | 14 | def to_xml 15 | builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| 16 | xml.send(:"bundle-app", name: @name, xmlns: "uri:oozie:bundle:0.1") do 17 | @coords.each do |coord| 18 | xml.coordinator(name: coord.name) do 19 | xml.send(:"app-path") do 20 | xml.text(coord.path) 21 | end 22 | end 23 | end 24 | end 25 | end 26 | 27 | builder.to_xml 28 | end 29 | 30 | def save! 31 | contents = to_xml 32 | @client.write(@path + @name, "bundle.xml", contents) 33 | end 34 | 35 | def submit! 36 | configuration = ActiveOozie::Configuration.new({"user.name" => "oozie", "oozie.bundle.application.path" => @path + @name}).to_xml 37 | 38 | @client.submit(configuration) 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /examples/workflow_with_coordinator_with_bundle.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'time' 4 | require 'ActiveOozie' 5 | require 'uri' 6 | 7 | # create the client that manages everything 8 | client = ActiveOozie::Client.new(URI(ENV['WEBHDFS_URL']), URI(ENV['OOZIE_URL'])) 9 | 10 | # make a basic workflow that does stuff 11 | wf = ActiveOozie::Workflow.new(client, "example", "/user/orenmazor/", "oren.mazor@gmail.com") 12 | wf.add(ActiveOozie::Actions::Shell.new("foo", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 13 | wf.add(ActiveOozie::Actions::Shell.new("foo2", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 14 | wf.add(ActiveOozie::Actions::Shell.new("foo3", "whoami", ENV['RESOURCE_MANAGER'], ENV['NAMENODE'])) 15 | 16 | wf.save! 17 | 18 | # create a coordinator to run the above once an hour from now until 20 days from now 19 | coord = ActiveOozie::Coordinator.new(client, "/user/orenmazor/", "example", Time.now, Time.now + (60 * 60 * 24 * 20), 60) 20 | coord.add(wf) 21 | 22 | coord.save! 23 | 24 | # create a bundle to RULE THEM ALL 25 | 26 | bundle = ActiveOozie::Bundle.new(client, "/user/orenmazor/", "example") 27 | bundle.add(coord) 28 | bundle.save! 29 | bundle.submit! 30 | 31 | 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Oren Mazor 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /lib/ActiveOozie.rb: -------------------------------------------------------------------------------- 1 | require "ActiveOozie/version" 2 | require "ActiveOozie/workflow" 3 | require "ActiveOozie/coordinator" 4 | require "ActiveOozie/bundle" 5 | require "ActiveOozie/Actions/shell" 6 | require "ActiveOozie/configuration" 7 | require 'webhdfs' 8 | require 'rest-client' 9 | require 'json' 10 | require 'byebug' 11 | 12 | module ActiveOozie 13 | class Client 14 | def initialize(hdfs_uri, oozie_uri) 15 | @fs = WebHDFS::Client.new(hdfs_uri.hostname, hdfs_uri.port, hdfs_uri.user) 16 | @oozie = oozie_uri 17 | end 18 | 19 | def bundles 20 | uri = @oozie.to_s + "/oozie/v1/jobs?jobtype=bundle" 21 | begin 22 | response = RestClient.get(uri) 23 | return JSON.parse(response.body) 24 | rescue RestClient::ExceptionWithResponse => err 25 | puts err.response.headers[:oozie_error_code] 26 | puts err.response.headers[:oozie_error_message] 27 | 28 | puts err.response.body 29 | end 30 | 31 | 32 | end 33 | 34 | def write(path, filename, contents) 35 | @fs.mkdir(path) 36 | @fs.create("#{path}/#{filename}", contents) 37 | end 38 | 39 | def submit(configuration, run = true) 40 | uri = @oozie.to_s + "/oozie/v1/jobs" 41 | if run 42 | uri += "?action=start" 43 | end 44 | 45 | begin 46 | response = RestClient.post(uri, configuration, {"Content-Type" => "application/xml"}) 47 | puts "[#{response.code}] #{response.body}" 48 | rescue RestClient::ExceptionWithResponse => err 49 | puts err.response.headers[:oozie_error_code] 50 | puts err.response.headers[:oozie_error_message] 51 | 52 | puts err.response.body 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/ActiveOozie/coordinator.rb: -------------------------------------------------------------------------------- 1 | module ActiveOozie 2 | class Coordinator 3 | attr_reader :name 4 | attr_reader :path 5 | def initialize(client, path, name, starttime, endtime, frequency) 6 | @path = path + name 7 | @client = client 8 | @name = name 9 | @starttime = starttime.strftime("%Y-%m-%dT%H:%MZ") 10 | @endtime = endtime.strftime("%Y-%m-%dT%H:%MZ") 11 | @frequency_in_minutes = frequency 12 | @workflows = [] 13 | end 14 | 15 | def add(wf) 16 | @workflows << wf 17 | end 18 | 19 | def to_xml 20 | builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| 21 | xml.send(:"coordinator-app", xmlns:"uri:oozie:coordinator:0.2", timezone:"America/New_York", name: @name, start: @starttime, end: @endtime, frequency: @frequency_in_minutes) do 22 | add_controls(xml) 23 | 24 | xml.action do 25 | @workflows.each do |wf| 26 | xml.workflow do 27 | xml.send(:"app-path") do 28 | xml.text(wf.path) 29 | end 30 | end 31 | end 32 | end 33 | end 34 | end 35 | 36 | builder.to_xml 37 | end 38 | 39 | def save! 40 | contents = to_xml 41 | @client.write(@path, "coordinator.xml", contents) 42 | end 43 | 44 | def submit! 45 | configuration = ActiveOozie::Configuration.new({"user.name" => "oozie", "oozie.coord.application.path" => @path}).to_xml 46 | 47 | @client.submit(configuration) 48 | end 49 | 50 | private 51 | 52 | def add_controls(xml) 53 | xml.controls do 54 | xml.timeout do 55 | xml.text("600") 56 | end 57 | 58 | xml.concurrency do 59 | xml.text("1") 60 | end 61 | 62 | xml.execution do 63 | xml.text("FIFO") 64 | end 65 | 66 | xml.throttle do 67 | xml.text("1") 68 | end 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/ActiveOozie/workflow.rb: -------------------------------------------------------------------------------- 1 | require 'nokogiri' 2 | 3 | module ActiveOozie 4 | class Workflow 5 | attr_reader :path 6 | def initialize(client, name, path, email) 7 | @client = client 8 | @actions = [] 9 | @path = path + name 10 | @name = name 11 | @notification_email = email 12 | end 13 | 14 | def add(action) 15 | @actions << action 16 | end 17 | 18 | def to_xml 19 | builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| 20 | xml.send(:"workflow-app", name: @name, xmlns: "uri:oozie:workflow:0.5", 'xmlns:sla'=> 'uri:oozie:sla:0.2') do 21 | # the first action is the start 22 | xml.start(to: @actions.first.name) 23 | 24 | add_actions(xml) 25 | 26 | # print the notification action 27 | add_notify(xml) 28 | add_kill(xml) 29 | add_end(xml) 30 | end 31 | end 32 | 33 | builder.to_xml 34 | end 35 | 36 | def save! 37 | contents = to_xml 38 | @client.write(@path, "workflow.xml", contents) 39 | end 40 | 41 | def submit!(run_now = false) 42 | configuration = ActiveOozie::Configuration.new({"user.name" => "oozie", "oozie.wf.application.path" => @path}).to_xml 43 | 44 | @client.submit(configuration) 45 | end 46 | 47 | 48 | private 49 | 50 | def add_end(xml) 51 | xml.end(name: "end") 52 | end 53 | 54 | def add_actions(xml) 55 | #print all the actions 56 | @actions.each_with_index do |action, index| 57 | xml.action(name: action.name) do 58 | action.to_xml(xml) 59 | 60 | # note where we go next 61 | if index + 1 == @actions.size 62 | xml.ok(to: "end") 63 | else 64 | xml.ok(to: @actions[index+1].name) 65 | end 66 | 67 | xml.error(to: "notify") 68 | end 69 | end 70 | end 71 | 72 | def add_kill(xml) 73 | xml.kill(name: "kill") do 74 | xml.message("${wf:errorMessage(wf:lastErrorNode())}") 75 | end 76 | end 77 | 78 | def add_notify(xml) 79 | xml.action(name: "notify") do 80 | xml.email(xmlns: "uri:oozie:email-action:0.1") do 81 | xml.to do 82 | xml.text(@notification_email) 83 | end 84 | 85 | xml.subject do 86 | xml.text("WF ${wf:name()} failed") 87 | end 88 | 89 | xml.body do 90 | xml.text("${wf:errorMessage(wf:lastErrorNode())}") 91 | end 92 | end 93 | 94 | xml.ok(to: "kill") 95 | xml.error(to: "kill") 96 | end 97 | end 98 | end 99 | end 100 | --------------------------------------------------------------------------------