master can delegate to workers. all tests green now

This commit is contained in:
Nick Gauthier 2010-01-29 13:01:53 -05:00
parent cbc5bb7a8e
commit e7db4fadee
7 changed files with 51 additions and 37 deletions

View File

@ -30,7 +30,7 @@ module Hydra #:nodoc:
child = Process.fork do child = Process.fork do
pipe.identify_as_child pipe.identify_as_child
# TODO num runners opt in next line # TODO num runners opt in next line
Hydra::Worker.new(pipe, 1) Hydra::Worker.new(:io => pipe, :runners => 1)
end end
pipe.identify_as_parent pipe.identify_as_parent
@workers << { :pid => child, :io => pipe, :idle => false } @workers << { :pid => child, :io => pipe, :idle => false }

View File

@ -3,22 +3,30 @@ module Hydra #:nodoc:
class Runner class Runner
# Boot up a runner. It takes an IO object (generally a pipe from its # Boot up a runner. It takes an IO object (generally a pipe from its
# parent) to send it messages on which files to execute. # parent) to send it messages on which files to execute.
def initialize(io) def initialize(opts = {})
@io = io @io = opts.fetch(:io) { raise "No IO Object" }
@verbose = opts.fetch(:verbose) { false }
@io.write Hydra::Messages::Runner::RequestFile.new @io.write Hydra::Messages::Runner::RequestFile.new
process_messages process_messages
end end
# The runner will continually read messages and handle them. # The runner will continually read messages and handle them.
def process_messages def process_messages
$stdout.write "RUNNER| Processing Messages\n" if @verbose
@running = true @running = true
while @running while @running
begin begin
message = @io.gets message = @io.gets
message.handle(self) if message if message
$stdout.write "RUNNER| Received message from worker\n" if @verbose
$stdout.write " | #{message.inspect}\n" if @verbose
message.handle(self)
else
@io.write Hydra::Messages::Runner::Ping.new @io.write Hydra::Messages::Runner::Ping.new
end
rescue IOError => ex rescue IOError => ex
$stderr.write "Runner lost Worker\n" $stderr.write "Runner lost Worker\n" if @verbose
@running = false @running = false
end end
end end

View File

@ -4,12 +4,16 @@ module Hydra #:nodoc:
# Create a new worker. # Create a new worker.
# * io: The IO object to use to communicate with the master # * io: The IO object to use to communicate with the master
# * num_runners: The number of runners to launch # * num_runners: The number of runners to launch
def initialize(io, num_runners) # TODO: options hash
@io = io def initialize(opts = {})
@verbose = opts.fetch(:verbose) { false }
@io = opts.fetch(:io) { raise "No IO Object" }
@runners = [] @runners = []
@listeners = [] @listeners = []
boot_runners(num_runners)
boot_runners(opts.fetch(:runners) { 1 })
process_messages process_messages
@runners.each{|r| Process.wait r[:pid] } @runners.each{|r| Process.wait r[:pid] }
end end
@ -46,28 +50,34 @@ module Hydra #:nodoc:
# processes if necessary. # processes if necessary.
def shutdown def shutdown
@running = false @running = false
$stdout.write "WORKER| Notifying #{@runners.size} Runners of Shutdown\n" if @verbose
@runners.each do |r| @runners.each do |r|
$stdout.write "WORKER| Sending Shutdown to Runner\n" if @verbose
$stdout.write " | #{r.inspect}\n" if @verbose
r[:io].write(Hydra::Messages::Runner::Shutdown.new) r[:io].write(Hydra::Messages::Runner::Shutdown.new)
Thread.exit
end end
Thread.exit
end end
private private
def boot_runners(num_runners) #:nodoc: def boot_runners(num_runners) #:nodoc:
$stdout.write "WORKER| Booting #{num_runners} Runners\n" if @verbose
num_runners.times do num_runners.times do
pipe = Hydra::Pipe.new pipe = Hydra::Pipe.new
child = Process.fork do child = Process.fork do
pipe.identify_as_child pipe.identify_as_child
Hydra::Runner.new(pipe) Hydra::Runner.new(:io => pipe, :verbose => @verbose)
end end
pipe.identify_as_parent pipe.identify_as_parent
@runners << { :pid => child, :io => pipe, :idle => false } @runners << { :pid => child, :io => pipe, :idle => false }
end end
$stdout.write "WORKER| #{@runners.size} Runners booted\n" if @verbose
end end
# Continuously process messages # Continuously process messages
def process_messages #:nodoc: def process_messages #:nodoc:
$stdout.write "WORKER| Processing Messages\n" if @verbose
@running = true @running = true
# Abort the worker if one of the runners has an exception # Abort the worker if one of the runners has an exception
@ -80,10 +90,15 @@ module Hydra #:nodoc:
while @running while @running
begin begin
message = @io.gets message = @io.gets
message.handle(self) if message if message
$stdout.write "WORKER| Received Message from Master\n" if @verbose
$stdout.write " | #{message.inspect}\n" if @verbose
message.handle(self)
else
@io.write Hydra::Messages::Worker::Ping.new @io.write Hydra::Messages::Worker::Ping.new
end
rescue IOError => ex rescue IOError => ex
$stderr.write "Worker lost Master\n" $stderr.write "Worker lost Master\n" if @verbose
Thread.exit Thread.exit
end end
end end
@ -96,10 +111,13 @@ module Hydra #:nodoc:
while @running while @running
begin begin
message = r[:io].gets message = r[:io].gets
message.handle(self, r) if message if message
$stdout.write "WORKER| Received Message from Runner\n" if @verbose
$stdout.write " | #{message.inspect}\n" if @verbose
message.handle(self, r)
end
rescue IOError => ex rescue IOError => ex
$stderr.write "Worker lost Runner [#{r.inspect}]\n" $stderr.write "Worker lost Runner [#{r.inspect}]\n"
@runners.delete(r)
Thread.exit Thread.exit
end end
end end
@ -107,6 +125,7 @@ module Hydra #:nodoc:
end end
@listeners.each{|l| l.join } @listeners.each{|l| l.join }
@io.close @io.close
$stdout.write "WORKER| Done processing messages\n" if @verbose
end end
# Get the next idle runner # Get the next idle runner

View File

@ -1,8 +1,5 @@
require File.join(File.dirname(__FILE__), 'test_helper') require File.join(File.dirname(__FILE__), 'test_helper')
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
class MasterTest < Test::Unit::TestCase class MasterTest < Test::Unit::TestCase
context "with a file to test and a destination to verify" do context "with a file to test and a destination to verify" do
setup do setup do

View File

@ -1,8 +1,5 @@
require File.join(File.dirname(__FILE__), 'test_helper') require File.join(File.dirname(__FILE__), 'test_helper')
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
class RunnerTest < Test::Unit::TestCase class RunnerTest < Test::Unit::TestCase
context "with a file to test and a destination to verify" do context "with a file to test and a destination to verify" do
setup do setup do
@ -55,14 +52,11 @@ class RunnerTest < Test::Unit::TestCase
# ensure it ran # ensure it ran
assert File.exists?(TARGET) assert File.exists?(TARGET)
assert_equal "HYDRA", File.read(TARGET) assert_equal "HYDRA", File.read(TARGET)
pipe.close
end end
def run_the_runner(pipe) def run_the_runner(pipe)
pipe.identify_as_child pipe.identify_as_child
Hydra::Runner.new(pipe) Hydra::Runner.new({:io => pipe})
pipe.close
end end
end end
include RunnerTestHelper include RunnerTestHelper

View File

@ -10,6 +10,9 @@ require 'hydra'
class Test::Unit::TestCase class Test::Unit::TestCase
end end
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
module Hydra #:nodoc: module Hydra #:nodoc:
module Messages #:nodoc: module Messages #:nodoc:
class TestMessage < Hydra::Message class TestMessage < Hydra::Message

View File

@ -1,8 +1,5 @@
require File.join(File.dirname(__FILE__), 'test_helper') require File.join(File.dirname(__FILE__), 'test_helper')
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
class WorkerTest < Test::Unit::TestCase class WorkerTest < Test::Unit::TestCase
context "with a file to test and a destination to verify" do context "with a file to test and a destination to verify" do
setup do setup do
@ -14,19 +11,18 @@ class WorkerTest < Test::Unit::TestCase
end end
# run the worker in the foreground and the requests in the background # run the worker in the foreground and the requests in the background
should "run a test" do should "run a test in the foreground" do
num_runners = 4 num_runners = 4
pipe = Hydra::Pipe.new pipe = Hydra::Pipe.new
child = Process.fork do child = Process.fork do
request_a_file_and_verify_completion(pipe, num_runners) request_a_file_and_verify_completion(pipe, num_runners)
pipe.close
end end
run_the_worker(pipe, num_runners) run_the_worker(pipe, num_runners)
Process.wait(child) Process.wait(child)
end end
# inverse of the above test to run the worker in the background # inverse of the above test to run the worker in the background
should "be able to tell a worker to run a test" do should "run a test in the background" do
num_runners = 4 num_runners = 4
pipe = Hydra::Pipe.new pipe = Hydra::Pipe.new
child = Process.fork do child = Process.fork do
@ -34,15 +30,13 @@ class WorkerTest < Test::Unit::TestCase
end end
request_a_file_and_verify_completion(pipe, num_runners) request_a_file_and_verify_completion(pipe, num_runners)
Process.wait(child) Process.wait(child)
pipe.close
end end
end end
module WorkerTestHelper module WorkerTestHelper
def run_the_worker(pipe, num_runners) def run_the_worker(pipe, num_runners)
pipe.identify_as_child pipe.identify_as_child
Hydra::Worker.new(pipe, num_runners) Hydra::Worker.new({:io => pipe, :runners => num_runners})
pipe.close
end end
def request_a_file_and_verify_completion(pipe, num_runners) def request_a_file_and_verify_completion(pipe, num_runners)
@ -52,8 +46,7 @@ class WorkerTest < Test::Unit::TestCase
end end
pipe.write(Hydra::Messages::Worker::RunFile.new(:file => TESTFILE)) pipe.write(Hydra::Messages::Worker::RunFile.new(:file => TESTFILE))
response = pipe.gets assert pipe.gets.is_a?(Hydra::Messages::Worker::Results)
assert response.is_a?(Hydra::Messages::Worker::Results)
pipe.write(Hydra::Messages::Worker::Shutdown.new) pipe.write(Hydra::Messages::Worker::Shutdown.new)