Fully-automated replica set tests.

Rakefile

@@ -74,7 +74,7 @@ namespace :test do
   end
 
   Rake::TestTask.new(:rs) do |t|
-    t.test_files = ['test/replica_sets/count_test.rb', 'test/replica_sets/connect_test.rb', 'test/replica_sets/insert_test.rb']
+    t.test_files = FileList['test/replica_sets/*_test.rb']
     t.verbose    = true
   end
 

lib/mongo/util/pool.rb

@@ -50,7 +50,12 @@ module Mongo
 
     def close
       @sockets.each do |sock|
+        begin
           sock.close
+        rescue IOError => ex
+          warn "IOError when attempting to close socket connected "
+           + "to #{@host}:#{@port}: #{ex.inspect}"
+        end
       end
       @host = @port = nil
       @sockets.clear

test/replica_sets/connect_test.rb

@@ -10,6 +10,10 @@ class ConnectTest < Test::Unit::TestCase
     RS.restart_killed_nodes
   end
 
+  def teardown
+    RS.restart_killed_nodes
+  end
+
   def test_connect_bad_name
     assert_raise_error(ReplicaSetConnectionError, "-wrong") do
       ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],

test/replica_sets/pooled_insert_test.rb

@@ -1,7 +1,5 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'mongo'
-require 'test/unit'
-require './test/test_helper'
+require './test/replica_sets/rs_test_helper'
 
 # NOTE: This test expects a replica set of three nodes to be running
 # on the local host.
@@ -16,13 +14,15 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
     @coll = @db.collection("test-sets")
   end
 
+  def teardown
+    RS.restart_killed_nodes
+  end
+
   def test_insert
     expected_results = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     @coll.save({:a => -1}, :safe => true)
 
     RS.kill_primary
-    puts "Please disconnect the current master."
-    gets
 
     threads = []
     10.times do |i|

test/replica_sets/query_secondaries.rb

@@ -1,7 +1,5 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'mongo'
-require 'test/unit'
-require './test/test_helper'
+require './test/replica_sets/rs_test_helper'
 
 # NOTE: This test expects a replica set of three nodes to be running
 # on the local host.
@@ -9,12 +7,16 @@ class ReplicaSetQuerySecondariesTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([TEST_HOST, TEST_PORT], :read_secondary => true)
+    @conn = ReplSetConnection.new([RS.host, RS.ports[0]], :read_secondary => true)
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets", :safe => {:w => 2, :wtimeout => 100})
   end
 
+  def teardown
+    RS.restart_killed_nodes
+  end
+
   def test_con
     assert @conn.primary_pool, "No primary pool!"
     assert @conn.read_pool, "No read pool!"
@@ -32,8 +34,7 @@ class ReplicaSetQuerySecondariesTest < Test::Unit::TestCase
     assert results.include?(30)
     assert results.include?(40)
 
-    puts "Please disconnect the current master."
-    gets
+    RS.kill_primary
 
     results = []
     rescue_connection_failure do

test/replica_sets/query_test.rb

@@ -1,7 +1,5 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'mongo'
-require 'test/unit'
-require './test/test_helper'
+require './test/replica_sets/rs_test_helper'
 
 # NOTE: This test expects a replica set of three nodes to be running
 # on the local host.
@@ -9,25 +7,27 @@ class ReplicaSetQueryTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([TEST_HOST, TEST_PORT], [TEST_HOST, TEST_PORT + 1],
-      [TEST_HOST, TEST_PORT + 2])
+    @conn = ReplSetConnection.new([RS.host, RS.ports[0]])
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")
   end
 
+  def teardown
+    RS.restart_killed_nodes
+  end
+
   def test_query
-    @coll.save({:a => 20})
-    @coll.save({:a => 30})
-    @coll.save({:a => 40})
+    @coll.save({:a => 20}, :safe => {:w => 3})
+    @coll.save({:a => 30}, :safe => {:w => 3})
+    @coll.save({:a => 40}, :safe => {:w => 3})
     results = []
     @coll.find.each {|r| results << r}
     [20, 30, 40].each do |a|
       assert results.any? {|r| r['a'] == a}, "Could not find record for a => #{a}"
     end
 
-    puts "Please disconnect the current master."
-    gets
+    RS.kill_primary
 
     results = []
     rescue_connection_failure do

test/replica_sets/replication_ack_test.rb

@@ -1,17 +1,14 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'mongo'
-require 'test/unit'
-require './test/test_helper'
+require './test/replica_sets/rs_test_helper'
 
 # NOTE: This test expects a replica set of three nodes to be running on local host.
 class ReplicaSetAckTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([TEST_HOST, TEST_PORT], [TEST_HOST, TEST_PORT + 1],
-      [TEST_HOST, TEST_PORT + 2])
+    RS.ensure_up
 
-    master = [@conn.primary_pool.host, @conn.primary_pool.port]
+    @conn = ReplSetConnection.new([RS.host, RS.ports[0]])
 
     @slave1 = Connection.new(@conn.secondary_pools[0].host,
       @conn.secondary_pools[0].port, :slave_ok => true)
@@ -34,31 +31,31 @@ class ReplicaSetAckTest < Test::Unit::TestCase
   end
 
   def test_safe_mode_replication_ack
-    @col.insert({:baz => "bar"}, :safe => {:w => 2, :wtimeout => 1000})
+    @col.insert({:baz => "bar"}, :safe => {:w => 2, :wtimeout => 5000})
 
-    assert @col.insert({:foo => "0" * 10000}, :safe => {:w => 2, :wtimeout => 1000})
+    assert @col.insert({:foo => "0" * 5000}, :safe => {:w => 2, :wtimeout => 5000})
     assert_equal 2, @slave1[MONGO_TEST_DB]["test-sets"].count
 
-    assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 2, :wtimeout => 1000})
+    assert @col.update({:baz => "bar"}, {:baz => "foo"}, :safe => {:w => 2, :wtimeout => 5000})
     assert @slave1[MONGO_TEST_DB]["test-sets"].find_one({:baz => "foo"})
 
-    assert @col.remove({}, :safe => {:w => 2, :wtimeout => 1000})
+    assert @col.remove({}, :safe => {:w => 2, :wtimeout => 5000})
     assert_equal 0, @slave1[MONGO_TEST_DB]["test-sets"].count
   end
 
   def test_last_error_responses
     20.times { @col.insert({:baz => "bar"}) }
-    response = @db.get_last_error(:w => 2, :wtimeout => 10000)
+    response = @db.get_last_error(:w => 2, :wtimeout => 5000)
     assert response['ok'] == 1
     assert response['lastOp']
 
     @col.update({}, {:baz => "foo"}, :multi => true)
-    response = @db.get_last_error(:w => 2, :wtimeout => 1000)
+    response = @db.get_last_error(:w => 2, :wtimeout => 5000)
     assert response['ok'] == 1
     assert response['lastOp']
 
     @col.remove({})
-    response =  @db.get_last_error(:w => 2, :wtimeout => 1000)
+    response =  @db.get_last_error(:w => 2, :wtimeout => 5000)
     assert response['ok'] == 1
     assert response['n'] == 20
     assert response['lastOp']

test/replica_sets/rs_test_helper.rb

@@ -19,7 +19,7 @@ class Test::Unit::TestCase
         yield
         success = true
       rescue Mongo::ConnectionFailure
-        puts "Rescue attempt #{tries}"
+        puts "Rescue attempt #{tries}\n"
         tries += 1
         sleep(1)
       end

test/tools/repl_set_manager.rb

@@ -151,7 +151,7 @@ class ReplSetManager
       con = get_connection
       status = con['admin'].command({'replSetGetStatus' => 1})
       print "."
-      if status['members'].all? { |m| [1, 2, 7].include?(m['state']) } &&
+      if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
          status['members'].any? { |m| m['state'] == 1 }
         print "all members up!\n\n"
         return status