Replica set automated tests

Rakefile

@@ -74,7 +74,7 @@ namespace :test do
   end
 
   Rake::TestTask.new(:rs) do |t|
-    t.test_files = FileList['test/replica_sets/*_test.rb']
+    t.test_files = ['test/replica_sets/count_test.rb', 'test/replica_sets/connect_test.rb', 'test/replica_sets/insert_test.rb']
     t.verbose    = true
   end
 

test/replica_sets/connect_test.rb

@@ -23,8 +23,8 @@ class ConnectTest < Test::Unit::TestCase
     assert @conn.connected?
 
     assert_equal RS.primary, @conn.primary
-    assert_equal RS.secondaries, @conn.secondaries
-    assert_equal RS.arbiters, @conn.arbiters
+    assert_equal RS.secondaries.sort, @conn.secondaries.sort
+    assert_equal RS.arbiters.sort, @conn.arbiters.sort
   end
 
   def test_connect_with_primary_node_killed

test/replica_sets/count_test.rb

@@ -14,7 +14,7 @@ class ReplicaSetCountTest < Test::Unit::TestCase
   end
 
   def teardown
-    RS.start(@node)
+    RS.restart_killed_nodes
   end
 
   def test_correct_count_after_insertion_reconnect

test/replica_sets/insert_test.rb

@@ -1,7 +1,5 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'mongo'
-require 'test/unit'
-require './test/test_helper'
+require './test/replica_sets/rs_test_helper'
 
 # NOTE: This test expects a replica set of three nodes to be running
 # on the local host.
@@ -9,17 +7,20 @@ class ReplicaSetInsertTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([[TEST_HOST, TEST_PORT], [TEST_HOST, TEST_PORT + 1],
-      [TEST_HOST, TEST_PORT + 2]])
+    @conn = ReplSetConnection.new([TEST_HOST, RS.ports[0]], [TEST_HOST, RS.ports[1]], [TEST_HOST, RS.ports[2]])
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")
   end
 
+  def teardown
+    RS.restart_killed_nodes
+  end
+
   def test_insert
     @coll.save({:a => 20}, :safe => true)
-    puts "Please disconnect the current master."
-    gets
+
+    RS.kill_primary
 
     rescue_connection_failure do
       @coll.save({:a => 30}, :safe => true)
@@ -30,9 +31,9 @@ class ReplicaSetInsertTest < Test::Unit::TestCase
     @coll.save({:a => 60}, :safe => true)
     @coll.save({:a => 70}, :safe => true)
 
-    puts "Please reconnect the old master to make sure that the new master " +
-         "has synced with the previous master. Note: this may have happened already."
-    gets
+    # Restart the old master and wait for sync
+    RS.restart_killed_nodes
+    sleep(1)
     results = []
 
     rescue_connection_failure do

test/replica_sets/node_type_test.rb

@@ -9,8 +9,7 @@ class ReplicaSetNodeTypeTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([TEST_HOST, TEST_PORT], [TEST_HOST, TEST_PORT + 1],
-      [TEST_HOST, TEST_PORT + 2])
+    @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]], [RS.host, RS.ports[2]])
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")
@@ -26,8 +25,7 @@ class ReplicaSetNodeTypeTest < Test::Unit::TestCase
     old_secondary = @conn.secondaries.first
     old_primary   = @conn.primary
 
-    puts "Please disconnect the current primary and reconnect so that it becomes secondary."
-    gets
+    RS.step_down_primary
 
     # Insert something to rescue the connection failure.
     rescue_connection_failure do

test/replica_sets/pooled_insert_test.rb

@@ -9,8 +9,8 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
   include Mongo
 
   def setup
-    @conn = ReplSetConnection.multi([TEST_HOST, TEST_PORT], [TEST_HOST, TEST_PORT + 1],
-      [TEST_HOST, TEST_PORT + 2], :pool_size => 10, :timeout => 5)
+    @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
+      [RS.host, RS.ports[2]], :pool_size => 10, :timeout => 5)
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")
@@ -19,6 +19,8 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
   def test_insert
     expected_results = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     @coll.save({:a => -1}, :safe => true)
+
+    RS.kill_primary
     puts "Please disconnect the current master."
     gets
 
@@ -31,12 +33,9 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
       end
     end
 
-    puts "Please reconnect the old master to make sure that the new master " +
-         "has synced with the previous master. Note: this may have happened already." +
-         "Note also that when connection with multiple threads, you may need to wait a few seconds" +
-         "after restarting the old master so that all the data has had a chance to sync." +
-         "This is a case of eventual consistency."
-    gets
+    # Restart the old master and wait for sync
+    RS.restart_killed_nodes
+    sleep(1)
     results = []
 
     rescue_connection_failure do

test/replica_sets/rs_test_helper.rb

@@ -19,7 +19,7 @@ class Test::Unit::TestCase
         yield
         success = true
       rescue Mongo::ConnectionFailure
-        puts "Rescuing attempt #{tries}"
+        puts "Rescue attempt #{tries}"
         tries += 1
         sleep(1)
       end

test/tools/repl_set_manager.rb

@@ -19,9 +19,9 @@ class ReplSetManager
     @config     = {"_id" => @name, "members" => []}
     @path       = File.join(File.expand_path(File.dirname(__FILE__)), "data")
 
-    @passive_count   = opts[:secondary_count] || 1
-    @arbiter_count   = opts[:arbiter_count]   || 1
-    @secondary_count = opts[:passive_count]   || 1
+    @arbiter_count   = opts[:arbiter_count]   || 2
+    @secondary_count = opts[:secondary_count] || 1
+    @passive_count   = opts[:passive_count] || 1
     @primary_count   = 1
 
     @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
@@ -33,7 +33,7 @@ class ReplSetManager
   end
 
   def start_set
-    puts "Starting a replica set with #{@count} nodes"
+    puts "** Starting a replica set with #{@count} nodes"
 
     system("killall mongod")
 
@@ -89,6 +89,8 @@ class ReplSetManager
   end
 
   def kill(node)
+    pid = @mongods[node]['pid']
+    puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
     system("kill -2 #{@mongods[node]['pid']}")
     @mongods[node]['up'] = false
     sleep(1)
@@ -137,13 +139,13 @@ class ReplSetManager
   def start(node)
     system(@mongods[node]['start'])
     @mongods[node]['up'] = true
-    sleep(1)
+    sleep(0.5)
     @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
   end
   alias :restart :start
 
   def ensure_up
-    print "Ensuring members are up..."
+    print "** Ensuring members are up..."
 
     attempt(Mongo::OperationFailure) do
       con = get_connection
@@ -151,7 +153,7 @@ class ReplSetManager
       print "."
       if status['members'].all? { |m| [1, 2, 7].include?(m['state']) } &&
          status['members'].any? { |m| m['state'] == 1 }
-        puts "All members up!"
+        print "all members up!\n\n"
         return status
       else
         raise Mongo::OperationFailure