RUBY-314 replica set connection and test cleanup

lib/mongo/connection.rb

@@ -605,6 +605,23 @@ module Mongo
       end
     end
 
+    # Log a message with the given level.
+    def log(level, message)
+      return unless @logger
+      case level
+        when :debug then
+          @logger.debug "MONGODB [DEBUG] #{msg}"
+        when :warn then
+          @logger.warn "MONGODB [WARNING] #{msg}"
+        when :error then
+          @logger.error "MONGODB [ERROR] #{msg}"
+        when :fatal then
+          @logger.fatal "MONGODB [FATAL] #{msg}"
+        else
+          @logger.info "MONGODB [INFO] #{msg}"
+        end
+    end
+
     # Execute the block and log the operation described by name
     # and payload.
     # TODO: Not sure if this should take a block.

lib/mongo/node.rb

@@ -27,6 +27,7 @@ module Mongo
     # return nil.
     def connect
       begin
+        socket = nil
         if self.connection.connect_timeout
           Mongo::TimeoutHandler.timeout(self.connection.connect_timeout, OperationTimeout) do
             socket = TCPSocket.new(self.host, self.port)
@@ -40,7 +41,9 @@ module Mongo
         else
           socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
         end
-      rescue OperationFailure, SocketError, SystemCallError, IOError => ex
+      rescue OperationTimeout, OperationFailure, SocketError, SystemCallError, IOError => ex
+        self.connection.log(:debug, "Failed connection to #{host_string} with #{ex.class}, #{ex.message}.")
+        socket.close if socket
         return nil
       end
 

lib/mongo/repl_set_connection.rb

@@ -21,7 +21,7 @@ module Mongo
   # Instantiates and manages connections to a MongoDB replica set.
   class ReplSetConnection < Connection
     attr_reader :nodes, :secondaries, :arbiters, :secondary_pools,
-      :replica_set_name, :read_pool
+      :replica_set_name, :read_pool, :seeds
 
     # Create a connection to a MongoDB replica set.
     #
@@ -163,6 +163,7 @@ module Mongo
       @seeds = manager.seeds
       @manager = manager
       @hosts = manager.hosts
+      @nodes = manager.nodes
     end
 
     # If ismaster doesn't match our current view
@@ -171,8 +172,13 @@ module Mongo
     # Then take out the connection lock and replace
     # our current values.
     def refresh
-      background_manager = PoolManager.new(self, @seeds)
+      return if !connected?
 
+      if !Thread.current[:background]
+        Thread.current[:background] = PoolManager.new(self, @seeds)
+      end
+
+      background_manager = Thread.current[:background]
       if update_struct = background_manager.update_required?(@hosts)
         @connection_lock.synchronize do
           background_manager.update(@manager, update_struct)
@@ -223,16 +229,20 @@ module Mongo
         @refresh_thread = nil
       end
 
+      if @nodes
         @nodes.each do |member|
           member.disconnect
         end
+      end
 
       @nodes = []
       @read_pool = nil
 
+      if @secondary_pools
         @secondary_pools.each do |pool|
           pool.close
         end
+      end
 
       @secondaries     = []
       @secondary_pools = []
@@ -272,6 +282,7 @@ module Mongo
     private
 
     def initiate_auto_refresh
+      return unless @auto_refresh
       return if @refresh_thread && @refresh_thread.alive?
       @refresh_thread = Thread.new do
         sleep(@refresh_interval)

lib/mongo/util/pool.rb

@@ -109,6 +109,7 @@ module Mongo
         socket = TCPSocket.new(@host, @port)
         socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
       rescue => ex
+        socket.close if socket
         raise ConnectionFailure, "Failed to connect to host #{@host} and port #{@port}: #{ex}"
       end
 

lib/mongo/util/pool_manager.rb

@@ -2,11 +2,12 @@ module Mongo
   class PoolManager
 
     attr_reader :connection, :seeds, :arbiters, :primary, :secondaries,
-      :primary_pool, :read_pool, :secondary_pools, :hosts
+      :primary_pool, :read_pool, :secondary_pools, :hosts, :nodes
 
     def initialize(connection, seeds)
       @connection = connection
       @seeds = seeds
+      @refresh_node = nil
     end
 
     def connect
@@ -14,6 +15,7 @@ module Mongo
       nodes = connect_to_members
       initialize_pools(nodes)
       update_seed_list(nodes)
+      @nodes = nodes
     end
 
     # Ensure that the view of the replica set is current by
@@ -26,15 +28,15 @@ module Mongo
     # If we're connected to nodes that are no longer part of the set,
     # remove these from our set of secondary pools.
     def update_required?(hosts)
-      node = Thread.current[:refresher_node]
+      if !@refresh_node || !@refresh_node.active?
-      if !node || !node.active?
         begin
-          node = get_valid_seed_node
+          @refresh_node = get_valid_seed_node
-          Thread.current[:refresher_node] = node
         rescue ConnectionFailure
-          warn "Could not refresh config because no valid seed node was unavailable."
+          warn "Could not refresh config because no valid seed node was available."
+          return
         end
       end
+      node = @refresh_node
 
       node_list = node.node_list
 
@@ -104,6 +106,7 @@ module Mongo
       @secondaries = []
       @secondary_pools = []
       @hosts = []
+      @nodes = []
     end
 
     def connected_nodes
@@ -204,6 +207,8 @@ module Mongo
         node = Mongo::Node.new(self.connection, seed)
         if node.connect && node.set_config
           return node
+        else
+          node.disconnect
         end
       end
 

test/replica_sets/connect_test.rb

@@ -6,12 +6,9 @@ require './test/replica_sets/rs_test_helper'
 class ConnectTest < Test::Unit::TestCase
   include Mongo
 
-  def setup
-    RS.restart_killed_nodes
-  end
-
   def teardown
     RS.restart_killed_nodes
+    @conn.close if defined?(@conn) && @conn
   end
 
   def test_connect_with_deprecated_multi
@@ -22,25 +19,25 @@ class ConnectTest < Test::Unit::TestCase
 
   def test_connect_bad_name
     assert_raise_error(ReplicaSetConnectionError, "-wrong") do
-      ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
+      @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
         [RS.host, RS.ports[2]], :rs_name => RS.name + "-wrong")
     end
   end
 
-  def test_connect_timeout
+ # def test_connect_timeout
-    passed = false
+ #   passed = false
-    timeout = 3
+ #   timeout = 3
-    begin
+ #   begin
-      t0 = Time.now
+ #     t0 = Time.now
-      ReplSetConnection.new(['192.169.169.1', 27017], :connect_timeout => timeout)
+ #     @conn = ReplSetConnection.new(['192.169.169.1', 27017], :connect_timeout => timeout)
-    rescue OperationTimeout
+ #   rescue OperationTimeout
-      passed = true
+ #     passed = true
-      t1 = Time.now
+ #     t1 = Time.now
-    end
+ #   end
 
-    assert passed
+ #   assert passed
-    assert t1 - t0 < timeout + 1
+ #   assert t1 - t0 < timeout + 1
-  end
+ # end
 
   def test_connect
     @conn = ReplSetConnection.new([RS.host, RS.ports[1]], [RS.host, RS.ports[0]],
@@ -84,16 +81,20 @@ class ConnectTest < Test::Unit::TestCase
   def test_connect_with_secondary_node_killed
     node = RS.kill_secondary
 
+    rescue_connection_failure do
       @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
         [RS.host, RS.ports[2]])
+    end
     assert @conn.connected?
   end
 
   def test_connect_with_third_node_killed
     RS.kill(RS.get_node_from_port(RS.ports[2]))
 
+    rescue_connection_failure do
       @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
         [RS.host, RS.ports[2]])
+    end
     assert @conn.connected?
   end
 

test/replica_sets/connection_string_test.rb

@@ -6,12 +6,9 @@ require './test/replica_sets/rs_test_helper'
 class ConnectionStringTest < Test::Unit::TestCase
   include Mongo
 
-  def setup
-    RS.restart_killed_nodes
-  end
-
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_connect_with_connection_string

test/replica_sets/count_test.rb

@@ -15,6 +15,7 @@ class ReplicaSetCountTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_correct_count_after_insertion_reconnect

test/replica_sets/insert_test.rb

@@ -15,6 +15,7 @@ class ReplicaSetInsertTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_insert

test/replica_sets/pooled_insert_test.rb

@@ -8,7 +8,7 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
 
   def setup
     @conn = ReplSetConnection.new([RS.host, RS.ports[0]], [RS.host, RS.ports[1]],
-      [RS.host, RS.ports[2]], :pool_size => 10, :timeout => 5)
+      [RS.host, RS.ports[2]], :pool_size => 5, :timeout => 5)
     @db = @conn.db(MONGO_TEST_DB)
     @db.drop_collection("test-sets")
     @coll = @db.collection("test-sets")
@@ -16,6 +16,7 @@ class ReplicaSetPooledInsertTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_insert

test/replica_sets/query_secondaries.rb

@@ -14,6 +14,7 @@ class ReplicaSetQuerySecondariesTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_read_primary

test/replica_sets/query_test.rb

@@ -15,6 +15,7 @@ class ReplicaSetQueryTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_query

test/replica_sets/reconfigure_test.rb

@@ -13,6 +13,7 @@ class ReplicaSetReconfigureTest < Test::Unit::TestCase
 
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_query

test/replica_sets/refresh_test.rb

@@ -7,12 +7,9 @@ require 'benchmark'
 class ReplicaSetRefreshTest < Test::Unit::TestCase
   include Mongo
 
-  def setup
-    #RS.restart_killed_nodes
-  end
-
   def teardown
     RS.restart_killed_nodes
+    @conn.close if @conn
   end
 
   def test_connect_and_manual_refresh_with_secondaries_down

test/replica_sets/replication_ack_test.rb

@@ -20,6 +20,11 @@ class ReplicaSetAckTest < Test::Unit::TestCase
     @col = @db.collection("test-sets")
   end
 
+  def teardown
+    RS.restart_killed_nodes
+    @conn.close if @conn
+  end
+
   def test_safe_mode_with_w_failure
     assert_raise_error OperationFailure, "timeout" do
       @col.insert({:foo => 1}, :safe => {:w => 4, :wtimeout => 1, :fsync => true})

test/replica_sets/rs_test_helper.rb

@@ -11,7 +11,7 @@ class Test::Unit::TestCase
 
   # Generic code for rescuing connection failures and retrying operations.
   # This could be combined with some timeout functionality.
-  def rescue_connection_failure(max_retries=60)
+  def rescue_connection_failure(max_retries=30)
     retries = 0
     begin
       yield
@@ -19,7 +19,7 @@ class Test::Unit::TestCase
       puts "Rescue attempt #{retries}: from #{ex}"
       retries += 1
       raise ex if retries > max_retries
-      sleep(1)
+      sleep(2)
       retry
     end
   end

test/tools/repl_set_manager.rb

@@ -17,7 +17,7 @@ class ReplSetManager
     @ports      = []
     @name       = opts[:name] || 'replica-set-foo'
     @host       = opts[:host]  || 'localhost'
-    @retries    = opts[:retries] || 60
+    @retries    = opts[:retries] || 30
     @config     = {"_id" => @name, "members" => []}
     @durable    = opts.fetch(:durable, false)
     @path       = File.join(File.expand_path(File.dirname(__FILE__)), "data")
@@ -110,8 +110,6 @@ class ReplSetManager
 
     config = con['local']['system.replset'].find_one
     @config['version'] = config['version'] + 1
-    p "Old config: #{config}"
-    p "New config: #{@config}"
 
     # We expect a connection failure on reconfigure here.
     begin
@@ -119,6 +117,7 @@ class ReplSetManager
     rescue Mongo::ConnectionFailure
     end
 
+    con.close
     ensure_up
   end
 
@@ -146,6 +145,7 @@ class ReplSetManager
       con['admin'].command({'replSetStepDown' => 90})
     rescue Mongo::ConnectionFailure
     end
+    con.close
   end
 
   def kill_secondary
@@ -194,11 +194,14 @@ class ReplSetManager
       con = get_connection
       status = con['admin'].command({'replSetGetStatus' => 1})
       print "."
-      if status['members'].all? { |m| m['health'] == 1 && [1, 2, 7].include?(m['state']) } &&
+      if status['members'].all? { |m| m['health'] == 1 &&
+         [1, 2, 7].include?(m['state']) } &&
          status['members'].any? { |m| m['state'] == 1 }
         print "all members up!\n\n"
+        con.close
         return status
       else
+        con.close
         raise Mongo::OperationFailure
       end
     end
@@ -235,6 +238,8 @@ class ReplSetManager
     attempt do
       con['admin'].command({'replSetInitiate' => @config})
     end
+
+    con.close
   end
 
   def get_all_nodes_with_state(state)
@@ -295,11 +300,12 @@ class ReplSetManager
       begin
         return yield
         rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
-          sleep(1)
+          sleep(2)
           count += 1
       end
     end
 
+    puts "NO MORE ATTEMPTS"
     raise ex
   end