From c70b1d1b30fa05e1a888a2083655b6065f75914b Mon Sep 17 00:00:00 2001 From: VvanGemert Date: Thu, 5 Jan 2012 16:27:05 +0100 Subject: [PATCH 1/7] Added support for keeping a limited amount of documents in GridFS and removed the delete_old option. The delete_old removed all the old files while having a versioning system you might want to keep a limited set of files. This small little patch does that. You can specify a number of versions to keep by using the new versions option. --- lib/mongo/gridfs/grid_file_system.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mongo/gridfs/grid_file_system.rb b/lib/mongo/gridfs/grid_file_system.rb index 3addfaa..41a18ca 100644 --- a/lib/mongo/gridfs/grid_file_system.rb +++ b/lib/mongo/gridfs/grid_file_system.rb @@ -97,7 +97,7 @@ module Mongo def open(filename, mode, opts={}) opts = opts.dup opts.merge!(default_grid_io_opts(filename)) - del = opts.delete(:delete_old) && mode == 'w' + versions = opts.delete(:versions) && mode == 'w' file = GridIO.new(@files, @chunks, filename, mode, opts) return file unless block_given? result = nil @@ -105,9 +105,9 @@ module Mongo result = yield file ensure id = file.close - if del + if versions self.delete do - @files.find({'filename' => filename, '_id' => {'$ne' => id}}, :fields => ['_id']) + @files.find({'filename' => filename, '_id' => {'$ne' => id}}, :fields => ['_id'], :sort => ['uploadDate', -1], :skip => (versions -1)) end end end From 417000da7bd00c83de9bae85b095811c87442be4 Mon Sep 17 00:00:00 2001 From: VvanGemert Date: Tue, 10 Jan 2012 17:37:25 +0100 Subject: [PATCH 2/7] Added back support for :delete_old option and wrote tests for :versions option on GridFilesystem. --- lib/mongo/gridfs/grid_file_system.rb | 7 ++++++- test/grid_file_system_test.rb | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/mongo/gridfs/grid_file_system.rb b/lib/mongo/gridfs/grid_file_system.rb index 41a18ca..2cd1ae4 100644 --- a/lib/mongo/gridfs/grid_file_system.rb +++ b/lib/mongo/gridfs/grid_file_system.rb @@ -97,7 +97,12 @@ module Mongo def open(filename, mode, opts={}) opts = opts.dup opts.merge!(default_grid_io_opts(filename)) - versions = opts.delete(:versions) && mode == 'w' + if mode == 'w' + versions = opts.delete(:versions) + if opts.delete(:delete_old) || (versions && versions < 1) + versions = 1 + end + end file = GridIO.new(@files, @chunks, filename, mode, opts) return file unless block_given? result = nil diff --git a/test/grid_file_system_test.rb b/test/grid_file_system_test.rb index bac67f4..0cae6b0 100644 --- a/test/grid_file_system_test.rb +++ b/test/grid_file_system_test.rb @@ -165,6 +165,18 @@ class GridFileSystemTest < Test::Unit::TestCase assert_equal 0, @db['fs.chunks'].find({'files_id' => {'$in' => @ids}}).count end + should "delete all versions which exceed the number of versions to keep specified by the option :versions" do + @versions = 1 + rand(4-1) + @grid.open('sample', 'w', :versions => @versions) do |f| + f.write @new_data + end + @new_ids = @db['fs.files'].find({'filename' => 'sample'}).map {|file| file['_id']} + assert_equal @versions, @new_ids.length + id = @new_ids.first + assert !@ids.include?(id) + assert_equal @versions, @db['fs.files'].find({'filename' => 'sample'}).count + end + should "delete old versions on write with :delete_old is passed in" do @grid.open('sample', 'w', :delete_old => true) do |f| f.write @new_data From 585e6722b4090a52d309ac9c91f49df2cd969f5d Mon Sep 17 00:00:00 2001 From: VvanGemert Date: Tue, 10 Jan 2012 17:52:13 +0100 Subject: [PATCH 3/7] Documented the option :versions in description of method GridFileSystem.open. --- lib/mongo/gridfs/grid_file_system.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/mongo/gridfs/grid_file_system.rb b/lib/mongo/gridfs/grid_file_system.rb index 2cd1ae4..c15c1fe 100644 --- a/lib/mongo/gridfs/grid_file_system.rb +++ b/lib/mongo/gridfs/grid_file_system.rb @@ -71,6 +71,9 @@ module Mongo # GridFileSystem#delete. # @option opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server # will be validated using an md5 hash. If validation fails, an exception will be raised. + # @option opts [Integer] :versions (false) deletes all versions which exceed the number specified to + # retain ordered by uploadDate. This option only works in 'w' mode. Certain precautions must be taken when + # deleting GridFS files. See the notes under GridFileSystem#delete. # # @example # From 6ab02373eaf51083c112c4f4a3d46c3cff072bc1 Mon Sep 17 00:00:00 2001 From: Cyril Mougel Date: Fri, 3 Feb 2012 17:29:30 +0100 Subject: [PATCH 4/7] add an option in connection to log duration --- lib/mongo/connection.rb | 9 ++++++--- test/connection_test.rb | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/mongo/connection.rb b/lib/mongo/connection.rb index b338a1e..a6a3291 100644 --- a/lib/mongo/connection.rb +++ b/lib/mongo/connection.rb @@ -36,7 +36,7 @@ module Mongo mongo_thread_local_accessor :connections - attr_reader :logger, :size, :auths, :primary, :safe, :host_to_try, + attr_reader :logger, :log_duration, :size, :auths, :primary, :safe, :host_to_try, :pool_size, :connect_timeout, :pool_timeout, :primary_pool, :socket_class, :op_timeout @@ -63,6 +63,8 @@ module Mongo # to a single, slave node. # @option opts [Logger, #debug] :logger (nil) A Logger instance for debugging driver ops. Note that # logging negatively impacts performance; therefore, it should not be used for high-performance apps. + # @option opts [Boolean, #debug] :log_duration (nil) A boolean to say if you want see information about log_duration on logger + # like logging, log_duration negatively impacts performance; therefore, it should not be used for high-performance apps. # @option opts [Integer] :pool_size (1) The maximum number of socket self.connections allowed per # connection pool. Note: this setting is relevant only for multi-threaded applications. # @option opts [Float] :pool_timeout (5.0) When all of the self.connections a pool are checked out, @@ -348,7 +350,7 @@ module Mongo self["admin"].command(oh) end - # Checks if a server is alive. This command will return immediately + # Checks if a server is alive. This command will return immediately # even if the server is in a lock. # # @return [Hash] @@ -383,7 +385,7 @@ module Mongo Thread.current[:socket_map] ||= {} Thread.current[:socket_map][self] ||= {} Thread.current[:socket_map][self][:writer] ||= checkout_writer - Thread.current[:socket_map][self][:reader] = + Thread.current[:socket_map][self][:reader] = Thread.current[:socket_map][self][:writer] end @@ -568,6 +570,7 @@ module Mongo if @logger write_logging_startup_message + @log_duration = opts[:log_duration] || false end should_connect = opts.fetch(:connect, true) diff --git a/test/connection_test.rb b/test/connection_test.rb index 3c1f4f3..f467673 100644 --- a/test/connection_test.rb +++ b/test/connection_test.rb @@ -148,6 +148,21 @@ class TestConnection < Test::Unit::TestCase assert output.string.include?('testing') end + def test_connection_log_duration_with_logger + output = StringIO.new + logger = Logger.new(output) + logger.level = Logger::DEBUG + connection = standard_connection(:logger => logger, :log_duration => true) + assert logger, connection.logger + assert connection.log_duration + end + + def test_connection_log_duration_without_logger + connection = standard_connection(:log_duration => true) + assert_nil connection.logger + assert !connection.log_duration + end + def test_drop_database db = @conn.db('ruby-mongo-will-be-deleted') coll = db.collection('temp') From 84c8e7cae241ec6050609347c533f09f304e39c1 Mon Sep 17 00:00:00 2001 From: Cyril Mougel Date: Fri, 3 Feb 2012 18:02:14 +0100 Subject: [PATCH 5/7] add log_duration system to add time spend in database --- lib/mongo/collection.rb | 11 ++++++----- lib/mongo/cursor.rb | 3 ++- lib/mongo/util/logging.rb | 11 ++++++++--- test/connection_test.rb | 9 +++++++++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/mongo/collection.rb b/lib/mongo/collection.rb index bbc5b72..9bdc14a 100644 --- a/lib/mongo/collection.rb +++ b/lib/mongo/collection.rb @@ -87,6 +87,7 @@ module Mongo @db, @name = db, name @connection = @db.connection @logger = @connection.logger + @log_duration = @connection.log_duration @cache_time = @db.cache_time @cache = Hash.new(0) unless pk_factory @@ -264,8 +265,8 @@ module Mongo # @return [OrderedHash, Nil] # a single document or nil if no result is found. # - # @param [Hash, ObjectId, Nil] spec_or_object_id a hash specifying elements - # which must be present for a document to be included in the result set or an + # @param [Hash, ObjectId, Nil] spec_or_object_id a hash specifying elements + # which must be present for a document to be included in the result set or an # instance of ObjectId to be used as the value for an _id query. # If nil, an empty selector, {}, will be used. # @@ -413,7 +414,7 @@ module Mongo # @option opts [Boolean] :upsert (+false+) if true, performs an upsert (update or insert) # @option opts [Boolean] :multi (+false+) update all documents matching the selector, as opposed to # just the first matching document. Note: only works in MongoDB 1.1.3 or later. - # @option opts [Boolean] :safe (+false+) + # @option opts [Boolean] :safe (+false+) # If true, check that the save succeeded. OperationFailure # will be raised on an error. Note that a safe check requires an extra # round-trip to the database. Safe options provided here will override any safe @@ -906,12 +907,12 @@ module Mongo if [Mongo::ASCENDING, Mongo::DESCENDING, Mongo::GEO2D, Mongo::GEOHAYSTACK].include?(f[1]) field_spec[f[0].to_s] = f[1] else - raise MongoArgumentError, "Invalid index field #{f[1].inspect}; " + + raise MongoArgumentError, "Invalid index field #{f[1].inspect}; " + "should be one of Mongo::ASCENDING (1), Mongo::DESCENDING (-1) or Mongo::GEO2D ('2d')." end end else - raise MongoArgumentError, "Invalid index specification #{spec.inspect}; " + + raise MongoArgumentError, "Invalid index specification #{spec.inspect}; " + "should be either a string, symbol, or an array of arrays." end field_spec diff --git a/lib/mongo/cursor.rb b/lib/mongo/cursor.rb index 452913d..9579562 100644 --- a/lib/mongo/cursor.rb +++ b/lib/mongo/cursor.rb @@ -43,6 +43,7 @@ module Mongo @collection = collection @connection = @db.connection @logger = @connection.logger + @log_duration = @connection.log_duration # Query selector @selector = opts[:selector] || {} @@ -199,7 +200,7 @@ module Mongo # This method overrides any sort order specified in the Collection#find # method, and only the last sort applied has an effect. # - # @param [Symbol, Array] key_or_list either 1) a key to sort by or 2) + # @param [Symbol, Array] key_or_list either 1) a key to sort by or 2) # an array of [key, direction] pairs to sort by. Direction should # be specified as Mongo::ASCENDING (or :ascending / :asc) or Mongo::DESCENDING (or :descending / :desc) # diff --git a/lib/mongo/util/logging.rb b/lib/mongo/util/logging.rb index 48a22a3..b6f303d 100644 --- a/lib/mongo/util/logging.rb +++ b/lib/mongo/util/logging.rb @@ -25,14 +25,15 @@ module Mongo # Execute the block and log the operation described by name and payload. def instrument(name, payload = {}, &blk) + time_elapse = @log_duration ? Time.now : nil res = yield - log_operation(name, payload) + log_operation(name, payload, time_elapse) res end protected - def log_operation(name, payload) + def log_operation(name, payload, time_elapse=nil) @logger ||= nil return unless @logger msg = "#{payload[:database]}['#{payload[:collection]}'].#{name}(" @@ -40,7 +41,11 @@ module Mongo msg += ".skip(#{payload[:skip]})" if payload[:skip] msg += ".limit(#{payload[:limit]})" if payload[:limit] msg += ".sort(#{payload[:order]})" if payload[:order] - @logger.debug "MONGODB #{msg}" + if time_elapse + @logger.debug "MONGODB (#{Time.now - time_elapse}s) #{msg}" + else + @logger.debug "MONGODB #{msg}" + end end end diff --git a/test/connection_test.rb b/test/connection_test.rb index f467673..8a64357 100644 --- a/test/connection_test.rb +++ b/test/connection_test.rb @@ -137,6 +137,15 @@ class TestConnection < Test::Unit::TestCase assert output.string.include?("admin['$cmd'].find") end + def test_logging_duration + output = StringIO.new + logger = Logger.new(output) + logger.level = Logger::DEBUG + connection = standard_connection(:logger => logger, :log_duration => true).db(MONGO_TEST_DB) + assert output.string.index(/\(0.\d+s\)/) + assert output.string.include?("admin['$cmd'].find") + end + def test_connection_logger output = StringIO.new logger = Logger.new(output) From 9eadb3c24a3b620fb756c44bbaeb8fdc612e86bf Mon Sep 17 00:00:00 2001 From: Ariel Salomon Date: Sun, 12 Feb 2012 13:14:18 -0800 Subject: [PATCH 6/7] Add read_only parameter to add_user --- lib/mongo/db.rb | 3 ++- test/auxillary/authentication_test.rb | 8 ++++++++ test/db_test.rb | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/mongo/db.rb b/lib/mongo/db.rb index f5613f5..0e4dd2d 100644 --- a/lib/mongo/db.rb +++ b/lib/mongo/db.rb @@ -175,10 +175,11 @@ module Mongo # @param [String] password # # @return [Hash] an object representing the user. - def add_user(username, password) + def add_user(username, password, read_only = false) users = self[SYSTEM_USER_COLLECTION] user = users.find_one({:user => username}) || {:user => username} user['pwd'] = Mongo::Support.hash_password(username, password) + user['readOnly'] = true if read_only; users.save(user) return user end diff --git a/test/auxillary/authentication_test.rb b/test/auxillary/authentication_test.rb index 83d33b4..a744d1e 100644 --- a/test/auxillary/authentication_test.rb +++ b/test/auxillary/authentication_test.rb @@ -26,6 +26,7 @@ class AuthenticationTest < Test::Unit::TestCase @admin.authenticate('bob', 'secret') @db1.add_user('user1', 'secret') @db2.add_user('user2', 'secret') + @db2.add_user('userRO', 'secret', true) # read-only @admin.logout assert_raise Mongo::OperationFailure do @@ -53,6 +54,7 @@ class AuthenticationTest < Test::Unit::TestCase assert @db1['stuff'].insert({:a => 2}, :safe => true) assert @db2['stuff'].insert({:a => 2}, :safe => true) + assert @db2['stuff'].find(:safe => true) @db1.logout assert_raise Mongo::OperationFailure do @@ -63,6 +65,12 @@ class AuthenticationTest < Test::Unit::TestCase assert_raise Mongo::OperationFailure do assert @db2['stuff'].insert({:a => 2}, :safe => true) end + + @db2.authenticate('userRO', 'secret') + assert @db2['stuff'].find(:safe => true) + assert_raise Mongo::OperationFailure do + assert @db2['stuff'].insert({:a => 2}, :safe => true) + end end end diff --git a/test/db_test.rb b/test/db_test.rb index 910d2c8..c14dd39 100644 --- a/test/db_test.rb +++ b/test/db_test.rb @@ -151,6 +151,13 @@ class DBTest < Test::Unit::TestCase @@db.remove_user('foo:bar') end + def test_authenticate_read_only + @@db.add_user('joebob', 'user', true) # read-only user + assert @@db.authenticate('joebob', 'user') + @@db.logout + @@db.remove_user('joebob') + end + def test_authenticate_with_connection_uri @@db.add_user('spongebob', 'squarepants') assert Mongo::Connection.from_uri("mongodb://spongebob:squarepants@#{host_port}/#{@@db.name}") From a89492d1f8c24db94c35be7f7ebb7e7209f9e9e2 Mon Sep 17 00:00:00 2001 From: Ariel Salomon Date: Sun, 12 Feb 2012 14:25:39 -0800 Subject: [PATCH 7/7] Add documentation for read_only parameter --- lib/mongo/db.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mongo/db.rb b/lib/mongo/db.rb index 0e4dd2d..72f0e8d 100644 --- a/lib/mongo/db.rb +++ b/lib/mongo/db.rb @@ -173,6 +173,8 @@ module Mongo # # @param [String] username # @param [String] password + # @param [Boolean] read_only + # Create a read-only user. # # @return [Hash] an object representing the user. def add_user(username, password, read_only = false)