diff --git a/Rakefile b/Rakefile index eefe0b0..04a52bc 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,5 @@ This.name = - "GridFs" + "Mongoid::GridFs" This.synopsis = "a mongoid 3/moped compatible implementation of the grid_fs specification" diff --git a/lib/mongoid-grid_fs.rb b/lib/mongoid-grid_fs.rb index fe974ce..24b5a5c 100644 --- a/lib/mongoid-grid_fs.rb +++ b/lib/mongoid-grid_fs.rb @@ -1,492 +1,493 @@ ## # - class GridFS - const_set :Version, '1.2.0' + module Mongoid + class GridFS + const_set :Version, '1.2.1' - class << GridFS - def version - const_get :Version - end - - def dependencies - { - 'mongoid' => [ 'mongoid' , ' >= 3.0.1' ] , - 'mime/types' => [ 'mime-types' , ' >= 1.19' ] , - } - end - - def libdir(*args, &block) - @libdir ||= File.expand_path(__FILE__).sub(/\.rb$/,'') - args.empty? ? @libdir : File.join(@libdir, *args) - ensure - if block - begin - $LOAD_PATH.unshift(@libdir) - block.call() - ensure - $LOAD_PATH.shift() - end - end - end - - def load(*libs) - libs = libs.join(' ').scan(/[^\s+]+/) - libdir{ libs.each{|lib| Kernel.load(lib) } } - end - end - - begin - require 'rubygems' - rescue LoadError - nil - end - - if defined?(gem) - dependencies.each do |lib, dependency| - gem(*dependency) - require(lib) - end - end - - require "digest/md5" - require "cgi" - end - -## -# - class GridFS - class << GridFS - attr_accessor :namespace - attr_accessor :file_model - attr_accessor :chunk_model - - def init! - GridFS.build_namespace_for(:Fs) - - GridFS.namespace = Fs - GridFS.file_model = Fs.file_model - GridFS.chunk_model = Fs.chunk_model - - const_set(:File, Fs.file_model) - const_set(:Chunk, Fs.chunk_model) - - to_delegate = %w( - put - get - delete - find - [] - []= - clear - ) - - to_delegate.each do |method| - class_eval <<-__ - def GridFS.#{ method }(*args, &block) - ::GridFS::Fs::#{ method }(*args, &block) - end - __ - end - end - end - - ## - # - def GridFS.namespace_for(prefix) - prefix = prefix.to_s.downcase - const = "::GridFS::#{ prefix.to_s.camelize }" - namespace = const.split(/::/).last - const_defined?(namespace) ? const_get(namespace) : build_namespace_for(namespace) - end - - ## - # - def GridFS.build_namespace_for(prefix) - prefix = prefix.to_s.downcase - const = prefix.camelize - - namespace = - Module.new do - module_eval(&NamespaceMixin) - self + class << GridFS + def version + const_get :Version end - const_set(const, namespace) - - file_model = build_file_model_for(namespace) - chunk_model = build_chunk_model_for(namespace) - - file_model.namespace = namespace - chunk_model.namespace = namespace - - file_model.chunk_model = chunk_model - chunk_model.file_model = file_model - - namespace.prefix = prefix - namespace.file_model = file_model - namespace.chunk_model = chunk_model - - namespace.send(:const_set, :File, file_model) - namespace.send(:const_set, :Chunk, chunk_model) - - #at_exit{ file_model.create_indexes rescue nil } - #at_exit{ chunk_model.create_indexes rescue nil } - - const_get(const) - end - - NamespaceMixin = proc do - class << self - attr_accessor :prefix - attr_accessor :file_model - attr_accessor :chunk_model - - def to_s - prefix + def dependencies + { + 'mongoid' => [ 'mongoid' , '>= 3.0.1' ] , + 'mime/types' => [ 'mime-types' , '>= 1.19' ] , + } end - def namespace - prefix - end - - def put(readable, attributes = {}) - chunks = [] - file = file_model.new - attributes.to_options! - - if attributes.has_key?(:id) - file.id = attributes.delete(:id) - end - - if attributes.has_key?(:_id) - file.id = attributes.delete(:_id) - end - - if attributes.has_key?(:content_type) - attributes[:contentType] = attributes.delete(:content_type) - end - - if attributes.has_key?(:upload_date) - attributes[:uploadDate] = attributes.delete(:upload_date) - end - - md5 = Digest::MD5.new - length = 0 - chunkSize = file.chunkSize - n = 0 - - GridFS.reading(readable) do |io| - - filename = - attributes[:filename] ||= - [file.id.to_s, GridFS.extract_basename(io)].join('/').squeeze('/') - - content_type = - attributes[:contentType] ||= - GridFS.extract_content_type(filename) || file.contentType - - GridFS.chunking(io, chunkSize) do |buf| - md5 << buf - length += buf.size - chunk = file.chunks.build - chunk.data = binary_for(buf) - chunk.n = n - n += 1 - chunk.save! - chunks.push(chunk) - end - - end - - attributes[:length] ||= length - attributes[:uploadDate] ||= Time.now.utc - attributes[:md5] ||= md5.hexdigest - - file.update_attributes(attributes) - - file.save! - file + def libdir(*args, &block) + @libdir ||= File.expand_path(__FILE__).sub(/\.rb$/,'') + args.empty? ? @libdir : File.join(@libdir, *args) ensure - chunks.each{|chunk| chunk.destroy rescue nil} if $! - end - - if defined?(Moped) - def binary_for(*buf) - Moped::BSON::Binary.new(:generic, buf.join) - end - else - def binary_for(buf) - BSON::Binary.new(buf.bytes.to_a) - end - end - - def get(id) - file_model.find(id) - end - - def delete(id) - file_model.find(id).destroy - rescue - nil - end - - def where(conditions = {}) - case conditions - when String - file_model.where(:filename => conditions) - else - file_model.where(conditions) - end - end - - def find(*args) - where(*args).first - end - - def [](filename) - file_model.where(:filename => filename.to_s).first - end - - def []=(filename, readable) - file = self[filename] - file.destroy if file - put(readable, :filename => filename.to_s) - end - - def clear - file_model.destroy_all - end - - # TODO - opening with a mode = 'w' should return a GridIO::IOProxy - # implementing a StringIO-like interface - # - def open(filename, mode = 'r', &block) - raise NotImplementedError - end - end - end - - ## - # - def GridFS.build_file_model_for(namespace) - prefix = namespace.name.split(/::/).last.downcase - file_model_name = "#{ namespace.name }::File" - chunk_model_name = "#{ namespace.name }::Chunk" - - Class.new do - include Mongoid::Document - - singleton_class = class << self; self; end - - singleton_class.instance_eval do - define_method(:name){ file_model_name } - attr_accessor :chunk_model - attr_accessor :namespace - end - - self.default_collection_name = "#{ prefix }.files" - - field(:filename, :type => String) - field(:contentType, :type => String, :default => 'application/octet-stream') - - field(:length, :type => Integer, :default => 0) - field(:chunkSize, :type => Integer, :default => (256 * (2 ** 20))) - field(:uploadDate, :type => Date, :default => Time.now.utc) - field(:md5, :type => String, :default => Digest::MD5.hexdigest('')) - - %w( filename contentType length chunkSize uploadDate md5 ).each do |f| - validates_presence_of(f) - end - validates_uniqueness_of(:filename) - - has_many(:chunks, :class_name => chunk_model_name, :inverse_of => :files, :dependent => :destroy, :order => [:n, :asc]) - - index({:filename => 1}, :unique => true) - - def path - filename - end - - def basename - ::File.basename(filename) - end - - def prefix - self.class.namespace.prefix - end - - def each(&block) - chunks.all.order_by([:n, :asc]).each do |chunk| - block.call(chunk.to_s) - end - end - - def data - data = '' - each{|chunk| data << chunk} - data - end - - def base64 - Array(to_s).pack('m') - end - - def data_uri(options = {}) - data = base64.chomp - "data:#{ content_type };base64,".concat(data) - end - - def bytes(&block) if block - each{|data| block.call(data)} - length - else - bytes = [] - each{|data| bytes.push(*data)} - bytes + begin + $LOAD_PATH.unshift(@libdir) + block.call() + ensure + $LOAD_PATH.shift() + end end end - def close - self - end - - def content_type - contentType - end - - def update_date - updateDate - end - - def created_at - updateDate - end - - def namespace - self.class.namespace + def load(*libs) + libs = libs.join(' ').scan(/[^\s+]+/) + libdir{ libs.each{|lib| Kernel.load(lib) } } end end - end - ## - # - def GridFS.build_chunk_model_for(namespace) - prefix = namespace.name.split(/::/).last.downcase - file_model_name = "#{ namespace.name }::File" - chunk_model_name = "#{ namespace.name }::Chunk" - - Class.new do - include Mongoid::Document - - singleton_class = class << self; self; end - - singleton_class.instance_eval do - define_method(:name){ chunk_model_name } - attr_accessor :file_model - attr_accessor :namespace - end - - self.default_collection_name = "#{ prefix }.chunks" - - field(:n, :type => Integer, :default => 0) - field(:data, :type => (defined?(Moped) ? Moped::BSON::Binary : BSON::Binary)) - - belongs_to(:file, :foreign_key => :files_id, :class_name => file_model_name) - - index({:files_id => 1, :n => -1}, :unique => true) - - def namespace - self.class.namespace - end - - def to_s - data.data - end - - alias_method 'to_str', 'to_s' - end - end - - ## - # - def GridFS.reading(arg, &block) - if arg.respond_to?(:read) - rewind(arg) do |io| - block.call(io) - end - else - open(arg.to_s) do |io| - block.call(io) - end - end - end - - def GridFS.chunking(io, chunk_size, &block) - if io.method(:read).arity == 0 - data = io.read - i = 0 - loop do - offset = i * chunk_size - length = i + chunk_size < data.size ? chunk_size : data.size - offset - - break if offset >= data.size - - buf = data[offset, length] - block.call(buf) - i += 1 - end - else - while((buf = io.read(chunk_size))) - block.call(buf) - end - end - end - - def GridFS.rewind(io, &block) begin - pos = io.pos - io.flush - io.rewind - rescue + require 'rubygems' + rescue LoadError nil end - begin - block.call(io) - ensure - begin - io.pos = pos - rescue - nil + if defined?(gem) + dependencies.each do |lib, dependency| + gem(*dependency) + require(lib) end end - end - def GridFS.extract_basename(object) - filename = nil - [:original_path, :original_filename, :path, :filename, :pathname].each do |msg| - if object.respond_to?(msg) - filename = object.send(msg) - break - end - end - filename ? cleanname(filename) : nil - end - - def GridFS.extract_content_type(filename) - content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first - content_type.to_s if content_type - end - - def GridFS.cleanname(pathname) - basename = ::File.basename(pathname.to_s) - CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_') + require "digest/md5" + require "cgi" end end ## # - GridFs = GridFS + module Mongoid + class GridFS + class << GridFS + attr_accessor :namespace + attr_accessor :file_model + attr_accessor :chunk_model - GridFS.init! + def init! + GridFS.build_namespace_for(:Fs) + + GridFS.namespace = Fs + GridFS.file_model = Fs.file_model + GridFS.chunk_model = Fs.chunk_model + + const_set(:File, Fs.file_model) + const_set(:Chunk, Fs.chunk_model) + + to_delegate = %w( + put + get + delete + find + [] + []= + clear + ) + + to_delegate.each do |method| + class_eval <<-__ + def GridFS.#{ method }(*args, &block) + ::GridFS::Fs::#{ method }(*args, &block) + end + __ + end + end + end + + ## + # + def GridFS.namespace_for(prefix) + prefix = prefix.to_s.downcase + const = "::GridFS::#{ prefix.to_s.camelize }" + namespace = const.split(/::/).last + const_defined?(namespace) ? const_get(namespace) : build_namespace_for(namespace) + end + + ## + # + def GridFS.build_namespace_for(prefix) + prefix = prefix.to_s.downcase + const = prefix.camelize + + namespace = + Module.new do + module_eval(&NamespaceMixin) + self + end + + const_set(const, namespace) + + file_model = build_file_model_for(namespace) + chunk_model = build_chunk_model_for(namespace) + + file_model.namespace = namespace + chunk_model.namespace = namespace + + file_model.chunk_model = chunk_model + chunk_model.file_model = file_model + + namespace.prefix = prefix + namespace.file_model = file_model + namespace.chunk_model = chunk_model + + namespace.send(:const_set, :File, file_model) + namespace.send(:const_set, :Chunk, chunk_model) + + #at_exit{ file_model.create_indexes rescue nil } + #at_exit{ chunk_model.create_indexes rescue nil } + + const_get(const) + end + + NamespaceMixin = proc do + class << self + attr_accessor :prefix + attr_accessor :file_model + attr_accessor :chunk_model + + def to_s + prefix + end + + def namespace + prefix + end + + def put(readable, attributes = {}) + chunks = [] + file = file_model.new + attributes.to_options! + + if attributes.has_key?(:id) + file.id = attributes.delete(:id) + end + + if attributes.has_key?(:_id) + file.id = attributes.delete(:_id) + end + + if attributes.has_key?(:content_type) + attributes[:contentType] = attributes.delete(:content_type) + end + + if attributes.has_key?(:upload_date) + attributes[:uploadDate] = attributes.delete(:upload_date) + end + + md5 = Digest::MD5.new + length = 0 + chunkSize = file.chunkSize + n = 0 + + GridFS.reading(readable) do |io| + + filename = + attributes[:filename] ||= + [file.id.to_s, GridFS.extract_basename(io)].join('/').squeeze('/') + + content_type = + attributes[:contentType] ||= + GridFS.extract_content_type(filename) || file.contentType + + GridFS.chunking(io, chunkSize) do |buf| + md5 << buf + length += buf.size + chunk = file.chunks.build + chunk.data = binary_for(buf) + chunk.n = n + n += 1 + chunk.save! + chunks.push(chunk) + end + + end + + attributes[:length] ||= length + attributes[:uploadDate] ||= Time.now.utc + attributes[:md5] ||= md5.hexdigest + + file.update_attributes(attributes) + + file.save! + file + ensure + chunks.each{|chunk| chunk.destroy rescue nil} if $! + end + + if defined?(Moped) + def binary_for(*buf) + Moped::BSON::Binary.new(:generic, buf.join) + end + else + def binary_for(buf) + BSON::Binary.new(buf.bytes.to_a) + end + end + + def get(id) + file_model.find(id) + end + + def delete(id) + file_model.find(id).destroy + rescue + nil + end + + def where(conditions = {}) + case conditions + when String + file_model.where(:filename => conditions) + else + file_model.where(conditions) + end + end + + def find(*args) + where(*args).first + end + + def [](filename) + file_model.where(:filename => filename.to_s).first + end + + def []=(filename, readable) + file = self[filename] + file.destroy if file + put(readable, :filename => filename.to_s) + end + + def clear + file_model.destroy_all + end + + # TODO - opening with a mode = 'w' should return a GridIO::IOProxy + # implementing a StringIO-like interface + # + def open(filename, mode = 'r', &block) + raise NotImplementedError + end + end + end + + ## + # + def GridFS.build_file_model_for(namespace) + prefix = namespace.name.split(/::/).last.downcase + file_model_name = "#{ namespace.name }::File" + chunk_model_name = "#{ namespace.name }::Chunk" + + Class.new do + include Mongoid::Document + + singleton_class = class << self; self; end + + singleton_class.instance_eval do + define_method(:name){ file_model_name } + attr_accessor :chunk_model + attr_accessor :namespace + end + + self.default_collection_name = "#{ prefix }.files" + + field(:filename, :type => String) + field(:contentType, :type => String, :default => 'application/octet-stream') + + field(:length, :type => Integer, :default => 0) + field(:chunkSize, :type => Integer, :default => (256 * (2 ** 20))) + field(:uploadDate, :type => Date, :default => Time.now.utc) + field(:md5, :type => String, :default => Digest::MD5.hexdigest('')) + + %w( filename contentType length chunkSize uploadDate md5 ).each do |f| + validates_presence_of(f) + end + validates_uniqueness_of(:filename) + + has_many(:chunks, :class_name => chunk_model_name, :inverse_of => :files, :dependent => :destroy, :order => [:n, :asc]) + + index({:filename => 1}, :unique => true) + + def path + filename + end + + def basename + ::File.basename(filename) + end + + def prefix + self.class.namespace.prefix + end + + def each(&block) + chunks.all.order_by([:n, :asc]).each do |chunk| + block.call(chunk.to_s) + end + end + + def data + data = '' + each{|chunk| data << chunk} + data + end + + def base64 + Array(to_s).pack('m') + end + + def data_uri(options = {}) + data = base64.chomp + "data:#{ content_type };base64,".concat(data) + end + + def bytes(&block) + if block + each{|data| block.call(data)} + length + else + bytes = [] + each{|data| bytes.push(*data)} + bytes + end + end + + def close + self + end + + def content_type + contentType + end + + def update_date + updateDate + end + + def created_at + updateDate + end + + def namespace + self.class.namespace + end + end + end + + ## + # + def GridFS.build_chunk_model_for(namespace) + prefix = namespace.name.split(/::/).last.downcase + file_model_name = "#{ namespace.name }::File" + chunk_model_name = "#{ namespace.name }::Chunk" + + Class.new do + include Mongoid::Document + + singleton_class = class << self; self; end + + singleton_class.instance_eval do + define_method(:name){ chunk_model_name } + attr_accessor :file_model + attr_accessor :namespace + end + + self.default_collection_name = "#{ prefix }.chunks" + + field(:n, :type => Integer, :default => 0) + field(:data, :type => (defined?(Moped) ? Moped::BSON::Binary : BSON::Binary)) + + belongs_to(:file, :foreign_key => :files_id, :class_name => file_model_name) + + index({:files_id => 1, :n => -1}, :unique => true) + + def namespace + self.class.namespace + end + + def to_s + data.data + end + + alias_method 'to_str', 'to_s' + end + end + + ## + # + def GridFS.reading(arg, &block) + if arg.respond_to?(:read) + rewind(arg) do |io| + block.call(io) + end + else + open(arg.to_s) do |io| + block.call(io) + end + end + end + + def GridFS.chunking(io, chunk_size, &block) + if io.method(:read).arity == 0 + data = io.read + i = 0 + loop do + offset = i * chunk_size + length = i + chunk_size < data.size ? chunk_size : data.size - offset + + break if offset >= data.size + + buf = data[offset, length] + block.call(buf) + i += 1 + end + else + while((buf = io.read(chunk_size))) + block.call(buf) + end + end + end + + def GridFS.rewind(io, &block) + begin + pos = io.pos + io.flush + io.rewind + rescue + nil + end + + begin + block.call(io) + ensure + begin + io.pos = pos + rescue + nil + end + end + end + + def GridFS.extract_basename(object) + filename = nil + [:original_path, :original_filename, :path, :filename, :pathname].each do |msg| + if object.respond_to?(msg) + filename = object.send(msg) + break + end + end + filename ? cleanname(filename) : nil + end + + def GridFS.extract_content_type(filename) + content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first + content_type.to_s if content_type + end + + def GridFS.cleanname(pathname) + basename = ::File.basename(pathname.to_s) + CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_') + end + end + + GridFs = GridFS + GridFS.init! + end diff --git a/test/mongoid-grid_fs_test.rb b/test/mongoid-grid_fs_test.rb index 8f8dd36..82f87e4 100644 --- a/test/mongoid-grid_fs_test.rb +++ b/test/mongoid-grid_fs_test.rb @@ -1,8 +1,12 @@ require_relative 'helper' -Testing GridFs do +Testing Mongoid::GridFs do ## # + GridFS = + GridFs = + Mongoid::GridFS + prepare do GridFS::File.destroy_all GridFS::Chunk.destroy_all