utf-8 encoding for 1.9 and checking for 1.8 in C

ext/cbson/cbson.c

@@ -43,6 +43,7 @@
 
 #include "version.h"
 #include "buffer.h"
+#include "encoding_helpers.h"
 
 #define SAFE_WRITE(buffer, data, size)                                  \
     if (buffer_write((buffer), (data), (size)) != 0)                    \
@@ -62,15 +63,32 @@ static VALUE Code;
 static VALUE RegexpOfHolding;
 static VALUE OrderedHash;
 static VALUE InvalidName;
+static VALUE InvalidStringEncoding;
 static VALUE DigestMD5;
 
 #if HAVE_RUBY_ENCODING_H
 #include "ruby/encoding.h"
 #define STR_NEW(p,n) rb_enc_str_new((p), (n), rb_utf8_encoding())
+/* MUST call TO_UTF8 before calling write_utf8. */
+#define TO_UTF8(string) rb_str_export_to_enc((string), rb_utf8_encoding())
+static void write_utf8(buffer_t buffer, VALUE string) {
+    SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string));
+}
 #else
 #define STR_NEW(p,n) rb_str_new((p), (n))
+/* MUST call TO_UTF8 before calling write_utf8. */
+#define TO_UTF8(string) (string)
+static void write_utf8(buffer_t buffer, VALUE string) {
+    if (!is_legal_utf8_string(RSTRING_PTR(string), RSTRING_LEN(string))) {
+        buffer_free(buffer);
+        rb_raise(InvalidStringEncoding, "String not valid UTF-8");
+    }
+    SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string));
+}
 #endif
 
+/* TODO free buffer on all exceptions! */
+
 // this sucks. but for some reason these moved around between 1.8 and 1.9
 #ifdef ONIGURUMA_H
 #define IGNORECASE ONIG_OPTION_IGNORECASE
@@ -118,7 +136,8 @@ static VALUE pack_extra(buffer_t buffer, VALUE check_keys) {
 
 static void write_name_and_type(buffer_t buffer, VALUE name, char type) {
     SAFE_WRITE(buffer, &type, 1);
-    SAFE_WRITE(buffer, RSTRING_PTR(name), RSTRING_LEN(name));
+    name = TO_UTF8(name);
+    write_utf8(buffer, name);
     SAFE_WRITE(buffer, &zero, 1);
 }
 
@@ -259,10 +278,12 @@ static int write_element_allow_id(VALUE key, VALUE value, VALUE extra, int allow
                 SAFE_WRITE_AT_POS(buffer, length_location, (const char*)&total_length, 4);
                 break;
             } else {
-                int length = RSTRING_LEN(value) + 1;
+                int length;
                 write_name_and_type(buffer, key, 0x02);
+                value = TO_UTF8(value);
+                length = RSTRING_LEN(value) + 1;
                 SAFE_WRITE(buffer, (char*)&length, 4);
-                SAFE_WRITE(buffer, RSTRING_PTR(value), length - 1);
+                write_utf8(buffer, value);
                 SAFE_WRITE(buffer, &zero, 1);
                 break;
             }
@@ -774,6 +795,7 @@ void Init_cbson() {
     RegexpOfHolding = rb_const_get(mongo, rb_intern("RegexpOfHolding"));
     rb_require("mongo/errors");
     InvalidName = rb_const_get(mongo, rb_intern("InvalidName"));
+    InvalidStringEncoding = rb_const_get(mongo, rb_intern("InvalidStringEncoding"));
     rb_require("mongo/util/ordered_hash");
     OrderedHash = rb_const_get(rb_cObject, rb_intern("OrderedHash"));
 

ext/cbson/encoding_helpers.c

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009 10gen, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright 2001 Unicode, Inc.
+ *
+ * Disclaimer
+ *
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ *
+ * Limitations on Rights to Redistribute This Code
+ *
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
+};
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
+ * This must be called with the length pre-determined by the first byte.
+ * The length can be set by:
+ *  length = trailingBytesForUTF8[*source]+1;
+ * and the sequence is illegal right away if there aren't that many bytes
+ * available.
+ * If presented with a length > 4, this returns 0.  The Unicode
+ * definition of UTF-8 goes up to 4-byte sequences.
+ */
+static unsigned char isLegalUTF8(const unsigned char* source, int length) {
+    unsigned char a;
+    const unsigned char* srcptr = source + length;
+    switch (length) {
+    default: return 0;
+        /* Everything else falls through when "true"... */
+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
+    case 2: if ((a = (*--srcptr)) > 0xBF) return 0;
+        switch (*source) {
+            /* no fall-through in this inner switch */
+            case 0xE0: if (a < 0xA0) return 0; break;
+            case 0xF0: if (a < 0x90) return 0; break;
+            case 0xF4: if (a > 0x8F) return 0; break;
+            default:  if (a < 0x80) return 0;
+        }
+        case 1: if (*source >= 0x80 && *source < 0xC2) return 0;
+        if (*source > 0xF4) return 0;
+    }
+    return 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
+ * Return whether a string containing UTF-8 is legal.
+ */
+unsigned char is_legal_utf8_string(const unsigned char* string, const int length) {
+    int position = 0;
+
+    while (position < length) {
+        int sequence_length = trailingBytesForUTF8[*(string + position)] + 1;
+        if ((position + sequence_length) > length) {
+            return 0;
+        }
+        if (!isLegalUTF8(string + position, sequence_length)) {
+            return 0;
+        }
+        position += sequence_length;
+    }
+    return 1;
+}

ext/cbson/encoding_helpers.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009 10gen, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ENCODING_HELPERS_H
+#define ENCODING_HELPERS_H
+
+unsigned char is_legal_utf8_string(const unsigned char* string, const int length);
+
+#endif