From 84da36c3804d86d42db54b2c985101289f1e293b Mon Sep 17 00:00:00 2001 From: Mike Dirolf Date: Wed, 2 Dec 2009 15:23:45 -0500 Subject: [PATCH] utf-8 encoding for 1.9 and checking for 1.8 in C --- ext/cbson/cbson.c | 28 ++++++++- ext/cbson/encoding_helpers.c | 107 +++++++++++++++++++++++++++++++++++ ext/cbson/encoding_helpers.h | 22 +++++++ 3 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 ext/cbson/encoding_helpers.c create mode 100644 ext/cbson/encoding_helpers.h diff --git a/ext/cbson/cbson.c b/ext/cbson/cbson.c index 115c195..9c75786 100644 --- a/ext/cbson/cbson.c +++ b/ext/cbson/cbson.c @@ -43,6 +43,7 @@ #include "version.h" #include "buffer.h" +#include "encoding_helpers.h" #define SAFE_WRITE(buffer, data, size) \ if (buffer_write((buffer), (data), (size)) != 0) \ @@ -62,15 +63,32 @@ static VALUE Code; static VALUE RegexpOfHolding; static VALUE OrderedHash; static VALUE InvalidName; +static VALUE InvalidStringEncoding; static VALUE DigestMD5; #if HAVE_RUBY_ENCODING_H #include "ruby/encoding.h" #define STR_NEW(p,n) rb_enc_str_new((p), (n), rb_utf8_encoding()) +/* MUST call TO_UTF8 before calling write_utf8. */ +#define TO_UTF8(string) rb_str_export_to_enc((string), rb_utf8_encoding()) +static void write_utf8(buffer_t buffer, VALUE string) { + SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string)); +} #else #define STR_NEW(p,n) rb_str_new((p), (n)) +/* MUST call TO_UTF8 before calling write_utf8. */ +#define TO_UTF8(string) (string) +static void write_utf8(buffer_t buffer, VALUE string) { + if (!is_legal_utf8_string(RSTRING_PTR(string), RSTRING_LEN(string))) { + buffer_free(buffer); + rb_raise(InvalidStringEncoding, "String not valid UTF-8"); + } + SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string)); +} #endif +/* TODO free buffer on all exceptions! */ + // this sucks. but for some reason these moved around between 1.8 and 1.9 #ifdef ONIGURUMA_H #define IGNORECASE ONIG_OPTION_IGNORECASE @@ -118,7 +136,8 @@ static VALUE pack_extra(buffer_t buffer, VALUE check_keys) { static void write_name_and_type(buffer_t buffer, VALUE name, char type) { SAFE_WRITE(buffer, &type, 1); - SAFE_WRITE(buffer, RSTRING_PTR(name), RSTRING_LEN(name)); + name = TO_UTF8(name); + write_utf8(buffer, name); SAFE_WRITE(buffer, &zero, 1); } @@ -259,10 +278,12 @@ static int write_element_allow_id(VALUE key, VALUE value, VALUE extra, int allow SAFE_WRITE_AT_POS(buffer, length_location, (const char*)&total_length, 4); break; } else { - int length = RSTRING_LEN(value) + 1; + int length; write_name_and_type(buffer, key, 0x02); + value = TO_UTF8(value); + length = RSTRING_LEN(value) + 1; SAFE_WRITE(buffer, (char*)&length, 4); - SAFE_WRITE(buffer, RSTRING_PTR(value), length - 1); + write_utf8(buffer, value); SAFE_WRITE(buffer, &zero, 1); break; } @@ -774,6 +795,7 @@ void Init_cbson() { RegexpOfHolding = rb_const_get(mongo, rb_intern("RegexpOfHolding")); rb_require("mongo/errors"); InvalidName = rb_const_get(mongo, rb_intern("InvalidName")); + InvalidStringEncoding = rb_const_get(mongo, rb_intern("InvalidStringEncoding")); rb_require("mongo/util/ordered_hash"); OrderedHash = rb_const_get(rb_cObject, rb_intern("OrderedHash")); diff --git a/ext/cbson/encoding_helpers.c b/ext/cbson/encoding_helpers.c new file mode 100644 index 0000000..6ec0f25 --- /dev/null +++ b/ext/cbson/encoding_helpers.c @@ -0,0 +1,107 @@ +/* + * Copyright 2009 10gen, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright 2001 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + +/* + * Index into the table below with the first byte of a UTF-8 sequence to + * get the number of trailing bytes that are supposed to follow it. + */ +static const char trailingBytesForUTF8[256] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 +}; + +/* --------------------------------------------------------------------- */ + +/* + * Utility routine to tell whether a sequence of bytes is legal UTF-8. + * This must be called with the length pre-determined by the first byte. + * The length can be set by: + * length = trailingBytesForUTF8[*source]+1; + * and the sequence is illegal right away if there aren't that many bytes + * available. + * If presented with a length > 4, this returns 0. The Unicode + * definition of UTF-8 goes up to 4-byte sequences. + */ +static unsigned char isLegalUTF8(const unsigned char* source, int length) { + unsigned char a; + const unsigned char* srcptr = source + length; + switch (length) { + default: return 0; + /* Everything else falls through when "true"... */ + case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; + case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0; + case 2: if ((a = (*--srcptr)) > 0xBF) return 0; + switch (*source) { + /* no fall-through in this inner switch */ + case 0xE0: if (a < 0xA0) return 0; break; + case 0xF0: if (a < 0x90) return 0; break; + case 0xF4: if (a > 0x8F) return 0; break; + default: if (a < 0x80) return 0; + } + case 1: if (*source >= 0x80 && *source < 0xC2) return 0; + if (*source > 0xF4) return 0; + } + return 1; +} + +/* --------------------------------------------------------------------- */ + +/* + * Return whether a string containing UTF-8 is legal. + */ +unsigned char is_legal_utf8_string(const unsigned char* string, const int length) { + int position = 0; + + while (position < length) { + int sequence_length = trailingBytesForUTF8[*(string + position)] + 1; + if ((position + sequence_length) > length) { + return 0; + } + if (!isLegalUTF8(string + position, sequence_length)) { + return 0; + } + position += sequence_length; + } + return 1; +} diff --git a/ext/cbson/encoding_helpers.h b/ext/cbson/encoding_helpers.h new file mode 100644 index 0000000..0c08727 --- /dev/null +++ b/ext/cbson/encoding_helpers.h @@ -0,0 +1,22 @@ +/* + * Copyright 2009 10gen, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENCODING_HELPERS_H +#define ENCODING_HELPERS_H + +unsigned char is_legal_utf8_string(const unsigned char* string, const int length); + +#endif