github-migration
/
mongo-ruby-driver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

utf-8 encoding for 1.9 and checking for 1.8 in C

This commit is contained in:
Mike Dirolf 2009-12-02 15:23:45 -05:00
parent 23bb6c2192
commit 84da36c380
3 changed files with 154 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ext/cbson/cbson.c
View File

@ -43,6 +43,7 @@
#include "version.h"
#include "buffer.h"
#include "encoding_helpers.h"
#define SAFE_WRITE(buffer, data, size) \
if (buffer_write((buffer), (data), (size)) != 0) \
@ -62,15 +63,32 @@ static VALUE Code;
static VALUE RegexpOfHolding;
static VALUE OrderedHash;
static VALUE InvalidName;
static VALUE InvalidStringEncoding;
static VALUE DigestMD5;
#if HAVE_RUBY_ENCODING_H
#include "ruby/encoding.h"
#define STR_NEW(p,n) rb_enc_str_new((p), (n), rb_utf8_encoding())
/* MUST call TO_UTF8 before calling write_utf8. */
#define TO_UTF8(string) rb_str_export_to_enc((string), rb_utf8_encoding())
static void write_utf8(buffer_t buffer, VALUE string) {
SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string));
}
#else
#define STR_NEW(p,n) rb_str_new((p), (n))
/* MUST call TO_UTF8 before calling write_utf8. */
#define TO_UTF8(string) (string)
static void write_utf8(buffer_t buffer, VALUE string) {
if (!is_legal_utf8_string(RSTRING_PTR(string), RSTRING_LEN(string))) {
buffer_free(buffer);
rb_raise(InvalidStringEncoding, "String not valid UTF-8");
}
SAFE_WRITE(buffer, RSTRING_PTR(string), RSTRING_LEN(string));
}
#endif
/* TODO free buffer on all exceptions! */
// this sucks. but for some reason these moved around between 1.8 and 1.9
#ifdef ONIGURUMA_H
#define IGNORECASE ONIG_OPTION_IGNORECASE
@ -118,7 +136,8 @@ static VALUE pack_extra(buffer_t buffer, VALUE check_keys) {
static void write_name_and_type(buffer_t buffer, VALUE name, char type) {
SAFE_WRITE(buffer, &type, 1);
SAFE_WRITE(buffer, RSTRING_PTR(name), RSTRING_LEN(name));
name = TO_UTF8(name);
write_utf8(buffer, name);
SAFE_WRITE(buffer, &zero, 1);
}
@ -259,10 +278,12 @@ static int write_element_allow_id(VALUE key, VALUE value, VALUE extra, int allow
SAFE_WRITE_AT_POS(buffer, length_location, (const char*)&total_length, 4);
break;
} else {
int length = RSTRING_LEN(value) + 1;
int length;
write_name_and_type(buffer, key, 0x02);
value = TO_UTF8(value);
length = RSTRING_LEN(value) + 1;
SAFE_WRITE(buffer, (char*)&length, 4);
SAFE_WRITE(buffer, RSTRING_PTR(value), length - 1);
write_utf8(buffer, value);
SAFE_WRITE(buffer, &zero, 1);
break;
}
@ -774,6 +795,7 @@ void Init_cbson() {
RegexpOfHolding = rb_const_get(mongo, rb_intern("RegexpOfHolding"));
rb_require("mongo/errors");
InvalidName = rb_const_get(mongo, rb_intern("InvalidName"));
InvalidStringEncoding = rb_const_get(mongo, rb_intern("InvalidStringEncoding"));
rb_require("mongo/util/ordered_hash");
OrderedHash = rb_const_get(rb_cObject, rb_intern("OrderedHash"));

107
ext/cbson/encoding_helpers.c Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright 2009 10gen, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Copyright 2001 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/
/*
* Index into the table below with the first byte of a UTF-8 sequence to
* get the number of trailing bytes that are supposed to follow it.
*/
static const char trailingBytesForUTF8[256] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};
/* --------------------------------------------------------------------- */
/*
* Utility routine to tell whether a sequence of bytes is legal UTF-8.
* This must be called with the length pre-determined by the first byte.
* The length can be set by:
* length = trailingBytesForUTF8[*source]+1;
* and the sequence is illegal right away if there aren't that many bytes
* available.
* If presented with a length > 4, this returns 0. The Unicode
* definition of UTF-8 goes up to 4-byte sequences.
*/
static unsigned char isLegalUTF8(const unsigned char* source, int length) {
unsigned char a;
const unsigned char* srcptr = source + length;
switch (length) {
default: return 0;
/* Everything else falls through when "true"... */
case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
case 2: if ((a = (*--srcptr)) > 0xBF) return 0;
switch (*source) {
/* no fall-through in this inner switch */
case 0xE0: if (a < 0xA0) return 0; break;
case 0xF0: if (a < 0x90) return 0; break;
case 0xF4: if (a > 0x8F) return 0; break;
default: if (a < 0x80) return 0;
}
case 1: if (*source >= 0x80 && *source < 0xC2) return 0;
if (*source > 0xF4) return 0;
}
return 1;
}
/* --------------------------------------------------------------------- */
/*
* Return whether a string containing UTF-8 is legal.
*/
unsigned char is_legal_utf8_string(const unsigned char* string, const int length) {
int position = 0;
while (position < length) {
int sequence_length = trailingBytesForUTF8[*(string + position)] + 1;
if ((position + sequence_length) > length) {
return 0;
}
if (!isLegalUTF8(string + position, sequence_length)) {
return 0;
}
position += sequence_length;
}
return 1;
}

22
ext/cbson/encoding_helpers.h Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2009 10gen, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ENCODING_HELPERS_H
#define ENCODING_HELPERS_H
unsigned char is_legal_utf8_string(const unsigned char* string, const int length);
#endif