Add bin and edit workflow
Gitea Actions Demo / Explore-Gitea-Actions (push) Failing after 9s

This commit is contained in:
2026-09-16 13:11:16 -06:00
parent c8ac4fcae5
commit 4cee170d66
17576 changed files with 895740 additions and 2 deletions
@@ -0,0 +1,3 @@
import '../../../lib/google/tasks/ffi.rake'
task default: ['ffi-protobuf:default']
@@ -0,0 +1,342 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Ruby <-> upb data conversion functions.
//
// This file Also contains a few other assorted algorithms on upb_MessageValue.
//
// None of the algorithms in this file require any access to the internal
// representation of Ruby or upb objects.
// -----------------------------------------------------------------------------
#include "convert.h"
#include "message.h"
#include "protobuf.h"
#include "shared_convert.h"
static upb_StringView Convert_StringData(VALUE str, upb_Arena* arena) {
upb_StringView ret;
if (arena) {
char* ptr = upb_Arena_Malloc(arena, RSTRING_LEN(str));
memcpy(ptr, RSTRING_PTR(str), RSTRING_LEN(str));
ret.data = ptr;
} else {
// Data is only needed temporarily (within map lookup).
ret.data = RSTRING_PTR(str);
}
ret.size = RSTRING_LEN(str);
return ret;
}
static bool is_ruby_num(VALUE value) {
return (TYPE(value) == T_FLOAT || TYPE(value) == T_FIXNUM ||
TYPE(value) == T_BIGNUM);
}
static void Convert_CheckInt(const char* name, upb_CType type, VALUE val) {
if (!is_ruby_num(val)) {
rb_raise(cTypeError,
"Expected number type for integral field '%s' (given %s).", name,
rb_class2name(CLASS_OF(val)));
}
// NUM2{INT,UINT,LL,ULL} macros do the appropriate range checks on upper
// bound; we just need to do precision checks (i.e., disallow rounding) and
// check for < 0 on unsigned types.
if (TYPE(val) == T_FLOAT) {
double dbl_val = NUM2DBL(val);
if (floor(dbl_val) != dbl_val) {
rb_raise(rb_eRangeError,
"Non-integral floating point value assigned to integer field "
"'%s' (given %s).",
name, rb_class2name(CLASS_OF(val)));
}
}
if (type == kUpb_CType_UInt32 || type == kUpb_CType_UInt64) {
if (NUM2DBL(val) < 0) {
rb_raise(
rb_eRangeError,
"Assigning negative value to unsigned integer field '%s' (given %s).",
name, rb_class2name(CLASS_OF(val)));
}
}
}
static int32_t Convert_ToEnum(VALUE value, const char* name,
const upb_EnumDef* e) {
int32_t val;
switch (TYPE(value)) {
case T_FLOAT:
case T_FIXNUM:
case T_BIGNUM:
Convert_CheckInt(name, kUpb_CType_Int32, value);
val = NUM2INT(value);
break;
case T_STRING: {
const upb_EnumValueDef* ev = upb_EnumDef_FindValueByNameWithSize(
e, RSTRING_PTR(value), RSTRING_LEN(value));
if (!ev) goto unknownval;
val = upb_EnumValueDef_Number(ev);
break;
}
case T_SYMBOL: {
const upb_EnumValueDef* ev =
upb_EnumDef_FindValueByName(e, rb_id2name(SYM2ID(value)));
if (!ev) goto unknownval;
val = upb_EnumValueDef_Number(ev);
break;
}
default:
rb_raise(cTypeError,
"Expected number or symbol type for enum field '%s'.", name);
}
return val;
unknownval:
rb_raise(rb_eRangeError, "Unknown symbol value for enum field '%s'.", name);
}
VALUE Convert_CheckStringUtf8(VALUE str) {
VALUE utf8 = rb_enc_from_encoding(rb_utf8_encoding());
if (rb_obj_encoding(str) == utf8) {
// Note: Just because a string is marked as having UTF-8 encoding does
// not mean that it is *valid* UTF-8. We have to check separately
// whether it is valid.
if (rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN) {
// TODO: For now
// we only warn for this case. We will remove the warning and throw an
// exception below in the 30.x release
rb_warn(
"String is invalid UTF-8. This will be an error in a future "
"version.");
// VALUE exc = rb_const_get_at(
// rb_cEncoding, rb_intern("InvalidByteSequenceError"));
// rb_raise(exc, "String is invalid UTF-8");
}
} else {
// Note: this will not duplicate underlying string data unless
// necessary.
//
// This will throw an exception if the conversion cannot be performed:
// - Encoding::UndefinedConversionError if certain characters cannot be
// converted to UTF-8.
// - Encoding::InvalidByteSequenceError if certain characters were invalid
// in the source encoding.
str = rb_str_encode(str, utf8, 0, Qnil);
PBRUBY_ASSERT(rb_enc_str_coderange(str) != ENC_CODERANGE_BROKEN);
}
return str;
}
upb_MessageValue Convert_RubyToUpb(VALUE value, const char* name,
TypeInfo type_info, upb_Arena* arena) {
upb_MessageValue ret;
switch (type_info.type) {
case kUpb_CType_Float:
if (!is_ruby_num(value)) {
rb_raise(cTypeError,
"Expected number type for float field '%s' (given %s).", name,
rb_class2name(CLASS_OF(value)));
}
ret.float_val = NUM2DBL(value);
break;
case kUpb_CType_Double:
if (!is_ruby_num(value)) {
rb_raise(cTypeError,
"Expected number type for double field '%s' (given %s).", name,
rb_class2name(CLASS_OF(value)));
}
ret.double_val = NUM2DBL(value);
break;
case kUpb_CType_Bool: {
if (value == Qtrue) {
ret.bool_val = 1;
} else if (value == Qfalse) {
ret.bool_val = 0;
} else {
rb_raise(cTypeError,
"Invalid argument for boolean field '%s' (given %s).", name,
rb_class2name(CLASS_OF(value)));
}
break;
}
case kUpb_CType_String:
if (rb_obj_class(value) == rb_cSymbol) {
value = rb_funcall(value, rb_intern("to_s"), 0);
} else if (!rb_obj_is_kind_of(value, rb_cString)) {
rb_raise(cTypeError,
"Invalid argument for string field '%s' (given %s).", name,
rb_class2name(CLASS_OF(value)));
}
value = Convert_CheckStringUtf8(value);
ret.str_val = Convert_StringData(value, arena);
break;
case kUpb_CType_Bytes: {
VALUE bytes = rb_enc_from_encoding(rb_ascii8bit_encoding());
if (rb_obj_class(value) != rb_cString) {
rb_raise(cTypeError,
"Invalid argument for bytes field '%s' (given %s).", name,
rb_class2name(CLASS_OF(value)));
}
if (rb_obj_encoding(value) != bytes) {
// Note: this will not duplicate underlying string data unless
// necessary.
// TODO: is this really necessary to get raw bytes?
value = rb_str_encode(value, bytes, 0, Qnil);
}
ret.str_val = Convert_StringData(value, arena);
break;
}
case kUpb_CType_Message:
ret.msg_val =
Message_GetUpbMessage(value, type_info.def.msgdef, name, arena);
break;
case kUpb_CType_Enum:
ret.int32_val = Convert_ToEnum(value, name, type_info.def.enumdef);
break;
case kUpb_CType_Int32:
case kUpb_CType_Int64:
case kUpb_CType_UInt32:
case kUpb_CType_UInt64:
Convert_CheckInt(name, type_info.type, value);
switch (type_info.type) {
case kUpb_CType_Int32:
ret.int32_val = NUM2INT(value);
break;
case kUpb_CType_Int64:
ret.int64_val = NUM2LL(value);
break;
case kUpb_CType_UInt32:
ret.uint32_val = NUM2UINT(value);
break;
case kUpb_CType_UInt64:
ret.uint64_val = NUM2ULL(value);
break;
default:
rb_raise(cTypeError, "Convert_RubyToUpb(): Unexpected type %d",
(int)type_info.type);
}
break;
default:
rb_raise(cTypeError,
"Convert_RubyToUpb(): Unexpected type %d", (int)type_info.type);
}
return ret;
}
VALUE Convert_UpbToRuby(upb_MessageValue upb_val, TypeInfo type_info,
VALUE arena) {
switch (type_info.type) {
case kUpb_CType_Float:
return DBL2NUM(upb_val.float_val);
case kUpb_CType_Double:
return DBL2NUM(upb_val.double_val);
case kUpb_CType_Bool:
return upb_val.bool_val ? Qtrue : Qfalse;
case kUpb_CType_Int32:
return INT2NUM(upb_val.int32_val);
case kUpb_CType_Int64:
return LL2NUM(upb_val.int64_val);
case kUpb_CType_UInt32:
return UINT2NUM(upb_val.uint32_val);
case kUpb_CType_UInt64:
return ULL2NUM(upb_val.int64_val);
case kUpb_CType_Enum: {
const upb_EnumValueDef* ev = upb_EnumDef_FindValueByNumber(
type_info.def.enumdef, upb_val.int32_val);
if (ev) {
return ID2SYM(rb_intern(upb_EnumValueDef_Name(ev)));
} else {
return INT2NUM(upb_val.int32_val);
}
}
case kUpb_CType_String: {
VALUE str_rb = rb_str_new(upb_val.str_val.data, upb_val.str_val.size);
rb_enc_associate(str_rb, rb_utf8_encoding());
rb_obj_freeze(str_rb);
return str_rb;
}
case kUpb_CType_Bytes: {
VALUE str_rb = rb_str_new(upb_val.str_val.data, upb_val.str_val.size);
rb_enc_associate(str_rb, rb_ascii8bit_encoding());
rb_obj_freeze(str_rb);
return str_rb;
}
case kUpb_CType_Message:
return Message_GetRubyWrapper((upb_Message*)upb_val.msg_val,
type_info.def.msgdef, arena);
default:
rb_raise(rb_eRuntimeError, "Convert_UpbToRuby(): Unexpected type %d",
(int)type_info.type);
}
}
upb_MessageValue Msgval_DeepCopy(upb_MessageValue msgval, TypeInfo type_info,
upb_Arena* arena) {
upb_MessageValue new_msgval;
switch (type_info.type) {
default:
memcpy(&new_msgval, &msgval, sizeof(msgval));
break;
case kUpb_CType_String:
case kUpb_CType_Bytes: {
size_t n = msgval.str_val.size;
char* mem = upb_Arena_Malloc(arena, n);
new_msgval.str_val.data = mem;
new_msgval.str_val.size = n;
memcpy(mem, msgval.str_val.data, n);
break;
}
case kUpb_CType_Message:
new_msgval.msg_val =
Message_deep_copy(msgval.msg_val, type_info.def.msgdef, arena);
break;
}
return new_msgval;
}
bool Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
TypeInfo type_info) {
upb_Status status;
upb_Status_Clear(&status);
bool return_value = shared_Msgval_IsEqual(val1, val2, type_info.type,
type_info.def.msgdef, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
rb_raise(rb_eRuntimeError, "Msgval_IsEqual(): %s",
upb_Status_ErrorMessage(&status));
}
}
uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info,
uint64_t seed) {
upb_Status status;
upb_Status_Clear(&status);
uint64_t return_value = shared_Msgval_GetHash(
val, type_info.type, type_info.def.msgdef, seed, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
rb_raise(rb_eRuntimeError, "Msgval_GetHash(): %s",
upb_Status_ErrorMessage(&status));
}
}
@@ -0,0 +1,50 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef RUBY_PROTOBUF_CONVERT_H_
#define RUBY_PROTOBUF_CONVERT_H_
#include "protobuf.h"
#include "ruby-upb.h"
// Converts |ruby_val| to a upb_MessageValue according to |type_info|.
//
// The |arena| parameter indicates the lifetime of the container where this
// value will be assigned. It is used as follows:
// - If type is string or bytes, the string data will be copied into |arena|.
// - If type is message, and we need to auto-construct a message due to implicit
// conversions (eg. Time -> Google::Protobuf::Timestamp), the new message
// will be created in |arena|.
// - If type is message and the Ruby value is a message instance, we will fuse
// the message's arena into |arena|, to ensure that this message outlives the
// container.
upb_MessageValue Convert_RubyToUpb(VALUE ruby_val, const char *name,
TypeInfo type_info, upb_Arena *arena);
// Converts |upb_val| to a Ruby VALUE according to |type_info|. This may involve
// creating a Ruby wrapper object.
//
// The |arena| parameter indicates the arena that owns the lifetime of
// |upb_val|. Any Ruby wrapper object that is created will reference |arena|
// and ensure it outlives the wrapper.
VALUE Convert_UpbToRuby(upb_MessageValue upb_val, TypeInfo type_info,
VALUE arena);
// Creates a deep copy of |msgval| in |arena|.
upb_MessageValue Msgval_DeepCopy(upb_MessageValue msgval, TypeInfo type_info,
upb_Arena *arena);
// Returns true if |val1| and |val2| are equal. Their type is given by
// |type_info|.
bool Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
TypeInfo type_info);
// Returns a hash value for the given upb_MessageValue.
uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info,
uint64_t seed);
#endif // RUBY_PROTOBUF_CONVERT_H_
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,82 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef RUBY_PROTOBUF_DEFS_H_
#define RUBY_PROTOBUF_DEFS_H_
#include "protobuf.h"
#include "ruby-upb.h"
// -----------------------------------------------------------------------------
// TypeInfo
// -----------------------------------------------------------------------------
// This bundles a upb_CType and msgdef/enumdef when appropriate. This is
// convenient for functions that need type information but cannot necessarily
// assume a upb_FieldDef will be available.
//
// For example, Google::Protobuf::Map and Google::Protobuf::RepeatedField can
// be constructed with type information alone:
//
// # RepeatedField will internally store the type information in a TypeInfo.
// Google::Protobuf::RepeatedField.new(:message, FooMessage)
typedef struct {
upb_CType type;
union {
const upb_MessageDef* msgdef; // When type == kUpb_CType_Message
const upb_EnumDef* enumdef; // When type == kUpb_CType_Enum
} def;
} TypeInfo;
static inline TypeInfo TypeInfo_get(const upb_FieldDef* f) {
TypeInfo ret = {upb_FieldDef_CType(f), {NULL}};
switch (ret.type) {
case kUpb_CType_Message:
ret.def.msgdef = upb_FieldDef_MessageSubDef(f);
break;
case kUpb_CType_Enum:
ret.def.enumdef = upb_FieldDef_EnumSubDef(f);
break;
default:
break;
}
return ret;
}
TypeInfo TypeInfo_FromClass(int argc, VALUE* argv, int skip_arg,
VALUE* type_class, VALUE* init_arg);
static inline TypeInfo TypeInfo_from_type(upb_CType type) {
TypeInfo ret = {type};
assert(type != kUpb_CType_Message && type != kUpb_CType_Enum);
return ret;
}
// -----------------------------------------------------------------------------
// Other utilities
// -----------------------------------------------------------------------------
VALUE Descriptor_DefToClass(const upb_MessageDef* m);
// Returns the underlying msgdef, enumdef, or symtab (respectively) for the
// given Descriptor, EnumDescriptor, or DescriptorPool Ruby object.
const upb_EnumDef* EnumDescriptor_GetEnumDef(VALUE enum_desc_rb);
const upb_DefPool* DescriptorPool_GetSymtab(VALUE desc_pool_rb);
const upb_MessageDef* Descriptor_GetMsgDef(VALUE desc_rb);
// Returns a upb field type for the given Ruby symbol
// (eg. :float => kUpb_CType_Float).
upb_CType ruby_to_fieldtype(VALUE type);
// The singleton generated pool (a DescriptorPool object).
extern VALUE generated_pool;
// Call at startup to register all types in this module.
void Defs_register(VALUE module);
#endif // RUBY_PROTOBUF_DEFS_H_
@@ -0,0 +1,28 @@
#!/usr/bin/ruby
require 'mkmf'
ext_name = "google/protobuf_c"
dir_config(ext_name)
if RUBY_PLATFORM =~ /darwin/ || RUBY_PLATFORM =~ /linux/ || RUBY_PLATFORM =~ /freebsd/
$CFLAGS += " -std=gnu99 -O3 -DNDEBUG -fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement"
else
$CFLAGS += " -std=gnu99 -O3 -DNDEBUG"
end
if RUBY_PLATFORM =~ /linux/
# Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper.
$LDFLAGS += " -Wl,-wrap,memcpy"
end
$VPATH << "$(srcdir)/third_party/utf8_range"
$INCFLAGS += " -I$(srcdir)/third_party/utf8_range"
$srcs = ["protobuf.c", "convert.c", "defs.c", "message.c",
"repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c",
"utf8_range.c", "shared_convert.c",
"shared_message.c"]
create_makefile(ext_name)
@@ -0,0 +1,72 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Exposing inlined UPB functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "ruby-upb.h"
upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); }
google_protobuf_FileDescriptorProto* FileDescriptorProto_parse(
const char* serialized_file_proto, size_t length, upb_Arena* arena) {
return google_protobuf_FileDescriptorProto_parse(serialized_file_proto,
length, arena);
}
char* EnumDescriptor_serialized_options(const upb_EnumDef* enumdef,
size_t* size, upb_Arena* arena) {
const google_protobuf_EnumOptions* opts = upb_EnumDef_Options(enumdef);
char* serialized = google_protobuf_EnumOptions_serialize(opts, arena, size);
return serialized;
}
char* FileDescriptor_serialized_options(const upb_FileDef* filedef,
size_t* size, upb_Arena* arena) {
const google_protobuf_FileOptions* opts = upb_FileDef_Options(filedef);
char* serialized = google_protobuf_FileOptions_serialize(opts, arena, size);
return serialized;
}
char* Descriptor_serialized_options(const upb_MessageDef* msgdef, size_t* size,
upb_Arena* arena) {
const google_protobuf_MessageOptions* opts = upb_MessageDef_Options(msgdef);
char* serialized =
google_protobuf_MessageOptions_serialize(opts, arena, size);
return serialized;
}
char* OneOfDescriptor_serialized_options(const upb_OneofDef* oneofdef,
size_t* size, upb_Arena* arena) {
const google_protobuf_OneofOptions* opts = upb_OneofDef_Options(oneofdef);
char* serialized = google_protobuf_OneofOptions_serialize(opts, arena, size);
return serialized;
}
char* FieldDescriptor_serialized_options(const upb_FieldDef* fielddef,
size_t* size, upb_Arena* arena) {
const google_protobuf_FieldOptions* opts = upb_FieldDef_Options(fielddef);
char* serialized = google_protobuf_FieldOptions_serialize(opts, arena, size);
return serialized;
}
char* ServiceDescriptor_serialized_options(const upb_ServiceDef* servicedef,
size_t* size, upb_Arena* arena) {
const google_protobuf_ServiceOptions* opts =
upb_ServiceDef_Options(servicedef);
char* serialized =
google_protobuf_ServiceOptions_serialize(opts, arena, size);
return serialized;
}
char* MethodDescriptor_serialized_options(const upb_MethodDef* methoddef,
size_t* size, upb_Arena* arena) {
const google_protobuf_MethodOptions* opts = upb_MethodDef_Options(methoddef);
char* serialized = google_protobuf_MethodOptions_serialize(opts, arena, size);
return serialized;
}
@@ -0,0 +1,731 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include "convert.h"
#include "defs.h"
#include "message.h"
#include "protobuf.h"
// -----------------------------------------------------------------------------
// Basic map operations on top of upb_Map.
//
// Note that we roll our own `Map` container here because, as for
// `RepeatedField`, we want a strongly-typed container. This is so that any user
// errors due to incorrect map key or value types are raised as close as
// possible to the error site, rather than at some deferred point (e.g.,
// serialization).
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// Map container type.
// -----------------------------------------------------------------------------
typedef struct {
const upb_Map* map; // Can convert to mutable when non-frozen.
upb_CType key_type;
TypeInfo value_type_info;
VALUE value_type_class;
VALUE arena;
} Map;
static void Map_mark(void* _self) {
Map* self = _self;
rb_gc_mark(self->value_type_class);
rb_gc_mark(self->arena);
}
static size_t Map_memsize(const void* _self) { return sizeof(Map); }
const rb_data_type_t Map_type = {
"Google::Protobuf::Map",
{Map_mark, RUBY_DEFAULT_FREE, Map_memsize},
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
};
VALUE cMap;
static Map* ruby_to_Map(VALUE _self) {
Map* self;
TypedData_Get_Struct(_self, Map, &Map_type, self);
return self;
}
static VALUE Map_alloc(VALUE klass) {
Map* self = ALLOC(Map);
self->map = NULL;
self->value_type_class = Qnil;
self->value_type_info.def.msgdef = NULL;
self->arena = Qnil;
return TypedData_Wrap_Struct(klass, &Map_type, self);
}
VALUE Map_GetRubyWrapper(const upb_Map* map, upb_CType key_type,
TypeInfo value_type, VALUE arena) {
PBRUBY_ASSERT(map);
PBRUBY_ASSERT(arena != Qnil);
VALUE val = ObjectCache_Get(map);
if (val == Qnil) {
val = Map_alloc(cMap);
Map* self;
TypedData_Get_Struct(val, Map, &Map_type, self);
self->map = map;
self->arena = arena;
self->key_type = key_type;
self->value_type_info = value_type;
if (self->value_type_info.type == kUpb_CType_Message) {
const upb_MessageDef* val_m = self->value_type_info.def.msgdef;
self->value_type_class = Descriptor_DefToClass(val_m);
}
return ObjectCache_TryAdd(map, val);
}
return val;
}
static VALUE Map_new_this_type(Map* from) {
VALUE arena_rb = Arena_new();
upb_Map* map = upb_Map_New(Arena_get(arena_rb), from->key_type,
from->value_type_info.type);
VALUE ret =
Map_GetRubyWrapper(map, from->key_type, from->value_type_info, arena_rb);
PBRUBY_ASSERT(ruby_to_Map(ret)->value_type_class == from->value_type_class);
return ret;
}
static TypeInfo Map_keyinfo(Map* self) {
TypeInfo ret;
ret.type = self->key_type;
ret.def.msgdef = NULL;
return ret;
}
static upb_Map* Map_GetMutable(VALUE _self) {
const upb_Map* map = ruby_to_Map(_self)->map;
Protobuf_CheckNotFrozen(_self, upb_Map_IsFrozen(map));
return (upb_Map*)map;
}
VALUE Map_CreateHash(const upb_Map* map, upb_CType key_type,
TypeInfo val_info) {
VALUE hash = rb_hash_new();
TypeInfo key_info = TypeInfo_from_type(key_type);
if (!map) return hash;
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(map, &key, &val, &iter)) {
VALUE key_val = Convert_UpbToRuby(key, key_info, Qnil);
VALUE val_val = Scalar_CreateHash(val, val_info);
rb_hash_aset(hash, key_val, val_val);
}
return hash;
}
VALUE Map_deep_copy(VALUE obj) {
Map* self = ruby_to_Map(obj);
VALUE new_arena_rb = Arena_new();
upb_Arena* arena = Arena_get(new_arena_rb);
upb_Map* new_map =
upb_Map_New(arena, self->key_type, self->value_type_info.type);
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
upb_MessageValue val_copy =
Msgval_DeepCopy(val, self->value_type_info, arena);
upb_Map_Set(new_map, key, val_copy, arena);
}
return Map_GetRubyWrapper(new_map, self->key_type, self->value_type_info,
new_arena_rb);
}
const upb_Map* Map_GetUpbMap(VALUE val, const upb_FieldDef* field,
upb_Arena* arena) {
const upb_FieldDef* key_field = map_field_key(field);
const upb_FieldDef* value_field = map_field_value(field);
TypeInfo value_type_info = TypeInfo_get(value_field);
Map* self;
if (!RB_TYPE_P(val, T_DATA) || !RTYPEDDATA_P(val) ||
RTYPEDDATA_TYPE(val) != &Map_type) {
rb_raise(cTypeError, "Expected Map instance");
}
self = ruby_to_Map(val);
if (self->key_type != upb_FieldDef_CType(key_field)) {
rb_raise(cTypeError, "Map key type does not match field's key type");
}
if (self->value_type_info.type != value_type_info.type) {
rb_raise(cTypeError, "Map value type does not match field's value type");
}
if (self->value_type_info.def.msgdef != value_type_info.def.msgdef) {
rb_raise(cTypeError, "Map value type has wrong message/enum class");
}
Arena_fuse(self->arena, arena);
return self->map;
}
void Map_Inspect(StringBuilder* b, const upb_Map* map, upb_CType key_type,
TypeInfo val_type) {
bool first = true;
TypeInfo key_type_info = {key_type};
StringBuilder_Printf(b, "{");
if (map) {
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(map, &key, &val, &iter)) {
if (first) {
first = false;
} else {
StringBuilder_Printf(b, ", ");
}
StringBuilder_PrintMsgval(b, key, key_type_info);
StringBuilder_Printf(b, "=>");
StringBuilder_PrintMsgval(b, val, val_type);
}
}
StringBuilder_Printf(b, "}");
}
static int merge_into_self_callback(VALUE key, VALUE val, VALUE _self) {
Map* self = ruby_to_Map(_self);
upb_Arena* arena = Arena_get(self->arena);
upb_MessageValue key_val =
Convert_RubyToUpb(key, "", Map_keyinfo(self), arena);
upb_MessageValue val_val =
Convert_RubyToUpb(val, "", self->value_type_info, arena);
upb_Map_Set(Map_GetMutable(_self), key_val, val_val, arena);
return ST_CONTINUE;
}
// Used only internally -- shared by #merge and #initialize.
static VALUE Map_merge_into_self(VALUE _self, VALUE hashmap) {
if (TYPE(hashmap) == T_HASH) {
rb_hash_foreach(hashmap, merge_into_self_callback, _self);
} else if (RB_TYPE_P(hashmap, T_DATA) && RTYPEDDATA_P(hashmap) &&
RTYPEDDATA_TYPE(hashmap) == &Map_type) {
Map* self = ruby_to_Map(_self);
Map* other = ruby_to_Map(hashmap);
upb_Arena* arena = Arena_get(self->arena);
upb_Map* self_map = Map_GetMutable(_self);
Arena_fuse(other->arena, arena);
if (self->key_type != other->key_type ||
self->value_type_info.type != other->value_type_info.type ||
self->value_type_class != other->value_type_class) {
rb_raise(rb_eArgError, "Attempt to merge Map with mismatching types");
}
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(other->map, &key, &val, &iter)) {
upb_Map_Set(self_map, key, val, arena);
}
} else {
rb_raise(rb_eArgError, "Unknown type merging into Map");
}
return _self;
}
/*
* call-seq:
* Map.new(key_type, value_type, value_typeclass = nil, init_hashmap = {})
* => new map
*
* Allocates a new Map container. This constructor may be called with 2, 3, or 4
* arguments. The first two arguments are always present and are symbols (taking
* on the same values as field-type symbols in message descriptors) that
* indicate the type of the map key and value fields.
*
* The supported key types are: :int32, :int64, :uint32, :uint64, :bool,
* :string, :bytes.
*
* The supported value types are: :int32, :int64, :uint32, :uint64, :bool,
* :string, :bytes, :enum, :message.
*
* The third argument, value_typeclass, must be present if value_type is :enum
* or :message. As in RepeatedField#new, this argument must be a message class
* (for :message) or enum module (for :enum).
*
* The last argument, if present, provides initial content for map. Note that
* this may be an ordinary Ruby hashmap or another Map instance with identical
* key and value types. Also note that this argument may be present whether or
* not value_typeclass is present (and it is unambiguously separate from
* value_typeclass because value_typeclass's presence is strictly determined by
* value_type). The contents of this initial hashmap or Map instance are
* shallow-copied into the new Map: the original map is unmodified, but
* references to underlying objects will be shared if the value type is a
* message type.
*/
static VALUE Map_init(int argc, VALUE* argv, VALUE _self) {
Map* self = ruby_to_Map(_self);
VALUE init_arg;
// We take either two args (:key_type, :value_type), three args (:key_type,
// :value_type, "ValueMessageType"), or four args (the above plus an initial
// hashmap).
if (argc < 2 || argc > 4) {
rb_raise(rb_eArgError, "Map constructor expects 2, 3 or 4 arguments.");
}
self->key_type = ruby_to_fieldtype(argv[0]);
self->value_type_info =
TypeInfo_FromClass(argc, argv, 1, &self->value_type_class, &init_arg);
self->arena = Arena_new();
// Check that the key type is an allowed type.
switch (self->key_type) {
case kUpb_CType_Int32:
case kUpb_CType_Int64:
case kUpb_CType_UInt32:
case kUpb_CType_UInt64:
case kUpb_CType_Bool:
case kUpb_CType_String:
case kUpb_CType_Bytes:
// These are OK.
break;
default:
rb_raise(rb_eArgError, "Invalid key type for map.");
}
self->map = upb_Map_New(Arena_get(self->arena), self->key_type,
self->value_type_info.type);
VALUE stored = ObjectCache_TryAdd(self->map, _self);
(void)stored;
PBRUBY_ASSERT(stored == _self);
if (init_arg != Qnil) {
Map_merge_into_self(_self, init_arg);
}
return Qnil;
}
/*
* call-seq:
* Map.each(&block)
*
* Invokes &block on each |key, value| pair in the map, in unspecified order.
* Note that Map also includes Enumerable; map thus acts like a normal Ruby
* sequence.
*/
static VALUE Map_each(VALUE _self) {
Map* self = ruby_to_Map(_self);
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
VALUE key_val = Convert_UpbToRuby(key, Map_keyinfo(self), self->arena);
VALUE val_val = Convert_UpbToRuby(val, self->value_type_info, self->arena);
rb_yield_values(2, key_val, val_val);
}
return Qnil;
}
/*
* call-seq:
* Map.keys => [list_of_keys]
*
* Returns the list of keys contained in the map, in unspecified order.
*/
static VALUE Map_keys(VALUE _self) {
Map* self = ruby_to_Map(_self);
size_t iter = kUpb_Map_Begin;
VALUE ret = rb_ary_new();
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
VALUE key_val = Convert_UpbToRuby(key, Map_keyinfo(self), self->arena);
rb_ary_push(ret, key_val);
}
return ret;
}
/*
* call-seq:
* Map.values => [list_of_values]
*
* Returns the list of values contained in the map, in unspecified order.
*/
static VALUE Map_values(VALUE _self) {
Map* self = ruby_to_Map(_self);
size_t iter = kUpb_Map_Begin;
VALUE ret = rb_ary_new();
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
VALUE val_val = Convert_UpbToRuby(val, self->value_type_info, self->arena);
rb_ary_push(ret, val_val);
}
return ret;
}
/*
* call-seq:
* Map.[](key) => value
*
* Accesses the element at the given key. Throws an exception if the key type is
* incorrect. Returns nil when the key is not present in the map.
*/
static VALUE Map_index(VALUE _self, VALUE key) {
Map* self = ruby_to_Map(_self);
upb_MessageValue key_upb =
Convert_RubyToUpb(key, "", Map_keyinfo(self), NULL);
upb_MessageValue val;
if (upb_Map_Get(self->map, key_upb, &val)) {
return Convert_UpbToRuby(val, self->value_type_info, self->arena);
} else {
return Qnil;
}
}
/*
* call-seq:
* Map.[]=(key, value) => value
*
* Inserts or overwrites the value at the given key with the given new value.
* Throws an exception if the key type is incorrect. Returns the new value that
* was just inserted.
*/
static VALUE Map_index_set(VALUE _self, VALUE key, VALUE val) {
Map* self = ruby_to_Map(_self);
upb_Arena* arena = Arena_get(self->arena);
upb_MessageValue key_upb =
Convert_RubyToUpb(key, "", Map_keyinfo(self), NULL);
upb_MessageValue val_upb =
Convert_RubyToUpb(val, "", self->value_type_info, arena);
upb_Map_Set(Map_GetMutable(_self), key_upb, val_upb, arena);
return val;
}
/*
* call-seq:
* Map.has_key?(key) => bool
*
* Returns true if the given key is present in the map. Throws an exception if
* the key has the wrong type.
*/
static VALUE Map_has_key(VALUE _self, VALUE key) {
Map* self = ruby_to_Map(_self);
upb_MessageValue key_upb =
Convert_RubyToUpb(key, "", Map_keyinfo(self), NULL);
if (upb_Map_Get(self->map, key_upb, NULL)) {
return Qtrue;
} else {
return Qfalse;
}
}
/*
* call-seq:
* Map.delete(key) => old_value
*
* Deletes the value at the given key, if any, returning either the old value or
* nil if none was present. Throws an exception if the key is of the wrong type.
*/
static VALUE Map_delete(VALUE _self, VALUE key) {
upb_Map* map = Map_GetMutable(_self);
Map* self = ruby_to_Map(_self);
upb_MessageValue key_upb =
Convert_RubyToUpb(key, "", Map_keyinfo(self), NULL);
upb_MessageValue val_upb;
if (upb_Map_Delete(map, key_upb, &val_upb)) {
return Convert_UpbToRuby(val_upb, self->value_type_info, self->arena);
} else {
return Qnil;
}
}
/*
* call-seq:
* Map.clear
*
* Removes all entries from the map.
*/
static VALUE Map_clear(VALUE _self) {
upb_Map_Clear(Map_GetMutable(_self));
return Qnil;
}
/*
* call-seq:
* Map.length
*
* Returns the number of entries (key-value pairs) in the map.
*/
static VALUE Map_length(VALUE _self) {
Map* self = ruby_to_Map(_self);
return ULL2NUM(upb_Map_Size(self->map));
}
/*
* call-seq:
* Map.dup => new_map
*
* Duplicates this map with a shallow copy. References to all non-primitive
* element objects (e.g., submessages) are shared.
*/
static VALUE Map_dup(VALUE _self) {
Map* self = ruby_to_Map(_self);
VALUE new_map_rb = Map_new_this_type(self);
Map* new_self = ruby_to_Map(new_map_rb);
size_t iter = kUpb_Map_Begin;
upb_Arena* arena = Arena_get(new_self->arena);
upb_Map* new_map = Map_GetMutable(new_map_rb);
Arena_fuse(self->arena, arena);
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
upb_Map_Set(new_map, key, val, arena);
}
return new_map_rb;
}
/*
* call-seq:
* Map.==(other) => boolean
*
* Compares this map to another. Maps are equal if they have identical key sets,
* and for each key, the values in both maps compare equal. Elements are
* compared as per normal Ruby semantics, by calling their :== methods (or
* performing a more efficient comparison for primitive types).
*
* Maps with dissimilar key types or value types/typeclasses are never equal,
* even if value comparison (for example, between integers and floats) would
* have otherwise indicated that every element has equal value.
*/
VALUE Map_eq(VALUE _self, VALUE _other) {
Map* self = ruby_to_Map(_self);
Map* other;
// Allow comparisons to Ruby hashmaps by converting to a temporary Map
// instance. Slow, but workable.
if (TYPE(_other) == T_HASH) {
VALUE other_map = Map_new_this_type(self);
Map_merge_into_self(other_map, _other);
_other = other_map;
}
other = ruby_to_Map(_other);
if (self == other) {
return Qtrue;
}
if (self->key_type != other->key_type ||
self->value_type_info.type != other->value_type_info.type ||
self->value_type_class != other->value_type_class) {
return Qfalse;
}
if (upb_Map_Size(self->map) != upb_Map_Size(other->map)) {
return Qfalse;
}
// For each member of self, check that an equal member exists at the same key
// in other.
size_t iter = kUpb_Map_Begin;
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
upb_MessageValue other_val;
if (!upb_Map_Get(other->map, key, &other_val)) {
// Not present in other map.
return Qfalse;
}
if (!Msgval_IsEqual(val, other_val, self->value_type_info)) {
// Present but different value.
return Qfalse;
}
}
return Qtrue;
}
/*
* call-seq:
* Map.frozen? => bool
*
* Returns true if the map is frozen in either Ruby or the underlying
* representation. Freezes the Ruby map object if it is not already frozen in
* Ruby but it is frozen in the underlying representation.
*/
VALUE Map_frozen(VALUE _self) {
Map* self = ruby_to_Map(_self);
if (!upb_Map_IsFrozen(self->map)) {
PBRUBY_ASSERT(!RB_OBJ_FROZEN(_self));
return Qfalse;
}
// Lazily freeze the Ruby wrapper.
if (!RB_OBJ_FROZEN(_self)) RB_OBJ_FREEZE(_self);
return Qtrue;
}
/*
* call-seq:
* Map.freeze => self
*
* Freezes the map object. We have to intercept this so we can freeze the
* underlying representation, not just the Ruby wrapper.
*/
VALUE Map_freeze(VALUE _self) {
Map* self = ruby_to_Map(_self);
if (RB_OBJ_FROZEN(_self)) {
PBRUBY_ASSERT(upb_Map_IsFrozen(self->map));
return _self;
}
if (!upb_Map_IsFrozen(self->map)) {
if (self->value_type_info.type == kUpb_CType_Message) {
upb_Map_Freeze(
Map_GetMutable(_self),
upb_MessageDef_MiniTable(self->value_type_info.def.msgdef));
} else {
upb_Map_Freeze(Map_GetMutable(_self), NULL);
}
}
RB_OBJ_FREEZE(_self);
return _self;
}
VALUE Map_EmptyFrozen(const upb_FieldDef* f) {
PBRUBY_ASSERT(upb_FieldDef_IsMap(f));
VALUE val = ObjectCache_Get(f);
if (val == Qnil) {
const upb_FieldDef* key_f = map_field_key(f);
const upb_FieldDef* val_f = map_field_value(f);
upb_CType key_type = upb_FieldDef_CType(key_f);
TypeInfo value_type_info = TypeInfo_get(val_f);
val = Map_alloc(cMap);
Map* self;
TypedData_Get_Struct(val, Map, &Map_type, self);
self->arena = Arena_new();
self->map =
upb_Map_New(Arena_get(self->arena), key_type, value_type_info.type);
self->key_type = key_type;
self->value_type_info = value_type_info;
if (self->value_type_info.type == kUpb_CType_Message) {
const upb_MessageDef* val_m = value_type_info.def.msgdef;
self->value_type_class = Descriptor_DefToClass(val_m);
}
return ObjectCache_TryAdd(f, Map_freeze(val));
}
PBRUBY_ASSERT(RB_OBJ_FROZEN(val));
PBRUBY_ASSERT(upb_Map_IsFrozen(ruby_to_Map(val)->map));
return val;
}
/*
* call-seq:
* Map.hash => hash_value
*
* Returns a hash value based on this map's contents.
*/
VALUE Map_hash(VALUE _self) {
Map* self = ruby_to_Map(_self);
uint64_t hash = 0;
size_t iter = kUpb_Map_Begin;
TypeInfo key_info = {self->key_type};
upb_MessageValue key, val;
while (upb_Map_Next(self->map, &key, &val, &iter)) {
hash = Msgval_GetHash(key, key_info, hash);
hash = Msgval_GetHash(val, self->value_type_info, hash);
}
return LL2NUM(hash);
}
/*
* call-seq:
* Map.to_h => {}
*
* Returns a Ruby Hash object containing all the values within the map
*/
VALUE Map_to_h(VALUE _self) {
Map* self = ruby_to_Map(_self);
return Map_CreateHash(self->map, self->key_type, self->value_type_info);
}
/*
* call-seq:
* Map.inspect => string
*
* Returns a string representing this map's elements. It will be formatted as
* "{key => value, key => value, ...}", with each key and value string
* representation computed by its own #inspect method.
*/
VALUE Map_inspect(VALUE _self) {
Map* self = ruby_to_Map(_self);
StringBuilder* builder = StringBuilder_New();
Map_Inspect(builder, self->map, self->key_type, self->value_type_info);
VALUE ret = StringBuilder_ToRubyString(builder);
StringBuilder_Free(builder);
return ret;
}
/*
* call-seq:
* Map.merge(other_map) => map
*
* Copies key/value pairs from other_map into a copy of this map. If a key is
* set in other_map and this map, the value from other_map overwrites the value
* in the new copy of this map. Returns the new copy of this map with merged
* contents.
*/
static VALUE Map_merge(VALUE _self, VALUE hashmap) {
VALUE dupped = Map_dup(_self);
return Map_merge_into_self(dupped, hashmap);
}
void Map_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "Map", rb_cObject);
rb_define_alloc_func(klass, Map_alloc);
rb_gc_register_address(&cMap);
cMap = klass;
rb_define_method(klass, "initialize", Map_init, -1);
rb_define_method(klass, "each", Map_each, 0);
rb_define_method(klass, "keys", Map_keys, 0);
rb_define_method(klass, "values", Map_values, 0);
rb_define_method(klass, "[]", Map_index, 1);
rb_define_method(klass, "[]=", Map_index_set, 2);
rb_define_method(klass, "has_key?", Map_has_key, 1);
rb_define_method(klass, "delete", Map_delete, 1);
rb_define_method(klass, "clear", Map_clear, 0);
rb_define_method(klass, "length", Map_length, 0);
rb_define_method(klass, "size", Map_length, 0);
rb_define_method(klass, "dup", Map_dup, 0);
// Also define #clone so that we don't inherit Object#clone.
rb_define_method(klass, "clone", Map_dup, 0);
rb_define_method(klass, "==", Map_eq, 1);
rb_define_method(klass, "freeze", Map_freeze, 0);
rb_define_method(klass, "frozen?", Map_frozen, 0);
rb_define_method(klass, "hash", Map_hash, 0);
rb_define_method(klass, "to_h", Map_to_h, 0);
rb_define_method(klass, "inspect", Map_inspect, 0);
rb_define_method(klass, "merge", Map_merge, 1);
rb_include_module(klass, rb_mEnumerable);
}
@@ -0,0 +1,48 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef RUBY_PROTOBUF_MAP_H_
#define RUBY_PROTOBUF_MAP_H_
#include "protobuf.h"
#include "ruby-upb.h"
// Returns a frozen sentinel Ruby wrapper object for an empty upb_Map with the
// key and value types specified by the field. Creates one if it doesn't exist.
VALUE Map_EmptyFrozen(const upb_FieldDef* f);
// Returns a Ruby wrapper object for the given map, which will be created if
// one does not exist already.
VALUE Map_GetRubyWrapper(const upb_Map *map, upb_CType key_type,
TypeInfo value_type, VALUE arena);
// Gets the underlying upb_Map for this Ruby map object, which must have
// key/value type that match |field|. If this is not a map or the type doesn't
// match, raises an exception.
const upb_Map *Map_GetUpbMap(VALUE val, const upb_FieldDef *field,
upb_Arena *arena);
// Implements #inspect for this map by appending its contents to |b|.
void Map_Inspect(StringBuilder *b, const upb_Map *map, upb_CType key_type,
TypeInfo val_type);
// Returns a new Hash object containing the contents of this Map.
VALUE Map_CreateHash(const upb_Map *map, upb_CType key_type, TypeInfo val_info);
// Returns a deep copy of this Map object.
VALUE Map_deep_copy(VALUE obj);
// Ruby class of Google::Protobuf::Map.
extern VALUE cMap;
// Call at startup to register all types in this module.
void Map_register(VALUE module);
// Recursively freeze map
VALUE Map_freeze(VALUE _self);
#endif // RUBY_PROTOBUF_MAP_H_
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,82 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef RUBY_PROTOBUF_MESSAGE_H_
#define RUBY_PROTOBUF_MESSAGE_H_
#include "protobuf.h"
#include "ruby-upb.h"
// Gets the underlying upb_Message* and upb_MessageDef for the given Ruby
// message wrapper. Requires that |value| is indeed a message object.
const upb_Message* Message_Get(VALUE value, const upb_MessageDef** m);
// Like Message_Get(), but checks that the object is not frozen and returns a
// mutable pointer.
upb_Message* Message_GetMutable(VALUE value, const upb_MessageDef** m);
// Returns the Arena object for this message.
VALUE Message_GetArena(VALUE value);
// Converts |value| into a upb_Message value of the expected upb_MessageDef
// type, raising an error if this is not possible. Used when assigning |value|
// to a field of another message, which means the message must be of a
// particular type.
//
// This will perform automatic conversions in some cases (for example, Time ->
// Google::Protobuf::Timestamp). If any new message is created, it will be
// created on |arena|, and any existing message will have its arena fused with
// |arena|.
const upb_Message* Message_GetUpbMessage(VALUE value, const upb_MessageDef* m,
const char* name, upb_Arena* arena);
// Gets or constructs a Ruby wrapper object for the given message. The wrapper
// object will reference |arena| and ensure that it outlives this object.
VALUE Message_GetRubyWrapper(const upb_Message* msg, const upb_MessageDef* m,
VALUE arena);
// Gets the given field from this message.
VALUE Message_getfield(VALUE _self, const upb_FieldDef* f);
// Implements #inspect for this message, printing the text to |b|.
void Message_PrintMessage(StringBuilder* b, const upb_Message* msg,
const upb_MessageDef* m);
// Returns a hash value for the given message.
uint64_t Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed);
// Returns a deep copy of the given message.
upb_Message* Message_deep_copy(const upb_Message* msg, const upb_MessageDef* m,
upb_Arena* arena);
// Checks that this Ruby object is a message, and raises an exception if not.
void Message_CheckClass(VALUE klass);
// Returns a new Hash object containing the contents of this message.
VALUE Scalar_CreateHash(upb_MessageValue val, TypeInfo type_info);
// Creates a message class or enum module for this descriptor, respectively.
VALUE build_class_from_descriptor(VALUE descriptor);
VALUE build_module_from_enumdesc(VALUE _enumdesc);
// Returns the Descriptor/EnumDescriptor for the given message class or enum
// module, respectively. Returns nil if this is not a message class or enum
// module.
VALUE MessageOrEnum_GetDescriptor(VALUE klass);
// Decodes a Message from a byte sequence.
VALUE Message_decode_bytes(int size, const char* bytes, int options,
VALUE klass, bool freeze);
// Recursively freeze message
VALUE Message_freeze(VALUE _self);
// Call at startup to register all types in this module.
void Message_register(VALUE protobuf);
#endif // RUBY_PROTOBUF_MESSAGE_H_
@@ -0,0 +1,356 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include "protobuf.h"
#include "defs.h"
#include "map.h"
#include "message.h"
#include "repeated_field.h"
VALUE cParseError;
VALUE cTypeError;
const upb_FieldDef *map_field_key(const upb_FieldDef *field) {
const upb_MessageDef *entry = upb_FieldDef_MessageSubDef(field);
return upb_MessageDef_FindFieldByNumber(entry, 1);
}
const upb_FieldDef *map_field_value(const upb_FieldDef *field) {
const upb_MessageDef *entry = upb_FieldDef_MessageSubDef(field);
return upb_MessageDef_FindFieldByNumber(entry, 2);
}
// -----------------------------------------------------------------------------
// StringBuilder, for inspect
// -----------------------------------------------------------------------------
struct StringBuilder {
size_t size;
size_t cap;
char *data;
};
typedef struct StringBuilder StringBuilder;
static size_t StringBuilder_SizeOf(size_t cap) {
return sizeof(StringBuilder) + cap;
}
StringBuilder *StringBuilder_New() {
const size_t cap = 128;
StringBuilder *builder = malloc(sizeof(*builder));
builder->size = 0;
builder->cap = cap;
builder->data = malloc(builder->cap);
return builder;
}
void StringBuilder_Free(StringBuilder *b) {
free(b->data);
free(b);
}
void StringBuilder_Printf(StringBuilder *b, const char *fmt, ...) {
size_t have = b->cap - b->size;
size_t n;
va_list args;
va_start(args, fmt);
n = vsnprintf(&b->data[b->size], have, fmt, args);
va_end(args);
if (have <= n) {
while (have <= n) {
b->cap *= 2;
have = b->cap - b->size;
}
b->data = realloc(b->data, StringBuilder_SizeOf(b->cap));
va_start(args, fmt);
n = vsnprintf(&b->data[b->size], have, fmt, args);
va_end(args);
PBRUBY_ASSERT(n < have);
}
b->size += n;
}
VALUE StringBuilder_ToRubyString(StringBuilder *b) {
VALUE ret = rb_str_new(b->data, b->size);
rb_enc_associate(ret, rb_utf8_encoding());
return ret;
}
static void StringBuilder_PrintEnum(StringBuilder *b, int32_t val,
const upb_EnumDef *e) {
const upb_EnumValueDef *ev = upb_EnumDef_FindValueByNumber(e, val);
if (ev) {
StringBuilder_Printf(b, ":%s", upb_EnumValueDef_Name(ev));
} else {
StringBuilder_Printf(b, "%" PRId32, val);
}
}
void StringBuilder_PrintMsgval(StringBuilder *b, upb_MessageValue val,
TypeInfo info) {
switch (info.type) {
case kUpb_CType_Bool:
StringBuilder_Printf(b, "%s", val.bool_val ? "true" : "false");
break;
case kUpb_CType_Float: {
VALUE str = rb_inspect(DBL2NUM(val.float_val));
StringBuilder_Printf(b, "%s", RSTRING_PTR(str));
break;
}
case kUpb_CType_Double: {
VALUE str = rb_inspect(DBL2NUM(val.double_val));
StringBuilder_Printf(b, "%s", RSTRING_PTR(str));
break;
}
case kUpb_CType_Int32:
StringBuilder_Printf(b, "%" PRId32, val.int32_val);
break;
case kUpb_CType_UInt32:
StringBuilder_Printf(b, "%" PRIu32, val.uint32_val);
break;
case kUpb_CType_Int64:
StringBuilder_Printf(b, "%" PRId64, val.int64_val);
break;
case kUpb_CType_UInt64:
StringBuilder_Printf(b, "%" PRIu64, val.uint64_val);
break;
case kUpb_CType_String:
StringBuilder_Printf(b, "\"%.*s\"", (int)val.str_val.size,
val.str_val.data);
break;
case kUpb_CType_Bytes:
StringBuilder_Printf(b, "\"%.*s\"", (int)val.str_val.size,
val.str_val.data);
break;
case kUpb_CType_Enum:
StringBuilder_PrintEnum(b, val.int32_val, info.def.enumdef);
break;
case kUpb_CType_Message:
Message_PrintMessage(b, val.msg_val, info.def.msgdef);
break;
}
}
// -----------------------------------------------------------------------------
// Arena
// -----------------------------------------------------------------------------
typedef struct {
upb_Arena *arena;
// IMPORTANT: WB_PROTECTED objects must only use the RB_OBJ_WRITE()
// macro to update VALUE references, as to trigger write barriers.
VALUE pinned_objs;
} Arena;
static void Arena_mark(void *data) {
Arena *arena = data;
rb_gc_mark(arena->pinned_objs);
}
static void Arena_free(void *data) {
Arena *arena = data;
upb_Arena_Free(arena->arena);
xfree(arena);
}
static size_t Arena_memsize(const void *data) {
const Arena *arena = data;
size_t fused_count;
size_t memsize = upb_Arena_SpaceAllocated(arena->arena, &fused_count);
if (fused_count > 1) {
// If other arena were fused we attribute an equal
// share of memory usage to each one.
memsize /= fused_count;
}
return memsize + sizeof(Arena);
}
static VALUE cArena;
const rb_data_type_t Arena_type = {
"Google::Protobuf::Internal::Arena",
{Arena_mark, Arena_free, Arena_memsize},
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED,
};
static void *ruby_upb_allocfunc(upb_alloc *alloc, void *ptr, size_t oldsize,
size_t size) {
if (size == 0) {
xfree(ptr);
return NULL;
} else {
return xrealloc(ptr, size);
}
}
upb_alloc ruby_upb_alloc = {&ruby_upb_allocfunc};
static VALUE Arena_alloc(VALUE klass) {
Arena *arena = ALLOC(Arena);
arena->arena = upb_Arena_Init(NULL, 0, &ruby_upb_alloc);
arena->pinned_objs = Qnil;
return TypedData_Wrap_Struct(klass, &Arena_type, arena);
}
upb_Arena *Arena_get(VALUE _arena) {
Arena *arena;
TypedData_Get_Struct(_arena, Arena, &Arena_type, arena);
return arena->arena;
}
void Arena_fuse(VALUE _arena, upb_Arena *other) {
Arena *arena;
TypedData_Get_Struct(_arena, Arena, &Arena_type, arena);
if (!upb_Arena_Fuse(arena->arena, other)) {
rb_raise(rb_eRuntimeError,
"Unable to fuse arenas. This should never happen since Ruby does "
"not use initial blocks");
}
}
VALUE Arena_new() { return Arena_alloc(cArena); }
void Arena_register(VALUE module) {
VALUE internal = rb_define_module_under(module, "Internal");
VALUE klass = rb_define_class_under(internal, "Arena", rb_cObject);
rb_define_alloc_func(klass, Arena_alloc);
rb_gc_register_address(&cArena);
cArena = klass;
}
// -----------------------------------------------------------------------------
// Object Cache
// -----------------------------------------------------------------------------
// Public ObjectCache API.
VALUE weak_obj_cache = Qnil;
ID item_get;
ID item_try_add;
static void ObjectCache_Init(VALUE protobuf) {
item_get = rb_intern("get");
item_try_add = rb_intern("try_add");
rb_gc_register_address(&weak_obj_cache);
VALUE internal = rb_const_get(protobuf, rb_intern("Internal"));
#if SIZEOF_LONG >= SIZEOF_VALUE
VALUE cache_class = rb_const_get(internal, rb_intern("ObjectCache"));
#else
VALUE cache_class = rb_const_get(internal, rb_intern("LegacyObjectCache"));
#endif
weak_obj_cache = rb_class_new_instance(0, NULL, cache_class);
rb_const_set(internal, rb_intern("OBJECT_CACHE"), weak_obj_cache);
rb_const_set(internal, rb_intern("SIZEOF_LONG"), INT2NUM(SIZEOF_LONG));
rb_const_set(internal, rb_intern("SIZEOF_VALUE"), INT2NUM(SIZEOF_VALUE));
}
static VALUE ObjectCache_GetKey(const void *key) {
VALUE key_val = (VALUE)key;
PBRUBY_ASSERT((key_val & 3) == 0);
// Ensure the key can be stored as a Fixnum since 1 bit is needed for
// FIXNUM_FLAG and 1 bit is needed for the sign bit.
VALUE new_key = LL2NUM(key_val >> 2);
PBRUBY_ASSERT(FIXNUM_P(new_key));
return new_key;
}
VALUE ObjectCache_TryAdd(const void *key, VALUE val) {
VALUE key_val = ObjectCache_GetKey(key);
return rb_funcall(weak_obj_cache, item_try_add, 2, key_val, val);
}
// Returns the cached object for this key, if any. Otherwise returns Qnil.
VALUE ObjectCache_Get(const void *key) {
VALUE key_val = ObjectCache_GetKey(key);
return rb_funcall(weak_obj_cache, item_get, 1, key_val);
}
/*
* call-seq:
* Google::Protobuf.discard_unknown(msg)
*
* Discard unknown fields in the given message object and recursively discard
* unknown fields in submessages.
*/
static VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb) {
const upb_MessageDef *m;
upb_Message *msg = Message_GetMutable(msg_rb, &m);
if (!upb_Message_DiscardUnknown(msg, m, 128)) {
rb_raise(rb_eRuntimeError, "Messages nested too deeply.");
}
return Qnil;
}
/*
* call-seq:
* Google::Protobuf.deep_copy(obj) => copy_of_obj
*
* Performs a deep copy of a RepeatedField instance, a Map instance, or a
* message object, recursively copying its members.
*/
VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj) {
VALUE klass = CLASS_OF(obj);
if (klass == cRepeatedField) {
return RepeatedField_deep_copy(obj);
} else if (klass == cMap) {
return Map_deep_copy(obj);
} else {
VALUE new_arena_rb = Arena_new();
upb_Arena *new_arena = Arena_get(new_arena_rb);
const upb_MessageDef *m;
const upb_Message *msg = Message_Get(obj, &m);
upb_Message *new_msg = Message_deep_copy(msg, m, new_arena);
return Message_GetRubyWrapper(new_msg, m, new_arena_rb);
}
}
// -----------------------------------------------------------------------------
// Initialization/entry point.
// -----------------------------------------------------------------------------
// This must be named "Init_protobuf_c" because the Ruby module is named
// "protobuf_c" -- the VM looks for this symbol in our .so.
__attribute__((visibility("default"))) void Init_protobuf_c() {
VALUE google = rb_define_module("Google");
VALUE protobuf = rb_define_module_under(google, "Protobuf");
ObjectCache_Init(protobuf);
Arena_register(protobuf);
Defs_register(protobuf);
RepeatedField_register(protobuf);
Map_register(protobuf);
Message_register(protobuf);
cParseError = rb_const_get(protobuf, rb_intern("ParseError"));
rb_gc_register_mark_object(cParseError);
cTypeError = rb_const_get(protobuf, rb_intern("TypeError"));
rb_gc_register_mark_object(cTypeError);
rb_define_singleton_method(protobuf, "discard_unknown",
Google_Protobuf_discard_unknown, 1);
rb_define_singleton_method(protobuf, "deep_copy", Google_Protobuf_deep_copy,
1);
}
// -----------------------------------------------------------------------------
// Utilities
// -----------------------------------------------------------------------------
// Raises a Ruby error if val is frozen in Ruby or UPB.
void Protobuf_CheckNotFrozen(VALUE val, bool upb_frozen) {
if (RB_UNLIKELY(rb_obj_frozen_p(val)||upb_frozen)) {
rb_error_frozen_object(val);
}
}
@@ -0,0 +1,108 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__
#define __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__
// Ruby 3+ defines NDEBUG itself, see: https://bugs.ruby-lang.org/issues/18777
#ifdef NDEBUG
#include <ruby.h>
#else
#include <ruby.h>
#undef NDEBUG
#endif
#include <ruby/version.h>
#if RUBY_API_VERSION_CODE < 20700
#error Protobuf requires Ruby >= 2.7
#endif
#include <assert.h> // Must be included after the NDEBUG logic above.
#include <ruby/encoding.h>
#include <ruby/vm.h>
#include "defs.h"
#include "ruby-upb.h"
// These operate on a map field (i.e., a repeated field of submessages whose
// submessage type is a map-entry msgdef).
const upb_FieldDef* map_field_key(const upb_FieldDef* field);
const upb_FieldDef* map_field_value(const upb_FieldDef* field);
// -----------------------------------------------------------------------------
// Arena
// -----------------------------------------------------------------------------
// A Ruby object that wraps an underlying upb_Arena. Any objects that are
// allocated from this arena should reference the Arena in rb_gc_mark(), to
// ensure that the object's underlying memory outlives any Ruby object that can
// reach it.
VALUE Arena_new();
upb_Arena* Arena_get(VALUE arena);
// Fuses this arena to another, throwing a Ruby exception if this is not
// possible.
void Arena_fuse(VALUE arena, upb_Arena* other);
// -----------------------------------------------------------------------------
// ObjectCache
// -----------------------------------------------------------------------------
// Global object cache from upb array/map/message/symtab to wrapper object.
//
// This is a conceptually "weak" cache, in that it does not prevent "val" from
// being collected (though in Ruby <2.7 is it effectively strong, due to
// implementation limitations).
// Tries to add a new entry to the cache, returning the newly installed value or
// the pre-existing entry.
VALUE ObjectCache_TryAdd(const void* key, VALUE val);
// Returns the cached object for this key, if any. Otherwise returns Qnil.
VALUE ObjectCache_Get(const void* key);
// -----------------------------------------------------------------------------
// StringBuilder, for inspect
// -----------------------------------------------------------------------------
struct StringBuilder;
typedef struct StringBuilder StringBuilder;
StringBuilder* StringBuilder_New();
void StringBuilder_Free(StringBuilder* b);
void StringBuilder_Printf(StringBuilder* b, const char* fmt, ...);
VALUE StringBuilder_ToRubyString(StringBuilder* b);
void StringBuilder_PrintMsgval(StringBuilder* b, upb_MessageValue val,
TypeInfo info);
// -----------------------------------------------------------------------------
// Utilities.
// -----------------------------------------------------------------------------
extern VALUE cTypeError;
#ifdef NDEBUG
#define PBRUBY_ASSERT(expr) \
do { \
} while (false && (expr))
#else
#define PBRUBY_ASSERT(expr) \
if (!(expr)) \
rb_bug("Assertion failed at %s:%d, expr: %s", __FILE__, __LINE__, #expr)
#endif
// Raises a Ruby error if val is frozen in Ruby or upb_frozen is true.
void Protobuf_CheckNotFrozen(VALUE val, bool upb_frozen);
#define PBRUBY_MAX(x, y) (((x) > (y)) ? (x) : (y))
#define UPB_UNUSED(var) (void)var
#endif // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__
@@ -0,0 +1,691 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2014 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include "repeated_field.h"
#include "convert.h"
#include "defs.h"
#include "message.h"
#include "protobuf.h"
// -----------------------------------------------------------------------------
// Repeated field container type.
// -----------------------------------------------------------------------------
typedef struct {
const upb_Array* array; // Can get as mutable when non-frozen.
TypeInfo type_info;
VALUE type_class; // To GC-root the msgdef/enumdef in type_info.
VALUE arena; // To GC-root the upb_Array.
} RepeatedField;
VALUE cRepeatedField;
static void RepeatedField_mark(void* _self) {
RepeatedField* self = (RepeatedField*)_self;
rb_gc_mark(self->type_class);
rb_gc_mark(self->arena);
}
const rb_data_type_t RepeatedField_type = {
"Google::Protobuf::RepeatedField",
{RepeatedField_mark, RUBY_DEFAULT_FREE, NULL},
.flags = RUBY_TYPED_FREE_IMMEDIATELY,
};
static RepeatedField* ruby_to_RepeatedField(VALUE _self) {
RepeatedField* self;
TypedData_Get_Struct(_self, RepeatedField, &RepeatedField_type, self);
return self;
}
static upb_Array* RepeatedField_GetMutable(VALUE _self) {
const upb_Array* array = ruby_to_RepeatedField(_self)->array;
Protobuf_CheckNotFrozen(_self, upb_Array_IsFrozen(array));
return (upb_Array*)array;
}
VALUE RepeatedField_alloc(VALUE klass) {
RepeatedField* self = ALLOC(RepeatedField);
self->arena = Qnil;
self->type_class = Qnil;
self->array = NULL;
return TypedData_Wrap_Struct(klass, &RepeatedField_type, self);
}
VALUE RepeatedField_EmptyFrozen(const upb_FieldDef* f) {
PBRUBY_ASSERT(upb_FieldDef_IsRepeated(f));
VALUE val = ObjectCache_Get(f);
if (val == Qnil) {
val = RepeatedField_alloc(cRepeatedField);
RepeatedField* self;
TypedData_Get_Struct(val, RepeatedField, &RepeatedField_type, self);
self->arena = Arena_new();
TypeInfo type_info = TypeInfo_get(f);
self->array = upb_Array_New(Arena_get(self->arena), type_info.type);
self->type_info = type_info;
if (self->type_info.type == kUpb_CType_Message) {
self->type_class = Descriptor_DefToClass(type_info.def.msgdef);
}
val = ObjectCache_TryAdd(f, RepeatedField_freeze(val));
}
PBRUBY_ASSERT(RB_OBJ_FROZEN(val));
PBRUBY_ASSERT(upb_Array_IsFrozen(ruby_to_RepeatedField(val)->array));
return val;
}
VALUE RepeatedField_GetRubyWrapper(const upb_Array* array, TypeInfo type_info,
VALUE arena) {
PBRUBY_ASSERT(array);
PBRUBY_ASSERT(arena != Qnil);
VALUE val = ObjectCache_Get(array);
if (val == Qnil) {
val = RepeatedField_alloc(cRepeatedField);
RepeatedField* self;
TypedData_Get_Struct(val, RepeatedField, &RepeatedField_type, self);
self->array = array;
self->arena = arena;
self->type_info = type_info;
if (self->type_info.type == kUpb_CType_Message) {
self->type_class = Descriptor_DefToClass(type_info.def.msgdef);
}
val = ObjectCache_TryAdd(array, val);
}
PBRUBY_ASSERT(ruby_to_RepeatedField(val)->type_info.type == type_info.type);
PBRUBY_ASSERT(ruby_to_RepeatedField(val)->type_info.def.msgdef ==
type_info.def.msgdef);
PBRUBY_ASSERT(ruby_to_RepeatedField(val)->array == array);
return val;
}
static VALUE RepeatedField_new_this_type(RepeatedField* from) {
VALUE arena_rb = Arena_new();
upb_Array* array = upb_Array_New(Arena_get(arena_rb), from->type_info.type);
VALUE ret = RepeatedField_GetRubyWrapper(array, from->type_info, arena_rb);
PBRUBY_ASSERT(ruby_to_RepeatedField(ret)->type_class == from->type_class);
return ret;
}
void RepeatedField_Inspect(StringBuilder* b, const upb_Array* array,
TypeInfo info) {
bool first = true;
StringBuilder_Printf(b, "[");
size_t n = array ? upb_Array_Size(array) : 0;
for (size_t i = 0; i < n; i++) {
if (first) {
first = false;
} else {
StringBuilder_Printf(b, ", ");
}
StringBuilder_PrintMsgval(b, upb_Array_Get(array, i), info);
}
StringBuilder_Printf(b, "]");
}
VALUE RepeatedField_deep_copy(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
VALUE new_rptfield = RepeatedField_new_this_type(self);
RepeatedField* new_self = ruby_to_RepeatedField(new_rptfield);
VALUE arena_rb = new_self->arena;
upb_Array* new_array = RepeatedField_GetMutable(new_rptfield);
upb_Arena* arena = Arena_get(arena_rb);
size_t elements = upb_Array_Size(self->array);
upb_Array_Resize(new_array, elements, arena);
size_t size = upb_Array_Size(self->array);
for (size_t i = 0; i < size; i++) {
upb_MessageValue msgval = upb_Array_Get(self->array, i);
upb_MessageValue copy = Msgval_DeepCopy(msgval, self->type_info, arena);
upb_Array_Set(new_array, i, copy);
}
return new_rptfield;
}
const upb_Array* RepeatedField_GetUpbArray(VALUE val, const upb_FieldDef* field,
upb_Arena* arena) {
RepeatedField* self;
TypeInfo type_info = TypeInfo_get(field);
if (!RB_TYPE_P(val, T_DATA) || !RTYPEDDATA_P(val) ||
RTYPEDDATA_TYPE(val) != &RepeatedField_type) {
rb_raise(cTypeError, "Expected repeated field array");
}
self = ruby_to_RepeatedField(val);
if (self->type_info.type != type_info.type) {
rb_raise(cTypeError, "Repeated field array has wrong element type");
}
if (self->type_info.def.msgdef != type_info.def.msgdef) {
rb_raise(cTypeError, "Repeated field array has wrong message/enum class");
}
Arena_fuse(self->arena, arena);
return self->array;
}
static int index_position(VALUE _index, RepeatedField* repeated_field) {
int index = NUM2INT(_index);
if (index < 0) index += upb_Array_Size(repeated_field->array);
return index;
}
static VALUE RepeatedField_subarray(RepeatedField* self, long beg, long len) {
size_t size = upb_Array_Size(self->array);
VALUE ary = rb_ary_new2(size);
long i;
for (i = beg; i < beg + len; i++) {
upb_MessageValue msgval = upb_Array_Get(self->array, i);
VALUE elem = Convert_UpbToRuby(msgval, self->type_info, self->arena);
rb_ary_push(ary, elem);
}
return ary;
}
/*
* call-seq:
* RepeatedField.each(&block)
*
* Invokes the block once for each element of the repeated field. RepeatedField
* also includes Enumerable; combined with this method, the repeated field thus
* acts like an ordinary Ruby sequence.
*/
static VALUE RepeatedField_each(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
int size = upb_Array_Size(self->array);
int i;
for (i = 0; i < size; i++) {
upb_MessageValue msgval = upb_Array_Get(self->array, i);
VALUE val = Convert_UpbToRuby(msgval, self->type_info, self->arena);
rb_yield(val);
}
return _self;
}
/*
* call-seq:
* RepeatedField.[](index) => value
*
* Accesses the element at the given index. Returns nil on out-of-bounds
*/
static VALUE RepeatedField_index(int argc, VALUE* argv, VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
long size = upb_Array_Size(self->array);
VALUE arg = argv[0];
long beg, len;
if (argc == 1) {
if (FIXNUM_P(arg)) {
/* standard case */
upb_MessageValue msgval;
int index = index_position(argv[0], self);
if (index < 0 || (size_t)index >= upb_Array_Size(self->array)) {
return Qnil;
}
msgval = upb_Array_Get(self->array, index);
return Convert_UpbToRuby(msgval, self->type_info, self->arena);
} else {
/* check if idx is Range */
switch (rb_range_beg_len(arg, &beg, &len, size, 0)) {
case Qfalse:
break;
case Qnil:
return Qnil;
default:
return RepeatedField_subarray(self, beg, len);
}
}
}
/* assume 2 arguments */
beg = NUM2LONG(argv[0]);
len = NUM2LONG(argv[1]);
if (beg < 0) {
beg += size;
}
if (beg >= size) {
return Qnil;
}
return RepeatedField_subarray(self, beg, len);
}
/*
* call-seq:
* RepeatedField.[]=(index, value)
*
* Sets the element at the given index. On out-of-bounds assignments, extends
* the array and fills the hole (if any) with default values.
*/
static VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) {
RepeatedField* self = ruby_to_RepeatedField(_self);
int size = upb_Array_Size(self->array);
upb_Array* array = RepeatedField_GetMutable(_self);
upb_Arena* arena = Arena_get(self->arena);
upb_MessageValue msgval = Convert_RubyToUpb(val, "", self->type_info, arena);
int index = index_position(_index, self);
if (index < 0 || index >= (INT_MAX - 1)) {
return Qnil;
}
if (index >= size) {
upb_Array_Resize(array, index + 1, arena);
upb_MessageValue fill;
memset(&fill, 0, sizeof(fill));
for (int i = size; i < index; i++) {
// Fill default values.
// TODO: should this happen at the upb level?
upb_Array_Set(array, i, fill);
}
}
upb_Array_Set(array, index, msgval);
return Qnil;
}
/*
* call-seq:
* RepeatedField.push(value, ...)
*
* Adds a new element to the repeated field.
*/
static VALUE RepeatedField_push_vararg(int argc, VALUE* argv, VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
upb_Arena* arena = Arena_get(self->arena);
upb_Array* array = RepeatedField_GetMutable(_self);
int i;
for (i = 0; i < argc; i++) {
upb_MessageValue msgval =
Convert_RubyToUpb(argv[i], "", self->type_info, arena);
upb_Array_Append(array, msgval, arena);
}
return _self;
}
/*
* call-seq:
* RepeatedField.<<(value)
*
* Adds a new element to the repeated field.
*/
static VALUE RepeatedField_push(VALUE _self, VALUE val) {
RepeatedField* self = ruby_to_RepeatedField(_self);
upb_Arena* arena = Arena_get(self->arena);
upb_Array* array = RepeatedField_GetMutable(_self);
upb_MessageValue msgval = Convert_RubyToUpb(val, "", self->type_info, arena);
upb_Array_Append(array, msgval, arena);
return _self;
}
/*
* Private ruby method, used by RepeatedField.pop
*/
static VALUE RepeatedField_pop_one(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
size_t size = upb_Array_Size(self->array);
upb_Array* array = RepeatedField_GetMutable(_self);
upb_MessageValue last;
VALUE ret;
if (size == 0) {
return Qnil;
}
last = upb_Array_Get(self->array, size - 1);
ret = Convert_UpbToRuby(last, self->type_info, self->arena);
upb_Array_Resize(array, size - 1, Arena_get(self->arena));
return ret;
}
/*
* call-seq:
* RepeatedField.replace(list)
*
* Replaces the contents of the repeated field with the given list of elements.
*/
static VALUE RepeatedField_replace(VALUE _self, VALUE list) {
RepeatedField* self = ruby_to_RepeatedField(_self);
upb_Array* array = RepeatedField_GetMutable(_self);
int i;
Check_Type(list, T_ARRAY);
upb_Array_Resize(array, 0, Arena_get(self->arena));
for (i = 0; i < RARRAY_LEN(list); i++) {
RepeatedField_push(_self, rb_ary_entry(list, i));
}
return list;
}
/*
* call-seq:
* RepeatedField.clear
*
* Clears (removes all elements from) this repeated field.
*/
static VALUE RepeatedField_clear(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
upb_Array* array = RepeatedField_GetMutable(_self);
upb_Array_Resize(array, 0, Arena_get(self->arena));
return _self;
}
/*
* call-seq:
* RepeatedField.length
*
* Returns the length of this repeated field.
*/
static VALUE RepeatedField_length(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
return INT2NUM(upb_Array_Size(self->array));
}
/*
* call-seq:
* RepeatedField.dup => repeated_field
*
* Duplicates this repeated field with a shallow copy. References to all
* non-primitive element objects (e.g., submessages) are shared.
*/
static VALUE RepeatedField_dup(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
VALUE new_rptfield = RepeatedField_new_this_type(self);
RepeatedField* new_rptfield_self = ruby_to_RepeatedField(new_rptfield);
upb_Array* new_array = RepeatedField_GetMutable(new_rptfield);
upb_Arena* arena = Arena_get(new_rptfield_self->arena);
int size = upb_Array_Size(self->array);
int i;
Arena_fuse(self->arena, arena);
for (i = 0; i < size; i++) {
upb_MessageValue msgval = upb_Array_Get(self->array, i);
upb_Array_Append(new_array, msgval, arena);
}
return new_rptfield;
}
/*
* call-seq:
* RepeatedField.to_ary => array
*
* Used when converted implicitly into array, e.g. compared to an Array.
* Also called as a fallback of Object#to_a
*/
VALUE RepeatedField_to_ary(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
int size = upb_Array_Size(self->array);
VALUE ary = rb_ary_new2(size);
int i;
for (i = 0; i < size; i++) {
upb_MessageValue msgval = upb_Array_Get(self->array, i);
VALUE val = Convert_UpbToRuby(msgval, self->type_info, self->arena);
rb_ary_push(ary, val);
}
return ary;
}
/*
* call-seq:
* RepeatedField.==(other) => boolean
*
* Compares this repeated field to another. Repeated fields are equal if their
* element types are equal, their lengths are equal, and each element is equal.
* Elements are compared as per normal Ruby semantics, by calling their :==
* methods (or performing a more efficient comparison for primitive types).
*
* Repeated fields with dissimilar element types are never equal, even if value
* comparison (for example, between integers and floats) would have otherwise
* indicated that every element has equal value.
*/
VALUE RepeatedField_eq(VALUE _self, VALUE _other) {
RepeatedField* self;
RepeatedField* other;
if (_self == _other) {
return Qtrue;
}
if (TYPE(_other) == T_ARRAY) {
VALUE self_ary = RepeatedField_to_ary(_self);
return rb_equal(self_ary, _other);
}
self = ruby_to_RepeatedField(_self);
other = ruby_to_RepeatedField(_other);
size_t n = upb_Array_Size(self->array);
if (self->type_info.type != other->type_info.type ||
self->type_class != other->type_class ||
upb_Array_Size(other->array) != n) {
return Qfalse;
}
for (size_t i = 0; i < n; i++) {
upb_MessageValue val1 = upb_Array_Get(self->array, i);
upb_MessageValue val2 = upb_Array_Get(other->array, i);
if (!Msgval_IsEqual(val1, val2, self->type_info)) {
return Qfalse;
}
}
return Qtrue;
}
/*
* call-seq:
* RepeatedField.frozen? => bool
*
* Returns true if the repeated field is frozen in either Ruby or the underlying
* representation. Freezes the Ruby repeated field object if it is not already
* frozen in Ruby but it is frozen in the underlying representation.
*/
VALUE RepeatedField_frozen(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
if (!upb_Array_IsFrozen(self->array)) {
PBRUBY_ASSERT(!RB_OBJ_FROZEN(_self));
return Qfalse;
}
// Lazily freeze the Ruby wrapper.
if (!RB_OBJ_FROZEN(_self)) RB_OBJ_FREEZE(_self);
return Qtrue;
}
/*
* call-seq:
* RepeatedField.freeze => self
*
* Freezes the repeated field object. We have to intercept this so we can freeze
* the underlying representation, not just the Ruby wrapper.
*/
VALUE RepeatedField_freeze(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
if (RB_OBJ_FROZEN(_self)) {
PBRUBY_ASSERT(upb_Array_IsFrozen(self->array));
return _self;
}
if (!upb_Array_IsFrozen(self->array)) {
if (self->type_info.type == kUpb_CType_Message) {
upb_Array_Freeze(RepeatedField_GetMutable(_self),
upb_MessageDef_MiniTable(self->type_info.def.msgdef));
} else {
upb_Array_Freeze(RepeatedField_GetMutable(_self), NULL);
}
}
RB_OBJ_FREEZE(_self);
return _self;
}
/*
* call-seq:
* RepeatedField.hash => hash_value
*
* Returns a hash value computed from this repeated field's elements.
*/
VALUE RepeatedField_hash(VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
uint64_t hash = 0;
size_t n = upb_Array_Size(self->array);
for (size_t i = 0; i < n; i++) {
upb_MessageValue val = upb_Array_Get(self->array, i);
hash = Msgval_GetHash(val, self->type_info, hash);
}
return LL2NUM(hash);
}
/*
* call-seq:
* RepeatedField.+(other) => repeated field
*
* Returns a new repeated field that contains the concatenated list of this
* repeated field's elements and other's elements. The other (second) list may
* be either another repeated field or a Ruby array.
*/
VALUE RepeatedField_plus(VALUE _self, VALUE list) {
VALUE dupped_ = RepeatedField_dup(_self);
if (TYPE(list) == T_ARRAY) {
int i;
for (i = 0; i < RARRAY_LEN(list); i++) {
VALUE elem = rb_ary_entry(list, i);
RepeatedField_push(dupped_, elem);
}
} else if (RB_TYPE_P(list, T_DATA) && RTYPEDDATA_P(list) &&
RTYPEDDATA_TYPE(list) == &RepeatedField_type) {
RepeatedField* self = ruby_to_RepeatedField(_self);
RepeatedField* list_rptfield = ruby_to_RepeatedField(list);
RepeatedField* dupped = ruby_to_RepeatedField(dupped_);
upb_Array* dupped_array = RepeatedField_GetMutable(dupped_);
upb_Arena* arena = Arena_get(dupped->arena);
Arena_fuse(list_rptfield->arena, arena);
int size = upb_Array_Size(list_rptfield->array);
int i;
if (self->type_info.type != list_rptfield->type_info.type ||
self->type_class != list_rptfield->type_class) {
rb_raise(rb_eArgError,
"Attempt to append RepeatedField with different element type.");
}
for (i = 0; i < size; i++) {
upb_MessageValue msgval = upb_Array_Get(list_rptfield->array, i);
upb_Array_Append(dupped_array, msgval, arena);
}
} else {
rb_raise(rb_eArgError, "Unknown type appending to RepeatedField");
}
return dupped_;
}
/*
* call-seq:
* RepeatedField.concat(other) => self
*
* concats the passed in array to self. Returns a Ruby array.
*/
VALUE RepeatedField_concat(VALUE _self, VALUE list) {
int i;
Check_Type(list, T_ARRAY);
for (i = 0; i < RARRAY_LEN(list); i++) {
RepeatedField_push(_self, rb_ary_entry(list, i));
}
return _self;
}
/*
* call-seq:
* RepeatedField.new(type, type_class = nil, initial_elems = [])
*
* Creates a new repeated field. The provided type must be a Ruby symbol, and
* can take on the same values as those accepted by FieldDescriptor#type=. If
* the type is :message or :enum, type_class must be non-nil, and must be the
* Ruby class or module returned by Descriptor#msgclass or
* EnumDescriptor#enummodule, respectively. An initial list of elements may also
* be provided.
*/
VALUE RepeatedField_init(int argc, VALUE* argv, VALUE _self) {
RepeatedField* self = ruby_to_RepeatedField(_self);
upb_Arena* arena;
VALUE ary = Qnil;
self->arena = Arena_new();
arena = Arena_get(self->arena);
if (argc < 1) {
rb_raise(rb_eArgError, "Expected at least 1 argument.");
}
self->type_info = TypeInfo_FromClass(argc, argv, 0, &self->type_class, &ary);
self->array = upb_Array_New(arena, self->type_info.type);
VALUE stored_val = ObjectCache_TryAdd(self->array, _self);
PBRUBY_ASSERT(stored_val == _self);
if (ary != Qnil) {
if (!RB_TYPE_P(ary, T_ARRAY)) {
rb_raise(rb_eArgError, "Expected array as initialize argument");
}
for (int i = 0; i < RARRAY_LEN(ary); i++) {
RepeatedField_push(_self, rb_ary_entry(ary, i));
}
}
return Qnil;
}
void RepeatedField_register(VALUE module) {
VALUE klass = rb_define_class_under(module, "RepeatedField", rb_cObject);
rb_define_alloc_func(klass, RepeatedField_alloc);
rb_gc_register_address(&cRepeatedField);
cRepeatedField = klass;
rb_define_method(klass, "initialize", RepeatedField_init, -1);
rb_define_method(klass, "each", RepeatedField_each, 0);
rb_define_method(klass, "[]", RepeatedField_index, -1);
rb_define_method(klass, "at", RepeatedField_index, -1);
rb_define_method(klass, "[]=", RepeatedField_index_set, 2);
rb_define_method(klass, "push", RepeatedField_push_vararg, -1);
rb_define_method(klass, "<<", RepeatedField_push, 1);
rb_define_private_method(klass, "pop_one", RepeatedField_pop_one, 0);
rb_define_method(klass, "replace", RepeatedField_replace, 1);
rb_define_method(klass, "clear", RepeatedField_clear, 0);
rb_define_method(klass, "length", RepeatedField_length, 0);
rb_define_method(klass, "size", RepeatedField_length, 0);
rb_define_method(klass, "dup", RepeatedField_dup, 0);
// Also define #clone so that we don't inherit Object#clone.
rb_define_method(klass, "clone", RepeatedField_dup, 0);
rb_define_method(klass, "==", RepeatedField_eq, 1);
rb_define_method(klass, "to_ary", RepeatedField_to_ary, 0);
rb_define_method(klass, "freeze", RepeatedField_freeze, 0);
rb_define_method(klass, "frozen?", RepeatedField_frozen, 0);
rb_define_method(klass, "hash", RepeatedField_hash, 0);
rb_define_method(klass, "+", RepeatedField_plus, 1);
rb_define_method(klass, "concat", RepeatedField_concat, 1);
rb_include_module(klass, rb_mEnumerable);
}
@@ -0,0 +1,45 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#ifndef RUBY_PROTOBUF_REPEATED_FIELD_H_
#define RUBY_PROTOBUF_REPEATED_FIELD_H_
#include "protobuf.h"
#include "ruby-upb.h"
// Returns a frozen sentinel Ruby wrapper object for an empty upb_Array of the
// type specified by the field. Creates one if it doesn't exist.
VALUE RepeatedField_EmptyFrozen(const upb_FieldDef* f);
// Returns a Ruby wrapper object for the given upb_Array, which will be created
// if one does not exist already.
VALUE RepeatedField_GetRubyWrapper(const upb_Array* msg, TypeInfo type_info,
VALUE arena);
// Gets the underlying upb_Array for this Ruby RepeatedField object, which must
// have a type that matches |f|. If this is not a repeated field or the type
// doesn't match, raises an exception.
const upb_Array* RepeatedField_GetUpbArray(VALUE value, const upb_FieldDef* f,
upb_Arena* arena);
// Implements #inspect for this repeated field by appending its contents to |b|.
void RepeatedField_Inspect(StringBuilder* b, const upb_Array* array,
TypeInfo info);
// Returns a deep copy of this RepeatedField object.
VALUE RepeatedField_deep_copy(VALUE obj);
// Ruby class of Google::Protobuf::RepeatedField.
extern VALUE cRepeatedField;
// Call at startup to register all types in this module.
void RepeatedField_register(VALUE module);
// Recursively freeze RepeatedField.
VALUE RepeatedField_freeze(VALUE _self);
#endif // RUBY_PROTOBUF_REPEATED_FIELD_H_
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,69 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Ruby <-> upb data conversion functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "shared_convert.h"
bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
upb_CType type, const upb_MessageDef* msgdef,
upb_Status* status) {
switch (type) {
case kUpb_CType_Bool:
return memcmp(&val1, &val2, 1) == 0;
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return memcmp(&val1, &val2, 4) == 0;
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return memcmp(&val1, &val2, 8) == 0;
case kUpb_CType_String:
case kUpb_CType_Bytes:
return val1.str_val.size == val2.str_val.size &&
memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) ==
0;
case kUpb_CType_Message: {
const upb_MiniTable* m = upb_MessageDef_MiniTable(msgdef);
const int options = 0;
return upb_Message_IsEqual(val1.msg_val, val2.msg_val, m, options);
}
default:
upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
return false;
}
}
uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
const upb_MessageDef* msgdef, uint64_t seed,
upb_Status* status) {
switch (type) {
case kUpb_CType_Bool:
return _upb_Hash(&val, 1, seed);
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return _upb_Hash(&val, 4, seed);
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return _upb_Hash(&val, 8, seed);
case kUpb_CType_String:
case kUpb_CType_Bytes:
return _upb_Hash(val.str_val.data, val.str_val.size, seed);
case kUpb_CType_Message:
return shared_Message_Hash(val.msg_val, msgdef, seed, status);
default:
upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
return 0;
}
}
@@ -0,0 +1,26 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Ruby <-> upb data conversion functions. Strictly free of dependencies on
// Ruby interpreter internals.
#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_
#define RUBY_PROTOBUF_SHARED_CONVERT_H_
#include "ruby-upb.h"
#include "shared_message.h"
bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
upb_CType type, const upb_MessageDef* msgdef,
upb_Status* status);
uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
const upb_MessageDef* msgdef, uint64_t seed,
upb_Status* status);
#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_
@@ -0,0 +1,37 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Ruby Message functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "shared_message.h"
// Support function for Message_Hash. Returns a hash value for the given
// message.
uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed, upb_Status* status) {
upb_Arena* arena = upb_Arena_New();
char* data;
size_t size;
// Hash a deterministically serialized payloads with no unknown fields.
upb_EncodeStatus encode_status = upb_Encode(
msg, upb_MessageDef_MiniTable(m),
kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena,
&data, &size);
if (encode_status == kUpb_EncodeStatus_Ok) {
uint64_t ret = _upb_Hash(data, size, seed);
upb_Arena_Free(arena);
return ret;
}
upb_Arena_Free(arena);
upb_Status_SetErrorMessage(status, "Error calculating hash");
return 0;
}
@@ -0,0 +1,21 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// -----------------------------------------------------------------------------
// Ruby Message functions. Strictly free of dependencies on
// Ruby interpreter internals.
#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_
#define RUBY_PROTOBUF_SHARED_MESSAGE_H_
#include "ruby-upb.h"
// Returns a hash value for the given message.
uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed, upb_Status* status);
#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_
@@ -0,0 +1,22 @@
MIT License
Copyright (c) 2019 Yibo Cai
Copyright 2022 Google LLC
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,467 @@
// Copyright 2023 Google LLC
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT.
/* This is a wrapper for the Google range-sse.cc algorithm which checks whether
* a sequence of bytes is a valid UTF-8 sequence and finds the longest valid
* prefix of the UTF-8 sequence.
*
* The key difference is that it checks for as much ASCII symbols as possible
* and then falls back to the range-sse.cc algorithm. The changes to the
* algorithm are cosmetic, mostly to trick the clang compiler to produce optimal
* code.
*
* For API see the utf8_validity.h header.
*/
#include "utf8_range.h"
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#ifdef __SSE4_1__
#include <emmintrin.h>
#include <smmintrin.h>
#include <tmmintrin.h>
#endif
#if defined(__GNUC__)
#define FORCE_INLINE_ATTR __attribute__((always_inline))
#elif defined(_MSC_VER)
#define FORCE_INLINE_ATTR __forceinline
#else
#define FORCE_INLINE_ATTR
#endif
static FORCE_INLINE_ATTR inline uint64_t utf8_range_UnalignedLoad64(
const void* p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static FORCE_INLINE_ATTR inline int utf8_range_AsciiIsAscii(unsigned char c) {
return c < 128;
}
static FORCE_INLINE_ATTR inline int utf8_range_IsTrailByteOk(const char c) {
return (int8_t)(c) <= (int8_t)(0xBF);
}
/* If return_position is false then it returns 1 if |data| is a valid utf8
* sequence, otherwise returns 0.
* If return_position is set to true, returns the length in bytes of the prefix
of |data| that is all structurally valid UTF-8.
*/
static size_t utf8_range_ValidateUTF8Naive(const char* data, const char* end,
int return_position) {
/* We return err_pos in the loop which is always 0 if !return_position */
size_t err_pos = 0;
size_t codepoint_bytes = 0;
/* The early check is done because of early continue's on codepoints of all
* sizes, i.e. we first check for ascii and if it is, we call continue, then
* for 2 byte codepoints, etc. This is done in order to reduce indentation and
* improve readability of the codepoint validity check.
*/
while (data + codepoint_bytes < end) {
if (return_position) {
err_pos += codepoint_bytes;
}
data += codepoint_bytes;
const size_t len = end - data;
const unsigned char byte1 = data[0];
/* We do not skip many ascii bytes at the same time as this function is
used for tail checking (< 16 bytes) and for non x86 platforms. We also
don't think that cases where non-ASCII codepoints are followed by ascii
happen often. For small strings it also introduces some penalty. For
purely ascii UTF8 strings (which is the overwhelming case) we call
SkipAscii function which is multiplatform and extremely fast.
*/
/* [00..7F] ASCII -> 1 byte */
if (utf8_range_AsciiIsAscii(byte1)) {
codepoint_bytes = 1;
continue;
}
/* [C2..DF], [80..BF] -> 2 bytes */
if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
utf8_range_IsTrailByteOk(data[1])) {
codepoint_bytes = 2;
continue;
}
if (len >= 3) {
const unsigned char byte2 = data[1];
const unsigned char byte3 = data[2];
/* Is byte2, byte3 between [0x80, 0xBF]
* Check for 0x80 was done above.
*/
if (!utf8_range_IsTrailByteOk(byte2) ||
!utf8_range_IsTrailByteOk(byte3)) {
return err_pos;
}
if (/* E0, A0..BF, 80..BF */
((byte1 == 0xE0 && byte2 >= 0xA0) ||
/* E1..EC, 80..BF, 80..BF */
(byte1 >= 0xE1 && byte1 <= 0xEC) ||
/* ED, 80..9F, 80..BF */
(byte1 == 0xED && byte2 <= 0x9F) ||
/* EE..EF, 80..BF, 80..BF */
(byte1 >= 0xEE && byte1 <= 0xEF))) {
codepoint_bytes = 3;
continue;
}
if (len >= 4) {
const unsigned char byte4 = data[3];
/* Is byte4 between 0x80 ~ 0xBF */
if (!utf8_range_IsTrailByteOk(byte4)) {
return err_pos;
}
if (/* F0, 90..BF, 80..BF, 80..BF */
((byte1 == 0xF0 && byte2 >= 0x90) ||
/* F1..F3, 80..BF, 80..BF, 80..BF */
(byte1 >= 0xF1 && byte1 <= 0xF3) ||
/* F4, 80..8F, 80..BF, 80..BF */
(byte1 == 0xF4 && byte2 <= 0x8F))) {
codepoint_bytes = 4;
continue;
}
}
}
return err_pos;
}
if (return_position) {
err_pos += codepoint_bytes;
}
/* if return_position is false, this returns 1.
* if return_position is true, this returns err_pos.
*/
return err_pos + (1 - return_position);
}
#ifdef __SSE4_1__
/* Returns the number of bytes needed to skip backwards to get to the first
byte of codepoint.
*/
static inline int utf8_range_CodepointSkipBackwards(int32_t codepoint_word) {
const int8_t* const codepoint = (const int8_t*)(&codepoint_word);
if (!utf8_range_IsTrailByteOk(codepoint[3])) {
return 1;
} else if (!utf8_range_IsTrailByteOk(codepoint[2])) {
return 2;
} else if (!utf8_range_IsTrailByteOk(codepoint[1])) {
return 3;
}
return 0;
}
#endif // __SSE4_1__
/* Skipping over ASCII as much as possible, per 8 bytes. It is intentional
as most strings to check for validity consist only of 1 byte codepoints.
*/
static inline const char* utf8_range_SkipAscii(const char* data,
const char* end) {
while (8 <= end - data &&
(utf8_range_UnalignedLoad64(data) & 0x8080808080808080) == 0) {
data += 8;
}
while (data < end && utf8_range_AsciiIsAscii(*data)) {
++data;
}
return data;
}
static FORCE_INLINE_ATTR inline size_t utf8_range_Validate(
const char* data, size_t len, int return_position) {
if (len == 0) return 1 - return_position;
const char* const end = data + len;
data = utf8_range_SkipAscii(data, end);
/* SIMD algorithm always outperforms the naive version for any data of
length >=16.
*/
if (end - data < 16) {
return (return_position ? (data - (end - len)) : 0) +
utf8_range_ValidateUTF8Naive(data, end, return_position);
}
#ifndef __SSE4_1__
return (return_position ? (data - (end - len)) : 0) +
utf8_range_ValidateUTF8Naive(data, end, return_position);
#else
/* This code checks that utf-8 ranges are structurally valid 16 bytes at once
* using superscalar instructions.
* The mapping between ranges of codepoint and their corresponding utf-8
* sequences is below.
*/
/*
* U+0000...U+007F 00...7F
* U+0080...U+07FF C2...DF 80...BF
* U+0800...U+0FFF E0 A0...BF 80...BF
* U+1000...U+CFFF E1...EC 80...BF 80...BF
* U+D000...U+D7FF ED 80...9F 80...BF
* U+E000...U+FFFF EE...EF 80...BF 80...BF
* U+10000...U+3FFFF F0 90...BF 80...BF 80...BF
* U+40000...U+FFFFF F1...F3 80...BF 80...BF 80...BF
* U+100000...U+10FFFF F4 80...8F 80...BF 80...BF
*/
/* First we compute the type for each byte, as given by the table below.
* This type will be used as an index later on.
*/
/*
* Index Min Max Byte Type
* 0 00 7F Single byte sequence
* 1,2,3 80 BF Second, third and fourth byte for many of the sequences.
* 4 A0 BF Second byte after E0
* 5 80 9F Second byte after ED
* 6 90 BF Second byte after F0
* 7 80 8F Second byte after F4
* 8 C2 F4 First non ASCII byte
* 9..15 7F 80 Invalid byte
*/
/* After the first step we compute the index for all bytes, then we permute
the bytes according to their indices to check the ranges from the range
table.
* The range for a given type can be found in the range_min_table and
range_max_table, the range for type/index X is in range_min_table[X] ...
range_max_table[X].
*/
/* Algorithm:
* Put index zero to all bytes.
* Find all non ASCII characters, give them index 8.
* For each tail byte in a codepoint sequence, give it an index corresponding
to the 1 based index from the end.
* If the first byte of the codepoint is in the [C0...DF] range, we write
index 1 in the following byte.
* If the first byte of the codepoint is in the range [E0...EF], we write
indices 2 and 1 in the next two bytes.
* If the first byte of the codepoint is in the range [F0...FF] we write
indices 3,2,1 into the next three bytes.
* For finding the number of bytes we need to look at high nibbles (4 bits)
and do the lookup from the table, it can be done with shift by 4 + shuffle
instructions. We call it `first_len`.
* Then we shift first_len by 8 bits to get the indices of the 2nd bytes.
* Saturating sub 1 and shift by 8 bits to get the indices of the 3rd bytes.
* Again to get the indices of the 4th bytes.
* Take OR of all that 4 values and check within range.
*/
/* For example:
* input C3 80 68 E2 80 20 A6 F0 A0 80 AC 20 F0 93 80 80
* first_len 1 0 0 2 0 0 0 3 0 0 0 0 3 0 0 0
* 1st byte 8 0 0 8 0 0 0 8 0 0 0 0 8 0 0 0
* 2nd byte 0 1 0 0 2 0 0 0 3 0 0 0 0 3 0 0 // Shift + sub
* 3rd byte 0 0 0 0 0 1 0 0 0 2 0 0 0 0 2 0 // Shift + sub
* 4th byte 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 // Shift + sub
* Index 8 1 0 8 2 1 0 8 3 2 1 0 8 3 2 1 // OR of results
*/
/* Checking for errors:
* Error checking is done by looking up the high nibble (4 bits) of each byte
against an error checking table.
* Because the lookup value for the second byte depends of the value of the
first byte in codepoint, we use saturated operations to adjust the index.
* Specifically we need to add 2 for E0, 3 for ED, 3 for F0 and 4 for F4 to
match the correct index.
* If we subtract from all bytes EF then EO -> 241, ED -> 254, F0 -> 1,
F4 -> 5
* Do saturating sub 240, then E0 -> 1, ED -> 14 and we can do lookup to
match the adjustment
* Add saturating 112, then F0 -> 113, F4 -> 117, all that were > 16 will
be more 128 and lookup in ef_fe_table will return 0 but for F0
and F4 it will be 4 and 5 accordingly
*/
/*
* Then just check the appropriate ranges with greater/smaller equal
instructions. Check tail with a naive algorithm.
* To save from previous 16 byte checks we just align previous_first_len to
get correct continuations of the codepoints.
*/
/*
* Map high nibble of "First Byte" to legal character length minus 1
* 0x00 ~ 0xBF --> 0
* 0xC0 ~ 0xDF --> 1
* 0xE0 ~ 0xEF --> 2
* 0xF0 ~ 0xFF --> 3
*/
const __m128i first_len_table =
_mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3);
/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
const __m128i first_range_table =
_mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8);
/*
* Range table, map range index to min and max values
*/
const __m128i range_min_table =
_mm_setr_epi8(0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, 0xC2, 0x7F,
0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F);
const __m128i range_max_table =
_mm_setr_epi8(0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, 0xF4, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80);
/*
* Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
* which the Second Byte are not 80~BF. It contains "range index adjustment".
* +------------+---------------+------------------+----------------+
* | First Byte | original range| range adjustment | adjusted range |
* +------------+---------------+------------------+----------------+
* | E0 | 2 | 2 | 4 |
* +------------+---------------+------------------+----------------+
* | ED | 2 | 3 | 5 |
* +------------+---------------+------------------+----------------+
* | F0 | 3 | 3 | 6 |
* +------------+---------------+------------------+----------------+
* | F4 | 4 | 4 | 8 |
* +------------+---------------+------------------+----------------+
*/
/* df_ee_table[1] -> E0, df_ee_table[14] -> ED as ED - E0 = 13 */
// The values represent the adjustment in the Range Index table for a correct
// index.
const __m128i df_ee_table =
_mm_setr_epi8(0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0);
/* ef_fe_table[1] -> F0, ef_fe_table[5] -> F4, F4 - F0 = 4 */
// The values represent the adjustment in the Range Index table for a correct
// index.
const __m128i ef_fe_table =
_mm_setr_epi8(0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
__m128i prev_input = _mm_set1_epi8(0);
__m128i prev_first_len = _mm_set1_epi8(0);
__m128i error = _mm_set1_epi8(0);
while (end - data >= 16) {
const __m128i input =
_mm_loadu_si128((const __m128i*)(data));
/* high_nibbles = input >> 4 */
const __m128i high_nibbles =
_mm_and_si128(_mm_srli_epi16(input, 4), _mm_set1_epi8(0x0F));
/* first_len = legal character length minus 1 */
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
/* first_len = first_len_table[high_nibbles] */
__m128i first_len = _mm_shuffle_epi8(first_len_table, high_nibbles);
/* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
/* range = first_range_table[high_nibbles] */
__m128i range = _mm_shuffle_epi8(first_range_table, high_nibbles);
/* Second Byte: set range index to first_len */
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
/* range |= (first_len, prev_first_len) << 1 byte */
range = _mm_or_si128(range, _mm_alignr_epi8(first_len, prev_first_len, 15));
/* Third Byte: set range index to saturate_sub(first_len, 1) */
/* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
__m128i tmp1;
__m128i tmp2;
/* tmp1 = saturate_sub(first_len, 1) */
tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(1));
/* tmp2 = saturate_sub(prev_first_len, 1) */
tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(1));
/* range |= (tmp1, tmp2) << 2 bytes */
range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 14));
/* Fourth Byte: set range index to saturate_sub(first_len, 2) */
/* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
/* tmp1 = saturate_sub(first_len, 2) */
tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(2));
/* tmp2 = saturate_sub(prev_first_len, 2) */
tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(2));
/* range |= (tmp1, tmp2) << 3 bytes */
range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 13));
/*
* Now we have below range indices calculated
* Correct cases:
* - 8 for C0~FF
* - 3 for 1st byte after F0~FF
* - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
* - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
* 3rd byte after F0~FF
* - 0 for others
* Error cases:
* >9 for non ascii First Byte overlapping
* E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
*/
/* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
/* Overlaps lead to index 9~15, which are illegal in range table */
__m128i shift1;
__m128i pos;
__m128i range2;
/* shift1 = (input, prev_input) << 1 byte */
shift1 = _mm_alignr_epi8(input, prev_input, 15);
pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
/*
* shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
* pos: | 0 1 15 | 16 17 239| 240 241 255|
* pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
* pos+112: | 112 113 127| >= 128 | >= 128 |
*/
tmp1 = _mm_subs_epu8(pos, _mm_set1_epi8(-16));
range2 = _mm_shuffle_epi8(df_ee_table, tmp1);
tmp2 = _mm_adds_epu8(pos, _mm_set1_epi8(112));
range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_table, tmp2));
range = _mm_add_epi8(range, range2);
/* Load min and max values per calculated range index */
__m128i min_range = _mm_shuffle_epi8(range_min_table, range);
__m128i max_range = _mm_shuffle_epi8(range_max_table, range);
/* Check value range */
if (return_position) {
error = _mm_cmplt_epi8(input, min_range);
error = _mm_or_si128(error, _mm_cmpgt_epi8(input, max_range));
/* 5% performance drop from this conditional branch */
if (!_mm_testz_si128(error, error)) {
break;
}
} else {
error = _mm_or_si128(error, _mm_cmplt_epi8(input, min_range));
error = _mm_or_si128(error, _mm_cmpgt_epi8(input, max_range));
}
prev_input = input;
prev_first_len = first_len;
data += 16;
}
/* If we got to the end, we don't need to skip any bytes backwards */
if (return_position && (data - (end - len)) == 0) {
return utf8_range_ValidateUTF8Naive(data, end, return_position);
}
/* Find previous codepoint (not 80~BF) */
data -= utf8_range_CodepointSkipBackwards(_mm_extract_epi32(prev_input, 3));
if (return_position) {
return (data - (end - len)) +
utf8_range_ValidateUTF8Naive(data, end, return_position);
}
/* Test if there was any error */
if (!_mm_testz_si128(error, error)) {
return 0;
}
/* Check the tail */
return utf8_range_ValidateUTF8Naive(data, end, return_position);
#endif
}
int utf8_range_IsValid(const char* data, size_t len) {
return utf8_range_Validate(data, len, /*return_position=*/0) != 0;
}
size_t utf8_range_ValidPrefix(const char* data, size_t len) {
return utf8_range_Validate(data, len, /*return_position=*/1);
}
@@ -0,0 +1,22 @@
#ifndef THIRD_PARTY_UTF8_RANGE_UTF8_RANGE_H_
#define THIRD_PARTY_UTF8_RANGE_UTF8_RANGE_H_
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
// Returns 1 if the sequence of characters is a valid UTF-8 sequence, otherwise
// 0.
int utf8_range_IsValid(const char* data, size_t len);
// Returns the length in bytes of the prefix of str that is all
// structurally valid UTF-8.
size_t utf8_range_ValidPrefix(const char* data, size_t len);
#ifdef __cplusplus
} // extern "C"
#endif
#endif // THIRD_PARTY_UTF8_RANGE_UTF8_RANGE_H_
@@ -0,0 +1,29 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2017 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
#include <string.h>
// On x86-64 Linux with glibc, we link against the 2.2.5 version of memcpy so
// that we avoid depending on the 2.14 version of the symbol. This way,
// distributions that are using pre-2.14 versions of glibc can successfully use
// the gem we distribute
// (https://github.com/protocolbuffers/protobuf/issues/2783).
//
// This wrapper is enabled by passing the linker flags -Wl,-wrap,memcpy in
// extconf.rb.
#ifdef __linux__
#if defined(__x86_64__) && defined(__GNU_LIBRARY__)
__asm__(".symver memcpy,memcpy@GLIBC_2.2.5");
void *__wrap_memcpy(void *dest, const void *src, size_t n) {
return memcpy(dest, src, n);
}
#else
void *__wrap_memcpy(void *dest, const void *src, size_t n) {
return memmove(dest, src, n);
}
#endif
#endif