diff --git a/benchmarks/csharp/BenchmarkModels.cs b/benchmarks/csharp/BenchmarkModels.cs index f3c2fc7c0f..abf945be57 100644 --- a/benchmarks/csharp/BenchmarkModels.cs +++ b/benchmarks/csharp/BenchmarkModels.cs @@ -26,35 +26,35 @@ namespace Apache.Fory.Benchmarks.CSharp; [ProtoContract] public sealed class NumericStruct { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public int F1 { get; set; } - [Field(Id = 2)] + [ForyField(Id = 2)] [ProtoMember(2)] public int F2 { get; set; } - [Field(Id = 3)] + [ForyField(Id = 3)] [ProtoMember(3)] public int F3 { get; set; } - [Field(Id = 4)] + [ForyField(Id = 4)] [ProtoMember(4)] public int F4 { get; set; } - [Field(Id = 5)] + [ForyField(Id = 5)] [ProtoMember(5)] public int F5 { get; set; } - [Field(Id = 6)] + [ForyField(Id = 6)] [ProtoMember(6)] public int F6 { get; set; } - [Field(Id = 7)] + [ForyField(Id = 7)] [ProtoMember(7)] public int F7 { get; set; } - [Field(Id = 8)] + [ForyField(Id = 8)] [ProtoMember(8)] public int F8 { get; set; } } @@ -64,7 +64,7 @@ public sealed class NumericStruct [ProtoContract] public sealed class StructList { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public List Values { get; set; } = []; } @@ -74,91 +74,91 @@ public sealed class StructList [ProtoContract] public sealed class Sample { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public int IntValue { get; set; } - [Field(Id = 2)] + [ForyField(Id = 2)] [ProtoMember(2)] public long LongValue { get; set; } - [Field(Id = 3)] + [ForyField(Id = 3)] [ProtoMember(3)] public float FloatValue { get; set; } - [Field(Id = 4)] + [ForyField(Id = 4)] [ProtoMember(4)] public double DoubleValue { get; set; } - [Field(Id = 5)] + [ForyField(Id = 5)] [ProtoMember(5)] public int ShortValue { get; set; } - [Field(Id = 6)] + [ForyField(Id = 6)] [ProtoMember(6)] public int CharValue { get; set; } - [Field(Id = 7)] + [ForyField(Id = 7)] [ProtoMember(7)] public bool BooleanValue { get; set; } - [Field(Id = 8)] + [ForyField(Id = 8)] [ProtoMember(8)] public int IntValueBoxed { get; set; } - [Field(Id = 9)] + [ForyField(Id = 9)] [ProtoMember(9)] public long LongValueBoxed { get; set; } - [Field(Id = 10)] + [ForyField(Id = 10)] [ProtoMember(10)] public float FloatValueBoxed { get; set; } - [Field(Id = 11)] + [ForyField(Id = 11)] [ProtoMember(11)] public double DoubleValueBoxed { get; set; } - [Field(Id = 12)] + [ForyField(Id = 12)] [ProtoMember(12)] public int ShortValueBoxed { get; set; } - [Field(Id = 13)] + [ForyField(Id = 13)] [ProtoMember(13)] public int CharValueBoxed { get; set; } - [Field(Id = 14)] + [ForyField(Id = 14)] [ProtoMember(14)] public bool BooleanValueBoxed { get; set; } - [Field(Id = 15)] + [ForyField(Id = 15)] [ProtoMember(15)] public int[] IntArray { get; set; } = []; - [Field(Id = 16)] + [ForyField(Id = 16)] [ProtoMember(16)] public long[] LongArray { get; set; } = []; - [Field(Id = 17)] + [ForyField(Id = 17)] [ProtoMember(17)] public float[] FloatArray { get; set; } = []; - [Field(Id = 18)] + [ForyField(Id = 18)] [ProtoMember(18)] public double[] DoubleArray { get; set; } = []; - [Field(Id = 19)] + [ForyField(Id = 19)] [ProtoMember(19)] public int[] ShortArray { get; set; } = []; - [Field(Id = 20)] + [ForyField(Id = 20)] [ProtoMember(20)] public int[] CharArray { get; set; } = []; - [Field(Id = 21)] + [ForyField(Id = 21)] [ProtoMember(21)] public bool[] BooleanArray { get; set; } = []; - [Field(Id = 22)] + [ForyField(Id = 22)] [ProtoMember(22)] public string String { get; set; } = string.Empty; } @@ -168,7 +168,7 @@ public sealed class Sample [ProtoContract] public sealed class SampleList { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public List Values { get; set; } = []; } @@ -198,51 +198,51 @@ public enum MediaSize [ProtoContract] public sealed class Media { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public string Uri { get; set; } = string.Empty; - [Field(Id = 2)] + [ForyField(Id = 2)] [ProtoMember(2)] public string Title { get; set; } = string.Empty; - [Field(Id = 3)] + [ForyField(Id = 3)] [ProtoMember(3)] public int Width { get; set; } - [Field(Id = 4)] + [ForyField(Id = 4)] [ProtoMember(4)] public int Height { get; set; } - [Field(Id = 5)] + [ForyField(Id = 5)] [ProtoMember(5)] public string Format { get; set; } = string.Empty; - [Field(Id = 6)] + [ForyField(Id = 6)] [ProtoMember(6)] public long Duration { get; set; } - [Field(Id = 7)] + [ForyField(Id = 7)] [ProtoMember(7)] public long Size { get; set; } - [Field(Id = 8)] + [ForyField(Id = 8)] [ProtoMember(8)] public int Bitrate { get; set; } - [Field(Id = 9)] + [ForyField(Id = 9)] [ProtoMember(9)] public bool HasBitrate { get; set; } - [Field(Id = 10)] + [ForyField(Id = 10)] [ProtoMember(10)] public List Persons { get; set; } = []; - [Field(Id = 11)] + [ForyField(Id = 11)] [ProtoMember(11)] public Player Player { get; set; } - [Field(Id = 12)] + [ForyField(Id = 12)] [ProtoMember(12)] public string Copyright { get; set; } = string.Empty; } @@ -252,23 +252,23 @@ public sealed class Media [ProtoContract] public sealed class Image { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public string Uri { get; set; } = string.Empty; - [Field(Id = 2)] + [ForyField(Id = 2)] [ProtoMember(2)] public string Title { get; set; } = string.Empty; - [Field(Id = 3)] + [ForyField(Id = 3)] [ProtoMember(3)] public int Width { get; set; } - [Field(Id = 4)] + [ForyField(Id = 4)] [ProtoMember(4)] public int Height { get; set; } - [Field(Id = 5)] + [ForyField(Id = 5)] [ProtoMember(5)] public MediaSize Size { get; set; } } @@ -278,11 +278,11 @@ public sealed class Image [ProtoContract] public sealed class MediaContent { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public Media Media { get; set; } = new(); - [Field(Id = 2)] + [ForyField(Id = 2)] [ProtoMember(2)] public List Images { get; set; } = []; } @@ -292,7 +292,7 @@ public sealed class MediaContent [ProtoContract] public sealed class MediaContentList { - [Field(Id = 1)] + [ForyField(Id = 1)] [ProtoMember(1)] public List Values { get; set; } = []; } diff --git a/compiler/fory_compiler/generators/csharp.py b/compiler/fory_compiler/generators/csharp.py index fc9fdcd817..8b49025396 100644 --- a/compiler/fory_compiler/generators/csharp.py +++ b/compiler/fory_compiler/generators/csharp.py @@ -59,8 +59,8 @@ class CSharpGenerator(BaseGenerator): PrimitiveKind.UINT64: "ulong", PrimitiveKind.VAR_UINT64: "ulong", PrimitiveKind.TAGGED_UINT64: "ulong", - PrimitiveKind.FLOAT16: "float", - PrimitiveKind.BFLOAT16: "float", + PrimitiveKind.FLOAT16: "Half", + PrimitiveKind.BFLOAT16: "BFloat16", PrimitiveKind.FLOAT32: "float", PrimitiveKind.FLOAT64: "double", PrimitiveKind.STRING: "string", @@ -401,6 +401,7 @@ def generate_file(self) -> GeneratedFile: lines.append("using System;") lines.append("using System.Collections.Generic;") lines.append("using Apache.Fory;") + lines.append("using S = Apache.Fory.Schema.Types;") lines.append("") lines.append(f"namespace {namespace_name};") lines.append("") @@ -573,20 +574,74 @@ def _default_initializer( return None - def _field_encoding(self, field: Field) -> Optional[str]: - field_type = field.field_type - if not isinstance(field_type, PrimitiveType): - return None - kind = field_type.kind + def _schema_type_hint( + self, field_type: FieldType, force: bool = False + ) -> Optional[str]: + if isinstance(field_type, PrimitiveType): + return self._primitive_schema_type_hint(field_type.kind, force) + + if isinstance(field_type, ListType): + element_hint = self._schema_type_hint(field_type.element_type, force=True) + if element_hint is None: + return None + return f"S.List<{element_hint}>" + + if isinstance(field_type, MapType): + key_hint = self._schema_type_hint(field_type.key_type, force=True) + value_hint = self._schema_type_hint(field_type.value_type, force=True) + if key_hint is None and value_hint is None: + return None + if key_hint is None: + key_hint = self._schema_type_hint(field_type.key_type, force=True) + if value_hint is None: + value_hint = self._schema_type_hint(field_type.value_type, force=True) + if key_hint is None or value_hint is None: + return None + return f"S.Map<{key_hint}, {value_hint}>" + + return None + + def _primitive_schema_type_hint( + self, kind: PrimitiveKind, force: bool + ) -> Optional[str]: + hints = { + PrimitiveKind.BOOL: "S.Bool", + PrimitiveKind.INT8: "S.Int8", + PrimitiveKind.INT16: "S.Int16", + PrimitiveKind.INT32: "S.Int32", + PrimitiveKind.VARINT32: "S.VarInt32", + PrimitiveKind.INT64: "S.Int64", + PrimitiveKind.VARINT64: "S.VarInt64", + PrimitiveKind.TAGGED_INT64: "S.TaggedInt64", + PrimitiveKind.UINT8: "S.UInt8", + PrimitiveKind.UINT16: "S.UInt16", + PrimitiveKind.UINT32: "S.UInt32", + PrimitiveKind.VAR_UINT32: "S.VarUInt32", + PrimitiveKind.UINT64: "S.UInt64", + PrimitiveKind.VAR_UINT64: "S.VarUInt64", + PrimitiveKind.TAGGED_UINT64: "S.TaggedUInt64", + PrimitiveKind.FLOAT16: "S.Float16", + PrimitiveKind.BFLOAT16: "S.BFloat16", + PrimitiveKind.FLOAT32: "S.Float32", + PrimitiveKind.FLOAT64: "S.Float64", + PrimitiveKind.STRING: "S.String", + PrimitiveKind.BYTES: "S.Binary", + PrimitiveKind.DATE: "S.Date", + PrimitiveKind.TIMESTAMP: "S.Timestamp", + PrimitiveKind.DURATION: "S.Duration", + PrimitiveKind.DECIMAL: "S.Decimal", + } + if force: + return hints.get(kind) if kind in { PrimitiveKind.INT32, PrimitiveKind.INT64, PrimitiveKind.UINT32, PrimitiveKind.UINT64, + PrimitiveKind.TAGGED_INT64, + PrimitiveKind.TAGGED_UINT64, }: - return "Fixed" - if kind in {PrimitiveKind.TAGGED_INT64, PrimitiveKind.TAGGED_UINT64}: - return "Tagged" + return hints[kind] return None def _type_reference_for_local( @@ -774,10 +829,10 @@ def generate_message( used_field_names: Set[str] = set() for field in message.fields: lines.append("") - encoding = self._field_encoding(field) - if encoding: + schema_type = self._schema_type_hint(field.field_type) + if schema_type: lines.append( - f"{ind}{self.indent_str}[Field(Encoding = FieldEncoding.{encoding})]" + f"{ind}{self.indent_str}[ForyField(Type = typeof({schema_type}))]" ) field_name = self._field_member_name(field, message, used_field_names) field_type = self.generate_type( diff --git a/compiler/fory_compiler/tests/test_csharp_generator.py b/compiler/fory_compiler/tests/test_csharp_generator.py index a0181eeff6..507cbc051a 100644 --- a/compiler/fory_compiler/tests/test_csharp_generator.py +++ b/compiler/fory_compiler/tests/test_csharp_generator.py @@ -101,11 +101,52 @@ def test_csharp_field_encoding_attributes(): """ ) - assert "[Field(Encoding = FieldEncoding.Fixed)]" in file.content - assert "[Field(Encoding = FieldEncoding.Tagged)]" in file.content + assert "[ForyField(Type = typeof(S.Int32))]" in file.content + assert "[ForyField(Type = typeof(S.TaggedUInt64))]" in file.content assert "public int Plain { get; set; }" in file.content +def test_csharp_nested_schema_type_attributes(): + file = generate( + """ + package example; + + message Nested { + map> values = 1; + } + """ + ) + + assert ( + "[ForyField(Type = typeof(S.Map>))]" + in file.content + ) + assert ( + "public Dictionary> Values { get; set; } = new();" + in file.content + ) + + +def test_csharp_reduced_precision_carriers(): + file = generate( + """ + package example; + + message Reduced { + float16 f16 = 1; + bfloat16 bf16 = 2; + list f16_values = 3; + list bf16_values = 4; + } + """ + ) + + assert "public Half F16 { get; set; }" in file.content + assert "public BFloat16 Bf16 { get; set; }" in file.content + assert "public List F16Values { get; set; } = new();" in file.content + assert "public List Bf16Values { get; set; } = new();" in file.content + + def test_csharp_imported_registration_calls_generated(): repo_root = Path(__file__).resolve().parents[3] idl_dir = repo_root / "integration_tests" / "idl_tests" / "idl" diff --git a/csharp/README.md b/csharp/README.md index f310a625ee..db4fbd406a 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -11,6 +11,7 @@ The C# implementation provides high-performance object graph serialization for . - High-performance binary serialization for .NET 8+ - Cross-language compatibility with Java, Python, C++, Go, Rust, and JavaScript - Source-generator-based serializers for `[ForyObject]` types +- Field-level schema descriptors with `[ForyField(Type = typeof(...))]` - Optional shared/circular reference tracking (`TrackRef(true)`) - Compatible mode for schema evolution - Reduced-precision carriers for `Half` / `BFloat16` scalars and `Half[]` / `List` / `BFloat16[]` / `List` array payloads diff --git a/csharp/src/Fory.Generator/ForyObjectGenerator.cs b/csharp/src/Fory.Generator/ForyObjectGenerator.cs index 3f9894e62f..c5dc09d5c6 100644 --- a/csharp/src/Fory.Generator/ForyObjectGenerator.cs +++ b/csharp/src/Fory.Generator/ForyObjectGenerator.cs @@ -46,10 +46,10 @@ public sealed class ForyObjectGenerator : IIncrementalGenerator defaultSeverity: DiagnosticSeverity.Error, isEnabledByDefault: true); - private static readonly DiagnosticDescriptor UnsupportedEncoding = new( + private static readonly DiagnosticDescriptor UnsupportedSchemaType = new( id: "FORY003", - title: "Unsupported Field encoding", - messageFormat: "Member '{0}' uses unsupported [Field] encoding for type '{1}'", + title: "Unsupported Fory field schema type", + messageFormat: "Member '{0}' uses unsupported [ForyField] schema descriptor for type '{1}'", category: "Fory", defaultSeverity: DiagnosticSeverity.Error, isEnabledByDefault: true); @@ -168,6 +168,14 @@ private static void EmitObjectSerializer(StringBuilder sb, TypeModel model) sb.AppendLine(" return nullable ? global::Apache.Fory.RefMode.NullOnly : global::Apache.Fory.RefMode.None;"); sb.AppendLine(" }"); sb.AppendLine(); + foreach (MemberModel member in model.SortedMembers) + { + if (member.FieldCodec is not null) + { + EmitFieldCodecMethods(sb, member); + } + } + sb.AppendLine(" private static bool __ForyCanReadCompatiblePrimitive(global::Apache.Fory.TypeId typeId)"); sb.AppendLine(" {"); sb.AppendLine(" return typeId switch"); @@ -625,6 +633,596 @@ private static void EmitObjectSerializer(StringBuilder sb, TypeModel model) sb.AppendLine("}"); } + private static void EmitFieldCodecMethods(StringBuilder sb, MemberModel member) + { + FieldCodecModel codec = member.FieldCodec!; + string memberId = Sanitize(member.Name); + sb.AppendLine( + $" private static void __ForyWrite{memberId}Field(global::Apache.Fory.WriteContext context, {member.TypeName} value, global::Apache.Fory.RefMode refMode)"); + sb.AppendLine(" {"); + sb.AppendLine(" if (refMode == global::Apache.Fory.RefMode.NullOnly)"); + sb.AppendLine(" {"); + if (member.IsNullableValueType) + { + sb.AppendLine(" if (!value.HasValue)"); + } + else + { + sb.AppendLine(" if (value is null)"); + } + + sb.AppendLine(" {"); + sb.AppendLine(" context.Writer.WriteInt8((sbyte)global::Apache.Fory.RefFlag.Null);"); + sb.AppendLine(" return;"); + sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" context.Writer.WriteInt8((sbyte)global::Apache.Fory.RefFlag.NotNullValue);"); + sb.AppendLine(" }"); + string writeValueExpr = member.IsNullableValueType ? "value.Value" : member.IsNullable ? "value!" : "value"; + int id = 0; + EmitWritePayload(sb, codec, writeValueExpr, 2, ref id); + sb.AppendLine(" }"); + sb.AppendLine(); + + sb.AppendLine( + $" private static {member.TypeName} __ForyRead{memberId}Field(global::Apache.Fory.ReadContext context, global::Apache.Fory.RefMode refMode)"); + sb.AppendLine(" {"); + sb.AppendLine(" if (refMode == global::Apache.Fory.RefMode.NullOnly)"); + sb.AppendLine(" {"); + sb.AppendLine(" sbyte refFlag = context.Reader.ReadInt8();"); + sb.AppendLine(" if (refFlag == (sbyte)global::Apache.Fory.RefFlag.Null)"); + sb.AppendLine(" {"); + sb.AppendLine($" return ({member.TypeName})default!;"); + sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" if (refFlag != (sbyte)global::Apache.Fory.RefFlag.NotNullValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" throw new global::Apache.Fory.InvalidDataException($\"invalid nullOnly ref flag {refFlag}\");"); + sb.AppendLine(" }"); + sb.AppendLine(" }"); + string resultVar = $"__{memberId}Value"; + id = 0; + EmitReadPayload(sb, codec, resultVar, 2, ref id); + sb.AppendLine($" return {resultVar};"); + sb.AppendLine(" }"); + sb.AppendLine(); + } + + private static void EmitWritePayload( + StringBuilder sb, + FieldCodecModel codec, + string valueExpr, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + switch (codec.Kind) + { + case FieldCodecKind.Scalar: + if (!TryBuildDirectPayloadWrite(codec.TypeId, valueExpr, out string? writeCode)) + { + sb.AppendLine($"{indent}context.TypeResolver.GetSerializer<{codec.TypeName}>().WriteData(context, {valueExpr}, false);"); + return; + } + + sb.AppendLine($"{indent}{writeCode}"); + return; + case FieldCodecKind.PackedArray: + EmitWritePackedArrayPayload(sb, codec, valueExpr, indentLevel, ref id); + return; + case FieldCodecKind.List: + EmitWriteCollectionPayload(sb, codec, valueExpr, indentLevel, ref id, isSet: false); + return; + case FieldCodecKind.Set: + EmitWriteCollectionPayload(sb, codec, valueExpr, indentLevel, ref id, isSet: true); + return; + case FieldCodecKind.Map: + EmitWriteMapPayload(sb, codec, valueExpr, indentLevel, ref id); + return; + } + } + + private static void EmitWritePackedArrayPayload( + StringBuilder sb, + FieldCodecModel codec, + string valueExpr, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + string valuesVar = $"__foryPacked{id++}"; + sb.AppendLine($"{indent}{codec.TypeName} {valuesVar} = {valueExpr} ?? [];"); + string countExpr = codec.CarrierKind == CarrierKind.Array ? $"{valuesVar}.Length" : $"{valuesVar}.Count"; + int width = PackedArrayElementWidth(codec.TypeId); + string lengthExpr = width == 1 ? countExpr : $"checked({countExpr} * {width})"; + sb.AppendLine($"{indent}context.Writer.WriteVarUInt32((uint){lengthExpr});"); + string packedIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{indent}for (int {packedIndexVar} = 0; {packedIndexVar} < {countExpr}; {packedIndexVar}++)"); + sb.AppendLine($"{indent}{{"); + string itemExpr = $"{valuesVar}[{packedIndexVar}]"; + uint elementTypeId = PackedArrayElementTypeId(codec.TypeId); + if (!TryBuildDirectPayloadWrite(elementTypeId, itemExpr, out string? writeCode)) + { + throw new InvalidOperationException($"unsupported packed array type id {codec.TypeId}"); + } + + sb.AppendLine($"{indent} {writeCode}"); + sb.AppendLine($"{indent}}}"); + } + + private static void EmitWriteCollectionPayload( + StringBuilder sb, + FieldCodecModel codec, + string valueExpr, + int indentLevel, + ref int id, + bool isSet) + { + string indent = new(' ', indentLevel * 4); + FieldCodecModel element = codec.Generics[0]; + string valuesVar = $"__foryCollection{id++}"; + sb.AppendLine($"{indent}{codec.TypeName} {valuesVar} = {valueExpr} ?? [];"); + string countExpr = codec.CarrierKind == CarrierKind.Array ? $"{valuesVar}.Length" : $"{valuesVar}.Count"; + sb.AppendLine($"{indent}int __foryCount{id} = {countExpr};"); + string countVar = $"__foryCount{id++}"; + sb.AppendLine($"{indent}context.Writer.WriteVarUInt32((uint){countVar});"); + sb.AppendLine($"{indent}if ({countVar} != 0)"); + sb.AppendLine($"{indent}{{"); + string innerIndent = indent + " "; + string hasNullVar = $"__foryHasNull{id++}"; + if (element.Nullable) + { + sb.AppendLine($"{innerIndent}bool {hasNullVar} = false;"); + if (isSet) + { + sb.AppendLine($"{innerIndent}foreach ({element.TypeName} __foryItem in {valuesVar})"); + sb.AppendLine($"{innerIndent}{{"); + sb.AppendLine($"{innerIndent} if (__foryItem is null)"); + sb.AppendLine($"{innerIndent} {{"); + sb.AppendLine($"{innerIndent} {hasNullVar} = true;"); + sb.AppendLine($"{innerIndent} break;"); + sb.AppendLine($"{innerIndent} }}"); + sb.AppendLine($"{innerIndent}}}"); + } + else + { + string scanIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{innerIndent}for (int {scanIndexVar} = 0; {scanIndexVar} < {countVar}; {scanIndexVar}++)"); + sb.AppendLine($"{innerIndent}{{"); + string itemExpr = $"{valuesVar}[{scanIndexVar}]"; + sb.AppendLine($"{innerIndent} if ({itemExpr} is null)"); + sb.AppendLine($"{innerIndent} {{"); + sb.AppendLine($"{innerIndent} {hasNullVar} = true;"); + sb.AppendLine($"{innerIndent} break;"); + sb.AppendLine($"{innerIndent} }}"); + sb.AppendLine($"{innerIndent}}}"); + } + } + else + { + sb.AppendLine($"{innerIndent}bool {hasNullVar} = false;"); + } + + string collectionHeaderVar = $"__foryHeader{id++}"; + sb.AppendLine($"{innerIndent}byte {collectionHeaderVar} = 0b0000_1000 | 0b0000_0100;"); + sb.AppendLine($"{innerIndent}if ({hasNullVar})"); + sb.AppendLine($"{innerIndent}{{"); + sb.AppendLine($"{innerIndent} {collectionHeaderVar} |= 0b0000_0010;"); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{innerIndent}context.Writer.WriteUInt8({collectionHeaderVar});"); + if (isSet) + { + sb.AppendLine($"{innerIndent}foreach ({element.TypeName} __foryItem in {valuesVar})"); + sb.AppendLine($"{innerIndent}{{"); + EmitWriteNullableElementPayload(sb, element, "__foryItem", indentLevel + 2, ref id, hasNullVar); + sb.AppendLine($"{innerIndent}}}"); + } + else + { + string writeIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{innerIndent}for (int {writeIndexVar} = 0; {writeIndexVar} < {countVar}; {writeIndexVar}++)"); + sb.AppendLine($"{innerIndent}{{"); + sb.AppendLine($"{innerIndent} {element.TypeName} __foryItem = {valuesVar}[{writeIndexVar}];"); + EmitWriteNullableElementPayload(sb, element, "__foryItem", indentLevel + 2, ref id, hasNullVar); + sb.AppendLine($"{innerIndent}}}"); + } + + sb.AppendLine($"{indent}}}"); + } + + private static void EmitWriteNullableElementPayload( + StringBuilder sb, + FieldCodecModel element, + string itemExpr, + int indentLevel, + ref int id, + string hasNullVar) + { + string indent = new(' ', indentLevel * 4); + if (!element.Nullable) + { + EmitWritePayload(sb, element, itemExpr, indentLevel, ref id); + return; + } + + sb.AppendLine($"{indent}if ({hasNullVar})"); + sb.AppendLine($"{indent}{{"); + sb.AppendLine($"{indent} if ({itemExpr} is null)"); + sb.AppendLine($"{indent} {{"); + sb.AppendLine($"{indent} context.Writer.WriteInt8((sbyte)global::Apache.Fory.RefFlag.Null);"); + sb.AppendLine($"{indent} continue;"); + sb.AppendLine($"{indent} }}"); + sb.AppendLine(); + sb.AppendLine($"{indent} context.Writer.WriteInt8((sbyte)global::Apache.Fory.RefFlag.NotNullValue);"); + string nonNullExpr = element.NullableValueType ? $"{itemExpr}.GetValueOrDefault()" : $"{itemExpr}!"; + EmitWritePayload(sb, element, nonNullExpr, indentLevel + 1, ref id); + sb.AppendLine($"{indent}}}"); + sb.AppendLine($"{indent}else"); + sb.AppendLine($"{indent}{{"); + EmitWritePayload(sb, element, element.NullableValueType ? $"{itemExpr}.GetValueOrDefault()" : $"{itemExpr}!", indentLevel + 1, ref id); + sb.AppendLine($"{indent}}}"); + } + + private static void EmitWriteMapPayload( + StringBuilder sb, + FieldCodecModel codec, + string valueExpr, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + FieldCodecModel key = codec.Generics[0]; + FieldCodecModel value = codec.Generics[1]; + string mapVar = $"__foryMap{id++}"; + sb.AppendLine($"{indent}{codec.TypeName} {mapVar} = {valueExpr} ?? [];"); + sb.AppendLine($"{indent}context.Writer.WriteVarUInt32((uint){mapVar}.Count);"); + sb.AppendLine($"{indent}foreach (global::System.Collections.Generic.KeyValuePair<{key.TypeName}, {value.TypeName}> __foryEntry in {mapVar})"); + sb.AppendLine($"{indent}{{"); + string innerIndent = indent + " "; + string keyNullVar = $"__foryKeyNull{id++}"; + string valueNullVar = $"__foryValueNull{id++}"; + if (key.Nullable) + { + sb.AppendLine($"{innerIndent}bool {keyNullVar} = __foryEntry.Key is null;"); + } + else + { + sb.AppendLine($"{innerIndent}bool {keyNullVar} = false;"); + } + + if (value.Nullable) + { + sb.AppendLine($"{innerIndent}bool {valueNullVar} = __foryEntry.Value is null;"); + } + else + { + sb.AppendLine($"{innerIndent}bool {valueNullVar} = false;"); + } + + string mapHeaderVar = $"__foryHeader{id++}"; + sb.AppendLine($"{innerIndent}byte {mapHeaderVar} = 0;"); + sb.AppendLine($"{innerIndent}if ({keyNullVar}) {mapHeaderVar} |= 0b0000_0010; else {mapHeaderVar} |= 0b0000_0100;"); + sb.AppendLine($"{innerIndent}if ({valueNullVar}) {mapHeaderVar} |= 0b0001_0000; else {mapHeaderVar} |= 0b0010_0000;"); + sb.AppendLine($"{innerIndent}context.Writer.WriteUInt8({mapHeaderVar});"); + sb.AppendLine($"{innerIndent}if (!{keyNullVar} && !{valueNullVar})"); + sb.AppendLine($"{innerIndent}{{"); + sb.AppendLine($"{innerIndent} context.Writer.WriteUInt8(1);"); + EmitWritePayload(sb, key, key.NullableValueType ? "__foryEntry.Key.GetValueOrDefault()" : "__foryEntry.Key!", indentLevel + 2, ref id); + EmitWritePayload(sb, value, value.NullableValueType ? "__foryEntry.Value.GetValueOrDefault()" : "__foryEntry.Value!", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent} continue;"); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{innerIndent}if (!{keyNullVar})"); + sb.AppendLine($"{innerIndent}{{"); + EmitWritePayload(sb, key, key.NullableValueType ? "__foryEntry.Key.GetValueOrDefault()" : "__foryEntry.Key!", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{innerIndent}if (!{valueNullVar})"); + sb.AppendLine($"{innerIndent}{{"); + EmitWritePayload(sb, value, value.NullableValueType ? "__foryEntry.Value.GetValueOrDefault()" : "__foryEntry.Value!", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{indent}}}"); + } + + private static void EmitReadPayload( + StringBuilder sb, + FieldCodecModel codec, + string targetVar, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + switch (codec.Kind) + { + case FieldCodecKind.Scalar: + if (TryBuildDirectPayloadRead(codec.TypeId, out string? readExpr)) + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = {readExpr};"); + } + else + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = context.TypeResolver.GetSerializer<{codec.TypeName}>().ReadData(context);"); + } + + return; + case FieldCodecKind.PackedArray: + EmitReadPackedArrayPayload(sb, codec, targetVar, indentLevel, ref id); + return; + case FieldCodecKind.List: + EmitReadCollectionPayload(sb, codec, targetVar, indentLevel, ref id, isSet: false); + return; + case FieldCodecKind.Set: + EmitReadCollectionPayload(sb, codec, targetVar, indentLevel, ref id, isSet: true); + return; + case FieldCodecKind.Map: + EmitReadMapPayload(sb, codec, targetVar, indentLevel, ref id); + return; + } + } + + private static void EmitReadPackedArrayPayload( + StringBuilder sb, + FieldCodecModel codec, + string targetVar, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + int width = PackedArrayElementWidth(codec.TypeId); + uint elementTypeId = PackedArrayElementTypeId(codec.TypeId); + string payloadSizeVar = $"__foryPayloadSize{id++}"; + string countVar = $"__foryPackedCount{id++}"; + sb.AppendLine($"{indent}int {payloadSizeVar} = checked((int)context.Reader.ReadVarUInt32());"); + if (width > 1) + { + int mask = width - 1; + sb.AppendLine($"{indent}if (({payloadSizeVar} & {mask}) != 0)"); + sb.AppendLine($"{indent}{{"); + sb.AppendLine($"{indent} throw new global::Apache.Fory.InvalidDataException(\"packed array payload size mismatch\");"); + sb.AppendLine($"{indent}}}"); + } + + sb.AppendLine($"{indent}int {countVar} = {payloadSizeVar}{(width == 1 ? string.Empty : $" / {width}")};"); + if (codec.CarrierKind == CarrierKind.Array) + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new {ElementTypeName(codec.TypeName)}[{countVar}];"); + } + else + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new({countVar});"); + } + + string packedIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{indent}for (int {packedIndexVar} = 0; {packedIndexVar} < {countVar}; {packedIndexVar}++)"); + sb.AppendLine($"{indent}{{"); + if (!TryBuildDirectPayloadRead(elementTypeId, out string? readExpr)) + { + throw new InvalidOperationException($"unsupported packed array type id {codec.TypeId}"); + } + + if (codec.CarrierKind == CarrierKind.Array) + { + sb.AppendLine($"{indent} {targetVar}[{packedIndexVar}] = {readExpr};"); + } + else + { + sb.AppendLine($"{indent} {targetVar}.Add({readExpr});"); + } + + sb.AppendLine($"{indent}}}"); + } + + private static void EmitReadCollectionPayload( + StringBuilder sb, + FieldCodecModel codec, + string targetVar, + int indentLevel, + ref int id, + bool isSet) + { + string indent = new(' ', indentLevel * 4); + FieldCodecModel element = codec.Generics[0]; + string lengthVar = $"__foryLength{id++}"; + string headerVar = $"__foryHeader{id++}"; + string hasNullVar = $"__foryHasNull{id++}"; + sb.AppendLine($"{indent}int {lengthVar} = checked((int)context.Reader.ReadVarUInt32());"); + if (isSet) + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new();"); + } + else if (codec.CarrierKind == CarrierKind.Array) + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new {ElementTypeName(codec.TypeName)}[{lengthVar}];"); + } + else + { + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new({lengthVar});"); + } + + sb.AppendLine($"{indent}if ({lengthVar} != 0)"); + sb.AppendLine($"{indent}{{"); + string innerIndent = indent + " "; + sb.AppendLine($"{innerIndent}byte {headerVar} = context.Reader.ReadUInt8();"); + sb.AppendLine($"{innerIndent}bool {hasNullVar} = ({headerVar} & 0b0000_0010) != 0;"); + string collectionIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{innerIndent}for (int {collectionIndexVar} = 0; {collectionIndexVar} < {lengthVar}; {collectionIndexVar}++)"); + sb.AppendLine($"{innerIndent}{{"); + EmitReadNullableElementPayload(sb, element, "__foryItem", indentLevel + 2, ref id, hasNullVar); + if (codec.CarrierKind == CarrierKind.Array) + { + sb.AppendLine($"{innerIndent} {targetVar}[{collectionIndexVar}] = __foryItem;"); + } + else + { + sb.AppendLine($"{innerIndent} {targetVar}.Add(__foryItem);"); + } + + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{indent}}}"); + } + + private static void EmitReadNullableElementPayload( + StringBuilder sb, + FieldCodecModel element, + string targetVar, + int indentLevel, + ref int id, + string hasNullVar) + { + string indent = new(' ', indentLevel * 4); + sb.AppendLine($"{indent}{element.TypeName} {targetVar};"); + if (element.Nullable) + { + sb.AppendLine($"{indent}if ({hasNullVar})"); + sb.AppendLine($"{indent}{{"); + sb.AppendLine($"{indent} sbyte __foryRefFlag = context.Reader.ReadInt8();"); + sb.AppendLine($"{indent} if (__foryRefFlag == (sbyte)global::Apache.Fory.RefFlag.Null)"); + sb.AppendLine($"{indent} {{"); + sb.AppendLine($"{indent} {targetVar} = ({element.TypeName})default!;"); + sb.AppendLine($"{indent} }}"); + sb.AppendLine($"{indent} else if (__foryRefFlag == (sbyte)global::Apache.Fory.RefFlag.NotNullValue)"); + sb.AppendLine($"{indent} {{"); + string nullableNonNullVar = $"__foryNonNull{id++}"; + EmitReadPayload(sb, NonNullableCodec(element), nullableNonNullVar, indentLevel + 2, ref id); + sb.AppendLine($"{indent} {targetVar} = {nullableNonNullVar};"); + sb.AppendLine($"{indent} }}"); + sb.AppendLine($"{indent} else"); + sb.AppendLine($"{indent} {{"); + sb.AppendLine($"{indent} throw new global::Apache.Fory.InvalidDataException($\"invalid collection null flag {{__foryRefFlag}}\");"); + sb.AppendLine($"{indent} }}"); + sb.AppendLine($"{indent}}}"); + sb.AppendLine($"{indent}else"); + sb.AppendLine($"{indent}{{"); + string nonNullVar = $"__foryNonNull{id++}"; + EmitReadPayload(sb, NonNullableCodec(element), nonNullVar, indentLevel + 1, ref id); + sb.AppendLine($"{indent} {targetVar} = {nonNullVar};"); + sb.AppendLine($"{indent}}}"); + return; + } + + string directNonNullVar = $"__foryNonNull{id++}"; + EmitReadPayload(sb, element, directNonNullVar, indentLevel, ref id); + sb.AppendLine($"{indent}{targetVar} = {directNonNullVar};"); + } + + private static void EmitReadMapPayload( + StringBuilder sb, + FieldCodecModel codec, + string targetVar, + int indentLevel, + ref int id) + { + string indent = new(' ', indentLevel * 4); + FieldCodecModel key = codec.Generics[0]; + FieldCodecModel value = codec.Generics[1]; + string totalVar = $"__foryTotal{id++}"; + sb.AppendLine($"{indent}int {totalVar} = checked((int)context.Reader.ReadVarUInt32());"); + sb.AppendLine($"{indent}{codec.TypeName} {targetVar} = new({totalVar});"); + sb.AppendLine($"{indent}int __foryRead = 0;"); + sb.AppendLine($"{indent}while (__foryRead < {totalVar})"); + sb.AppendLine($"{indent}{{"); + string innerIndent = indent + " "; + sb.AppendLine($"{innerIndent}byte __foryHeader = context.Reader.ReadUInt8();"); + sb.AppendLine($"{innerIndent}bool __foryKeyNull = (__foryHeader & 0b0000_0010) != 0;"); + sb.AppendLine($"{innerIndent}bool __foryValueNull = (__foryHeader & 0b0001_0000) != 0;"); + sb.AppendLine($"{innerIndent}if (__foryKeyNull || __foryValueNull)"); + sb.AppendLine($"{innerIndent}{{"); + sb.AppendLine($"{innerIndent} {key.TypeName} __foryKey = ({key.TypeName})default!;"); + sb.AppendLine($"{innerIndent} {value.TypeName} __foryValue = ({value.TypeName})default!;"); + sb.AppendLine($"{innerIndent} if (!__foryKeyNull)"); + sb.AppendLine($"{innerIndent} {{"); + EmitReadPayload(sb, NonNullableCodec(key), "__foryReadKey", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent} __foryKey = __foryReadKey;"); + sb.AppendLine($"{innerIndent} }}"); + sb.AppendLine($"{innerIndent} if (!__foryValueNull)"); + sb.AppendLine($"{innerIndent} {{"); + EmitReadPayload(sb, NonNullableCodec(value), "__foryReadValue", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent} __foryValue = __foryReadValue;"); + sb.AppendLine($"{innerIndent} }}"); + if (codec.CarrierKind == CarrierKind.NullableKeyDictionary) + { + sb.AppendLine($"{innerIndent} {targetVar}[__foryKey] = __foryValue;"); + } + else + { + sb.AppendLine($"{innerIndent} if (!__foryKeyNull)"); + sb.AppendLine($"{innerIndent} {{"); + sb.AppendLine($"{innerIndent} {targetVar}[__foryKey] = __foryValue;"); + sb.AppendLine($"{innerIndent} }}"); + } + + sb.AppendLine($"{innerIndent} __foryRead++;"); + sb.AppendLine($"{innerIndent} continue;"); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{innerIndent}int __foryChunkSize = context.Reader.ReadUInt8();"); + string mapIndexVar = $"__foryIndex{id++}"; + sb.AppendLine($"{innerIndent}for (int {mapIndexVar} = 0; {mapIndexVar} < __foryChunkSize; {mapIndexVar}++)"); + sb.AppendLine($"{innerIndent}{{"); + EmitReadPayload(sb, NonNullableCodec(key), "__foryKey", indentLevel + 2, ref id); + EmitReadPayload(sb, NonNullableCodec(value), "__foryValue", indentLevel + 2, ref id); + sb.AppendLine($"{innerIndent} {targetVar}[__foryKey] = __foryValue;"); + sb.AppendLine($"{innerIndent}}}"); + sb.AppendLine($"{innerIndent}__foryRead += __foryChunkSize;"); + sb.AppendLine($"{indent}}}"); + } + + private static FieldCodecModel NonNullableCodec(FieldCodecModel codec) + { + if (!codec.Nullable) + { + return codec; + } + + return new FieldCodecModel( + codec.Kind, + codec.TypeId, + codec.NullableValueType && codec.TypeName.EndsWith("?", StringComparison.Ordinal) + ? codec.TypeName.Substring(0, codec.TypeName.Length - 1) + : codec.TypeName, + false, + false, + codec.CarrierKind, + codec.Generics); + } + + private static string ElementTypeName(string arrayTypeName) + { + return arrayTypeName.EndsWith("[]", StringComparison.Ordinal) + ? arrayTypeName.Substring(0, arrayTypeName.Length - 2) + : "object"; + } + + private static int PackedArrayElementWidth(uint typeId) + { + return typeId switch + { + 41 or 43 or 44 => 1, + 45 or 49 or 53 or 54 => 2, + 46 or 50 or 55 => 4, + 47 or 51 or 56 => 8, + _ => throw new InvalidOperationException($"unsupported packed array type id {typeId}"), + }; + } + + private static uint PackedArrayElementTypeId(uint typeId) + { + return typeId switch + { + 41 => 9, + 43 => 1, + 44 => 2, + 45 => 3, + 46 => 4, + 47 => 6, + 49 => 10, + 50 => 11, + 51 => 13, + 53 => 17, + 54 => 18, + 55 => 19, + 56 => 20, + _ => throw new InvalidOperationException($"unsupported packed array type id {typeId}"), + }; + } + private static void EmitWriteMember(StringBuilder sb, MemberModel member, bool compatibleMode) { string refModeExpr = BuildWriteRefModeExpression(member); @@ -646,6 +1244,13 @@ private static void EmitWriteMember(StringBuilder sb, MemberModel member, bool c throw new InvalidOperationException($"unsupported dynamic any kind {member.DynamicAnyKind}"); } + if (member.FieldCodec is not null) + { + sb.AppendLine( + $" __ForyWrite{Sanitize(member.Name)}Field(context, {memberAccess}, {refModeExpr});"); + return; + } + if (member.UseDictionaryTypeInfoCache) { EmitWriteDictionaryWithTypeInfoCache( @@ -754,6 +1359,13 @@ private static void EmitReadMemberAssignment( throw new InvalidOperationException($"unsupported dynamic any kind {member.DynamicAnyKind}"); } + if (member.FieldCodec is not null) + { + sb.AppendLine( + $"{indent}{assignmentTarget} = __ForyRead{Sanitize(member.Name)}Field(context, {refModeExpr});"); + return; + } + if (allowDirectRead && !member.IsNullable && TryBuildDirectFieldRead(member, out string? directReadExpr)) { sb.AppendLine($"{indent}{assignmentTarget} = {directReadExpr};"); @@ -1219,16 +1831,22 @@ private static string BuildFieldTypeInfoLiteral(MemberModel member) MemberDeclKind memberDeclKind) { (bool isOptional, ITypeSymbol unwrappedType) = UnwrapNullable(memberType); - FieldEncoding fieldEncoding = FieldEncoding.None; short? fieldId = null; + SchemaTypeModel? schemaType = null; foreach (AttributeData attribute in memberSymbol.GetAttributes()) { string? attrName = attribute.AttributeClass?.ToDisplayString(); - if (!string.Equals(attrName, "Apache.Fory.FieldAttribute", StringComparison.Ordinal)) + if (!string.Equals(attrName, "Apache.Fory.ForyFieldAttribute", StringComparison.Ordinal)) { continue; } + if (attribute.ConstructorArguments.Length == 1 && + TryGetNonNegativeShort(attribute.ConstructorArguments[0], out short ctorFieldId)) + { + fieldId = ctorFieldId; + } + foreach (KeyValuePair namedArg in attribute.NamedArguments) { if (string.Equals(namedArg.Key, "Id", StringComparison.Ordinal)) @@ -1241,20 +1859,20 @@ private static string BuildFieldTypeInfoLiteral(MemberModel member) continue; } - if (!string.Equals(namedArg.Key, "Encoding", StringComparison.Ordinal)) + if (!string.Equals(namedArg.Key, "Type", StringComparison.Ordinal)) { continue; } - if (namedArg.Value.Value is int encoding) + if (namedArg.Value.Value is ITypeSymbol schemaSymbol) { - fieldEncoding = (FieldEncoding)encoding; + schemaType = TryParseSchemaType(schemaSymbol); } } } DynamicAnyKind dynamicAnyKind = ResolveDynamicAnyKind(unwrappedType); - TypeResolution resolution = ResolveTypeResolution(unwrappedType, fieldEncoding); + TypeResolution resolution = ResolveTypeResolution(unwrappedType, schemaType); if (!resolution.Supported) { return null; @@ -1288,7 +1906,9 @@ private static string BuildFieldTypeInfoLiteral(MemberModel member) memberType, isOptional, dynamicAnyKind, - resolution.Classification.TypeId); + resolution.Classification.TypeId, + schemaType); + FieldCodecModel? fieldCodec = BuildFieldCodecModel(memberType, typeMeta, schemaType, classification); return new MemberModel( name, @@ -1307,28 +1927,31 @@ memberType is INamedTypeSymbol nts && !unwrappedType.IsValueType && classification.TypeId != 21, FieldNeedsTypeInfo(classification, dynamicAnyKind, unwrappedType), dynamicAnyKind == DynamicAnyKind.None ? DynamicAnyKind.None : dynamicAnyKind, - typeMeta); + typeMeta, + fieldCodec); } private static TypeMetaFieldTypeModel BuildTypeMetaFieldTypeModel( ITypeSymbol memberType, bool nullable, DynamicAnyKind dynamicAnyKind, - uint explicitTypeId) + uint explicitTypeId, + SchemaTypeModel? schemaType = null) { (bool _, ITypeSymbol unwrapped) = UnwrapNullable(memberType); + if (schemaType is not null) + { + return BuildSchemaTypeMetaFieldTypeModel(memberType, nullable, schemaType); + } + if (TryGetListElementType(unwrapped, out ITypeSymbol? listElementType)) { bool elementNullable = GenericNullable(listElementType!); if (!elementNullable && TryResolvePackedArrayTypeIdForElement(listElementType!) is uint packedArrayTypeId && - explicitTypeId == packedArrayTypeId) + (explicitTypeId == packedArrayTypeId || explicitTypeId == 22)) { - // Align compatible TypeMeta with C++ vector arithmetic handling: - // when the wire type is already classified as a packed array (e.g. int[]), - // use the specialized array TypeId directly instead of LIST. - // This keeps schema/type-meta bytes consistent across languages. return new TypeMetaFieldTypeModel( packedArrayTypeId.ToString(), nullable, @@ -1428,6 +2051,227 @@ private static TypeMetaFieldTypeModel BuildTypeMetaFieldTypeModel( ImmutableArray.Empty); } + private static TypeMetaFieldTypeModel BuildSchemaTypeMetaFieldTypeModel( + ITypeSymbol carrierType, + bool nullable, + SchemaTypeModel schemaType) + { + (bool _, ITypeSymbol unwrapped) = UnwrapNullable(carrierType); + switch (schemaType.Kind) + { + case SchemaTypeKind.List: + if (!TryGetListElementType(unwrapped, out ITypeSymbol? listElementType)) + { + return new TypeMetaFieldTypeModel( + schemaType.TypeId.ToString(), + nullable, + false, + ImmutableArray.Empty); + } + + bool elementNullable = GenericNullable(listElementType!); + return new TypeMetaFieldTypeModel( + "(uint)global::Apache.Fory.TypeId.List", + nullable, + false, + ImmutableArray.Create( + BuildSchemaTypeMetaFieldTypeModel( + listElementType!, + elementNullable, + schemaType.Generics[0]))); + case SchemaTypeKind.Set: + if (!TryGetSetElementType(unwrapped, out ITypeSymbol? setElementType)) + { + return new TypeMetaFieldTypeModel( + schemaType.TypeId.ToString(), + nullable, + false, + ImmutableArray.Empty); + } + + bool setElementNullable = GenericNullable(setElementType!); + return new TypeMetaFieldTypeModel( + "(uint)global::Apache.Fory.TypeId.Set", + nullable, + false, + ImmutableArray.Create( + BuildSchemaTypeMetaFieldTypeModel( + setElementType!, + setElementNullable, + schemaType.Generics[0]))); + case SchemaTypeKind.Map: + if (!TryGetMapTypeArguments(unwrapped, out ITypeSymbol? keyType, out ITypeSymbol? valueType)) + { + return new TypeMetaFieldTypeModel( + schemaType.TypeId.ToString(), + nullable, + false, + ImmutableArray.Empty); + } + + bool keyNullable = GenericNullable(keyType!); + bool valueNullable = GenericNullable(valueType!); + return new TypeMetaFieldTypeModel( + "(uint)global::Apache.Fory.TypeId.Map", + nullable, + false, + ImmutableArray.Create( + BuildSchemaTypeMetaFieldTypeModel(keyType!, keyNullable, schemaType.Generics[0]), + BuildSchemaTypeMetaFieldTypeModel(valueType!, valueNullable, schemaType.Generics[1]))); + default: + return new TypeMetaFieldTypeModel( + schemaType.TypeId.ToString(), + nullable, + false, + ImmutableArray.Empty); + } + } + + private static FieldCodecModel? BuildFieldCodecModel( + ITypeSymbol carrierType, + TypeMetaFieldTypeModel typeMeta, + SchemaTypeModel? schemaType, + TypeClassification classification) + { + (bool nullable, ITypeSymbol unwrapped) = UnwrapNullable(carrierType); + bool nullableValueType = carrierType is INamedTypeSymbol nts && + nts.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T; + + if (schemaType is not null) + { + FieldCodecModel codec = BuildFieldCodecFromSchema(carrierType, nullable, nullableValueType, schemaType); + return codec.Kind == FieldCodecKind.Scalar ? null : codec; + } + + _ = typeMeta; + _ = classification; + return null; + } + + private static FieldCodecModel BuildFieldCodecFromSchema( + ITypeSymbol carrierType, + bool nullable, + bool nullableValueType, + SchemaTypeModel schemaType) + { + (bool _, ITypeSymbol unwrapped) = UnwrapNullable(carrierType); + switch (schemaType.Kind) + { + case SchemaTypeKind.List: + { + ITypeSymbol elementType = TryGetListElementType(unwrapped, out ITypeSymbol? listElementType) + ? listElementType! + : carrierType; + FieldCodecModel element = BuildFieldCodecFromSchema( + elementType, + GenericNullable(elementType), + elementType is INamedTypeSymbol elementNamed && + elementNamed.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T, + schemaType.Generics[0]); + return new FieldCodecModel( + FieldCodecKind.List, + schemaType.TypeId, + carrierType.ToDisplayString(FullNameFormat), + nullable, + nullableValueType, + GetCarrierKind(unwrapped), + ImmutableArray.Create(element)); + } + case SchemaTypeKind.Set: + { + ITypeSymbol elementType = TryGetSetElementType(unwrapped, out ITypeSymbol? setElementType) + ? setElementType! + : carrierType; + FieldCodecModel element = BuildFieldCodecFromSchema( + elementType, + GenericNullable(elementType), + elementType is INamedTypeSymbol elementNamed && + elementNamed.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T, + schemaType.Generics[0]); + return new FieldCodecModel( + FieldCodecKind.Set, + schemaType.TypeId, + carrierType.ToDisplayString(FullNameFormat), + nullable, + nullableValueType, + GetCarrierKind(unwrapped), + ImmutableArray.Create(element)); + } + case SchemaTypeKind.Map: + { + ITypeSymbol keyType = carrierType; + ITypeSymbol valueType = carrierType; + if (TryGetMapTypeArguments(unwrapped, out ITypeSymbol? parsedKeyType, out ITypeSymbol? parsedValueType)) + { + keyType = parsedKeyType!; + valueType = parsedValueType!; + } + + FieldCodecModel key = BuildFieldCodecFromSchema( + keyType, + GenericNullable(keyType), + keyType is INamedTypeSymbol keyNamed && + keyNamed.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T, + schemaType.Generics[0]); + FieldCodecModel value = BuildFieldCodecFromSchema( + valueType, + GenericNullable(valueType), + valueType is INamedTypeSymbol valueNamed && + valueNamed.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T, + schemaType.Generics[1]); + return new FieldCodecModel( + FieldCodecKind.Map, + schemaType.TypeId, + carrierType.ToDisplayString(FullNameFormat), + nullable, + nullableValueType, + GetCarrierKind(unwrapped), + ImmutableArray.Create(key, value)); + } + case SchemaTypeKind.PackedArray: + return new FieldCodecModel( + FieldCodecKind.PackedArray, + schemaType.TypeId, + carrierType.ToDisplayString(FullNameFormat), + nullable, + nullableValueType, + GetCarrierKind(unwrapped), + ImmutableArray.Empty); + default: + return new FieldCodecModel( + FieldCodecKind.Scalar, + schemaType.TypeId, + carrierType.ToDisplayString(FullNameFormat), + nullable, + nullableValueType, + GetCarrierKind(unwrapped), + ImmutableArray.Empty); + } + } + + private static CarrierKind GetCarrierKind(ITypeSymbol unwrappedType) + { + if (unwrappedType is IArrayTypeSymbol) + { + return CarrierKind.Array; + } + + if (unwrappedType is not INamedTypeSymbol named) + { + return CarrierKind.Value; + } + + string genericName = named.ConstructedFrom.ToDisplayString(); + return genericName switch + { + "System.Collections.Generic.List" => CarrierKind.List, + "System.Collections.Generic.HashSet" => CarrierKind.HashSet, + "System.Collections.Generic.Dictionary" => CarrierKind.Dictionary, + "Apache.Fory.NullableKeyDictionary" => CarrierKind.NullableKeyDictionary, + _ => CarrierKind.Value, + }; + } + private static bool TryGetNonNegativeShort(TypedConstant value, out short fieldId) { fieldId = default; @@ -1569,74 +2413,227 @@ private static bool HasAccessibleParameterlessCtor(INamedTypeSymbol type) return false; } - private static TypeResolution ResolveTypeResolution(ITypeSymbol type, FieldEncoding encoding) + private static SchemaTypeModel? TryParseSchemaType(ITypeSymbol symbol) { - TypeClassification baseType = ClassifyType(type); - if (encoding == FieldEncoding.None) + if (symbol is not INamedTypeSymbol named) { - return new TypeResolution(true, baseType); + return null; } - bool isInt32 = type.SpecialType == SpecialType.System_Int32; - bool isUInt32 = type.SpecialType == SpecialType.System_UInt32; - bool isInt64 = type.SpecialType == SpecialType.System_Int64; - bool isUInt64 = type.SpecialType == SpecialType.System_UInt64; + string fullName = named.ConstructedFrom.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + fullName = fullName.StartsWith("global::", StringComparison.Ordinal) + ? fullName.Substring("global::".Length) + : fullName; - if (isInt32) + if (fullName == "Apache.Fory.Schema.Types.List") { - return encoding switch + if (named.TypeArguments.Length != 1 || + TryParseSchemaType(named.TypeArguments[0]) is not SchemaTypeModel element) { - FieldEncoding.Varint => new TypeResolution(true, baseType), - FieldEncoding.Fixed => new TypeResolution( - true, - new TypeClassification(4, true, true, false, false, false, 4)), - _ => new TypeResolution(false, baseType), - }; + return null; + } + + return new SchemaTypeModel(22, SchemaTypeKind.List, ImmutableArray.Create(element)); } - if (isUInt32) + if (fullName == "Apache.Fory.Schema.Types.Set") { - return encoding switch + if (named.TypeArguments.Length != 1 || + TryParseSchemaType(named.TypeArguments[0]) is not SchemaTypeModel element) { - FieldEncoding.Varint => new TypeResolution(true, baseType), - FieldEncoding.Fixed => new TypeResolution( - true, - new TypeClassification(11, true, true, false, false, false, 4)), - _ => new TypeResolution(false, baseType), - }; + return null; + } + + return new SchemaTypeModel(23, SchemaTypeKind.Set, ImmutableArray.Create(element)); } - if (isInt64) + if (fullName == "Apache.Fory.Schema.Types.Map") { - return encoding switch + if (named.TypeArguments.Length != 2 || + TryParseSchemaType(named.TypeArguments[0]) is not SchemaTypeModel key || + TryParseSchemaType(named.TypeArguments[1]) is not SchemaTypeModel value) { - FieldEncoding.Varint => new TypeResolution(true, baseType), - FieldEncoding.Fixed => new TypeResolution( - true, - new TypeClassification(6, true, true, false, false, false, 8)), - FieldEncoding.Tagged => new TypeResolution( - true, - new TypeClassification(8, true, true, false, false, true, 8)), - _ => new TypeResolution(false, baseType), - }; + return null; + } + + return new SchemaTypeModel(24, SchemaTypeKind.Map, ImmutableArray.Create(key, value)); } - if (isUInt64) + return TryResolveSchemaTypeId(fullName, out uint typeId, out SchemaTypeKind kind) + ? new SchemaTypeModel(typeId, kind, ImmutableArray.Empty) + : null; + } + + private static bool TryResolveSchemaTypeId(string fullName, out uint typeId, out SchemaTypeKind kind) + { + kind = SchemaTypeKind.Scalar; + switch (fullName) { - return encoding switch - { - FieldEncoding.Varint => new TypeResolution(true, baseType), - FieldEncoding.Fixed => new TypeResolution( - true, - new TypeClassification(13, true, true, false, false, false, 8)), - FieldEncoding.Tagged => new TypeResolution( - true, - new TypeClassification(15, true, true, false, false, true, 8)), - _ => new TypeResolution(false, baseType), - }; + case "Apache.Fory.Schema.Types.Bool": + typeId = 1; + return true; + case "Apache.Fory.Schema.Types.Int8": + typeId = 2; + return true; + case "Apache.Fory.Schema.Types.Int16": + typeId = 3; + return true; + case "Apache.Fory.Schema.Types.Int32": + typeId = 4; + return true; + case "Apache.Fory.Schema.Types.VarInt32": + typeId = 5; + return true; + case "Apache.Fory.Schema.Types.Int64": + typeId = 6; + return true; + case "Apache.Fory.Schema.Types.VarInt64": + typeId = 7; + return true; + case "Apache.Fory.Schema.Types.TaggedInt64": + typeId = 8; + return true; + case "Apache.Fory.Schema.Types.UInt8": + typeId = 9; + return true; + case "Apache.Fory.Schema.Types.UInt16": + typeId = 10; + return true; + case "Apache.Fory.Schema.Types.UInt32": + typeId = 11; + return true; + case "Apache.Fory.Schema.Types.VarUInt32": + typeId = 12; + return true; + case "Apache.Fory.Schema.Types.UInt64": + typeId = 13; + return true; + case "Apache.Fory.Schema.Types.VarUInt64": + typeId = 14; + return true; + case "Apache.Fory.Schema.Types.TaggedUInt64": + typeId = 15; + return true; + case "Apache.Fory.Schema.Types.Float16": + typeId = 17; + return true; + case "Apache.Fory.Schema.Types.BFloat16": + typeId = 18; + return true; + case "Apache.Fory.Schema.Types.Float32": + typeId = 19; + return true; + case "Apache.Fory.Schema.Types.Float64": + typeId = 20; + return true; + case "Apache.Fory.Schema.Types.String": + typeId = 21; + return true; + case "Apache.Fory.Schema.Types.Binary": + typeId = 41; + return true; + case "Apache.Fory.Schema.Types.Duration": + typeId = 37; + return true; + case "Apache.Fory.Schema.Types.Timestamp": + typeId = 38; + return true; + case "Apache.Fory.Schema.Types.Date": + typeId = 39; + return true; + case "Apache.Fory.Schema.Types.Decimal": + typeId = 42; + return true; + case "Apache.Fory.Schema.Types.BoolArray": + typeId = 43; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Int8Array": + typeId = 44; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Int16Array": + typeId = 45; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Int32Array": + typeId = 46; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Int64Array": + typeId = 47; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.UInt8Array": + typeId = 41; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.UInt16Array": + typeId = 49; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.UInt32Array": + typeId = 50; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.UInt64Array": + typeId = 51; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Float16Array": + typeId = 53; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.BFloat16Array": + typeId = 54; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Float32Array": + typeId = 55; + kind = SchemaTypeKind.PackedArray; + return true; + case "Apache.Fory.Schema.Types.Float64Array": + typeId = 56; + kind = SchemaTypeKind.PackedArray; + return true; + default: + typeId = 0; + return false; } + } - return new TypeResolution(false, baseType); + private static TypeResolution ResolveTypeResolution(ITypeSymbol type, SchemaTypeModel? schemaType) + { + TypeClassification baseType = ClassifyType(type); + if (schemaType is null) + { + return new TypeResolution(true, baseType); + } + + bool isPrimitive = schemaType.Kind == SchemaTypeKind.Scalar; + bool isCollection = schemaType.Kind == SchemaTypeKind.List || + schemaType.Kind == SchemaTypeKind.Set || + schemaType.Kind == SchemaTypeKind.PackedArray; + bool isMap = schemaType.Kind == SchemaTypeKind.Map; + bool isCompressedNumeric = schemaType.TypeId is 5 or 7 or 8 or 12 or 14 or 15; + int primitiveSize = schemaType.TypeId switch + { + 1 or 2 or 9 => 1, + 3 or 10 or 17 or 18 => 2, + 4 or 5 or 11 or 12 or 19 => 4, + 6 or 7 or 8 or 13 or 14 or 15 or 20 => 8, + _ => 0, + }; + return new TypeResolution( + true, + new TypeClassification( + schemaType.TypeId, + isPrimitive, + true, + isCollection, + isMap, + isCompressedNumeric, + primitiveSize)); } private static TypeClassification ClassifyType(ITypeSymbol type) @@ -1743,6 +2740,13 @@ private static TypeClassification ClassifyType(ITypeSymbol type) if (TryGetListElementType(type, out _)) { + if (GetCarrierKind(type) == CarrierKind.List && + TryGetListElementType(type, out ITypeSymbol? elementType) && + TryResolvePackedArrayTypeIdForElement(elementType!) is uint packedArrayTypeId) + { + return new TypeClassification(packedArrayTypeId, false, true, true, false, false, 0); + } + return new TypeClassification(22, false, true, true, false, false, 0); } @@ -2053,6 +3057,52 @@ public TypeMetaFieldTypeModel( public ImmutableArray Generics { get; } } + private sealed class SchemaTypeModel + { + public SchemaTypeModel( + uint typeId, + SchemaTypeKind kind, + ImmutableArray generics) + { + TypeId = typeId; + Kind = kind; + Generics = generics; + } + + public uint TypeId { get; } + public SchemaTypeKind Kind { get; } + public ImmutableArray Generics { get; } + } + + private sealed class FieldCodecModel + { + public FieldCodecModel( + FieldCodecKind kind, + uint typeId, + string typeName, + bool nullable, + bool nullableValueType, + CarrierKind carrierKind, + ImmutableArray generics) + { + Kind = kind; + TypeId = typeId; + TypeName = typeName; + Nullable = nullable; + NullableValueType = nullableValueType; + CarrierKind = carrierKind; + Generics = generics; + } + + public FieldCodecKind Kind { get; } + public uint TypeId { get; } + public string TypeName { get; } + public bool Nullable { get; } + public bool NullableValueType { get; } + public CarrierKind CarrierKind { get; } + public ImmutableArray Generics { get; } + } + private sealed class TypeModel { public TypeModel( @@ -2094,7 +3144,8 @@ public MemberModel( bool isRefType, bool needsFieldTypeInfo, DynamicAnyKind dynamicAnyKind, - TypeMetaFieldTypeModel typeMeta) + TypeMetaFieldTypeModel typeMeta, + FieldCodecModel? fieldCodec) { Name = name; FieldIdentifier = fieldIdentifier; @@ -2112,6 +3163,7 @@ public MemberModel( NeedsFieldTypeInfo = needsFieldTypeInfo; DynamicAnyKind = dynamicAnyKind; TypeMeta = typeMeta; + FieldCodec = fieldCodec; } public string Name { get; } @@ -2130,6 +3182,7 @@ public MemberModel( public bool NeedsFieldTypeInfo { get; } public DynamicAnyKind DynamicAnyKind { get; } public TypeMetaFieldTypeModel TypeMeta { get; } + public FieldCodecModel? FieldCodec { get; } } private enum MemberDeclKind @@ -2152,11 +3205,31 @@ private enum DynamicAnyKind AnyValue, } - private enum FieldEncoding + private enum SchemaTypeKind + { + Scalar, + PackedArray, + List, + Set, + Map, + } + + private enum FieldCodecKind + { + Scalar, + PackedArray, + List, + Set, + Map, + } + + private enum CarrierKind { - None = -1, - Varint = 0, - Fixed = 1, - Tagged = 2, + Value, + Array, + List, + HashSet, + Dictionary, + NullableKeyDictionary, } } diff --git a/csharp/src/Fory/Attributes.cs b/csharp/src/Fory/Attributes.cs index 6c4ce727f4..e18be220d5 100644 --- a/csharp/src/Fory/Attributes.cs +++ b/csharp/src/Fory/Attributes.cs @@ -29,39 +29,58 @@ public sealed class ForyObjectAttribute : Attribute public bool Evolving { get; set; } = true; } -/// -/// Specifies field-level integer/number encoding strategy for generated serializers. -/// -public enum FieldEncoding -{ - /// - /// Variable-length integer encoding. - /// - Varint, - /// - /// Fixed-width integer encoding. - /// - Fixed, - /// - /// Tagged field encoding for schema-evolution scenarios. - /// - Tagged, -} - /// /// Overrides generated serializer behavior for a field or property. /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] -public sealed class FieldAttribute : Attribute +public sealed class ForyFieldAttribute : Attribute { + private short id = -1; + + public ForyFieldAttribute() + { + } + + public ForyFieldAttribute(short id) + { + ValidateId(id); + this.id = id; + } + + public ForyFieldAttribute(int id) + { + if (id is < 0 or > short.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(id)); + } + + this.id = (short)id; + } + /// /// Optional stable field tag id used for compatible metadata dispatch. /// Use a non-negative value to emit numeric field ids instead of field names. /// - public short Id { get; set; } = -1; + public short Id + { + get => id; + set + { + ValidateId(value); + id = value; + } + } /// - /// Gets or sets the field encoding strategy used by generated serializers. + /// Optional Fory schema descriptor type from Apache.Fory.Schema.Types. /// - public FieldEncoding Encoding { get; set; } = FieldEncoding.Varint; + public Type? Type { get; set; } + + private static void ValidateId(short id) + { + if (id < 0) + { + throw new ArgumentOutOfRangeException(nameof(id)); + } + } } diff --git a/csharp/src/Fory/FieldSkipper.cs b/csharp/src/Fory/FieldSkipper.cs index 652d59cfac..87ba06be5c 100644 --- a/csharp/src/Fory/FieldSkipper.cs +++ b/csharp/src/Fory/FieldSkipper.cs @@ -21,104 +21,251 @@ public static class FieldSkipper { public static void SkipFieldValue(ReadContext context, TypeMetaFieldType fieldType) { - _ = ReadFieldValue(context, fieldType); + SkipValue(context, fieldType, RefModeExtensions.From(fieldType.Nullable, fieldType.TrackRef)); } - private static uint? ReadEnumOrdinal(ReadContext context, RefMode refMode) + private static void SkipValue(ReadContext context, TypeMetaFieldType fieldType, RefMode refMode) { - return refMode switch + switch (refMode) { - RefMode.None => context.Reader.ReadVarUInt32(), - RefMode.NullOnly => ReadNullableEnumOrdinal(context), - RefMode.Tracking => throw new InvalidDataException("enum tracking ref mode is not supported"), - _ => throw new InvalidDataException($"unsupported ref mode {refMode}"), - }; - } + case RefMode.None: + SkipPayload(context, fieldType); + return; + case RefMode.NullOnly: + { + sbyte flag = context.Reader.ReadInt8(); + if (flag == (sbyte)RefFlag.Null) + { + return; + } - private static uint? ReadNullableEnumOrdinal(ReadContext context) - { - sbyte flag = context.Reader.ReadInt8(); - if (flag == (sbyte)RefFlag.Null) - { - return null; - } + if (flag != (sbyte)RefFlag.NotNullValue) + { + throw new InvalidDataException($"unexpected nullOnly flag {flag}"); + } - if (flag != (sbyte)RefFlag.NotNullValue) - { - throw new InvalidDataException($"unexpected enum nullOnly flag {flag}"); + SkipPayload(context, fieldType); + return; + } + case RefMode.Tracking: + _ = ReadTrackedValue(context, fieldType); + return; + default: + throw new InvalidDataException($"unsupported ref mode {refMode}"); } + } - return context.Reader.ReadVarUInt32(); + private static object? ReadTrackedValue(ReadContext context, TypeMetaFieldType fieldType) + { + return fieldType.TypeId switch + { + (uint)TypeId.String => context.TypeResolver.GetSerializer().Read(context, RefMode.Tracking, false), + (uint)TypeId.List => context.TypeResolver.GetSerializer>().Read(context, RefMode.Tracking, false), + (uint)TypeId.Set => context.TypeResolver.GetSerializer>().Read(context, RefMode.Tracking, false), + (uint)TypeId.Map => context.TypeResolver.GetSerializer>().Read(context, RefMode.Tracking, false), + (uint)TypeId.Union or + (uint)TypeId.TypedUnion or + (uint)TypeId.NamedUnion => context.TypeResolver.GetSerializer().Read(context, RefMode.Tracking, false), + _ => throw new InvalidDataException($"unsupported tracked skip field type id {fieldType.TypeId}"), + }; } - private static object? ReadFieldValue(ReadContext context, TypeMetaFieldType fieldType) + private static void SkipPayload(ReadContext context, TypeMetaFieldType fieldType) { - RefMode refMode = RefModeExtensions.From(fieldType.Nullable, fieldType.TrackRef); switch (fieldType.TypeId) { case (uint)TypeId.Bool: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); case (uint)TypeId.Int8: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + case (uint)TypeId.UInt8: + context.Reader.Skip(1); + return; case (uint)TypeId.Int16: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); - case (uint)TypeId.VarInt32: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); - case (uint)TypeId.VarInt64: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + case (uint)TypeId.UInt16: case (uint)TypeId.Float16: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); case (uint)TypeId.BFloat16: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); - case (uint)TypeId.Float16Array: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); - case (uint)TypeId.BFloat16Array: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + context.Reader.Skip(2); + return; + case (uint)TypeId.Int32: + case (uint)TypeId.UInt32: case (uint)TypeId.Float32: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + case (uint)TypeId.Date: + context.Reader.Skip(4); + return; + case (uint)TypeId.Int64: + case (uint)TypeId.UInt64: case (uint)TypeId.Float64: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + case (uint)TypeId.Timestamp: + case (uint)TypeId.Duration: + context.Reader.Skip(8); + return; + case (uint)TypeId.VarInt32: + _ = context.Reader.ReadVarInt32(); + return; + case (uint)TypeId.VarUInt32: + _ = context.Reader.ReadVarUInt32(); + return; + case (uint)TypeId.VarInt64: + _ = context.Reader.ReadVarInt64(); + return; + case (uint)TypeId.VarUInt64: + _ = context.Reader.ReadVarUInt64(); + return; + case (uint)TypeId.TaggedInt64: + _ = context.Reader.ReadTaggedInt64(); + return; + case (uint)TypeId.TaggedUInt64: + _ = context.Reader.ReadTaggedUInt64(); + return; case (uint)TypeId.String: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + _ = StringSerializer.ReadString(context); + return; case (uint)TypeId.Decimal: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); + _ = context.TypeResolver.GetSerializer().ReadData(context); + return; + case (uint)TypeId.Binary: + case (uint)TypeId.BoolArray: + case (uint)TypeId.Int8Array: + case (uint)TypeId.Int16Array: + case (uint)TypeId.Int32Array: + case (uint)TypeId.Int64Array: + case (uint)TypeId.UInt8Array: + case (uint)TypeId.UInt16Array: + case (uint)TypeId.UInt32Array: + case (uint)TypeId.UInt64Array: + case (uint)TypeId.Float16Array: + case (uint)TypeId.BFloat16Array: + case (uint)TypeId.Float32Array: + case (uint)TypeId.Float64Array: + SkipPackedArray(context); + return; case (uint)TypeId.List: - { - if (fieldType.Generics.Count != 1 || fieldType.Generics[0].TypeId != (uint)TypeId.String) - { - throw new InvalidDataException("unsupported compatible list element type"); - } - - return context.TypeResolver.GetSerializer>().Read(context, refMode, false); - } case (uint)TypeId.Set: + SkipListOrSet(context, fieldType); + return; + case (uint)TypeId.Map: + SkipMap(context, fieldType); + return; + case (uint)TypeId.Enum: + case (uint)TypeId.NamedEnum: + _ = context.Reader.ReadVarUInt32(); + return; + case (uint)TypeId.Union: + case (uint)TypeId.TypedUnion: + case (uint)TypeId.NamedUnion: + _ = context.TypeResolver.GetSerializer().ReadData(context); + return; + default: + throw new InvalidDataException($"unsupported compatible field type id {fieldType.TypeId}"); + } + } + + private static void SkipPackedArray(ReadContext context) + { + int payloadSize = checked((int)context.Reader.ReadVarUInt32()); + context.Reader.Skip(payloadSize); + } + + private static void SkipListOrSet(ReadContext context, TypeMetaFieldType fieldType) + { + if (fieldType.Generics.Count != 1) + { + throw new InvalidDataException("list/set field metadata must have one element type"); + } + + int length = checked((int)context.Reader.ReadVarUInt32()); + if (length == 0) + { + return; + } + + TypeMetaFieldType elementType = fieldType.Generics[0]; + byte header = context.Reader.ReadUInt8(); + bool trackRef = (header & CollectionBits.TrackingRef) != 0; + bool hasNull = (header & CollectionBits.HasNull) != 0; + bool declared = (header & CollectionBits.DeclaredElementType) != 0; + bool sameType = (header & CollectionBits.SameType) != 0; + if (!sameType) + { + throw new InvalidDataException("dynamic compatible list/set skip is not supported"); + } + + if (!declared) + { + _ = context.TypeResolver.ReadAnyTypeInfo(context); + } + + RefMode elementRefMode = trackRef ? RefMode.Tracking : hasNull ? RefMode.NullOnly : RefMode.None; + for (int i = 0; i < length; i++) + { + SkipValue(context, elementType, elementRefMode); + } + } + + private static void SkipMap(ReadContext context, TypeMetaFieldType fieldType) + { + if (fieldType.Generics.Count != 2) + { + throw new InvalidDataException("map field metadata must have key/value types"); + } + + TypeMetaFieldType keyType = fieldType.Generics[0]; + TypeMetaFieldType valueType = fieldType.Generics[1]; + int totalLength = checked((int)context.Reader.ReadVarUInt32()); + int readCount = 0; + while (readCount < totalLength) + { + byte header = context.Reader.ReadUInt8(); + bool trackKeyRef = (header & DictionaryBits.TrackingKeyRef) != 0; + bool keyNull = (header & DictionaryBits.KeyNull) != 0; + bool keyDeclared = (header & DictionaryBits.DeclaredKeyType) != 0; + bool trackValueRef = (header & DictionaryBits.TrackingValueRef) != 0; + bool valueNull = (header & DictionaryBits.ValueNull) != 0; + bool valueDeclared = (header & DictionaryBits.DeclaredValueType) != 0; + + if (keyNull || valueNull) + { + if (!keyNull) { - if (fieldType.Generics.Count != 1 || fieldType.Generics[0].TypeId != (uint)TypeId.String) + if (!keyDeclared) { - throw new InvalidDataException("unsupported compatible set element type"); + _ = context.TypeResolver.ReadAnyTypeInfo(context); } - return context.TypeResolver.GetSerializer>().Read(context, refMode, false); + SkipValue(context, keyType, trackKeyRef ? RefMode.Tracking : RefMode.None); } - case (uint)TypeId.Map: + + if (!valueNull) { - if (fieldType.Generics.Count != 2 || - fieldType.Generics[0].TypeId != (uint)TypeId.String || - fieldType.Generics[1].TypeId != (uint)TypeId.String) + if (!valueDeclared) { - throw new InvalidDataException("unsupported compatible map key/value type"); + _ = context.TypeResolver.ReadAnyTypeInfo(context); } - return context.TypeResolver.GetSerializer>().Read(context, refMode, false); + SkipValue(context, valueType, trackValueRef ? RefMode.Tracking : RefMode.None); } - case (uint)TypeId.Enum: - return ReadEnumOrdinal(context, refMode); - case (uint)TypeId.Union: - case (uint)TypeId.TypedUnion: - case (uint)TypeId.NamedUnion: - return context.TypeResolver.GetSerializer().Read(context, refMode, false); - default: - throw new InvalidDataException($"unsupported compatible field type id {fieldType.TypeId}"); + + readCount++; + continue; + } + + int chunkSize = context.Reader.ReadUInt8(); + if (!keyDeclared) + { + _ = context.TypeResolver.ReadAnyTypeInfo(context); + } + + if (!valueDeclared) + { + _ = context.TypeResolver.ReadAnyTypeInfo(context); + } + + for (int i = 0; i < chunkSize; i++) + { + SkipValue(context, keyType, trackKeyRef ? RefMode.Tracking : RefMode.None); + SkipValue(context, valueType, trackValueRef ? RefMode.Tracking : RefMode.None); + } + + readCount += chunkSize; } } } diff --git a/csharp/src/Fory/SchemaTypes.cs b/csharp/src/Fory/SchemaTypes.cs new file mode 100644 index 0000000000..35eedfa291 --- /dev/null +++ b/csharp/src/Fory/SchemaTypes.cs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Fory.Schema.Types; + +public sealed class Bool { private Bool() { } } +public sealed class Int8 { private Int8() { } } +public sealed class Int16 { private Int16() { } } +public sealed class Int32 { private Int32() { } } +public sealed class VarInt32 { private VarInt32() { } } +public sealed class Int64 { private Int64() { } } +public sealed class VarInt64 { private VarInt64() { } } +public sealed class TaggedInt64 { private TaggedInt64() { } } +public sealed class UInt8 { private UInt8() { } } +public sealed class UInt16 { private UInt16() { } } +public sealed class UInt32 { private UInt32() { } } +public sealed class VarUInt32 { private VarUInt32() { } } +public sealed class UInt64 { private UInt64() { } } +public sealed class VarUInt64 { private VarUInt64() { } } +public sealed class TaggedUInt64 { private TaggedUInt64() { } } +public sealed class Float16 { private Float16() { } } +public sealed class BFloat16 { private BFloat16() { } } +public sealed class Float32 { private Float32() { } } +public sealed class Float64 { private Float64() { } } +public sealed class String { private String() { } } +public sealed class Binary { private Binary() { } } +public sealed class Date { private Date() { } } +public sealed class Timestamp { private Timestamp() { } } +public sealed class Duration { private Duration() { } } +public sealed class Decimal { private Decimal() { } } + +public sealed class BoolArray { private BoolArray() { } } +public sealed class Int8Array { private Int8Array() { } } +public sealed class Int16Array { private Int16Array() { } } +public sealed class Int32Array { private Int32Array() { } } +public sealed class Int64Array { private Int64Array() { } } +public sealed class UInt8Array { private UInt8Array() { } } +public sealed class UInt16Array { private UInt16Array() { } } +public sealed class UInt32Array { private UInt32Array() { } } +public sealed class UInt64Array { private UInt64Array() { } } +public sealed class Float16Array { private Float16Array() { } } +public sealed class BFloat16Array { private BFloat16Array() { } } +public sealed class Float32Array { private Float32Array() { } } +public sealed class Float64Array { private Float64Array() { } } + +public sealed class List { private List() { } } +public sealed class Set { private Set() { } } +public sealed class Map { private Map() { } } diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 355fcb6285..2c6f89bb13 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -21,6 +21,7 @@ using System.Threading.Tasks; using Apache.Fory; using ForyRuntime = Apache.Fory.Fory; +using S = Apache.Fory.Schema.Types; namespace Apache.Fory.Tests; @@ -69,15 +70,44 @@ public sealed class FieldOrder } [ForyObject] -public sealed class EncodedNumbers +public sealed class SchemaNumbers { - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint U32Fixed { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong U64Tagged { get; set; } } +[ForyObject] +public sealed class NestedSchemaByName +{ + [ForyField(Type = typeof(S.Map>))] + public Dictionary?> Values { get; set; } = []; +} + +[ForyObject] +public sealed class NestedSchemaById +{ + [ForyField(3, Type = typeof(S.Map>))] + public Dictionary?> Values { get; set; } = []; +} + +[ForyObject] +public sealed class NestedSchemaSkipWriter +{ + [ForyField(Type = typeof(S.Map>))] + public Dictionary?> Values { get; set; } = []; + + public int Tail { get; set; } +} + +[ForyObject] +public sealed class NestedSchemaSkipReader +{ + public int Tail { get; set; } +} + [ForyObject] public sealed class OneStringField { @@ -796,34 +826,108 @@ public void MacroFieldOrderFollowsForyRules() } [Fact] - public void MacroFieldEncodingOverridesForUnsignedTypes() + public void ForyFieldSchemaTypeOverridesForUnsignedTypes() { ForyRuntime fory = ForyRuntime.Builder().Build(); - fory.Register(301); + fory.Register(301); - EncodedNumbers value = new() + SchemaNumbers value = new() { U32Fixed = 0x11223344u, U64Tagged = (ulong)int.MaxValue + 99UL, }; - EncodedNumbers decoded = fory.Deserialize(fory.Serialize(value)); + SchemaNumbers decoded = fory.Deserialize(fory.Serialize(value)); Assert.Equal(value.U32Fixed, decoded.U32Fixed); Assert.Equal(value.U64Tagged, decoded.U64Tagged); } + [Fact] + public void ForyFieldTypeWithoutIdUsesNameBasedNestedSchema() + { + TypeResolver resolver = new(); + resolver.GetTypeInfo(); + TypeMetaFieldInfo field = Assert.Single(resolver.GetTypeInfo().TypeMetaFields(false)); + + Assert.Null(field.FieldId); + Assert.Equal("values", field.FieldName); + Assert.Equal((uint)TypeId.Map, field.FieldType.TypeId); + Assert.Equal((uint)TypeId.UInt32, field.FieldType.Generics[0].TypeId); + Assert.Equal((uint)TypeId.List, field.FieldType.Generics[1].TypeId); + Assert.True(field.FieldType.Generics[1].Nullable); + Assert.Equal((uint)TypeId.TaggedUInt64, field.FieldType.Generics[1].Generics[0].TypeId); + Assert.True(field.FieldType.Generics[1].Generics[0].Nullable); + } + + [Fact] + public void ForyFieldTypeWithIdUsesAssignedIdNestedSchema() + { + TypeResolver resolver = new(); + resolver.GetTypeInfo(); + TypeMetaFieldInfo field = Assert.Single(resolver.GetTypeInfo().TypeMetaFields(false)); + + Assert.Equal((short)3, field.FieldId); + Assert.Equal((uint)TypeId.Map, field.FieldType.TypeId); + Assert.Equal((uint)TypeId.UInt32, field.FieldType.Generics[0].TypeId); + Assert.Equal((uint)TypeId.TaggedUInt64, field.FieldType.Generics[1].Generics[0].TypeId); + } + + [Fact] + public void NestedSchemaAnnotationControlsMapAndListPayload() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + fory.Register(303); + + NestedSchemaByName value = new() + { + Values = + { + [4_000_000_000u] = [7UL, 1_000_000_000UL, null], + [3u] = [42UL], + }, + }; + + NestedSchemaByName decoded = fory.Deserialize(fory.Serialize(value)); + Assert.Equal(value.Values.Count, decoded.Values.Count); + Assert.Equal(value.Values[4_000_000_000u], decoded.Values[4_000_000_000u]); + Assert.Equal(value.Values[3u], decoded.Values[3u]); + } + + [Fact] + public void CompatibleSkipUsesRemoteNestedSchemaMetadata() + { + ForyRuntime writer = ForyRuntime.Builder().Compatible(true).Build(); + writer.Register(304); + + ForyRuntime reader = ForyRuntime.Builder().Compatible(true).Build(); + reader.Register(304); + + NestedSchemaSkipWriter value = new() + { + Values = + { + [4_000_000_000u] = [7UL, 1_000_000_000UL], + [3u] = [42UL], + }, + Tail = 99, + }; + + NestedSchemaSkipReader decoded = reader.Deserialize(writer.Serialize(value)); + Assert.Equal(99, decoded.Tail); + } + [Fact] public void TaggedUnsignedFieldUsesCompactBoundary() { ForyRuntime fory = ForyRuntime.Builder().Build(); - fory.Register(301); + fory.Register(301); - EncodedNumbers compact = new() + SchemaNumbers compact = new() { U32Fixed = 0x11223344u, U64Tagged = (ulong)int.MaxValue, }; - EncodedNumbers wide = new() + SchemaNumbers wide = new() { U32Fixed = 0x11223344u, U64Tagged = (ulong)int.MaxValue + 1UL, @@ -833,8 +937,8 @@ public void TaggedUnsignedFieldUsesCompactBoundary() byte[] widePayload = fory.Serialize(wide); Assert.Equal(5, widePayload.Length - compactPayload.Length); - Assert.Equal(compact.U64Tagged, fory.Deserialize(compactPayload).U64Tagged); - Assert.Equal(wide.U64Tagged, fory.Deserialize(widePayload).U64Tagged); + Assert.Equal(compact.U64Tagged, fory.Deserialize(compactPayload).U64Tagged); + Assert.Equal(wide.U64Tagged, fory.Deserialize(widePayload).U64Tagged); } [Theory] diff --git a/csharp/tests/Fory.XlangPeer/Program.cs b/csharp/tests/Fory.XlangPeer/Program.cs index 80030eaf22..dfcf7fccbd 100644 --- a/csharp/tests/Fory.XlangPeer/Program.cs +++ b/csharp/tests/Fory.XlangPeer/Program.cs @@ -20,6 +20,7 @@ using System.Text; using Apache.Fory; using ForyRuntime = Apache.Fory.Fory; +using S = Apache.Fory.Schema.Types; namespace Apache.Fory.XlangPeer; @@ -236,6 +237,8 @@ private static byte[] ExecuteCase(string caseName, byte[] input) "test_unsigned_schema_consistent_simple" => CaseUnsignedSchemaConsistentSimple(input), "test_unsigned_schema_consistent" => CaseUnsignedSchemaConsistent(input), "test_unsigned_schema_compatible" => CaseUnsignedSchemaCompatible(input), + "test_nested_annotated_container_schema_consistent" => CaseNestedAnnotatedContainerSchemaConsistent(input), + "test_nested_annotated_container_compatible" => CaseNestedAnnotatedContainerCompatible(input), _ => throw new InvalidOperationException($"unknown test case {caseName}"), }; } @@ -995,6 +998,20 @@ private static byte[] CaseUnsignedSchemaCompatible(byte[] input) return RoundTripSingle(input, fory); } + private static byte[] CaseNestedAnnotatedContainerSchemaConsistent(byte[] input) + { + ForyRuntime fory = BuildFory(compatible: false); + fory.Register(801); + return RoundTripSingle(input, fory); + } + + private static byte[] CaseNestedAnnotatedContainerCompatible(byte[] input) + { + ForyRuntime fory = BuildFory(compatible: true); + fory.Register(802); + return RoundTripSingle(input, fory); + } + private static byte[] RoundTripSingle(byte[] input, ForyRuntime fory) { ReadOnlySequence sequence = new(input); @@ -1357,10 +1374,10 @@ public sealed class CircularRefStruct [ForyObject] public sealed class UnsignedSchemaConsistentSimple { - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong U64Tagged { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong? U64TaggedNullable { get; set; } } @@ -1371,30 +1388,30 @@ public sealed class UnsignedSchemaConsistent public ushort U16Field { get; set; } public uint U32VarField { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint U32FixedField { get; set; } public ulong U64VarField { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt64))] public ulong U64FixedField { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong U64TaggedField { get; set; } public byte? U8NullableField { get; set; } public ushort? U16NullableField { get; set; } public uint? U32VarNullableField { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint? U32FixedNullableField { get; set; } public ulong? U64VarNullableField { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt64))] public ulong? U64FixedNullableField { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong? U64TaggedNullableField { get; set; } } @@ -1405,29 +1422,45 @@ public sealed class UnsignedSchemaCompatible public ushort? U16Field1 { get; set; } public uint? U32VarField1 { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint? U32FixedField1 { get; set; } public ulong? U64VarField1 { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt64))] public ulong? U64FixedField1 { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong? U64TaggedField1 { get; set; } public byte U8Field2 { get; set; } public ushort U16Field2 { get; set; } public uint U32VarField2 { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint U32FixedField2 { get; set; } public ulong U64VarField2 { get; set; } - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt64))] public ulong U64FixedField2 { get; set; } - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong U64TaggedField2 { get; set; } } + +#pragma warning disable CS8714 +[ForyObject] +public sealed class NestedAnnotatedContainerSchemaConsistent +{ + [ForyField(Type = typeof(S.Map>))] + public NullableKeyDictionary?> Values { get; set; } = new(); +} + +[ForyObject] +public sealed class NestedAnnotatedContainerCompatible +{ + [ForyField(Type = typeof(S.Map>))] + public NullableKeyDictionary?> Values { get; set; } = new(); +} +#pragma warning restore CS8714 diff --git a/docs/guide/csharp/field-configuration.md b/docs/guide/csharp/field-configuration.md index 3fc502ee8e..0e53eacf27 100644 --- a/docs/guide/csharp/field-configuration.md +++ b/docs/guide/csharp/field-configuration.md @@ -21,47 +21,64 @@ license: | This page covers field-level serializer configuration for C# generated serializers. -## `[ForyObject]` and `[Field]` +## `[ForyObject]` and `[ForyField]` -Use `[ForyObject]` to enable source-generated serializers. Use `[Field]` to override integer encoding for a specific field. +Use `[ForyObject]` to enable source-generated serializers. Use `[ForyField]` to assign an optional stable field id or to override the Fory schema type used for a field. ```csharp using Apache.Fory; +using S = Apache.Fory.Schema.Types; [ForyObject] public sealed class Metrics { - // Fixed-width 32-bit encoding - [Field(Encoding = FieldEncoding.Fixed)] + [ForyField(Type = typeof(S.UInt32))] public uint Count { get; set; } - // Tagged 64-bit encoding - [Field(Encoding = FieldEncoding.Tagged)] + [ForyField(Type = typeof(S.TaggedUInt64))] public ulong TraceId { get; set; } - // Default (varint) encoding public long LatencyMicros { get; set; } } ``` -## Available Encodings +`Id` is optional. When it is omitted, compatible mode still matches the field by name. -| Encoding | Meaning | -| ---------------------- | ----------------------------------------------- | -| `FieldEncoding.Varint` | Variable-length integer encoding (default) | -| `FieldEncoding.Fixed` | Fixed-width integer encoding | -| `FieldEncoding.Tagged` | Tagged integer encoding (`long` / `ulong` only) | +```csharp +using Apache.Fory; +using S = Apache.Fory.Schema.Types; + +[ForyObject] +public sealed class NestedMetrics +{ + [ForyField(Type = typeof(S.Map>))] + public Dictionary?> Values { get; set; } = []; + + [ForyField(3, Type = typeof(S.UInt64))] + public ulong StableCount { get; set; } +} +``` + +## Schema Descriptor Types + +Schema descriptors live under `Apache.Fory.Schema.Types` and are metadata only. They do not replace normal C# carrier types. + +Common scalar descriptors include: + +- `S.Int32`, `S.VarInt32`, `S.UInt32`, `S.VarUInt32` +- `S.Int64`, `S.VarInt64`, `S.TaggedInt64` +- `S.UInt64`, `S.VarUInt64`, `S.TaggedUInt64` +- `S.Float16`, `S.BFloat16`, `S.Float32`, `S.Float64` -## Supported Field Types for Encoding Override +Container descriptors are composable: -`[Field(Encoding = ...)]` currently applies to: +- `S.List` +- `S.Set` +- `S.Map` -- `int` -- `uint` -- `long` -- `ulong` +Packed array descriptors such as `S.Int32Array`, `S.UInt32Array`, `S.Float16Array`, and `S.BFloat16Array` are available when the field should use the packed array wire type. -Nullable value variants (for example `long?`) are also handled by generated serializers. +Nullability comes from the C# carrier type. Use `List` for nullable list elements and `NullableKeyDictionary` when a map needs nullable keys. ## Nullability and Reference Tracking diff --git a/docs/guide/csharp/index.md b/docs/guide/csharp/index.md index 95bcac2ac2..f0a899c855 100644 --- a/docs/guide/csharp/index.md +++ b/docs/guide/csharp/index.md @@ -83,19 +83,19 @@ User decoded = fory.Deserialize(payload); ## Documentation -| Topic | Description | -| --------------------------------------------- | ------------------------------------------------ | -| [Configuration](configuration.md) | Builder options and runtime modes | -| [Basic Serialization](basic-serialization.md) | Typed and dynamic serialization APIs | -| [Type Registration](type-registration.md) | Registering user types and custom serializers | -| [Custom Serializers](custom-serializers.md) | Implementing `Serializer` | -| [Field Configuration](field-configuration.md) | `[Field]` attribute and integer encoding options | -| [References](references.md) | Shared/circular reference handling | -| [Schema Evolution](schema-evolution.md) | Compatible mode behavior | -| [Cross-Language](cross-language.md) | Interoperability guidance | -| [Supported Types](supported-types.md) | Built-in and generated type support | -| [Thread Safety](thread-safety.md) | `Fory` vs `ThreadSafeFory` usage | -| [Troubleshooting](troubleshooting.md) | Common errors and debugging steps | +| Topic | Description | +| --------------------------------------------- | --------------------------------------------- | +| [Configuration](configuration.md) | Builder options and runtime modes | +| [Basic Serialization](basic-serialization.md) | Typed and dynamic serialization APIs | +| [Type Registration](type-registration.md) | Registering user types and custom serializers | +| [Custom Serializers](custom-serializers.md) | Implementing `Serializer` | +| [Field Configuration](field-configuration.md) | `[ForyField]` ids and schema type descriptors | +| [References](references.md) | Shared/circular reference handling | +| [Schema Evolution](schema-evolution.md) | Compatible mode behavior | +| [Cross-Language](cross-language.md) | Interoperability guidance | +| [Supported Types](supported-types.md) | Built-in and generated type support | +| [Thread Safety](thread-safety.md) | `Fory` vs `ThreadSafeFory` usage | +| [Troubleshooting](troubleshooting.md) | Common errors and debugging steps | ## Related Resources diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index c1f320d8a9..ded3f3923b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -210,6 +210,11 @@ public void testStructWithMap(boolean enableCodegen) throws java.io.IOException super.testStructWithMap(enableCodegen); } + @Test(groups = "xlang", dataProvider = "enableCodegenParallel") + public void testCollectionElementRefOverride(boolean enableCodegen) throws java.io.IOException { + super.testCollectionElementRefOverride(enableCodegen); + } + @Test(groups = "xlang", dataProvider = "enableCodegenParallel") public void testNestedAnnotatedContainerSchemaConsistent(boolean enableCodegen) throws java.io.IOException { @@ -222,11 +227,6 @@ public void testNestedAnnotatedContainerCompatible(boolean enableCodegen) super.testNestedAnnotatedContainerCompatible(enableCodegen); } - @Test(groups = "xlang", dataProvider = "enableCodegenParallel") - public void testCollectionElementRefOverride(boolean enableCodegen) throws java.io.IOException { - super.testCollectionElementRefOverride(enableCodegen); - } - @Test(groups = "xlang", dataProvider = "enableCodegenParallel") public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { super.testSkipIdCustom(enableCodegen); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/CSharpXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CSharpXlangTest.java index 42f1a4f5ab..92c196ed5b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/CSharpXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CSharpXlangTest.java @@ -355,4 +355,16 @@ public void testUnsignedSchemaConsistentSimple(boolean enableCodegen) throws jav public void testUnsignedSchemaCompatible(boolean enableCodegen) throws java.io.IOException { super.testUnsignedSchemaCompatible(enableCodegen); } + + @Test(groups = "xlang", dataProvider = "enableCodegenParallel") + public void testNestedAnnotatedContainerSchemaConsistent(boolean enableCodegen) + throws java.io.IOException { + super.testNestedAnnotatedContainerSchemaConsistent(enableCodegen); + } + + @Test(groups = "xlang", dataProvider = "enableCodegenParallel") + public void testNestedAnnotatedContainerCompatible(boolean enableCodegen) + throws java.io.IOException { + super.testNestedAnnotatedContainerCompatible(enableCodegen); + } }