From 6355c2c409aa90692563539ece3dcc40898a85ff Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Tue, 7 Oct 2025 01:26:02 +0530 Subject: [PATCH 01/37] fix: remove commented content from git commit message --- scripts/remove-commented-commit.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scripts/remove-commented-commit.sh diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh new file mode 100644 index 0000000000..f1564efc38 --- /dev/null +++ b/scripts/remove-commented-commit.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# remove-commented-commit.sh +# Remove commented lines (# or ) from Git commit messages +# Usage: ./remove-commented-commit.sh COMMIT_MSG_FILE + +COMMIT_MSG_FILE="$1" + +if [ ! -f "$COMMIT_MSG_FILE" ]; then + echo "Commit message file not found!" + exit 1 +fi + +# Remove lines starting with '#' (Git comments) +sed -i '/^#/d' "$COMMIT_MSG_FILE" + +# Remove HTML-style comment blocks () +sed -i '/^$/d' "$COMMIT_MSG_FILE" From f93d34b62c478249b6dd3691973f227b89b5bdc7 Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Tue, 7 Oct 2025 21:32:24 +0530 Subject: [PATCH 02/37] chore(ci): invoke remove-commented-commit.sh in CI workflow --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ scripts/remove-commented-commit.sh | 0 2 files changed, 23 insertions(+) mode change 100644 => 100755 scripts/remove-commented-commit.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bcf1a32ec..c21282d739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,3 +344,26 @@ jobs: node-version: 20.x - name: Check code style run: python ./ci/run_ci.py format + + + cleanup_commit_message: + name: Cleanup commit message + runs-on: ubuntu-latest + # only run on pull_request events + if: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + # ensure the full commit history is available + fetch-depth: 0 + + - name: Get last commit message + run: git --no-pager log -1 --pretty=%B > commit_msg.txt + + - name: Remove commented content from commit message + run: bash scripts/remove-commented-commit.sh commit_msg.txt + + - name: Show cleaned commit message + run: cat commit_msg.txt + diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh old mode 100644 new mode 100755 From 33868bf19b615b3da20e5f1377f3867f0d9d096c Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Tue, 7 Oct 2025 21:51:07 +0530 Subject: [PATCH 03/37] fix: corrected indentation in cleanup_commit_message job --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c21282d739..a4d9737e7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -346,16 +346,14 @@ jobs: run: python ./ci/run_ci.py format - cleanup_commit_message: + cleanup_commit_message: name: Cleanup commit message runs-on: ubuntu-latest - # only run on pull_request events if: ${{ github.event_name == 'pull_request' }} steps: - name: Checkout repository uses: actions/checkout@v5 with: - # ensure the full commit history is available fetch-depth: 0 - name: Get last commit message @@ -366,4 +364,3 @@ jobs: - name: Show cleaned commit message run: cat commit_msg.txt - From c5f4f4fb872ee4badbe3d009276dc9605bdcf750 Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:24:47 +0530 Subject: [PATCH 04/37] chore(ci): invoke remove-commented-commit.sh in CI workflow --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4d9737e7b..1cdf42238f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,7 +345,6 @@ jobs: - name: Check code style run: python ./ci/run_ci.py format - cleanup_commit_message: name: Cleanup commit message runs-on: ubuntu-latest From 97bdfa8976e25e5ba6ee6f85576473a56bda7dc6 Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Thu, 9 Oct 2025 00:49:12 +0530 Subject: [PATCH 05/37] fix: add Apache license header to remove-commented-commit.sh --- scripts/remove-commented-commit.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh index f1564efc38..e1df86d348 100755 --- a/scripts/remove-commented-commit.sh +++ b/scripts/remove-commented-commit.sh @@ -1,4 +1,25 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Script to remove commented lines (# or HTML comments) from commit messages. + +echo "Running remove-commented-commit.sh" +#!/bin/bash # remove-commented-commit.sh # Remove commented lines (# or ) from Git commit messages # Usage: ./remove-commented-commit.sh COMMIT_MSG_FILE From 2cc8a5077682dd0a5f2af9677857dbef68a68f78 Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Thu, 9 Oct 2025 01:00:04 +0530 Subject: [PATCH 06/37] fix: add Apache license header to remove-commented-commit.sh --- scripts/remove-commented-commit.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh index e1df86d348..d5fd0e121b 100755 --- a/scripts/remove-commented-commit.sh +++ b/scripts/remove-commented-commit.sh @@ -19,7 +19,6 @@ # Script to remove commented lines (# or HTML comments) from commit messages. echo "Running remove-commented-commit.sh" -#!/bin/bash # remove-commented-commit.sh # Remove commented lines (# or ) from Git commit messages # Usage: ./remove-commented-commit.sh COMMIT_MSG_FILE From 60f43a9a8368f1c9aa698345da59b2149597938c Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Thu, 9 Oct 2025 01:20:32 +0530 Subject: [PATCH 07/37] fix: add Apache license header to remove-commented-commit.sh --- scripts/remove-commented-commit.sh | 32 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh index d5fd0e121b..960d3988a4 100755 --- a/scripts/remove-commented-commit.sh +++ b/scripts/remove-commented-commit.sh @@ -15,23 +15,29 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -# Script to remove commented lines (# or HTML comments) from commit messages. -echo "Running remove-commented-commit.sh" -# remove-commented-commit.sh -# Remove commented lines (# or ) from Git commit messages -# Usage: ./remove-commented-commit.sh COMMIT_MSG_FILE +# Script to remove commented content from git commit messages +# This ensures clean and readable commit history by filtering out +# lines starting with # or wrapped in -COMMIT_MSG_FILE="$1" +# Read the commit message file +COMMIT_MSG_FILE=$1 -if [ ! -f "$COMMIT_MSG_FILE" ]; then - echo "Commit message file not found!" +if [ -z "$COMMIT_MSG_FILE" ]; then + echo "Error: No commit message file provided" exit 1 fi -# Remove lines starting with '#' (Git comments) -sed -i '/^#/d' "$COMMIT_MSG_FILE" +# Create a temporary file +TEMP_FILE=$(mktemp) + +# Remove commented lines and HTML comments +# - Lines starting with # (after optional whitespace) +# - HTML comment blocks +grep -v '^\s*#' "$COMMIT_MSG_FILE" | \ + sed '//d' > "$TEMP_FILE" + +# Replace original file with cleaned content +mv "$TEMP_FILE" "$COMMIT_MSG_FILE" -# Remove HTML-style comment blocks () -sed -i '/^$/d' "$COMMIT_MSG_FILE" +exit 0 From 031059ffddc608ae499fd880b31cd4bde55744c8 Mon Sep 17 00:00:00 2001 From: urlyy Date: Sat, 11 Oct 2025 11:08:06 +0800 Subject: [PATCH 08/37] feat(Rust): support context_pool to reduce context allocation && support se/de in multi-thread (#2737) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What does this PR do? 1. Remove the lifetime annotations from `WriteContext` and `ReadContext` because managing their lifetimes was too troublesome. The **trade-off** is that, previously, `MetaWriterResolver` held references to the corresponding `TypeDef` obtained from `TypeResolver`, and both contexts also fetched references to `Harness` objects managed by `TypeResolver`. Now, all these references have been replaced with `clone()` to simplify lifetime management. This may introduce a non-negligible performance overhead, but it was the only practical solution I could implement.🥲 2. Remove `&fory` from the member variables of `Context`. Instead, `&fory` is now passed in as a function parameter rather than being retrieved via `context.get_fory()`. This change modified many API signatures. 3. Add two pools to `Fory`’s member variables to allow reuse of both types of contexts. Tests confirmed that the same context addresses were reused multiple times. However, since automatic recycling would require `Arc`, only manual recycling has been implemented so far — this operation is handled internally(within `serialize()/deserialize()`), and users don’t need to recycle contexts manually. Also, for the `de/serialize_with_context(context)` functions, if users call them directly, the user's manually managed contexts will **not** be returned to the pool. 4. Add `reset()` methods to be executed on `Context objects` before recycling. 5. Modified `TypeResolver::sorted_field_names_map` from `RefCell<...>` to `RwLock<...>`, and changed `MetaReaderResolver::reading_type_defs` from `Vec>` to `Vec>`. Split `Fory::MetaStringResolver` to `ReadContext::MetaStringResolver` and `WriteContext::MetaStringResolver`. In addition, `RefReader` was unsafely marked as `Send` and `Sync`. These changes allow `Fory` to support serialization across multiple threads. A concrete example can be found in `rust/tests/tests/test_multi_thread.rs`. However, I’m not sure whether using `Lock` and `Arc` will impact single-thread performance. But it may be troublesome to write another `ThreadSafeFory`. 6. The reason why I `unsafe impl Send/Sync for RefReader {}`, is that , `RefReader` contains `Box`, which by default is not `Send`/`Sync`. But In our usage, each ref_reader is only accessed within a context, by `one` thread at a time, so it is safe to implement `Send`/`Sync` manually using `unsafe`. ## Related issues - close #2717 ## Does this PR introduce any user-facing change? yes, like implementing in EXT, need to pass a extra parameter `fory`. --- rust/benches/README.md | 22 ++ rust/fory-core/benches/simd_bench.rs | 1 - rust/fory-core/src/buffer.rs | 101 ++++++--- rust/fory-core/src/fory.rs | 61 +++--- rust/fory-core/src/meta/meta_string.rs | 26 ++- rust/fory-core/src/meta/string_util.rs | 14 +- rust/fory-core/src/meta/type_meta.rs | 36 ++-- rust/fory-core/src/resolver/context.rs | 134 ++++++++---- rust/fory-core/src/resolver/meta_resolver.rs | 21 +- .../src/resolver/metastring_resolver.rs | 194 ++++++++---------- rust/fory-core/src/resolver/ref_resolver.rs | 8 +- rust/fory-core/src/resolver/type_resolver.rs | 108 +++++----- rust/fory-core/src/serializer/any.rs | 115 ++++++----- rust/fory-core/src/serializer/arc.rs | 35 ++-- rust/fory-core/src/serializer/bool.rs | 16 +- rust/fory-core/src/serializer/box_.rs | 20 +- rust/fory-core/src/serializer/collection.rs | 48 +++-- rust/fory-core/src/serializer/datetime.rs | 32 +-- rust/fory-core/src/serializer/enum_.rs | 53 +++-- rust/fory-core/src/serializer/heap.rs | 16 +- rust/fory-core/src/serializer/list.rs | 48 +++-- rust/fory-core/src/serializer/map.rs | 120 ++++++----- rust/fory-core/src/serializer/mod.rs | 94 ++++----- rust/fory-core/src/serializer/mutex.rs | 28 +-- rust/fory-core/src/serializer/number.rs | 16 +- rust/fory-core/src/serializer/option.rs | 20 +- rust/fory-core/src/serializer/rc.rs | 35 ++-- rust/fory-core/src/serializer/refcell.rs | 28 +-- rust/fory-core/src/serializer/set.rs | 32 +-- rust/fory-core/src/serializer/skip.rs | 38 ++-- rust/fory-core/src/serializer/string.rs | 18 +- rust/fory-core/src/serializer/struct_.rs | 66 +++--- rust/fory-core/src/serializer/trait_object.rs | 101 ++++----- rust/fory-core/src/serializer/weak.rs | 66 +++--- rust/fory-derive/src/object/derive_enum.rs | 10 +- rust/fory-derive/src/object/misc.rs | 11 +- rust/fory-derive/src/object/read.rs | 82 ++++---- rust/fory-derive/src/object/serializer.rs | 16 +- rust/fory-derive/src/object/util.rs | 52 ++--- rust/fory-derive/src/object/write.rs | 32 +-- rust/fory/src/lib.rs | 4 +- .../tests/tests/compatible/test_basic_type.rs | 24 +-- rust/tests/tests/compatible/test_container.rs | 54 ++--- .../tests/compatible/test_struct_enum.rs | 42 ++-- rust/tests/tests/test_cross_language.rs | 55 ++--- rust/tests/tests/test_ext.rs | 12 +- rust/tests/tests/test_multi_thread.rs | 102 +++++++++ rust/tests/tests/test_ref_resolver.rs | 4 +- 48 files changed, 1341 insertions(+), 930 deletions(-) create mode 100644 rust/tests/tests/test_multi_thread.rs diff --git a/rust/benches/README.md b/rust/benches/README.md index 53108cc3a0..9db37f871e 100644 --- a/rust/benches/README.md +++ b/rust/benches/README.md @@ -4,8 +4,30 @@ cargo flamegraph --bin fory_profiler -- --operation deserialize --serializer fory ``` +detailed command: + +```bash +cd benches +rm -rf cargo-flamegraph.trace +export CARGO_PROFILE_RELEASE_DEBUG=true && +cargo flamegraph \ + --inverted \ + --deterministic \ + --palette rust \ + --min-width 0.05 \ + --bin fory_profiler -- \ + --operation deserialize \ + --serializer fory +``` + ## How to run benchmarks ```bash cargo bench ``` + +To run only a specific benchmark group, you can use a command like + +```bash +cargo bench --bench serialization_bench -- simple_struct +``` diff --git a/rust/fory-core/benches/simd_bench.rs b/rust/fory-core/benches/simd_bench.rs index 3f8b7daac0..7fc0e422a8 100644 --- a/rust/fory-core/benches/simd_bench.rs +++ b/rust/fory-core/benches/simd_bench.rs @@ -219,7 +219,6 @@ fn benchmark_read_utf16(c: &mut Criterion) { for &size in &sizes { let s = test_string.repeat(size / test_string.len() + 1); - // 构造 UTF-16 LE 字节数据 let mut data: Vec = Vec::with_capacity(s.len() * 2); for u in s.encode_utf16() { let lo = (u & 0x00FF) as u8; diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index b7fefe179c..ec5ea22941 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -20,6 +20,7 @@ use crate::meta::buffer_rw_string::{ write_utf8_simd, }; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; +use std::slice; #[derive(Default)] pub struct Writer { @@ -28,6 +29,11 @@ pub struct Writer { } impl Writer { + pub fn reset(&mut self) { + // keep capacity and reset len to 0 + self.bf.clear(); + } + pub fn dump(&self) -> Vec { self.bf.clone() } @@ -304,22 +310,49 @@ impl Writer { } } -pub struct Reader<'de> { - pub(crate) bf: &'de [u8], +pub struct Reader { + pub(crate) bf: *const u8, + len: usize, pub(crate) cursor: usize, } -impl<'bf> Reader<'bf> { - pub fn new(bf: &'bf [u8]) -> Reader<'bf> { - Reader { bf, cursor: 0 } +impl Reader { + pub fn new(bf: &[u8]) -> Reader { + Reader { + bf: bf.as_ptr(), + len: bf.len(), + cursor: 0, + } + } + + pub fn init(&mut self, bf: &[u8]) { + self.bf = bf.as_ptr(); + self.len = bf.len(); + self.cursor = 0; + } + + pub fn reset(&mut self) { + self.bf = std::ptr::null(); + self.len = 0; + self.cursor = 0; } pub(crate) fn move_next(&mut self, additional: usize) { self.cursor += additional; } + #[inline] + unsafe fn ptr_at(&self, offset: usize) -> *const u8 { + self.bf.add(offset) + } + pub fn slice_after_cursor(&self) -> &[u8] { - &self.bf[self.cursor..self.bf.len()] + let remaining = self.len - self.cursor; + if self.bf.is_null() || remaining == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(self.bf.add(self.cursor), remaining) } + } } pub fn get_cursor(&self) -> usize { @@ -327,15 +360,13 @@ impl<'bf> Reader<'bf> { } pub fn read_u8(&mut self) -> u8 { - let result = self.bf[self.cursor]; + let result = unsafe { *self.ptr_at(self.cursor) }; self.move_next(1); result } pub fn read_i8(&mut self) -> i8 { - let result = self.bf[self.cursor]; - self.move_next(1); - result as i8 + self.read_u8() as i8 } pub fn read_u16(&mut self) -> u16 { @@ -388,35 +419,35 @@ impl<'bf> Reader<'bf> { pub fn read_varuint32(&mut self) -> u32 { let start = self.cursor; - let b0 = self.bf[start] as u32; + let b0 = unsafe { *self.bf.add(start) as u32 }; if b0 < 0x80 { self.cursor += 1; return b0; } let mut encoded = b0 & 0x7F; - let b1 = self.bf[start + 1] as u32; + let b1 = unsafe { *self.bf.add(start + 1) as u32 }; encoded |= (b1 & 0x7F) << 7; if b1 < 0x80 { self.cursor += 2; return encoded; } - let b2 = self.bf[start + 2] as u32; + let b2 = unsafe { *self.bf.add(start + 2) as u32 }; encoded |= (b2 & 0x7F) << 14; if b2 < 0x80 { self.cursor += 3; return encoded; } - let b3 = self.bf[start + 3] as u32; + let b3 = unsafe { *self.bf.add(start + 3) as u32 }; encoded |= (b3 & 0x7F) << 21; if b3 < 0x80 { self.cursor += 4; return encoded; } - let b4 = self.bf[start + 4] as u32; + let b4 = unsafe { *self.bf.add(start + 4) as u32 }; encoded |= b4 << 28; self.cursor += 5; encoded @@ -429,63 +460,63 @@ impl<'bf> Reader<'bf> { pub fn read_varuint64(&mut self) -> u64 { let start = self.cursor; - let b0 = self.bf[start] as u64; + let b0 = unsafe { *self.bf.add(start) } as u64; if b0 < 0x80 { self.cursor += 1; return b0; } let mut var64 = b0 & 0x7F; - let b1 = self.bf[start + 1] as u64; + let b1 = unsafe { *self.bf.add(start + 1) } as u64; var64 |= (b1 & 0x7F) << 7; if b1 < 0x80 { self.cursor += 2; return var64; } - let b2 = self.bf[start + 2] as u64; + let b2 = unsafe { *self.bf.add(start + 2) } as u64; var64 |= (b2 & 0x7F) << 14; if b2 < 0x80 { self.cursor += 3; return var64; } - let b3 = self.bf[start + 3] as u64; + let b3 = unsafe { *self.bf.add(start + 3) } as u64; var64 |= (b3 & 0x7F) << 21; if b3 < 0x80 { self.cursor += 4; return var64; } - let b4 = self.bf[start + 4] as u64; + let b4 = unsafe { *self.bf.add(start + 4) } as u64; var64 |= (b4 & 0x7F) << 28; if b4 < 0x80 { self.cursor += 5; return var64; } - let b5 = self.bf[start + 5] as u64; + let b5 = unsafe { *self.bf.add(start + 5) } as u64; var64 |= (b5 & 0x7F) << 35; if b5 < 0x80 { self.cursor += 6; return var64; } - let b6 = self.bf[start + 6] as u64; + let b6 = unsafe { *self.bf.add(start + 6) } as u64; var64 |= (b6 & 0x7F) << 42; if b6 < 0x80 { self.cursor += 7; return var64; } - let b7 = self.bf[start + 7] as u64; + let b7 = unsafe { *self.bf.add(start + 7) } as u64; var64 |= (b7 & 0x7F) << 49; if b7 < 0x80 { self.cursor += 8; return var64; } - let b8 = self.bf[start + 8] as u64; + let b8 = unsafe { *self.bf.add(start + 8) } as u64; var64 |= (b8 & 0xFF) << 56; self.cursor += 9; var64 @@ -538,7 +569,7 @@ impl<'bf> Reader<'bf> { // slow path let mut result = 0u64; let mut shift = 0; - while self.cursor < self.bf.len() { + while self.cursor < self.len { let b = self.read_u8(); result |= ((b & 0x7F) as u64) << shift; if (b & 0x80) == 0 { @@ -554,13 +585,17 @@ impl<'bf> Reader<'bf> { } pub fn get_slice(&self) -> &[u8] { - self.bf + if self.bf.is_null() || self.len == 0 { + &[] + } else { + unsafe { slice::from_raw_parts(self.bf, self.len) } + } } - pub fn read_bytes(&mut self, len: usize) -> &'bf [u8] { - let result = &self.bf[self.cursor..self.cursor + len]; + pub fn read_bytes(&mut self, len: usize) -> &[u8] { + let s = unsafe { slice::from_raw_parts(self.bf.add(self.cursor), len) }; self.move_next(len); - result + s } pub fn reset_cursor_to_here(&self) -> impl FnOnce(&mut Self) { @@ -571,6 +606,12 @@ impl<'bf> Reader<'bf> { } pub fn aligned(&self) -> bool { - unsafe { (self.bf.as_ptr().add(self.cursor) as usize) % std::mem::align_of::() == 0 } + if self.bf.is_null() { + return false; + } + unsafe { (self.bf.add(self.cursor) as usize) % std::mem::align_of::() == 0 } } } + +unsafe impl Send for Reader {} +unsafe impl Sync for Reader {} diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index d4d2aa8916..a4c2b940b9 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -18,9 +18,8 @@ use crate::buffer::{Reader, Writer}; use crate::ensure; use crate::error::Error; -use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; -use crate::resolver::metastring_resolver::MetaStringResolver; +use crate::resolver::context::{Pool, ReadContext}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::ForyDefault; use crate::serializer::{Serializer, StructSerializer}; @@ -31,8 +30,6 @@ use crate::types::{ }; use crate::util::get_ext_actual_type_id; use anyhow::anyhow; -use std::cell::RefCell; -use std::rc::Rc; static EMPTY_STRING: String = String::new(); @@ -84,21 +81,32 @@ pub struct Fory { xlang: bool, share_meta: bool, type_resolver: TypeResolver, - metastring_resolver: Rc>, compress_string: bool, max_dyn_depth: u32, + write_context_pool: Pool, + read_context_pool: Pool, } impl Default for Fory { fn default() -> Self { + let write_context_constructor = || { + let writer = Writer::default(); + WriteContext::new(writer) + }; + let read_context_constructor = || { + let reader = Reader::new(&[]); + // when context is popped out, max_dyn_depth will be assigned a valid value + ReadContext::new(reader, 0) + }; Fory { mode: Mode::SchemaConsistent, xlang: true, share_meta: false, type_resolver: TypeResolver::default(), - metastring_resolver: Rc::from(RefCell::from(MetaStringResolver::default())), compress_string: false, max_dyn_depth: 5, + write_context_pool: Pool::new(write_context_constructor), + read_context_pool: Pool::new(read_context_constructor), } } } @@ -277,15 +285,6 @@ impl Fory { &self.type_resolver } - /// Returns a cloned reference to the meta string resolver. - /// - /// # Returns - /// - /// An `Rc>` for meta string compression and decompression. - pub fn get_metastring_resolver(&self) -> Rc> { - Rc::clone(&self.metastring_resolver) - } - pub fn write_head(&self, is_none: bool, writer: &mut Writer) { const HEAD_SIZE: usize = 10; writer.reserve(T::fory_reserved_space() + SIZE_OF_REF_AND_TYPE + HEAD_SIZE); @@ -382,12 +381,14 @@ impl Fory { /// let deserialized: Point = fory.deserialize(&bytes).unwrap(); /// ``` pub fn deserialize(&self, bf: &[u8]) -> Result { - let reader = Reader::new(bf); - let mut context = ReadContext::new(self, reader, self.max_dyn_depth); + let mut context = self.read_context_pool.get(); + context.init(bf, self.max_dyn_depth); let result = self.deserialize_with_context(&mut context); if result.is_ok() { assert_eq!(context.reader.slice_after_cursor().len(), 0); } + context.reset(); + self.read_context_pool.put(context); result } @@ -406,7 +407,7 @@ impl Fory { bytes_to_skip = context.load_meta(meta_offset as usize); } } - let result = ::fory_read(context, false); + let result = ::fory_read(self, context, false); if bytes_to_skip > 0 { context.reader.skip(bytes_to_skip as u32); } @@ -442,9 +443,11 @@ impl Fory { /// let bytes = fory.serialize(&point); /// ``` pub fn serialize(&self, record: &T) -> Vec { - let mut writer = Writer::default(); - let mut context: WriteContext<'_> = WriteContext::new(self, &mut writer); - self.serialize_with_context(record, &mut context) + let mut context = self.write_context_pool.get(); + let result = self.serialize_with_context(record, &mut context); + context.reset(); + self.write_context_pool.put(context); + result } pub fn serialize_with_context( @@ -453,13 +456,13 @@ impl Fory { context: &mut WriteContext, ) -> Vec { let is_none = record.fory_is_none(); - self.write_head::(is_none, context.writer); + self.write_head::(is_none, &mut context.writer); let meta_start_offset = context.writer.len(); if !is_none { if self.mode == Mode::Compatible { context.writer.write_i32(-1); }; - ::fory_write(record, context, false); + ::fory_write(record, self, context, false); if self.mode == Mode::Compatible && !context.empty() { context.write_meta(meta_start_offset); } @@ -658,13 +661,19 @@ impl Fory { } } -pub fn write_data(this: &T, context: &mut WriteContext, is_field: bool) { - T::fory_write_data(this, context, is_field); +pub fn write_data( + this: &T, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) { + T::fory_write_data(this, fory, context, is_field); } pub fn read_data( + fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result { - T::fory_read_data(context, is_field) + T::fory_read_data(fory, context, is_field) } diff --git a/rust/fory-core/src/meta/meta_string.rs b/rust/fory-core/src/meta/meta_string.rs index 513232fe46..9fb3d1df2f 100644 --- a/rust/fory-core/src/meta/meta_string.rs +++ b/rust/fory-core/src/meta/meta_string.rs @@ -365,9 +365,29 @@ impl MetaStringEncoder { } pub fn encode_first_to_lower_special(&self, input: &str) -> Result, Error> { - let mut chars: Vec = input.chars().collect(); - chars[0] = chars[0].to_lowercase().next().unwrap(); - self.encode_generic(&chars.iter().collect::(), 5) + if input.is_empty() { + return self.encode_generic("", 5); + } + + let mut iter = input.char_indices(); + let (first_idx, first_char) = iter.next().unwrap(); + + let lower = first_char.to_lowercase().to_string(); + + // Fast path: if lowercase has the same byte length and is ASCII, + // we can modify the first byte directly without rebuilding the string. + if lower.len() == first_char.len_utf8() && first_char.is_ascii() { + let mut bytes = input.as_bytes().to_owned(); + bytes[first_idx] = lower.as_bytes()[0]; + return self.encode_generic(std::str::from_utf8(&bytes).unwrap(), 5); + } + + // rebuild only the necessary prefix + suffix (still efficient). + let (_, rest) = input.split_at(first_char.len_utf8()); + let mut result = String::with_capacity(input.len() + lower.len() - first_char.len_utf8()); + result.push_str(&lower); + result.push_str(rest); + self.encode_generic(&result, 5) } pub fn encode_all_to_lower_special( diff --git a/rust/fory-core/src/meta/string_util.rs b/rust/fory-core/src/meta/string_util.rs index ba77c63e0d..b7ffcdd777 100644 --- a/rust/fory-core/src/meta/string_util.rs +++ b/rust/fory-core/src/meta/string_util.rs @@ -568,7 +568,7 @@ pub mod buffer_rw_string { #[inline] pub fn read_latin1_standard(reader: &mut Reader, len: usize) -> String { - let slice = &reader.bf[reader.cursor..reader.cursor + len]; + let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let result: String = slice.iter().map(|&b| b as char).collect(); reader.move_next(len); result @@ -576,8 +576,8 @@ pub mod buffer_rw_string { #[inline] pub fn read_utf8_standard(reader: &mut Reader, len: usize) -> String { - let result = - String::from_utf8_lossy(&reader.bf[reader.cursor..reader.cursor + len]).to_string(); + let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; + let result = String::from_utf8_lossy(slice).to_string(); reader.move_next(len); result } @@ -585,7 +585,7 @@ pub mod buffer_rw_string { #[inline] pub fn read_utf16_standard(reader: &mut Reader, len: usize) -> String { assert!(len % 2 == 0, "UTF-16 length must be even"); - let slice = &reader.bf[reader.cursor..reader.cursor + len]; + let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let units: Vec = slice .chunks(2) // little endian @@ -773,7 +773,7 @@ pub mod buffer_rw_string { if len == 0 { return String::new(); } - let src = &reader.bf[reader.cursor..reader.cursor + len]; + let src = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let mut out: Vec = Vec::with_capacity(len + len / 4); @@ -873,7 +873,7 @@ pub mod buffer_rw_string { return String::new(); } - let src = &reader.bf[reader.cursor..reader.cursor + len]; + let src = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let mut result = String::with_capacity(len); unsafe { @@ -999,7 +999,7 @@ pub mod buffer_rw_string { String::from_utf16(&units).unwrap_or_else(|_| String::new()) } - let slice = &reader.bf[reader.cursor..reader.cursor + len]; + let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; #[cfg(target_arch = "x86_64")] { if std::arch::is_x86_feature_detected!("avx2") { diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 2d04083336..2d56a21926 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -70,20 +70,16 @@ pub struct NullableFieldType { } impl NullableFieldType { - pub fn from(node: FieldType) -> Self { + pub fn from(node: &FieldType) -> Self { if node.type_id == TypeId::ForyNullable as u32 { - let inner = NullableFieldType::from(node.generics.into_iter().next().unwrap()); + let inner = NullableFieldType::from(&node.generics[0]); NullableFieldType { type_id: inner.type_id, generics: inner.generics, nullable: true, } } else { - let generics = node - .generics - .into_iter() - .map(NullableFieldType::from) - .collect(); + let generics = node.generics.iter().map(NullableFieldType::from).collect(); NullableFieldType { type_id: node.type_id, generics, @@ -385,7 +381,8 @@ impl TypeMetaLayer { Ok(writer.dump()) } - fn sort_field_infos(field_infos: &[FieldInfo]) -> Vec { + fn sort_field_infos(field_infos: Vec) -> Vec { + let fields_len = field_infos.len(); // group let mut primitive_fields = Vec::new(); let mut nullable_primitive_fields = Vec::new(); @@ -394,27 +391,29 @@ impl TypeMetaLayer { let mut unknown_fields = Vec::new(); let mut collection_fields = Vec::new(); let mut map_fields = Vec::new(); - for field_info in field_infos.iter() { + + for field_info in field_infos.into_iter() { let mut type_id = field_info.field_type.type_id; if type_id == TypeId::ForyNullable as u32 { type_id = field_info.field_type.generics.first().unwrap().type_id; if PRIMITIVE_TYPES.contains(&type_id) { - nullable_primitive_fields.push(field_info.clone()); + nullable_primitive_fields.push(field_info); continue; } } + let internal_id = type_id & 0xff; if PRIMITIVE_TYPES.contains(&type_id) { - primitive_fields.push(field_info.clone()); + primitive_fields.push(field_info); } else if PRIMITIVE_ARRAY_TYPES.contains(&type_id) || FINAL_TYPES.contains(&type_id) || [TypeId::ENUM as u32, TypeId::NAMED_ENUM as u32].contains(&internal_id) { - final_fields.push(field_info.clone()); + final_fields.push(field_info); } else if [TypeId::LIST as u32, TypeId::SET as u32].contains(&type_id) { - collection_fields.push(field_info.clone()); + collection_fields.push(field_info); } else if TypeId::MAP as u32 == type_id { - map_fields.push(field_info.clone()); + map_fields.push(field_info); } else if [ TypeId::COMPATIBLE_STRUCT as u32, TypeId::NAMED_COMPATIBLE_STRUCT as u32, @@ -423,13 +422,14 @@ impl TypeMetaLayer { ] .contains(&internal_id) { - other_fields.push(field_info.clone()); + other_fields.push(field_info); } else if internal_id == TypeId::UNKNOWN as u32 { - unknown_fields.push(field_info.clone()); + unknown_fields.push(field_info); } else { unreachable!("type_id: {type_id}"); } } + fn sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { let a_id = if a.field_type.type_id == TypeId::ForyNullable as u32 { a.field_type.generics.first().unwrap().type_id @@ -500,7 +500,7 @@ impl TypeMetaLayer { unknown_fields.sort_by(sorter); collection_fields.sort_by(sorter); map_fields.sort_by(sorter); - let mut sorted_field_infos = Vec::with_capacity(field_infos.len()); + let mut sorted_field_infos = Vec::with_capacity(fields_len); sorted_field_infos.extend(primitive_fields); sorted_field_infos.extend(nullable_primitive_fields); sorted_field_infos.extend(final_fields); @@ -535,7 +535,7 @@ impl TypeMetaLayer { for _ in 0..num_fields { field_infos.push(FieldInfo::from_bytes(reader)); } - let sorted_field_infos = Self::sort_field_infos(&field_infos); + let sorted_field_infos = Self::sort_field_infos(field_infos); TypeMetaLayer::new( type_id, namespace, diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index b8415d2daa..b8e0311797 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -18,26 +18,28 @@ use crate::buffer::{Reader, Writer}; use crate::fory::Fory; -use crate::meta::TypeMeta; +use crate::meta::{MetaString, TypeMeta}; use crate::resolver::meta_resolver::{MetaReaderResolver, MetaWriterResolver}; +use crate::resolver::metastring_resolver::{ + MetaStringBytes, MetaStringReaderResolver, MetaStringWriterResolver, +}; use crate::resolver::ref_resolver::{RefReader, RefWriter}; use crate::resolver::type_resolver::Harness; -use std::any::TypeId; -use std::rc::Rc; +use std::sync::{Arc, Mutex}; -pub struct WriteContext<'se> { - pub writer: &'se mut Writer, - fory: &'se Fory, - meta_resolver: MetaWriterResolver<'se>, +pub struct WriteContext { + pub writer: Writer, + meta_resolver: MetaWriterResolver, + meta_string_resolver: MetaStringWriterResolver, pub ref_writer: RefWriter, } -impl<'se> WriteContext<'se> { - pub fn new(fory: &'se Fory, writer: &'se mut Writer) -> WriteContext<'se> { +impl WriteContext { + pub fn new(writer: Writer) -> WriteContext { WriteContext { writer, - fory, meta_resolver: MetaWriterResolver::default(), + meta_string_resolver: MetaStringWriterResolver::default(), ref_writer: RefWriter::new(), } } @@ -46,8 +48,8 @@ impl<'se> WriteContext<'se> { self.meta_resolver.empty() } - pub fn push_meta(&mut self, type_id: std::any::TypeId) -> usize { - self.meta_resolver.push(type_id, self.fory) + pub fn push_meta(&mut self, fory: &Fory, type_id: std::any::TypeId) -> usize { + self.meta_resolver.push(type_id, fory) } pub fn write_meta(&mut self, offset: usize) { @@ -55,38 +57,38 @@ impl<'se> WriteContext<'se> { offset, &((self.writer.len() - offset - 4) as u32).to_le_bytes(), ); - self.meta_resolver.to_bytes(self.writer).unwrap() + self.meta_resolver.to_bytes(&mut self.writer).unwrap() } - pub fn get_fory(&self) -> &Fory { - self.fory - } - - pub fn write_any_typeinfo(&mut self, concrete_type_id: TypeId) -> &'se Harness { + pub fn write_any_typeinfo( + &mut self, + fory: &Fory, + concrete_type_id: std::any::TypeId, + ) -> Arc { use crate::types::TypeId as ForyTypeId; - let type_resolver = self.fory.get_type_resolver(); + let type_resolver = fory.get_type_resolver(); let type_info = type_resolver.get_type_info(concrete_type_id); let fory_type_id = type_info.get_type_id(); if type_info.is_registered_by_name() { if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - if self.fory.is_share_meta() { - let meta_index = self.push_meta(concrete_type_id) as u32; + if fory.is_share_meta() { + let meta_index = self.push_meta(fory, concrete_type_id) as u32; self.writer.write_varuint32(meta_index); } else { - type_info.get_namespace().write_to(self.writer); - type_info.get_type_name().write_to(self.writer); + type_info.get_namespace().write_to(&mut self.writer); + type_info.get_type_name().write_to(&mut self.writer); } } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(concrete_type_id) as u32; + let meta_index = self.push_meta(fory, concrete_type_id) as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(u32::MAX); - type_info.get_namespace().write_to(self.writer); - type_info.get_type_name().write_to(self.writer); + type_info.get_namespace().write_to(&mut self.writer); + type_info.get_type_name().write_to(&mut self.writer); } type_resolver .get_name_harness(type_info.get_namespace(), type_info.get_type_name()) @@ -94,7 +96,7 @@ impl<'se> WriteContext<'se> { } else { if fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(concrete_type_id) as u32; + let meta_index = self.push_meta(fory, concrete_type_id) as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(fory_type_id); @@ -104,34 +106,47 @@ impl<'se> WriteContext<'se> { .expect("ID harness not found") } } + + pub fn write_meta_string_bytes(&mut self, ms: &MetaString) { + self.meta_string_resolver + .write_meta_string_bytes(&mut self.writer, ms); + } + + pub fn reset(&mut self) { + self.meta_resolver.reset(); + self.ref_writer.reset(); + self.writer.reset(); + } } -pub struct ReadContext<'de, 'bf: 'de> { - pub reader: Reader<'bf>, - fory: &'de Fory, +pub struct ReadContext { + pub reader: Reader, pub meta_resolver: MetaReaderResolver, + meta_string_resolver: MetaStringReaderResolver, pub ref_reader: RefReader, max_dyn_depth: u32, current_depth: u32, } -impl<'de, 'bf: 'de> ReadContext<'de, 'bf> { - pub fn new(fory: &'de Fory, reader: Reader<'bf>, max_dyn_depth: u32) -> ReadContext<'de, 'bf> { +impl ReadContext { + pub fn new(reader: Reader, max_dyn_depth: u32) -> ReadContext { ReadContext { reader, - fory, meta_resolver: MetaReaderResolver::default(), + meta_string_resolver: MetaStringReaderResolver::default(), ref_reader: RefReader::new(), max_dyn_depth, current_depth: 0, } } - pub fn get_fory(&self) -> &Fory { - self.fory + pub fn init(&mut self, bytes: &[u8], max_dyn_depth: u32) { + self.reader.init(bytes); + self.max_dyn_depth = max_dyn_depth; + self.current_depth = 0; } - pub fn get_meta(&self, type_index: usize) -> &Rc { + pub fn get_meta(&self, type_index: usize) -> &Arc { self.meta_resolver.get(type_index) } @@ -141,11 +156,11 @@ impl<'de, 'bf: 'de> ReadContext<'de, 'bf> { )) } - pub fn read_any_typeinfo(&mut self) -> &'de Harness { + pub fn read_any_typeinfo(&mut self, fory: &Fory) -> Arc { use crate::types::TypeId as ForyTypeId; let fory_type_id = self.reader.read_varuint32(); - let type_resolver = self.fory.get_type_resolver(); + let type_resolver = fory.get_type_resolver(); if fory_type_id == u32::MAX { let namespace = self.meta_resolver.read_metastring(&mut self.reader); @@ -154,7 +169,7 @@ impl<'de, 'bf: 'de> ReadContext<'de, 'bf> { .get_name_harness(&namespace, &type_name) .expect("Name harness not found") } else if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { - if self.fory.is_share_meta() { + if fory.is_share_meta() { let _meta_index = self.reader.read_varuint32(); } else { let namespace = self.meta_resolver.read_metastring(&mut self.reader); @@ -180,6 +195,11 @@ impl<'de, 'bf: 'de> ReadContext<'de, 'bf> { } } + pub fn read_meta_string_bytes(&mut self) -> MetaStringBytes { + self.meta_string_resolver + .read_meta_string_bytes(&mut self.reader) + } + pub fn inc_depth(&mut self) -> Result<(), crate::error::Error> { self.current_depth += 1; if self.current_depth > self.max_dyn_depth { @@ -198,4 +218,40 @@ impl<'de, 'bf: 'de> ReadContext<'de, 'bf> { pub fn dec_depth(&mut self) { self.current_depth = self.current_depth.saturating_sub(1); } + + pub fn reset(&mut self) { + self.reader.reset(); + self.meta_resolver.reset(); + self.ref_reader.reset(); + } +} + +pub struct Pool { + items: Mutex>, + factory: fn() -> T, +} + +impl Pool { + pub fn new(factory: fn() -> T) -> Self { + Pool { + items: Mutex::new(vec![]), + factory, + } + } + + pub fn get(&self) -> T { + let item = self + .items + .lock() + .unwrap() + .pop() + .unwrap_or_else(|| (self.factory)()); + // println!("Object address: {:p}", &item); + item + } + + // put back manually + pub fn put(&self, item: T) { + self.items.lock().unwrap().push(item); + } } diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index fc13edb0e2..6d12d744d9 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -20,15 +20,15 @@ use crate::error::Error; use crate::fory::Fory; use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER}; use std::collections::HashMap; -use std::rc::Rc; +use std::sync::Arc; #[derive(Default)] pub struct MetaReaderResolver { - pub reading_type_defs: Vec>, + pub reading_type_defs: Vec>, } impl MetaReaderResolver { - pub fn get(&self, index: usize) -> &Rc { + pub fn get(&self, index: usize) -> &Arc { unsafe { self.reading_type_defs.get_unchecked(index) } } @@ -37,7 +37,7 @@ impl MetaReaderResolver { // self.reading_type_defs.reserve(meta_size as usize); for _ in 0..meta_size { let type_meta = TypeMeta::from_bytes(reader); - self.reading_type_defs.push(Rc::new(type_meta)); + self.reading_type_defs.push(Arc::new(type_meta)); } reader.get_cursor() } @@ -66,17 +66,21 @@ impl MetaReaderResolver { }; NAMESPACE_DECODER.decode(bytes, encoding).unwrap() } + + pub fn reset(&mut self) { + self.reading_type_defs.clear(); + } } #[derive(Default)] -pub struct MetaWriterResolver<'a> { - type_defs: Vec<&'a Vec>, +pub struct MetaWriterResolver { + type_defs: Vec>>, type_id_index_map: HashMap, } #[allow(dead_code)] -impl<'a> MetaWriterResolver<'a> { - pub fn push<'b: 'a>(&mut self, type_id: std::any::TypeId, fory: &'a Fory) -> usize { +impl MetaWriterResolver { + pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize { match self.type_id_index_map.get(&type_id) { None => { let index = self.type_defs.len(); @@ -106,5 +110,6 @@ impl<'a> MetaWriterResolver<'a> { pub fn reset(&mut self) { self.type_defs.clear(); + self.type_id_index_map.clear(); } } diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs b/rust/fory-core/src/resolver/metastring_resolver.rs index 71489d0eac..6e5f63cfd2 100644 --- a/rust/fory-core/src/resolver/metastring_resolver.rs +++ b/rust/fory-core/src/resolver/metastring_resolver.rs @@ -21,7 +21,7 @@ use std::convert::TryInto; use crate::buffer::Writer; use crate::meta::murmurhash3_x64_128; use crate::meta::{Encoding, MetaString}; -use crate::resolver::context::{ReadContext, WriteContext}; +use crate::Reader; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct MetaStringBytes { @@ -79,7 +79,7 @@ impl MetaStringBytes { } pub(crate) fn from_metastring(meta_string: &MetaString) -> Self { - let bytes = meta_string.bytes.to_vec(); + let mut bytes = meta_string.bytes.to_vec(); let mut hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64; hash_code = hash_code.abs(); if hash_code == 0 { @@ -91,47 +91,33 @@ impl MetaStringBytes { hash_code |= header; let header = (hash_code & HEADER_MASK) as u8; let encoding = byte_to_encoding(header); - let mut data = bytes.clone(); - if data.len() < 16 { - data.resize(16, 0); + + if bytes.len() < 16 { + bytes.resize(16, 0); } - let first8 = u64::from_le_bytes(data[0..8].try_into().unwrap()); - let second8 = u64::from_le_bytes(data[8..16].try_into().unwrap()); + let first8 = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let second8 = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + Self::new(bytes, hash_code, encoding, first8, second8) } } #[derive(Default)] -pub struct MetaStringResolver { - meta_string_bytes_to_string: HashMap, - - hash_to_meta: HashMap, - - small_map: HashMap<(u64, u64, u8), MetaStringBytes>, - +pub struct MetaStringWriterResolver { meta_string_to_bytes: HashMap, - dynamic_written: Vec>, - dynamic_read: Vec>, - dynamic_write_id: usize, - dynamic_read_id: usize, } -impl MetaStringResolver { +impl MetaStringWriterResolver { const INITIAL_CAPACITY: usize = 8; const SMALL_STRING_THRESHOLD: usize = 16; pub fn new() -> Self { - MetaStringResolver { - meta_string_bytes_to_string: HashMap::with_capacity(Self::INITIAL_CAPACITY), - hash_to_meta: HashMap::with_capacity(Self::INITIAL_CAPACITY), - small_map: HashMap::with_capacity(Self::INITIAL_CAPACITY), + Self { meta_string_to_bytes: HashMap::with_capacity(Self::INITIAL_CAPACITY), dynamic_written: vec![None; 32], - dynamic_read: vec![None; 32], dynamic_write_id: 0, - dynamic_read_id: 0, } } @@ -139,7 +125,7 @@ impl MetaStringResolver { if let Some(b) = self.meta_string_to_bytes.get(m) { return b.clone(); } - let bytes = m.bytes.clone(); + let bytes = m.bytes.to_vec(); let hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64; let encoding = m.encoding; let mut first8: u64 = 0; @@ -152,7 +138,7 @@ impl MetaStringResolver { second8 |= (bytes[8 + j] as u64) << (8 * j); } } - let msb = MetaStringBytes::new(bytes.clone(), hash_code, encoding, first8, second8); + let msb = MetaStringBytes::new(bytes, hash_code, encoding, first8, second8); self.meta_string_to_bytes.insert(m.clone(), msb.clone()); msb } @@ -160,21 +146,17 @@ impl MetaStringResolver { pub fn write_meta_string_bytes_with_flag(&mut self, w: &mut Writer, mut mb: MetaStringBytes) { let id = mb.dynamic_write_id; if id == MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID { - // allocate id let id_usize = self.dynamic_write_id; self.dynamic_write_id += 1; mb.dynamic_write_id = id_usize as i16; - // grow dynamic_written if needed if id_usize >= self.dynamic_written.len() { self.dynamic_written.resize(id_usize * 2, None); } self.dynamic_written[id_usize] = Some(mb.clone()); let len = mb.bytes.len(); - // last bit `1` indicates class is written by name instead of registered id. let header = ((len as u32) << 2) | 0b1; w.write_varuint32(header); - if len > Self::SMALL_STRING_THRESHOLD { w.write_i64(mb.hash_code); } else { @@ -182,13 +164,12 @@ impl MetaStringResolver { } w.write_bytes(&mb.bytes); } else { - // write id reference: ((id + 1) << 2) | 0b11 let header = ((id as u32 + 1) << 2) | 0b11; w.write_varuint32(header); } } - pub fn write_meta_string_bytes(&mut self, context: &mut WriteContext, ms: &MetaString) { + pub fn write_meta_string_bytes(&mut self, writer: &mut Writer, ms: &MetaString) { let mut mb = MetaStringBytes::from_metastring(ms); let id = mb.dynamic_write_id; if id == MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID { @@ -201,34 +182,69 @@ impl MetaStringResolver { self.dynamic_written[id_usize] = Some(mb.clone()); let len = mb.bytes.len(); - context.writer.write_varuint32((len as u32) << 1); + writer.write_varuint32((len as u32) << 1); if len > Self::SMALL_STRING_THRESHOLD { - context.writer.write_i64(mb.hash_code); + writer.write_i64(mb.hash_code); } else { - context.writer.write_u8(mb.encoding as i16 as u8); + writer.write_u8(mb.encoding as i16 as u8); } - context.writer.write_bytes(&mb.bytes); + writer.write_bytes(&mb.bytes); } else { let header = ((id as u32 + 1) << 1) | 1; - context.writer.write_varuint32(header); + writer.write_varuint32(header); + } + } + + pub fn reset_write(&mut self) { + if self.dynamic_write_id != 0 { + for i in 0..self.dynamic_write_id { + if let Some(ref mut mb) = self.dynamic_written[i] { + mb.dynamic_write_id = MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID; + } + self.dynamic_written[i] = None; + } + self.dynamic_write_id = 0; + } + } +} + +#[derive(Default)] +pub struct MetaStringReaderResolver { + meta_string_bytes_to_string: HashMap, + hash_to_meta: HashMap, + small_map: HashMap<(u64, u64, u8), MetaStringBytes>, + dynamic_read: Vec>, + dynamic_read_id: usize, +} + +impl MetaStringReaderResolver { + const INITIAL_CAPACITY: usize = 8; + const SMALL_STRING_THRESHOLD: usize = 16; + + pub fn new() -> Self { + Self { + meta_string_bytes_to_string: HashMap::with_capacity(Self::INITIAL_CAPACITY), + hash_to_meta: HashMap::with_capacity(Self::INITIAL_CAPACITY), + small_map: HashMap::with_capacity(Self::INITIAL_CAPACITY), + dynamic_read: vec![None; 32], + dynamic_read_id: 0, } } pub fn read_meta_string_bytes_with_flag( &mut self, - context: &mut ReadContext, + reader: &mut Reader, header: u32, ) -> MetaStringBytes { let len = (header >> 2) as usize; if (header & 0b10) == 0 { - // by-name path if len <= Self::SMALL_STRING_THRESHOLD { - let mb = self.read_small_meta_string_bytes(context, len); + let mb = self.read_small_meta_string_bytes(reader, len); self.update_dynamic_string(mb.clone()); mb } else { - let hash_code = context.reader.read_i64(); - let mb = self.read_big_meta_string_bytes(context, len, hash_code); + let hash_code = reader.read_i64(); + let mb = self.read_big_meta_string_bytes(reader, len, hash_code); self.update_dynamic_string(mb.clone()); mb } @@ -241,17 +257,17 @@ impl MetaStringResolver { } } - pub fn read_meta_string_bytes(&mut self, context: &mut ReadContext) -> MetaStringBytes { - let header = context.reader.read_varuint32(); + pub fn read_meta_string_bytes(&mut self, reader: &mut Reader) -> MetaStringBytes { + let header = reader.read_varuint32(); let len = (header >> 1) as usize; if (header & 0b1) == 0 { if len > Self::SMALL_STRING_THRESHOLD { - let hash_code = context.reader.read_i64(); - let mb = self.read_big_meta_string_bytes(context, len, hash_code); + let hash_code = reader.read_i64(); + let mb = self.read_big_meta_string_bytes(reader, len, hash_code); self.update_dynamic_string(mb.clone()); mb } else { - let mb = self.read_small_meta_string_bytes(context, len); + let mb = self.read_small_meta_string_bytes(reader, len); self.update_dynamic_string(mb.clone()); mb } @@ -266,53 +282,43 @@ impl MetaStringResolver { fn read_big_meta_string_bytes( &mut self, - context: &mut ReadContext, + reader: &mut Reader, len: usize, hash_code: i64, ) -> MetaStringBytes { if let Some(existing) = self.hash_to_meta.get(&hash_code) { - // skip bytes - context.reader.skip(len as u32); + reader.skip(len as u32); existing.clone() } else { - let slice = context.reader.read_bytes(len); - let bytes = slice.to_vec(); - let mb = { - // compute first8/second8 like createSmall - let mut first8: u64 = 0; - let mut second8: u64 = 0; - for (i, b) in bytes.iter().enumerate().take(8) { - first8 |= (*b as u64) << (8 * i); - } - if bytes.len() > 8 { - for j in 0..usize::min(8, bytes.len() - 8) { - second8 |= (bytes[8 + j] as u64) << (8 * j); - } + let bytes = reader.read_bytes(len).to_vec(); + let mut first8 = 0; + let mut second8 = 0; + for (i, b) in bytes.iter().enumerate().take(8) { + first8 |= (*b as u64) << (8 * i); + } + if bytes.len() > 8 { + for j in 0..usize::min(8, bytes.len() - 8) { + second8 |= (bytes[8 + j] as u64) << (8 * j); } - MetaStringBytes::new(bytes, hash_code, Encoding::Utf8, first8, second8) - }; + } + let mb = MetaStringBytes::new(bytes, hash_code, Encoding::Utf8, first8, second8); self.hash_to_meta.insert(hash_code, mb.clone()); mb } } - fn read_small_meta_string_bytes( - &mut self, - context: &mut ReadContext, - len: usize, - ) -> MetaStringBytes { - let encoding_val = context.reader.read_u8(); + fn read_small_meta_string_bytes(&mut self, reader: &mut Reader, len: usize) -> MetaStringBytes { + let encoding_val = reader.read_u8(); if len == 0 { - // assert encoding is UTF-8 debug_assert_eq!(encoding_val, Encoding::Utf8 as i16 as u8); return MetaStringBytes::EMPTY.clone(); } let (v1, v2) = if len <= 8 { - let v1 = Self::read_bytes_as_u64(context, len); - (v1, 0u64) + let v1 = Self::read_bytes_as_u64(reader, len); + (v1, 0) } else { - let v1 = context.reader.read_i64() as u64; - let v2 = Self::read_bytes_as_u64(context, len - 8); + let v1 = reader.read_i64() as u64; + let v2 = Self::read_bytes_as_u64(reader, len - 8); (v1, v2) }; let key = (v1, v2, encoding_val); @@ -328,13 +334,12 @@ impl MetaStringResolver { data.push(((v2 >> (8 * j)) & 0xFF) as u8); } } - let mut data_trimmed = data; - data_trimmed.truncate(len); - let hash_code = (murmurhash3_x64_128(&data_trimmed, 47).0 as i64).abs(); + data.truncate(len); + let hash_code = (murmurhash3_x64_128(&data, 47).0 as i64).abs(); let hash_code = (hash_code as u64 & 0xffffffffffffff00_u64) as i64 | (encoding_val as i64); let mb = MetaStringBytes::new( - data_trimmed.clone(), + data.clone(), hash_code, byte_to_encoding(encoding_val), v1, @@ -345,9 +350,9 @@ impl MetaStringResolver { } } - fn read_bytes_as_u64(context: &mut ReadContext, len: usize) -> u64 { - let mut v: u64 = 0; - let slice = context.reader.read_bytes(len); + fn read_bytes_as_u64(reader: &mut Reader, len: usize) -> u64 { + let mut v = 0; + let slice = reader.read_bytes(len); for (i, b) in slice.iter().take(len).enumerate() { v |= (*b as u64) << (8 * i); } @@ -363,11 +368,6 @@ impl MetaStringResolver { self.dynamic_read[id] = Some(mb); } - pub fn reset(&mut self) { - self.reset_read(); - self.reset_write(); - } - pub fn reset_read(&mut self) { if self.dynamic_read_id != 0 { for i in 0..self.dynamic_read_id { @@ -377,20 +377,8 @@ impl MetaStringResolver { } } - pub fn reset_write(&mut self) { - if self.dynamic_write_id != 0 { - for i in 0..self.dynamic_write_id { - if let Some(ref mut mb) = self.dynamic_written[i] { - mb.dynamic_write_id = MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID; - } - self.dynamic_written[i] = None; - } - self.dynamic_write_id = 0; - } - } - - pub fn read_meta_string(&mut self, context: &mut ReadContext) -> String { - let mb = self.read_meta_string_bytes(context); + pub fn read_meta_string(&mut self, reader: &mut Reader) -> String { + let mb = self.read_meta_string_bytes(reader); if let Some(s) = self.meta_string_bytes_to_string.get(&mb) { s.clone() } else { diff --git a/rust/fory-core/src/resolver/ref_resolver.rs b/rust/fory-core/src/resolver/ref_resolver.rs index 215b10832b..77433d48b5 100644 --- a/rust/fory-core/src/resolver/ref_resolver.rs +++ b/rust/fory-core/src/resolver/ref_resolver.rs @@ -130,7 +130,7 @@ impl RefWriter { /// Clear all stored references. /// /// This is useful for reusing the RefWriter for multiple serialization operations. - pub fn clear(&mut self) { + pub fn reset(&mut self) { self.refs.clear(); self.next_ref_id = 0; } @@ -167,6 +167,10 @@ pub struct RefReader { callbacks: Vec, } +// danger but useful for multi-thread +unsafe impl Send for RefReader {} +unsafe impl Sync for RefReader {} + impl RefReader { /// Creates a new RefReader instance. pub fn new() -> Self { @@ -322,7 +326,7 @@ impl RefReader { /// Clear all stored references and callbacks. /// /// This is useful for reusing the RefReader for multiple deserialization operations. - pub fn clear(&mut self) { + pub fn reset(&mut self) { self.resolve_callbacks(); self.refs.clear(); self.callbacks.clear(); diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index 2d17aa760b..ab3c57735c 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -23,17 +23,22 @@ use crate::meta::{ TYPE_NAME_ENCODINGS, }; use crate::serializer::{ForyDefault, Serializer, StructSerializer}; -use std::cell::RefCell; +use std::sync::{Arc, RwLock}; use std::{any::Any, collections::HashMap}; -type WriteFn = fn(&dyn Any, &mut WriteContext, is_field: bool); -type ReadFn = - fn(&mut ReadContext, is_field: bool, skip_ref_flag: bool) -> Result, Error>; +type WriteFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool); +type ReadFn = fn( + fory: &Fory, + &mut ReadContext, + is_field: bool, + skip_ref_flag: bool, +) -> Result, Error>; -type WriteDataFn = fn(&dyn Any, &mut WriteContext, is_field: bool); -type ReadDataFn = fn(&mut ReadContext, is_field: bool) -> Result, Error>; +type WriteDataFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool); +type ReadDataFn = fn(fory: &Fory, &mut ReadContext, is_field: bool) -> Result, Error>; type ToSerializerFn = fn(Box) -> Result, Error>; +#[derive(Clone)] pub struct Harness { write_fn: WriteFn, read_fn: ReadFn, @@ -82,7 +87,7 @@ impl Harness { #[derive(Clone, Debug)] pub struct TypeInfo { - type_def: Vec, + type_def: Arc>, type_id: u32, namespace: MetaString, type_name: MetaString, @@ -104,13 +109,13 @@ impl TypeInfo { .encode_with_encodings(type_name, TYPE_NAME_ENCODINGS) .unwrap(); TypeInfo { - type_def: T::fory_type_def( + type_def: Arc::from(T::fory_type_def( fory, type_id, namespace_metastring.clone(), type_name_metastring.clone(), register_by_name, - ), + )), type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -140,7 +145,7 @@ impl TypeInfo { ); let type_def = meta.to_bytes().unwrap(); TypeInfo { - type_def, + type_def: Arc::from(type_def), type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -160,8 +165,8 @@ impl TypeInfo { &self.type_name } - pub fn get_type_def(&self) -> &Vec { - &self.type_def + pub fn get_type_def(&self) -> Arc> { + self.type_def.clone() } pub fn is_registered_by_name(&self) -> bool { @@ -170,14 +175,14 @@ impl TypeInfo { } pub struct TypeResolver { - serializer_map: HashMap, - name_serializer_map: HashMap<(MetaString, MetaString), Harness>, + serializer_map: HashMap>, + name_serializer_map: HashMap<(MetaString, MetaString), Arc>, type_id_map: HashMap, type_name_map: HashMap, type_info_cache: HashMap, // Fast lookup by numeric ID for common types type_id_index: Vec, - sorted_field_names_map: RefCell>>, + sorted_field_names_map: RwLock>>>, } const NO_TYPE_ID: u32 = 1000000000; @@ -191,7 +196,7 @@ impl Default for TypeResolver { type_name_map: HashMap::new(), type_info_cache: HashMap::new(), type_id_index: Vec::new(), - sorted_field_names_map: RefCell::new(HashMap::new()), + sorted_field_names_map: RwLock::new(HashMap::new()), }; resolver.register_builtin_types(); resolver @@ -205,7 +210,7 @@ impl TypeResolver { macro_rules! register_basic_type { ($ty:ty, $type_id:expr) => {{ let type_info = TypeInfo { - type_def: vec![], + type_def: Arc::from(vec![]), type_id: $type_id as u32, namespace: NAMESPACE_ENCODER .encode_with_encodings("", NAMESPACE_ENCODINGS) @@ -267,16 +272,17 @@ impl TypeResolver { ) { fn write( this: &dyn Any, + fory: &Fory, context: &mut WriteContext, is_field: bool, ) { let this = this.downcast_ref::(); match this { Some(v) => { - let skip_ref_flag = - crate::serializer::get_skip_ref_flag::(context.get_fory()); + let skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory); crate::serializer::write_ref_info_data( v, + fory, context, is_field, skip_ref_flag, @@ -288,11 +294,13 @@ impl TypeResolver { } fn read( + fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, ) -> Result, Error> { match crate::serializer::read_ref_info_data::( + fory, context, is_field, skip_ref_flag, @@ -305,23 +313,25 @@ impl TypeResolver { fn write_data( this: &dyn Any, + fory: &Fory, context: &mut WriteContext, is_field: bool, ) { let this = this.downcast_ref::(); match this { Some(v) => { - T2::fory_write_data(v, context, is_field); + T2::fory_write_data(v, fory, context, is_field); } None => todo!(), } } fn read_data( + fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(context, is_field) { + match T2::fory_read_data(fory, context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -364,13 +374,13 @@ impl TypeResolver { self.type_name_map.insert(rs_type_id, key.clone()); self.name_serializer_map.insert( key, - Harness::new( + Arc::from(Harness::new( write::, read::, write_data::, read_data::, to_serializer::, - ), + )), ); } else { let type_id = type_info.type_id; @@ -380,13 +390,13 @@ impl TypeResolver { self.type_id_map.insert(rs_type_id, type_id); self.serializer_map.insert( type_id, - Harness::new( + Arc::from(Harness::new( write::, read::, write_data::, read_data::, to_serializer::, - ), + )), ); } } @@ -394,30 +404,32 @@ impl TypeResolver { pub fn register_serializer(&mut self, type_info: &TypeInfo) { fn write( this: &dyn Any, + fory: &Fory, context: &mut WriteContext, is_field: bool, ) { let this = this.downcast_ref::(); match this { Some(v) => { - v.fory_write(context, is_field); + v.fory_write(fory, context, is_field); } None => todo!(), } } fn read( + fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, ) -> Result, Error> { if skip_ref_flag { - match T2::fory_read_data(context, is_field) { + match T2::fory_read_data(fory, context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } } else { - match T2::fory_read(context, is_field) { + match T2::fory_read(fory, context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -426,23 +438,25 @@ impl TypeResolver { fn write_data( this: &dyn Any, + fory: &Fory, context: &mut WriteContext, is_field: bool, ) { let this = this.downcast_ref::(); match this { Some(v) => { - T2::fory_write_data(v, context, is_field); + T2::fory_write_data(v, fory, context, is_field); } None => todo!(), } } fn read_data( + fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(context, is_field) { + match T2::fory_read_data(fory, context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -450,9 +464,9 @@ impl TypeResolver { fn to_serializer( boxed_any: Box, - ) -> Result, Error> { + ) -> Result, Error> { match boxed_any.downcast::() { - Ok(concrete) => Ok(Box::new(*concrete) as Box), + Ok(concrete) => Ok(Box::new(*concrete) as Box), Err(_) => Err(Error::Other(anyhow::anyhow!( "Failed to downcast to concrete type" ))), @@ -477,13 +491,13 @@ impl TypeResolver { self.type_name_map.insert(rs_type_id, key.clone()); self.name_serializer_map.insert( key, - Harness::new( + Arc::from(Harness::new( write::, read::, write_data::, read_data::, to_serializer::, - ), + )), ); } else { let type_id = type_info.type_id; @@ -493,32 +507,28 @@ impl TypeResolver { self.type_id_map.insert(rs_type_id, type_id); self.serializer_map.insert( type_id, - Harness::new( + Arc::from(Harness::new( write::, read::, write_data::, read_data::, to_serializer::, - ), + )), ); } } - pub fn get_harness_by_type(&self, type_id: std::any::TypeId) -> Option<&Harness> { - self.get_harness(*self.type_id_map.get(&type_id).unwrap()) - } - - pub fn get_harness(&self, id: u32) -> Option<&Harness> { - self.serializer_map.get(&id) + pub fn get_harness(&self, id: u32) -> Option> { + self.serializer_map.get(&id).cloned() } pub fn get_name_harness( &self, namespace: &MetaString, type_name: &MetaString, - ) -> Option<&Harness> { + ) -> Option> { let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map.get(&key) + self.name_serializer_map.get(&key).cloned() } pub fn get_ext_harness(&self, id: u32) -> &Harness { @@ -537,14 +547,14 @@ impl TypeResolver { pub fn get_sorted_field_names( &self, type_id: std::any::TypeId, - ) -> Option> { - let map = self.sorted_field_names_map.borrow(); + ) -> Option>> { + let map = self.sorted_field_names_map.read().unwrap(); map.get(&type_id).cloned() } - pub fn set_sorted_field_names(&self, field_names: &[String]) { - let mut map = self.sorted_field_names_map.borrow_mut(); - map.insert(std::any::TypeId::of::(), field_names.to_owned()); + pub fn set_sorted_field_names(&self, field_names: Arc>) { + let mut map = self.sorted_field_names_map.write().unwrap(); + map.insert(std::any::TypeId::of::(), field_names); } pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index d82b162a73..745c687b3d 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -25,17 +25,22 @@ use std::rc::Rc; use std::sync::Arc; /// Helper function to serialize a `Box` -pub fn serialize_any_box(any_box: &Box, context: &mut WriteContext, is_field: bool) { +pub fn serialize_any_box( + any_box: &Box, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) { context.writer.write_i8(RefFlag::NotNullValue as i8); let concrete_type_id = (**any_box).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id); let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**any_box, context, is_field); + serializer_fn(&**any_box, fory, context, is_field); } /// Helper function to deserialize to `Box` -pub fn deserialize_any_box(context: &mut ReadContext) -> Result, Error> { +pub fn deserialize_any_box(fory: &Fory, context: &mut ReadContext) -> Result, Error> { context.inc_depth()?; let ref_flag = context.reader.read_i8(); if ref_flag != RefFlag::NotNullValue as i8 { @@ -43,9 +48,9 @@ pub fn deserialize_any_box(context: &mut ReadContext) -> Result, Er "Expected NotNullValue for Box" ))); } - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let result = deserializer_fn(context, true); + let result = deserializer_fn(fory, context, true); context.dec_depth(); result } @@ -57,20 +62,24 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - serialize_any_box(self, context, is_field); + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + serialize_any_box(self, fory, context, is_field); } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - serialize_any_box(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + serialize_any_box(self, fory, context, is_field); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { - deserialize_any_box(context) + fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { + deserialize_any_box(fory, context) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - deserialize_any_box(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + deserialize_any_box(fory, context) } fn fory_get_type_id(_fory: &Fory) -> u32 { @@ -88,11 +97,11 @@ impl Serializer for Box { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { // Box is polymorphic - type info is written per element } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { // Box is polymorphic - type info is read per element } @@ -108,20 +117,23 @@ impl ForyDefault for Rc { } impl Serializer for Rc { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - if !context.ref_writer.try_write_rc_ref(context.writer, self) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + if !context + .ref_writer + .try_write_rc_ref(&mut context.writer, self) + { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id); let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, context, is_field); + serializer_fn(&**self, fory, context, is_field); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - self.fory_write(context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + self.fory_write(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { @@ -137,17 +149,17 @@ impl Serializer for Rc { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; + let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); Ok(Rc::::from(boxed)) } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; + let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); let rc: Rc = Rc::from(boxed); context.ref_reader.store_rc_ref(rc.clone()); @@ -156,8 +168,12 @@ impl Serializer for Rc { } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Self::fory_read(fory, context, is_field) } fn fory_get_type_id(_fory: &Fory) -> u32 { @@ -175,11 +191,11 @@ impl Serializer for Rc { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { // Rc is polymorphic - type info is written per element } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { // Rc is polymorphic - type info is read per element } @@ -195,20 +211,23 @@ impl ForyDefault for Arc { } impl Serializer for Arc { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - if !context.ref_writer.try_write_arc_ref(context.writer, self) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + if !context + .ref_writer + .try_write_arc_ref(&mut context.writer, self) + { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id); let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, context, is_field); + serializer_fn(&**self, fory, context, is_field); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - self.fory_write(context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + self.fory_write(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { @@ -224,17 +243,17 @@ impl Serializer for Arc { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; + let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); Ok(Arc::::from(boxed)) } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; + let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); let arc: Arc = Arc::from(boxed); context.ref_reader.store_arc_ref(arc.clone()); @@ -243,8 +262,12 @@ impl Serializer for Arc { } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Self::fory_read(fory, context, is_field) } fn fory_get_type_id(_fory: &Fory) -> u32 { @@ -262,11 +285,11 @@ impl Serializer for Arc { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { // Arc is polymorphic - type info is written per element } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { // Arc is polymorphic - type info is read per element } diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index f62b2e1191..cc0a0bc875 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -28,24 +28,27 @@ impl Serializer for Arc true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - if !context.ref_writer.try_write_arc_ref(context.writer, self) { - T::fory_write_data(self.as_ref(), context, is_field); + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + if !context + .ref_writer + .try_write_arc_ref(&mut context.writer, self) + { + T::fory_write_data(self.as_ref(), fory, context, is_field); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // When Arc is nested inside another shared ref (like Rc>), // the outer ref calls fory_write_data on the inner Arc. // We still need to track the Arc's own references here. - self.fory_write(context, is_field); + self.fory_write(fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { @@ -58,12 +61,12 @@ impl Serializer for Arc .ok_or_else(|| anyhow!("Arc reference {} not found", ref_id).into()) } RefFlag::NotNullValue => { - let inner = T::fory_read_data(context, is_field)?; + let inner = T::fory_read_data(fory, context, is_field)?; Ok(Arc::new(inner)) } RefFlag::RefValue => { let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(context, is_field)?; + let inner = T::fory_read_data(fory, context, is_field)?; let arc = Arc::new(inner); context.ref_reader.store_arc_ref_at(ref_id, arc.clone()); Ok(arc) @@ -71,14 +74,18 @@ impl Serializer for Arc } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { // When Arc is nested inside another shared ref, fory_read_data is called. // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(context, is_field) + Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index aedc14ff0a..c3a71449d8 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -24,11 +24,15 @@ use crate::types::TypeId; use std::mem; impl Serializer for bool { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { context.writer.write_u8(if *self { 1 } else { 0 }); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { Ok(context.reader.read_u8() == 1) } @@ -48,12 +52,12 @@ impl Serializer for bool { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } diff --git a/rust/fory-core/src/serializer/box_.rs b/rust/fory-core/src/serializer/box_.rs index 3715f802ca..88c4c1893a 100644 --- a/rust/fory-core/src/serializer/box_.rs +++ b/rust/fory-core/src/serializer/box_.rs @@ -22,20 +22,24 @@ use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; impl Serializer for Box { - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Box::new(T::fory_read_data(context, is_field)?)) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Ok(Box::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - T::fory_write_data(self.as_ref(), context, is_field) + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_data(self.as_ref(), fory, context, is_field) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index b9fdda3041..488bc84ee0 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -20,6 +20,7 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; use crate::types::PRIMITIVE_ARRAY_TYPES; +use crate::Fory; // const TRACKING_REF: u8 = 0b1; @@ -42,28 +43,30 @@ pub fn write_collection_type_info( context.writer.write_varuint32(collection_type_id); } -pub fn write_collection<'a, T: Serializer + 'a, I: IntoIterator>( - iter: I, - context: &mut WriteContext, - is_field: bool, -) { - let items: Vec<&T> = iter.into_iter().collect(); - let len = items.len(); +pub fn write_collection<'a, T, I>(iter: I, fory: &Fory, context: &mut WriteContext, is_field: bool) +where + T: Serializer + 'a, + I: IntoIterator, + I::IntoIter: ExactSizeIterator + Clone, +{ + let iter = iter.into_iter(); + let len = iter.len(); context.writer.write_varuint32(len as u32); if len == 0 { return; } let mut header = 0; let mut has_null = false; + let is_same_type = !T::fory_is_polymorphic(); if T::fory_is_option() { - for item in &items { + // iter.clone() is zero-copy + for item in iter.clone() { if item.fory_is_none() { has_null = true; break; } } } - let is_same_type = !T::fory_is_polymorphic(); if has_null { header |= HAS_NULL; } @@ -74,18 +77,25 @@ pub fn write_collection<'a, T: Serializer + 'a, I: IntoIterator>( header |= IS_SAME_TYPE; } context.writer.write_u8(header); - T::fory_write_type_info(context, is_field); + T::fory_write_type_info(fory, context, is_field); // context.writer.reserve((T::reserved_space() + SIZE_OF_REF_AND_TYPE) * len); if T::fory_is_polymorphic() || T::fory_is_shared_ref() { // TOTO: make it xlang compatible - for item in &items { - item.fory_write(context, is_field); + for item in iter { + item.fory_write(fory, context, is_field); } } else { // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); let skip_ref_flag = is_same_type && !has_null; - for item in &items { - crate::serializer::write_ref_info_data(*item, context, is_field, skip_ref_flag, true); + for item in iter { + crate::serializer::write_ref_info_data( + item, + fory, + context, + is_field, + skip_ref_flag, + true, + ); } } } @@ -106,7 +116,7 @@ pub fn read_collection_type_info( assert_eq!(remote_collection_type_id, collection_type_id); } -pub fn read_collection(context: &mut ReadContext) -> Result +pub fn read_collection(fory: &Fory, context: &mut ReadContext) -> Result where T: Serializer + ForyDefault, C: FromIterator, @@ -117,18 +127,20 @@ where } let header = context.reader.read_u8(); let declared = (header & DECL_ELEMENT_TYPE) != 0; - T::fory_read_type_info(context, declared); + T::fory_read_type_info(fory, context, declared); let has_null = (header & HAS_NULL) != 0; let is_same_type = (header & IS_SAME_TYPE) != 0; if T::fory_is_polymorphic() || T::fory_is_shared_ref() { (0..len) - .map(|_| T::fory_read(context, declared)) + .map(|_| T::fory_read(fory, context, declared)) .collect::>() } else { let skip_ref_flag = is_same_type && !has_null; // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); (0..len) - .map(|_| crate::serializer::read_ref_info_data(context, declared, skip_ref_flag, true)) + .map(|_| { + crate::serializer::read_ref_info_data(fory, context, declared, skip_ref_flag, true) + }) .collect::>() } } diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs index 52bc2c714c..d709227e30 100644 --- a/rust/fory-core/src/serializer/datetime.rs +++ b/rust/fory-core/src/serializer/datetime.rs @@ -28,13 +28,17 @@ use chrono::{DateTime, Days, NaiveDate, NaiveDateTime}; use std::mem; impl Serializer for NaiveDateTime { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { let dt = self.and_utc(); let micros = dt.timestamp() * 1_000_000 + dt.timestamp_subsec_micros() as i64; context.writer.write_i64(micros); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { let micros = context.reader.read_i64(); let seconds = micros / 1_000_000; let subsec_micros = (micros % 1_000_000) as u32; @@ -62,22 +66,26 @@ impl Serializer for NaiveDateTime { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } impl Serializer for NaiveDate { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { let days_since_epoch = self.signed_duration_since(EPOCH).num_days(); context.writer.write_i32(days_since_epoch as i32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { let days = context.reader.read_i32(); EPOCH .checked_add_days(Days::new(days as u64)) @@ -102,12 +110,12 @@ impl Serializer for NaiveDate { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index c1ae7c7e33..fa3c4c245b 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -44,73 +44,68 @@ pub fn type_def( } #[inline(always)] -pub fn write_type_info(context: &mut WriteContext, is_field: bool) { +pub fn write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { if is_field { return; } - let type_id = T::fory_get_type_id(context.get_fory()); + let type_id = T::fory_get_type_id(fory); context.writer.write_varuint32(type_id); let is_named_enum = type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { return; } let rs_type_id = std::any::TypeId::of::(); - if context.get_fory().is_share_meta() { - let meta_index = context.push_meta(rs_type_id) as u32; + if fory.is_share_meta() { + let meta_index = context.push_meta(fory, rs_type_id) as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = context - .get_fory() - .get_type_resolver() - .get_type_info(rs_type_id); + let type_info = fory.get_type_resolver().get_type_info(rs_type_id); let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - let resolver = context.get_fory().get_metastring_resolver(); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &namespace); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &type_name); + context.write_meta_string_bytes(&namespace); + context.write_meta_string_bytes(&type_name); } } #[inline(always)] -pub fn read_type_info(context: &mut ReadContext, is_field: bool) { +pub fn read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { if is_field { return; } - let local_type_id = T::fory_get_type_id(context.get_fory()); + let local_type_id = T::fory_get_type_id(fory); let remote_type_id = context.reader.read_varuint32(); assert_eq!(local_type_id, remote_type_id); let is_named_enum = local_type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { return; } - if context.get_fory().is_share_meta() { + if fory.is_share_meta() { let _meta_index = context.reader.read_varuint32(); } else { - let resolver = context.get_fory().get_metastring_resolver(); - resolver.borrow_mut().read_meta_string_bytes(context); - resolver.borrow_mut().read_meta_string_bytes(context); + let _namespace_msb = context.read_meta_string_bytes(); + let _type_name_msb = context.read_meta_string_bytes(); } } #[inline(always)] -pub fn read_compatible(context: &mut ReadContext) -> Result { - T::fory_read_type_info(context, true); - T::fory_read_data(context, true) +pub fn read_compatible( + fory: &Fory, + context: &mut ReadContext, +) -> Result { + T::fory_read_type_info(fory, context, true); + T::fory_read_data(fory, context, true) } #[inline(always)] -pub fn write(this: &T, context: &mut WriteContext, is_field: bool) { +pub fn write(this: &T, fory: &Fory, context: &mut WriteContext, is_field: bool) { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, is_field); - this.fory_write_data(context, is_field); + T::fory_write_type_info(fory, context, is_field); + this.fory_write_data(fory, context, is_field); } #[inline(always)] pub fn read( + fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result { @@ -118,8 +113,8 @@ pub fn read( if ref_flag == RefFlag::Null as i8 { Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { - T::fory_read_type_info(context, false); - T::fory_read_data(context, is_field) + T::fory_read_type_info(fory, context, false); + T::fory_read_data(fory, context, is_field) } else { unimplemented!() } diff --git a/rust/fory-core/src/serializer/heap.rs b/rust/fory-core/src/serializer/heap.rs index 6e9d56d45f..f61b6095c2 100644 --- a/rust/fory-core/src/serializer/heap.rs +++ b/rust/fory-core/src/serializer/heap.rs @@ -29,19 +29,23 @@ use std::collections::BinaryHeap; use std::mem; impl Serializer for BinaryHeap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_collection(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_collection(self, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { write_collection_type_info(context, is_field, TypeId::SET as u32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + read_collection(fory, context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { read_collection_type_info(context, is_field, TypeId::SET as u32) } diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs index f60d9641fb..61b6d99042 100644 --- a/rust/fory-core/src/serializer/list.rs +++ b/rust/fory-core/src/serializer/list.rs @@ -44,18 +44,18 @@ fn check_primitive() -> Option { } impl Serializer for Vec { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { match check_primitive::() { Some(_) => { primitive_list::fory_write_data(self, context); } None => { - write_collection(self, context, is_field); + write_collection(self, fory, context, is_field); } } } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { match check_primitive::() { Some(type_id) => { primitive_list::fory_write_type_info(context, is_field, type_id); @@ -66,14 +66,18 @@ impl Serializer for Vec { } } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { match check_primitive::() { Some(_) => primitive_list::fory_read_data(context), - None => read_collection(context), + None => read_collection(fory, context), } } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { match check_primitive::() { Some(type_id) => primitive_list::fory_read_type_info(context, is_field, type_id), None => read_collection_type_info(context, is_field, TypeId::LIST as u32), @@ -116,19 +120,23 @@ impl ForyDefault for Vec { } impl Serializer for VecDeque { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_collection(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_collection(self, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { write_collection_type_info(context, is_field, TypeId::LIST as u32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + read_collection(fory, context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { read_collection_type_info(context, is_field, TypeId::LIST as u32); } @@ -156,19 +164,23 @@ impl ForyDefault for VecDeque { } impl Serializer for LinkedList { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_collection(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_collection(self, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { write_collection_type_info(context, is_field, TypeId::LIST as u32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + read_collection(fory, context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { read_collection_type_info(context, is_field, TypeId::LIST as u32); } diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index 7db4fc8f99..f7b650a82f 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -35,7 +35,8 @@ const TRACKING_VALUE_REF: u8 = 0b1000; pub const VALUE_NULL: u8 = 0b10000; const DECL_VALUE_TYPE: u8 = 0b100000; -fn check_and_write_null( +fn check_and_write_null( + fory: &Fory, context: &mut WriteContext, is_field: bool, key: &K, @@ -49,7 +50,7 @@ fn check_and_write_null( let mut chunk_header = KEY_NULL; let skip_ref_flag; if is_field { - skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); + skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory); chunk_header |= DECL_VALUE_TYPE; } else { skip_ref_flag = false; @@ -57,7 +58,7 @@ fn check_and_write_null( } context.writer.write_u8(chunk_header); - write_ref_info_data(value, context, is_field, skip_ref_flag, false); + write_ref_info_data(value, fory, context, is_field, skip_ref_flag, false); return true; } if value.fory_is_none() { @@ -72,7 +73,7 @@ fn check_and_write_null( chunk_header |= TRACKING_KEY_REF; } context.writer.write_u8(chunk_header); - write_ref_info_data(key, context, is_field, skip_ref_flag, false); + write_ref_info_data(key, fory, context, is_field, skip_ref_flag, false); return true; } false @@ -82,9 +83,14 @@ fn write_chunk_size(context: &mut WriteContext, header_offset: usize, size: u8) context.writer.set_bytes(header_offset + 1, &[size]); } -fn write_map_data<'a, K, V, I>(iter: I, length: usize, context: &mut WriteContext, is_field: bool) -where - K: Serializer + ForyDefault + 'a, +fn write_map_data<'a, K, V, I>( + iter: I, + length: usize, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) where + K: Serializer + ForyDefault + 'a + Eq + std::hash::Hash, V: Serializer + ForyDefault + 'a, I: Iterator, { @@ -103,7 +109,7 @@ where let mut skip_val_ref_flag = false; for (key, value) in iter { if need_write_header { - if check_and_write_null(context, is_field, key, value) { + if check_and_write_null(fory, context, is_field, key, value) { continue; } header_offset = context.writer.len(); @@ -120,8 +126,8 @@ where if !skip_val_ref_flag { chunk_header |= TRACKING_VALUE_REF; } - K::fory_write_type_info(context, is_field); - V::fory_write_type_info(context, is_field); + K::fory_write_type_info(fory, context, is_field); + V::fory_write_type_info(fory, context, is_field); context.writer.set_bytes(header_offset, &[chunk_header]); need_write_header = false; } @@ -129,18 +135,18 @@ where write_chunk_size(context, header_offset, pair_counter); pair_counter = 0; need_write_header = true; - check_and_write_null(context, is_field, key, value); + check_and_write_null(fory, context, is_field, key, value); continue; } if K::fory_is_polymorphic() || K::fory_is_shared_ref() { - key.fory_write(context, is_field); + key.fory_write(fory, context, is_field); } else { - write_ref_info_data(key, context, is_field, skip_key_ref_flag, true); + write_ref_info_data(key, fory, context, is_field, skip_key_ref_flag, true); } if V::fory_is_polymorphic() || V::fory_is_shared_ref() { - value.fory_write(context, is_field); + value.fory_write(fory, context, is_field); } else { - write_ref_info_data(value, context, is_field, skip_val_ref_flag, true); + write_ref_info_data(value, fory, context, is_field, skip_val_ref_flag, true); } pair_counter += 1; if pair_counter == MAX_CHUNK_SIZE { @@ -157,11 +163,15 @@ where impl Serializer for HashMap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_map_data(self.iter(), self.len(), context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_map_data(self.iter(), self.len(), fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { let len = context.reader.read_varuint32(); let mut map = HashMap::::with_capacity(len as usize); if len == 0 { @@ -182,40 +192,43 @@ impl(context.get_fory()) + crate::serializer::get_skip_ref_flag::(fory) } else { false }; - let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; + let value = + read_ref_info_data(fory, context, value_declared, skip_ref_flag, false)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(context.get_fory()) + crate::serializer::get_skip_ref_flag::(fory) } else { false }; - let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; + let key = read_ref_info_data(fory, context, key_declared, skip_ref_flag, false)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8(); - K::fory_read_type_info(context, key_declared); - V::fory_read_type_info(context, value_declared); + K::fory_read_type_info(fory, context, key_declared); + V::fory_read_type_info(fory, context, value_declared); assert!(len_counter + chunk_size as u32 <= len); for _ in 0..chunk_size { let key = if K::fory_is_polymorphic() { - K::fory_read(context, key_declared)? + K::fory_read(fory, context, key_declared)? } else { - read_ref_info_data(context, key_declared, true, true)? + // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); + read_ref_info_data(fory, context, key_declared, true, true)? }; let value = if V::fory_is_polymorphic() { - V::fory_read(context, value_declared)? + V::fory_read(fory, context, value_declared)? } else { - read_ref_info_data(context, value_declared, true, true)? + // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); + read_ref_info_data(fory, context, value_declared, true, true)? }; map.insert(key, value); } @@ -240,12 +253,12 @@ impl(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } @@ -255,12 +268,18 @@ impl ForyDefault for HashMap { } } -impl Serializer for BTreeMap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_map_data(self.iter(), self.len(), context, is_field); +impl Serializer + for BTreeMap +{ + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_map_data(self.iter(), self.len(), fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { let len = context.reader.read_varuint32(); let mut map = BTreeMap::::new(); if len == 0 { @@ -281,40 +300,41 @@ impl Serializer let value_declared = (header & DECL_VALUE_TYPE) != 0; if header & KEY_NULL != 0 { let skip_ref_flag = if value_declared { - crate::serializer::get_skip_ref_flag::(context.get_fory()) + crate::serializer::get_skip_ref_flag::(fory) } else { false }; - let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; + let value = + read_ref_info_data(fory, context, value_declared, skip_ref_flag, false)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(context.get_fory()) + crate::serializer::get_skip_ref_flag::(fory) } else { false }; - let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; + let key = read_ref_info_data(fory, context, key_declared, skip_ref_flag, false)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8(); - K::fory_read_type_info(context, key_declared); - V::fory_read_type_info(context, value_declared); + K::fory_read_type_info(fory, context, key_declared); + V::fory_read_type_info(fory, context, value_declared); assert!(len_counter + chunk_size as u32 <= len); for _ in 0..chunk_size { let key = if K::fory_is_polymorphic() { - K::fory_read(context, key_declared)? + K::fory_read(fory, context, key_declared)? } else { - read_ref_info_data(context, key_declared, true, true)? + read_ref_info_data(fory, context, key_declared, true, true)? }; let value = if V::fory_is_polymorphic() { - V::fory_read(context, value_declared)? + V::fory_read(fory, context, value_declared)? } else { - read_ref_info_data(context, value_declared, true, true)? + read_ref_info_data(fory, context, value_declared, true, true)? }; map.insert(key, value); } @@ -339,12 +359,12 @@ impl Serializer self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 10d300c69f..0f3221f7f0 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -22,6 +22,7 @@ use crate::resolver::context::{ReadContext, WriteContext}; use crate::types::{Mode, RefFlag, TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; use std::any::Any; +use std::sync::Arc; pub mod any; mod arc; @@ -48,6 +49,7 @@ pub mod weak; pub fn write_ref_info_data( record: &T, + fory: &Fory, context: &mut WriteContext, is_field: bool, skip_ref_flag: bool, @@ -60,13 +62,14 @@ pub fn write_ref_info_data( context.writer.write_i8(RefFlag::NotNullValue as i8); } if !skip_type_info { - T::fory_write_type_info(context, is_field); + T::fory_write_type_info(fory, context, is_field); } - record.fory_write_data(context, is_field); + record.fory_write_data(fory, context, is_field); } } pub fn read_ref_info_data( + fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, @@ -78,15 +81,15 @@ pub fn read_ref_info_data( Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { if !skip_type_info { - T::fory_read_type_info(context, is_field); + T::fory_read_type_info(fory, context, is_field); } - T::fory_read_data(context, is_field) + T::fory_read_data(fory, context, is_field) } else if ref_flag == (RefFlag::RefValue as i8) { // First time seeing this referenceable object if !skip_type_info { - T::fory_read_type_info(context, is_field); + T::fory_read_type_info(fory, context, is_field); } - T::fory_read_data(context, is_field) + T::fory_read_data(fory, context, is_field) } else if ref_flag == (RefFlag::Ref as i8) { // This is a reference to a previously deserialized object // For now, just return default - this should be handled by specific types @@ -96,25 +99,25 @@ pub fn read_ref_info_data( } } else { if !skip_type_info { - T::fory_read_type_info(context, is_field); + T::fory_read_type_info(fory, context, is_field); } - T::fory_read_data(context, is_field) + T::fory_read_data(fory, context, is_field) } } -fn write_type_info(context: &mut WriteContext, is_field: bool) { +fn write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { if is_field { return; } - let type_id = T::fory_get_type_id(context.get_fory()); + let type_id = T::fory_get_type_id(fory); context.writer.write_varuint32(type_id); } -fn read_type_info(context: &mut ReadContext, is_field: bool) { +fn read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { if is_field { return; } - let local_type_id = T::fory_get_type_id(context.get_fory()); + let local_type_id = T::fory_get_type_id(fory); let remote_type_id = context.reader.read_varuint32(); assert_eq!(local_type_id, remote_type_id); } @@ -139,18 +142,18 @@ pub trait ForyDefault: Sized { pub trait Serializer: 'static { /// Entry point of the serialization. - fn fory_write(&self, context: &mut WriteContext, is_field: bool) + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) where Self: Sized, { - write_ref_info_data(self, context, is_field, false, false); + write_ref_info_data(self, fory, context, is_field, false, false); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - read_ref_info_data(context, is_field, false, false) + read_ref_info_data(fory, context, is_field, false, false) } fn fory_is_option() -> bool @@ -198,93 +201,86 @@ pub trait Serializer: 'static { 0 } - fn fory_write_type_info(context: &mut WriteContext, _is_field: bool) + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, _is_field: bool) where Self: Sized, { // default implementation only for ext/named_ext - let type_id = Self::fory_get_type_id(context.get_fory()); + let type_id = Self::fory_get_type_id(fory); context.writer.write_varuint32(type_id); if type_id & 0xff == TypeId::EXT as u32 { return; } let rs_type_id = std::any::TypeId::of::(); - if context.get_fory().is_share_meta() { - let meta_index = context.push_meta(rs_type_id) as u32; + if fory.is_share_meta() { + let meta_index = context.push_meta(fory, rs_type_id) as u32; context.writer.write_varuint32(meta_index); } else { let type_info = { - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); type_resolver.get_type_info(rs_type_id) }; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - let resolver = context.get_fory().get_metastring_resolver(); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &namespace); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &type_name); + context.write_meta_string_bytes(&namespace); + context.write_meta_string_bytes(&type_name); } } - fn fory_read_type_info(context: &mut ReadContext, _is_field: bool) + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, _is_field: bool) where Self: Sized, { // default implementation only for ext/named_ext - let local_type_id = Self::fory_get_type_id(context.get_fory()); + let local_type_id = Self::fory_get_type_id(fory); let remote_type_id = context.reader.read_varuint32(); assert_eq!(local_type_id, remote_type_id); if local_type_id & 0xff == TypeId::EXT as u32 { return; } - if context.get_fory().is_share_meta() { + if fory.is_share_meta() { let _meta_index = context.reader.read_varuint32(); } else { - let resolver = context.get_fory().get_metastring_resolver(); - resolver.borrow_mut().read_meta_string_bytes(context); - resolver.borrow_mut().read_meta_string_bytes(context); + let _namespace_msb = context.read_meta_string_bytes(); + let _type_name_msb = context.read_meta_string_bytes(); } } // only used by struct/enum/ext - fn fory_read_compatible(context: &mut ReadContext) -> Result + fn fory_read_compatible(fory: &Fory, context: &mut ReadContext) -> Result where Self: Sized, { // default logic only for ext/named_ext let remote_type_id = context.reader.read_varuint32(); - let local_type_id = Self::fory_get_type_id(context.get_fory()); + let local_type_id = Self::fory_get_type_id(fory); assert_eq!(remote_type_id, local_type_id); if local_type_id & 0xff == TypeId::EXT as u32 { - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); type_resolver .get_ext_harness(local_type_id) - .get_read_data_fn()(context, true) + .get_read_data_fn()(fory, context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) .map_err(|_| anyhow!("downcast to Self failed").into()) }) } else { - let (namespace, type_name) = if context.get_fory().is_share_meta() { + let (namespace, type_name) = if fory.is_share_meta() { let meta_index = context.reader.read_varuint32(); let type_def = context.get_meta(meta_index as usize); (type_def.get_namespace(), type_def.get_type_name()) } else { - let resolver = context.get_fory().get_metastring_resolver(); - let nsb = resolver.borrow_mut().read_meta_string_bytes(context); - let tsb = resolver.borrow_mut().read_meta_string_bytes(context); + let nsb = context.read_meta_string_bytes(); + let tsb = context.read_meta_string_bytes(); let ns = NAMESPACE_DECODER.decode(&nsb.bytes, nsb.encoding)?; let ts = TYPE_NAME_DECODER.decode(&tsb.bytes, tsb.encoding)?; (ns, ts) }; - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); type_resolver .get_ext_name_harness(&namespace, &type_name) - .get_read_data_fn()(context, true) + .get_read_data_fn()(fory, context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) @@ -294,9 +290,13 @@ pub trait Serializer: 'static { } /// Write/Read the data into the buffer. Need to be implemented. - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool); - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result where Self: Sized + ForyDefault; @@ -325,7 +325,7 @@ pub trait StructSerializer: Serializer + 'static { struct_::actual_type_id(type_id, register_by_name, mode) } - fn fory_get_sorted_field_names(_fory: &Fory) -> Vec { + fn fory_get_sorted_field_names(_fory: &Fory) -> Arc> { unimplemented!() } } diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index b7d8c540e7..a16e22f717 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -52,21 +52,21 @@ use std::sync::Mutex; /// Simply delegates to the serializer for `T`, allowing thread-safe interior mutable /// containers to be included in serialized graphs. impl Serializer for Mutex { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // Don't add ref tracking for Mutex itself, just delegate to inner type // The inner type will handle its own ref tracking let guard = self.lock().unwrap(); - T::fory_write(&*guard, context, is_field); + T::fory_write(&*guard, fory, context, is_field); } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // When called from Rc/Arc, just delegate to inner type's data serialization let guard = self.lock().unwrap(); - T::fory_write_data(&*guard, context, is_field); + T::fory_write_data(&*guard, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { @@ -74,19 +74,23 @@ impl Serializer for Mutex { T::fory_reserved_space() } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - Ok(Mutex::new(T::fory_read(context, is_field)?)) + Ok(Mutex::new(T::fory_read(fory, context, is_field)?)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Mutex::new(T::fory_read_data(context, is_field)?)) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Ok(Mutex::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } fn fory_get_type_id(fory: &Fory) -> u32 { diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index a830fc3612..429afc4de4 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -26,11 +26,15 @@ use crate::types::TypeId; macro_rules! impl_num_serializer { ($ty:ty, $writer:expr, $reader:expr, $field_type:expr) => { impl Serializer for $ty { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { $writer(&mut context.writer, *self); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { Ok($reader(&mut context.reader)) } @@ -50,12 +54,12 @@ macro_rules! impl_num_serializer { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } impl ForyDefault for $ty { diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 8726c94b5d..78211b16a2 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -22,24 +22,28 @@ use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; impl Serializer for Option { - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Some(T::fory_read_data(context, is_field)?)) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Ok(Some(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { if let Some(v) = self { - T::fory_write_data(v, context, is_field) + T::fory_write_data(v, fory, context, is_field) } else { unreachable!("write should be call by serialize") } } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index b0cafc96d2..5d3c3716ff 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -28,24 +28,27 @@ impl Serializer for Rc { true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - if !context.ref_writer.try_write_rc_ref(context.writer, self) { - T::fory_write_data(self.as_ref(), context, is_field); + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + if !context + .ref_writer + .try_write_rc_ref(&mut context.writer, self) + { + T::fory_write_data(self.as_ref(), fory, context, is_field); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // When Rc is nested inside another shared ref (like Arc>), // the outer ref calls fory_write_data on the inner Rc. // We still need to track the Rc's own references here. - self.fory_write(context, is_field); + self.fory_write(fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { @@ -58,12 +61,12 @@ impl Serializer for Rc { .ok_or_else(|| anyhow!("Rc reference {} not found", ref_id).into()) } RefFlag::NotNullValue => { - let inner = T::fory_read_data(context, is_field)?; + let inner = T::fory_read_data(fory, context, is_field)?; Ok(Rc::new(inner)) } RefFlag::RefValue => { let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(context, is_field)?; + let inner = T::fory_read_data(fory, context, is_field)?; let rc = Rc::new(inner); context.ref_reader.store_rc_ref_at(ref_id, rc.clone()); Ok(rc) @@ -71,14 +74,18 @@ impl Serializer for Rc { } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { // When Rc is nested inside another shared ref, fory_read_data is called. // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(context, is_field) + Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index f10978c340..9fff58be7b 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -43,34 +43,38 @@ use std::cell::RefCell; /// Simply delegates to the serializer for `T`, allowing interior mutable /// containers to be included in serialized graphs. impl Serializer for RefCell { - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - Ok(RefCell::new(T::fory_read(context, is_field)?)) + Ok(RefCell::new(T::fory_read(fory, context, is_field)?)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(RefCell::new(T::fory_read_data(context, is_field)?)) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Ok(RefCell::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // Don't add ref tracking for RefCell itself, just delegate to inner type // The inner type will handle its own ref tracking - T::fory_write(&*self.borrow(), context, is_field); + T::fory_write(&*self.borrow(), fory, context, is_field); } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { // When called from Rc, just delegate to inner type's data serialization - T::fory_write_data(&*self.borrow(), context, is_field) + T::fory_write_data(&*self.borrow(), fory, context, is_field) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs index 47ebe0b0ba..7755cbaf7e 100644 --- a/rust/fory-core/src/serializer/set.rs +++ b/rust/fory-core/src/serializer/set.rs @@ -29,19 +29,23 @@ use std::collections::{BTreeSet, HashSet}; use std::mem; impl Serializer for HashSet { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_collection(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_collection(self, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { write_collection_type_info(context, is_field, TypeId::SET as u32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + read_collection(fory, context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -69,19 +73,23 @@ impl ForyDefault for HashSet { } impl Serializer for BTreeSet { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_collection(self, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_collection(self, fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { write_collection_type_info(context, is_field, TypeId::SET as u32); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { + read_collection(fory, context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { read_collection_type_info(context, is_field, TypeId::SET as u32) } diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index c1538dbfee..dda7ff175a 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -21,6 +21,7 @@ use crate::resolver::context::ReadContext; use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE}; use crate::serializer::Serializer; use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPES}; +use crate::Fory; use chrono::{NaiveDate, NaiveDateTime}; pub fn get_read_ref_flag(field_type: &NullableFieldType) -> bool { @@ -29,11 +30,11 @@ pub fn get_read_ref_flag(field_type: &NullableFieldType) -> bool { } macro_rules! basic_type_deserialize { - ($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { + ($fory:expr, $tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { $( if $tid == TypeId::$id { - <$ty as Serializer>::fory_read_type_info($context, true); - <$ty as Serializer>::fory_read_data($context, true)?; + <$ty as Serializer>::fory_read_type_info($fory, $context, true); + <$ty as Serializer>::fory_read_data($fory, $context, true)?; return Ok(()); } )+else { @@ -45,6 +46,7 @@ macro_rules! basic_type_deserialize { // call when is_field && is_compatible_mode #[allow(unreachable_code)] pub fn skip_field_value( + fory: &Fory, context: &mut ReadContext, field_type: &NullableFieldType, read_ref_flag: bool, @@ -59,7 +61,7 @@ pub fn skip_field_value( match TypeId::try_from(type_id_num as i16) { Ok(type_id) => { if BASIC_TYPES.contains(&type_id) { - basic_type_deserialize!(type_id, context; + basic_type_deserialize!(fory, type_id, context; (bool, BOOL), (i8, INT8), (i16, INT16), @@ -91,7 +93,7 @@ pub fn skip_field_value( let elem_type = field_type.generics.first().unwrap(); context.inc_depth()?; for _ in 0..length { - skip_field_value(context, elem_type, !skip_ref_flag)?; + skip_field_value(fory, context, elem_type, !skip_ref_flag)?; } context.dec_depth(); } else if type_id == TypeId::MAP { @@ -116,7 +118,7 @@ pub fn skip_field_value( if header & crate::serializer::map::KEY_NULL != 0 { // let read_ref_flag = get_read_ref_flag(value_type); context.inc_depth()?; - skip_field_value(context, value_type, false)?; + skip_field_value(fory, context, value_type, false)?; context.dec_depth(); len_counter += 1; continue; @@ -124,7 +126,7 @@ pub fn skip_field_value( if header & crate::serializer::map::VALUE_NULL != 0 { // let read_ref_flag = get_read_ref_flag(key_type); context.inc_depth()?; - skip_field_value(context, key_type, false)?; + skip_field_value(fory, context, key_type, false)?; context.dec_depth(); len_counter += 1; continue; @@ -133,9 +135,9 @@ pub fn skip_field_value( context.inc_depth()?; for _ in (0..chunk_size).enumerate() { // let read_ref_flag = get_read_ref_flag(key_type); - skip_field_value(context, key_type, false)?; + skip_field_value(fory, context, key_type, false)?; // let read_ref_flag = get_read_ref_flag(value_type); - skip_field_value(context, value_type, false)?; + skip_field_value(fory, context, value_type, false)?; } context.dec_depth(); len_counter += chunk_size as u32; @@ -153,10 +155,9 @@ pub fn skip_field_value( let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let nullable_field_type = - NullableFieldType::from(field_info.field_type.clone()); + let nullable_field_type = NullableFieldType::from(&field_info.field_type); let read_ref_flag = get_read_ref_flag(&nullable_field_type); - skip_field_value(context, &nullable_field_type, read_ref_flag)?; + skip_field_value(fory, context, &nullable_field_type, read_ref_flag)?; } context.dec_depth(); Ok(()) @@ -165,10 +166,10 @@ pub fn skip_field_value( assert_eq!(type_id_num, remote_type_id); let meta_index = context.reader.read_varuint32(); let type_meta = context.get_meta(meta_index as usize); - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); type_resolver .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name()) - .get_read_data_fn()(context, true)?; + .get_read_data_fn()(fory, context, true)?; Ok(()) } else { unreachable!("unimplemented type: {:?}", type_id); @@ -187,10 +188,9 @@ pub fn skip_field_value( let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let nullable_field_type = - NullableFieldType::from(field_info.field_type.clone()); + let nullable_field_type = NullableFieldType::from(&field_info.field_type); let read_ref_flag = get_read_ref_flag(&nullable_field_type); - skip_field_value(context, &nullable_field_type, read_ref_flag)?; + skip_field_value(fory, context, &nullable_field_type, read_ref_flag)?; } context.dec_depth(); } else if internal_id == ENUM_ID { @@ -199,10 +199,10 @@ pub fn skip_field_value( let remote_type_id = context.reader.read_varuint32(); assert_eq!(remote_type_id, type_id_num); context.inc_depth()?; - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); type_resolver .get_ext_harness(type_id_num) - .get_read_data_fn()(context, true)?; + .get_read_data_fn()(fory, context, true)?; context.dec_depth(); } else { unreachable!("unimplemented skipped type: {:?}", type_id_num); diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index 65c2eb77b1..9fb68d69b1 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -31,13 +31,13 @@ enum StrEncoding { } impl Serializer for String { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, _is_field: bool) { let mut len = get_latin1_length(self); if len >= 0 { let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64; context.writer.write_varuint36_small(bitor); context.writer.write_latin1_string(self); - } else if context.get_fory().is_compress_string() { + } else if fory.is_compress_string() { // todo: support `writeNumUtf16BytesForUtf8Encoding` like in java len = self.len() as i32; let bitor = (len as u64) << 2 | StrEncoding::Utf8 as u64; @@ -51,7 +51,11 @@ impl Serializer for String { } } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { let bitor = context.reader.read_varuint36small(); let len = bitor >> 2; let encoding = bitor & 0b11; @@ -87,12 +91,12 @@ impl Serializer for String { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - write_type_info::(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_type_info::(fory, context, is_field); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - read_type_info::(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + read_type_info::(fory, context, is_field); } } diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index b8b553c727..a171d18596 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -43,14 +43,21 @@ pub fn type_def( namespace: MetaString, type_name: MetaString, register_by_name: bool, - field_infos: &[FieldInfo], + mut field_infos: Vec, ) -> Vec { let sorted_field_names = T::fory_get_sorted_field_names(fory); - let mut sorted_field_infos = Vec::with_capacity(field_infos.len()); - for name in &sorted_field_names { - if let Some(info) = field_infos.iter().find(|f| &f.field_name == name) { - sorted_field_infos.push(info.clone()); - } else { + let mut sorted_field_infos: Vec = Vec::with_capacity(field_infos.len()); + for name in sorted_field_names.iter() { + let mut found = false; + for i in 0..field_infos.len() { + if &field_infos[i].field_name == name { + // swap_remove is faster + sorted_field_infos.push(field_infos.swap_remove(i)); + found = true; + break; + } + } + if !found { panic!("Field {} not found in field_infos", name); } } @@ -65,51 +72,42 @@ pub fn type_def( } #[inline(always)] -pub fn write_type_info(context: &mut WriteContext, _is_field: bool) { - let type_id = T::fory_get_type_id(context.get_fory()); +pub fn write_type_info(fory: &Fory, context: &mut WriteContext, _is_field: bool) { + let type_id = T::fory_get_type_id(fory); context.writer.write_varuint32(type_id); let rs_type_id = std::any::TypeId::of::(); if type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if context.get_fory().is_share_meta() { - let meta_index = context.push_meta(rs_type_id) as u32; + if fory.is_share_meta() { + let meta_index = context.push_meta(fory, rs_type_id) as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = context - .get_fory() - .get_type_resolver() - .get_type_info(rs_type_id); + let type_info = fory.get_type_resolver().get_type_info(rs_type_id); let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - let resolver = context.get_fory().get_metastring_resolver(); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &namespace); - resolver - .borrow_mut() - .write_meta_string_bytes(context, &type_name); + context.write_meta_string_bytes(&namespace); + context.write_meta_string_bytes(&type_name); } } else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { - let meta_index = context.push_meta(rs_type_id) as u32; + let meta_index = context.push_meta(fory, rs_type_id) as u32; context.writer.write_varuint32(meta_index); } } #[inline(always)] -pub fn read_type_info(context: &mut ReadContext, _is_field: bool) { +pub fn read_type_info(fory: &Fory, context: &mut ReadContext, _is_field: bool) { let remote_type_id = context.reader.read_varuint32(); - let local_type_id = T::fory_get_type_id(context.get_fory()); + let local_type_id = T::fory_get_type_id(fory); assert_eq!(remote_type_id, local_type_id); if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if context.get_fory().is_share_meta() { + if fory.is_share_meta() { let _meta_index = context.reader.read_varuint32(); } else { - let resolver = context.get_fory().get_metastring_resolver(); - resolver.borrow_mut().read_meta_string_bytes(context); - resolver.borrow_mut().read_meta_string_bytes(context); + let _namespace_msb = context.read_meta_string_bytes(); + let _type_name_msb = context.read_meta_string_bytes(); } } else if local_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || local_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 @@ -119,18 +117,18 @@ pub fn read_type_info(context: &mut ReadContext, _is_field: bool) } #[inline(always)] -pub fn write(this: &T, context: &mut WriteContext, _is_field: bool) { - match context.get_fory().get_mode() { +pub fn write(this: &T, fory: &Fory, context: &mut WriteContext, _is_field: bool) { + match fory.get_mode() { // currently same Mode::SchemaConsistent => { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, false); - this.fory_write_data(context, true); + T::fory_write_type_info(fory, context, false); + this.fory_write_data(fory, context, true); } Mode::Compatible => { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, false); - this.fory_write_data(context, true); + T::fory_write_type_info(fory, context, false); + this.fory_write_data(fory, context, true); } } } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index d4ce4c39a6..29ea1a7bdc 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -24,6 +24,7 @@ use crate::serializer::{ForyDefault, Serializer}; /// /// Writes common trait object headers (ref flag, type ID, compatibility metadata) pub fn write_trait_object_headers( + fory: &Fory, context: &mut WriteContext, fory_type_id: u32, concrete_type_id: std::any::TypeId, @@ -33,17 +34,17 @@ pub fn write_trait_object_headers( context.writer.write_i8(RefFlag::NotNullValue as i8); context.writer.write_varuint32(fory_type_id); - if context.get_fory().get_mode() == &Mode::Compatible + if fory.get_mode() == &Mode::Compatible && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { - let meta_index = context.push_meta(concrete_type_id) as u32; + let meta_index = context.push_meta(fory, concrete_type_id) as u32; context.writer.write_varuint32(meta_index); } } /// Reads common trait object headers and returns the type ID -pub fn read_trait_object_headers(context: &mut ReadContext) -> Result { +pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Result { use crate::types::{Mode, RefFlag, TypeId}; let ref_flag = context.reader.read_i8(); @@ -56,7 +57,7 @@ pub fn read_trait_object_headers(context: &mut ReadContext) -> Result Result {{ + ($any_ref:expr, $fory:expr, $context:expr, $is_field:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( if $any_ref.type_id() == std::any::TypeId::of::<$impl_type>() { if let Some(concrete) = $any_ref.downcast_ref::<$impl_type>() { - concrete.fory_write_data($context, $is_field); + concrete.fory_write_data($fory, $context, $is_field); return; } } @@ -85,11 +86,11 @@ macro_rules! downcast_and_serialize { /// Helper macro for common type resolution and deserialization pattern #[macro_export] macro_rules! resolve_and_deserialize { - ($fory_type_id:expr, $context:expr, $is_field:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ + ($fory_type_id:expr, $fory:expr, $context:expr, $is_field:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( - if let Some(registered_type_id) = $context.get_fory().get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { + if let Some(registered_type_id) = $fory.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { if $fory_type_id == registered_type_id { - let concrete_obj = <$impl_type as $crate::serializer::Serializer>::fory_read_data($context, $is_field)?; + let concrete_obj = <$impl_type as $crate::serializer::Serializer>::fory_read_data($fory, $context, $is_field)?; return Ok($constructor(concrete_obj)); } } @@ -200,21 +201,21 @@ macro_rules! register_trait_type { // 4. Serializer implementation for Box (existing functionality) impl $crate::serializer::Serializer for Box { - fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); - if let Some(fory_type_id) = context.get_fory().get_type_resolver().get_fory_type_id(concrete_type_id) { - $crate::serializer::trait_object::write_trait_object_headers(context, fory_type_id, concrete_type_id); - $crate::downcast_and_serialize!(any_ref, context, is_field, $trait_name, $($impl_type),+); + if let Some(fory_type_id) = fory.get_type_resolver().get_fory_type_id(concrete_type_id) { + $crate::serializer::trait_object::write_trait_object_headers(fory, context, fory_type_id, concrete_type_id); + $crate::downcast_and_serialize!(any_ref, fory, context, is_field, $trait_name, $($impl_type),+); } else { panic!("Type {:?} not registered for Box serialization", concrete_type_id, stringify!($trait_name)); } } - fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { // Delegate to fory_write since this handles the polymorphic dispatch - self.fory_write(context, is_field); + self.fory_write(fory, context, is_field); } fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> u32 { @@ -229,19 +230,19 @@ macro_rules! register_trait_type { true } - fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { + fn fory_write_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { // Box is polymorphic - type info is written per element } - fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { + fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { // Box is polymorphic - type info is read per element } - fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { context.inc_depth()?; - let fory_type_id = $crate::serializer::trait_object::read_trait_object_headers(context)?; + let fory_type_id = $crate::serializer::trait_object::read_trait_object_headers(fory, context)?; let result = $crate::resolve_and_deserialize!( - fory_type_id, context, is_field, + fory_type_id, fory, context, is_field, |obj| Box::new(obj) as Box, $trait_name, $($impl_type),+ ); @@ -249,7 +250,7 @@ macro_rules! register_trait_type { result } - fn fory_read_data(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result { + fn fory_read_data(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result { // This should not be called for polymorphic types like Box // The fory_read method handles the polymorphic dispatch panic!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)); @@ -404,22 +405,22 @@ macro_rules! generate_smart_pointer_wrapper { macro_rules! impl_smart_pointer_serializer { ($wrapper_name:ident, $pointer_type:ty, $constructor_expr:expr, $trait_name:ident, $try_write_ref:ident, $get_ref:ident, $store_ref:ident, $($impl_type:ty),+) => { impl $crate::serializer::Serializer for $wrapper_name { - fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { - if !context.ref_writer.$try_write_ref(context.writer, &self.0) { + fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + if !context.ref_writer.$try_write_ref(&mut context.writer, &self.0) { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); - let harness = context.write_any_typeinfo(concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id); let serializer_fn = harness.get_write_data_fn(); - serializer_fn(any_obj, context, is_field); + serializer_fn(any_obj, fory, context, is_field); } } - fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { let any_obj = ::as_any(&*self.0); - $crate::downcast_and_serialize!(any_obj, context, is_field, $trait_name, $($impl_type),+); + $crate::downcast_and_serialize!(any_obj, fory, context, is_field, $trait_name, $($impl_type),+); } - fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { use $crate::types::RefFlag; let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); @@ -438,9 +439,9 @@ macro_rules! impl_smart_pointer_serializer { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(context, is_field)?; + let boxed_any = deserializer_fn(fory, context, is_field)?; context.dec_depth(); $( @@ -458,9 +459,9 @@ macro_rules! impl_smart_pointer_serializer { } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(); + let harness = context.read_any_typeinfo(fory); let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(context, is_field)?; + let boxed_any = deserializer_fn(fory, context, is_field)?; context.dec_depth(); $( @@ -479,10 +480,10 @@ macro_rules! impl_smart_pointer_serializer { } } } - fn fory_read_data(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read_data(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { let concrete_fory_type_id = context.reader.read_varuint32(); $crate::resolve_and_deserialize!( - concrete_fory_type_id, context, is_field, + concrete_fory_type_id, fory, context, is_field, |obj| { let pointer = $constructor_expr(obj) as $pointer_type; Self::from(pointer) @@ -495,10 +496,10 @@ macro_rules! impl_smart_pointer_serializer { $crate::types::TypeId::STRUCT as u32 } - fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { } - fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { + fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { } fn fory_is_polymorphic() -> bool { @@ -538,15 +539,15 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) { - let fory_type_id = (**self).fory_type_id_dyn(context.get_fory()); + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + let fory_type_id = (**self).fory_type_id_dyn(fory); let concrete_type_id = (**self).fory_concrete_type_id(); - write_trait_object_headers(context, fory_type_id, concrete_type_id); - (**self).fory_write_data(context, is_field); + write_trait_object_headers(fory, context, fory_type_id, concrete_type_id); + (**self).fory_write_data(fory, context, is_field); } - fn fory_write_data(&self, _context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, _fory: &Fory, _context: &mut WriteContext, _is_field: bool) { panic!("fory_write_data should not be called directly on Box"); } @@ -562,23 +563,23 @@ impl Serializer for Box { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { // Box is polymorphic - type info is written per element } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { // Box is polymorphic - type info is read per element } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { - let fory_type_id = read_trait_object_headers(context)?; + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { + let fory_type_id = read_trait_object_headers(fory, context)?; context.inc_depth()?; - let type_resolver = context.get_fory().get_type_resolver(); + let type_resolver = fory.get_type_resolver(); if let Some(harness) = type_resolver.get_harness(fory_type_id) { let deserializer_fn = harness.get_read_fn(); let to_serializer_fn = harness.get_to_serializer(); - let boxed_any = deserializer_fn(context, is_field, true)?; + let boxed_any = deserializer_fn(fory, context, is_field, true)?; let trait_object = to_serializer_fn(boxed_any)?; context.dec_depth(); Ok(trait_object) @@ -613,7 +614,11 @@ impl Serializer for Box { } } } - fn fory_read_data(_context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + _fory: &Fory, + _context: &mut ReadContext, + _is_field: bool, + ) -> Result { panic!("fory_read_data should not be called directly on Box"); } } diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 57b722e375..f0c8e3fbab 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -313,33 +313,36 @@ impl Serializer for RcWeak { true } - fn fory_write(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { if let Some(rc) = self.upgrade() { - if context.ref_writer.try_write_rc_ref(context.writer, &rc) { + if context + .ref_writer + .try_write_rc_ref(&mut context.writer, &rc) + { return; } - T::fory_write_data(&*rc, context, _is_field); + T::fory_write_data(&*rc, fory, context, is_field); } else { context.writer.write_i8(RefFlag::Null as i8); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - self.fory_write(context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + self.fory_write(fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { RefFlag::Null => Ok(RcWeak::new()), RefFlag::RefValue => { context.inc_depth()?; - let data = T::fory_read_data(context, _is_field)?; + let data = T::fory_read_data(fory, context, is_field)?; context.dec_depth(); let rc = Rc::new(data); let ref_id = context.ref_reader.store_rc_ref(rc); @@ -368,12 +371,16 @@ impl Serializer for RcWeak { } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { @@ -409,36 +416,39 @@ impl Serializer for ArcWeak true } - fn fory_write(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { if let Some(arc) = self.upgrade() { // IMPORTANT: If the target Arc was serialized already, just write a ref - if context.ref_writer.try_write_arc_ref(context.writer, &arc) { + if context + .ref_writer + .try_write_arc_ref(&mut context.writer, &arc) + { // Already seen, wrote Ref flag + id, we're done return; } // First time seeing this object, write RefValue and then its data - T::fory_write_data(&*arc, context, _is_field); + T::fory_write_data(&*arc, fory, context, is_field); } else { context.writer.write_i8(RefFlag::Null as i8); } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - self.fory_write(context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + self.fory_write(fory, context, is_field); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(context, is_field); + fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { + T::fory_write_type_info(fory, context, is_field); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); match ref_flag { RefFlag::Null => Ok(ArcWeak::new()), RefFlag::RefValue => { context.inc_depth()?; - let data = T::fory_read_data(context, _is_field)?; + let data = T::fory_read_data(fory, context, _is_field)?; context.dec_depth(); let arc = Arc::new(data); let ref_id = context.ref_reader.store_arc_ref(arc); @@ -469,12 +479,16 @@ impl Serializer for ArcWeak _ => Err(anyhow!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into()), } } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { + Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(context, is_field); + fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { + T::fory_read_type_info(fory, context, is_field); } fn fory_reserved_space() -> usize { diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 876a97c8e1..df35fdd4ac 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -39,13 +39,13 @@ pub fn gen_reserved_space() -> TokenStream { pub fn gen_write_type_info() -> TokenStream { quote! { - fory_core::serializer::enum_::write_type_info::(context, is_field) + fory_core::serializer::enum_::write_type_info::(fory, context, is_field) } } pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::enum_::read_type_info::(context, is_field) + fory_core::serializer::enum_::read_type_info::(fory, context, is_field) } } @@ -79,18 +79,18 @@ pub fn gen_read_data(data_enum: &DataEnum) -> TokenStream { pub fn gen_read_compatible() -> TokenStream { quote! { - fory_core::serializer::enum_::read_compatible::(context) + fory_core::serializer::enum_::read_compatible::(fory, context) } } pub fn gen_write(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::write::(self, context, is_field) + fory_core::serializer::enum_::write::(self, fory, context, is_field) } } pub fn gen_read(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::read::(context, is_field) + fory_core::serializer::enum_::read::(fory, context, is_field) } } diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index ec05f0f4c1..80fedeb201 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -71,8 +71,9 @@ pub fn gen_get_sorted_field_names(fields: &[&Field]) -> TokenStream { Some(result) => result, None => { #create_sorted_field_names - fory.get_type_resolver().set_sorted_field_names::(&sorted_field_names); - sorted_field_names + let arc_sorted_field_names = std::sync::Arc::new(sorted_field_names); + fory.get_type_resolver().set_sorted_field_names::(arc_sorted_field_names.clone()); + arc_sorted_field_names } }; sorted_field_names @@ -86,7 +87,7 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { match classify_trait_object_field(ty) { StructField::None => { let generic_tree = parse_generic_tree(ty); - let generic_token = generic_tree_to_tokens(&generic_tree, false); + let generic_token = generic_tree_to_tokens(&generic_tree); quote! { fory_core::meta::FieldInfo::new(#name, #generic_token) } @@ -104,7 +105,7 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { } StructField::HashMapRc(key_ty, _) | StructField::HashMapArc(key_ty, _) => { let key_generic_tree = parse_generic_tree(key_ty.as_ref()); - let key_generic_token = generic_tree_to_tokens(&key_generic_tree, false); + let key_generic_token = generic_tree_to_tokens(&key_generic_tree); quote! { fory_core::meta::FieldInfo::new(#name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::MAP as u32, @@ -130,6 +131,6 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { }); quote! { let field_infos: Vec = vec![#(#field_infos),*]; - fory_core::serializer::struct_::type_def::(fory, type_id, namespace, type_name, register_by_name, &field_infos) + fory_core::serializer::struct_::type_def::(fory, type_id, namespace, type_name, register_by_name, field_infos) } } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index b41e05de06..c87e1bfda8 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -100,13 +100,12 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let fory_type_id = context.reader.read_varuint32(); - let harness = context.get_fory() - .get_type_resolver() + let harness = fory.get_type_resolver() .get_harness(fory_type_id) .expect("Type not registered for trait object field"); let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(context, true, false)?; + let any_box = deserializer_fn(fory, context, true, false)?; let base_type_id = fory_type_id >> 8; #private_ident = #helper_mod::#from_any_fn(any_box, base_type_id)?; @@ -119,7 +118,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = std::rc::Rc::::from(wrapper); } } @@ -130,7 +129,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = std::sync::Arc::::from(wrapper); } } @@ -141,7 +140,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = Some(wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect()); @@ -154,7 +153,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = Some(wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect()); @@ -167,7 +166,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect()); @@ -180,7 +179,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let trait_ident = types.trait_ident; quote! { #name_str => { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; #private_ident = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect()); @@ -190,15 +189,15 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { StructField::Forward => { quote! { #name_str => { - #private_ident = Some(fory_core::serializer::Serializer::fory_read(context, true)?); + #private_ident = Some(fory_core::serializer::Serializer::fory_read(fory, context, true)?); } } } _ => { quote! { #name_str => { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory()); - #private_ident = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, skip_ref_flag, false)?); + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); + #private_ident = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); } } } @@ -207,7 +206,7 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::read_type_info::(context, is_field) + fory_core::serializer::struct_::read_type_info::(fory, context, is_field) } } @@ -275,7 +274,8 @@ pub fn gen_read_data(fields: &[&Field]) -> TokenStream { let loop_ts = get_fields_loop_ts(fields); quote! { #(#declare_var_ts)* - let field_names = ::fory_get_sorted_field_names(context.get_fory()); + let field_names = ::fory_get_sorted_field_names(fory); + let field_names = field_names.as_ref(); #loop_ts } }; @@ -324,12 +324,11 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream panic!("Expected NotNullValue for trait object field"); } let fory_type_id = context.reader.read_varuint32(); - let harness = context.get_fory() - .get_type_resolver() + let harness = fory.get_type_resolver() .get_harness(fory_type_id) .expect("Type not registered for trait object field"); let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(context, true, false).unwrap(); + let any_box = deserializer_fn(fory, context, true, false).unwrap(); let base_type_id = fory_type_id >> 8; #var_name = #helper_mod::#from_any_fn(any_box, base_type_id).unwrap(); } @@ -341,7 +340,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(std::rc::Rc::::from(wrapper)); } } @@ -352,7 +351,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(std::sync::Arc::::from(wrapper)); } } @@ -363,7 +362,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect()); @@ -376,7 +375,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect()); @@ -389,7 +388,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect()); @@ -402,7 +401,7 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let trait_ident = types.trait_ident; quote! { if _field.field_name.as_str() == #field_name_str { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true).unwrap(); + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect()); @@ -412,21 +411,21 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream StructField::ContainsTraitObject => { quote! { if _field.field_name.as_str() == #field_name_str { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory()); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, skip_ref_flag, false)?); + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); } } } StructField::Forward => { quote! { if _field.field_name.as_str() == #field_name_str { - #var_name = Some(fory_core::serializer::Serializer::fory_read(context, true).unwrap()); + #var_name = Some(fory_core::serializer::Serializer::fory_read(fory, context, true).unwrap()); } } } StructField::None => { let generic_tree = parse_generic_tree(ty); - let generic_token = generic_tree_to_tokens(&generic_tree, true); + let generic_token = generic_tree_to_tokens(&generic_tree); let read_nullable_fn_name = create_read_nullable_fn_name(field); let _base_ty = match &ty { @@ -437,22 +436,23 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream if _field.field_name.as_str() == #field_name_str { let local_field_type = #generic_token; if &_field.field_type == &local_field_type { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory()); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, skip_ref_flag, false).unwrap_or_else(|_err| { + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false).unwrap_or_else(|_err| { panic!("Err at deserializing {:?}: {:?}", #field_name_str, _err); })); } else { - let local_nullable_type = fory_core::meta::NullableFieldType::from(local_field_type.clone()); - let remote_nullable_type = fory_core::meta::NullableFieldType::from(_field.field_type.clone()); + let local_nullable_type = fory_core::meta::NullableFieldType::from(&local_field_type); + let remote_nullable_type = fory_core::meta::NullableFieldType::from(&_field.field_type); if local_nullable_type != remote_nullable_type { println!("Type not match, just skip: {}", #field_name_str); let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&remote_nullable_type); - fory_core::serializer::skip::skip_field_value(context, &remote_nullable_type, read_ref_flag).unwrap(); + fory_core::serializer::skip::skip_field_value(fory, context, &remote_nullable_type, read_ref_flag).unwrap(); #var_name = Some(<#ty as fory_core::serializer::ForyDefault>::fory_default()); } else { println!("Try to deserialize_compatible: {}", #field_name_str); #var_name = Some( Self::#read_nullable_fn_name( + fory, context, &local_nullable_type, &remote_nullable_type @@ -472,13 +472,13 @@ pub fn gen_read(struct_ident: &Ident) -> TokenStream { quote! { let ref_flag = context.reader.read_i8(); if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::types::RefFlag::RefValue as i8) { - match context.get_fory().get_mode() { + match fory.get_mode() { fory_core::types::Mode::SchemaConsistent => { - ::fory_read_type_info(context, false); - ::fory_read_data(context, false) + ::fory_read_type_info(fory, context, false); + ::fory_read_data(fory, context, false) }, fory_core::types::Mode::Compatible => { - <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(context) + <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) }, _ => unreachable!() } @@ -512,20 +512,21 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { }; #(#declare_ts)* - let local_type_def = context.get_fory().get_type_resolver().get_type_info(std::any::TypeId::of::()).get_type_def(); + let local_type_def = fory.get_type_resolver().get_type_info(std::any::TypeId::of::()).get_type_def(); let high_bytes = &local_type_def[..8]; let local_type_hash = i64::from_le_bytes(high_bytes.try_into().unwrap()); if meta.get_hash() == local_type_hash { // fast path - let field_names = ::fory_get_sorted_field_names(context.get_fory()); + let field_names = ::fory_get_sorted_field_names(fory); + let field_names = field_names.as_ref(); #consistent_fields_loop_ts } else { for _field in fields.iter() { #(#pattern_items else)* { println!("skip {:?}:{:?}", _field.field_name.as_str(), _field.field_type); - let nullable_field_type = fory_core::meta::NullableFieldType::from(_field.field_type.clone()); + let nullable_field_type = fory_core::meta::NullableFieldType::from(&_field.field_type); let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&nullable_field_type); - fory_core::serializer::skip::skip_field_value(context, &nullable_field_type, read_ref_flag).unwrap(); + fory_core::serializer::skip::skip_field_value(fory, context, &nullable_field_type, read_ref_flag).unwrap(); } } } @@ -548,6 +549,7 @@ pub fn gen_read_nullable(fields: &[&Field]) -> TokenStream { let read_tokens = nullable_generic_tree.to_read_tokens(&vec![], true); Some(quote! { fn #fn_name( + fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, local_nullable_type: &fory_core::meta::NullableFieldType, remote_nullable_type: &fory_core::meta::NullableFieldType diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index fc76ba9be8..20f0421c4a 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -135,7 +135,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #actual_type_id_ts } - fn fory_get_sorted_field_names(fory: &fory_core::fory::Fory) -> Vec { + fn fory_get_sorted_field_names(fory: &fory_core::fory::Fory) -> std::sync::Arc> { #get_sorted_field_names_ts } @@ -160,31 +160,31 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #reserved_space_ts } - fn fory_write_type_info(context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { #write_type_info_ts } - fn fory_read_type_info(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) { + fn fory_read_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) { #read_type_info_ts } - fn fory_write_data(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { #write_data_ts } - fn fory_read_data(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read_data(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { #read_data_ts } - fn fory_write(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { #write_ts } - fn fory_read(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { #read_ts } - fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext) -> Result { + fn fory_read_compatible(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext) -> Result { #read_compatible_ts } } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 6eee2120e1..8626bf4629 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -230,15 +230,15 @@ macro_rules! basic_type_deserialize { $ty_str => { if $nullable { quote! { - <$ty as fory_core::serializer::Serializer>::fory_read_type_info(context, true); - let res1 = Some(<$ty as fory_core::serializer::Serializer>::fory_read_data(context, true) + <$ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); + let res1 = Some(<$ty as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) .map_err(fory_core::error::Error::from)?); Ok::, fory_core::error::Error>(res1) } } else { quote! { - <$ty as fory_core::serializer::Serializer>::fory_read_type_info(context, true); - let res2 = <$ty as fory_core::serializer::Serializer>::fory_read_data(context, true) + <$ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); + let res2 = <$ty as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) .map_err(fory_core::error::Error::from)?; Ok::<$ty, fory_core::error::Error>(res2) } @@ -310,8 +310,8 @@ impl NullableTypeNode { let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { None } else { - <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(context, true); - Some(<#ty_type as fory_core::serializer::Serializer>::fory_read_data(context, true) + <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); + Some(<#ty_type as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) .map_err(fory_core::error::Error::from)?) }; Ok::, fory_core::error::Error>(res1) @@ -321,8 +321,8 @@ impl NullableTypeNode { let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { Vec::default() } else { - <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(context, true); - <#ty_type as fory_core::serializer::Serializer>::fory_read_data(context, true) + <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); + <#ty_type as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) .map_err(fory_core::error::Error::from)? }; Ok::<#ty_type, fory_core::error::Error>(res2) @@ -490,8 +490,8 @@ impl NullableTypeNode { continue; } let chunk_size = context.reader.read_u8(); - <#key_ty as fory_core::serializer::Serializer>::fory_read_type_info(context, true); - <#val_ty as fory_core::serializer::Serializer>::fory_read_type_info(context, true); + <#key_ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); + <#val_ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); for _ in (0..chunk_size).enumerate() { let key: #key_ty = {#key_tokens}?; let value: #val_ty = {#val_tokens}?; @@ -543,7 +543,7 @@ impl NullableTypeNode { || internal_id == EXT_ID || internal_id == NAMED_EXT_ID { - <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(context) + <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) .map_err(fory_core::error::Error::from)? } else { unimplemented!() @@ -566,7 +566,7 @@ impl NullableTypeNode { || internal_id == EXT_ID || internal_id == NAMED_EXT_ID { - <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(context) + <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) .map_err(fory_core::error::Error::from)? } else { unimplemented!() @@ -741,7 +741,7 @@ pub(super) fn parse_generic_tree(ty: &Type) -> TypeNode { TypeNode { name, generics } } -pub(super) fn generic_tree_to_tokens(node: &TypeNode, have_context: bool) -> TokenStream { +pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { if node.name == "Option" { if let Some(first_generic) = node.generics.first() { if first_generic.name == "Option" { @@ -758,23 +758,11 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode, have_context: bool) -> Tok let primitive_vec = try_primitive_vec_type(node); let children_tokens: Vec = if primitive_vec.is_none() { - node.generics - .iter() - .map(|child| generic_tree_to_tokens(child, have_context)) - .collect() + node.generics.iter().map(generic_tree_to_tokens).collect() } else { vec![] }; let ty: syn::Type = syn::parse_str(&node.to_string()).unwrap(); - let param = if have_context { - quote! { - context.get_fory() - } - } else { - quote! { - fory - } - }; let get_type_id = if node.name == "Option" { let option_type_id = TypeId::ForyNullable as u32; quote! { #option_type_id } @@ -782,7 +770,7 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode, have_context: bool) -> Tok ts } else { quote! { - <#ty as fory_core::serializer::Serializer>::fory_get_type_id(#param) + <#ty as fory_core::serializer::Serializer>::fory_get_type_id(fory) } }; quote! { @@ -1055,8 +1043,7 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { }, quote! { final_fields.sort_by(#sorter_ts); - let final_field_names: Vec = final_fields.iter().map(|(_, name)| name.clone()).collect(); - sorted_field_names.extend(final_field_names); + for (_, name) in final_fields.drain(..) { sorted_field_names.push(name); } }, ) } @@ -1071,8 +1058,7 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { }, quote! { other_fields.sort_by(#sorter_ts); - let other_field_names: Vec = other_fields.iter().map(|(_, name)| name.clone()).collect(); - sorted_field_names.extend(other_field_names); + for (_, name) in other_fields.drain(..) { sorted_field_names.push(name); } }, ) } @@ -1138,6 +1124,8 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { let (final_declare, final_extend) = final_fields_declare_extend_ts; let (other_declare, other_extend) = other_fields_declare_extend_ts; + let fields_len = fields.len(); + quote! { let sorted_field_names = { #all_primitive_declare @@ -1148,7 +1136,7 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { #trait_object_fields_ts #group_sort_enum_other_fields - let mut sorted_field_names: Vec = Vec::new(); + let mut sorted_field_names: Vec = Vec::with_capacity(#fields_len); #all_primitive_extend #final_extend #other_extend diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 5aa8620841..6adba6780c 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -94,7 +94,7 @@ pub fn gen_reserved_space(fields: &[&Field]) -> TokenStream { pub fn gen_write_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::write_type_info::(context, is_field) + fory_core::serializer::struct_::write_type_info::(fory, context, is_field) } } @@ -109,21 +109,20 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { #name_str => { let any_ref = self.#ident.as_any(); let concrete_type_id = any_ref.type_id(); - let fory_type_id = context.get_fory() - .get_type_resolver() + let fory_type_id = fory.get_type_resolver() .get_fory_type_id(concrete_type_id) .expect("Type not registered for trait object field"); context.writer.write_i8(fory_core::types::RefFlag::NotNullValue as i8); context.writer.write_varuint32(fory_type_id); - let harness = context.get_fory() + let harness = fory .get_type_resolver() .get_harness(fory_type_id) .expect("Harness not found for trait object field"); let serializer_fn = harness.get_write_fn(); - serializer_fn(any_ref, context, true); + serializer_fn(any_ref, fory, context, true); } } } @@ -134,7 +133,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { quote! { #name_str => { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); - fory_core::serializer::Serializer::fory_write(&wrapper, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); } } } @@ -145,7 +144,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { quote! { #name_str => { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); - fory_core::serializer::Serializer::fory_write(&wrapper, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); } } } @@ -158,7 +157,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true); } } } @@ -171,7 +170,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true); } } } @@ -184,7 +183,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true); } } } @@ -197,22 +196,22 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true); } } } StructField::Forward => { quote! { #name_str => { - fory_core::serializer::Serializer::fory_write(&self.#ident, context, true); + fory_core::serializer::Serializer::fory_write(&self.#ident, fory, context, true); } } } _ => { quote! { #name_str => { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory()); - fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, context, true, skip_ref_flag, false); + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); + fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, skip_ref_flag, false); } } } @@ -253,7 +252,8 @@ pub fn gen_write_data(fields: &[&Field]) -> TokenStream { } }; quote! { - let sorted_field_names = ::fory_get_sorted_field_names(context.get_fory()); + let sorted_field_names = ::fory_get_sorted_field_names(fory); + let sorted_field_names = sorted_field_names.as_ref(); #loop_ts } }; @@ -264,6 +264,6 @@ pub fn gen_write_data(fields: &[&Field]) -> TokenStream { pub fn gen_write() -> TokenStream { quote! { - fory_core::serializer::struct_::write::(self, context, is_field) + fory_core::serializer::struct_::write::(self, fory, context, is_field) } } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index faaae0d266..c5d90d6283 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -717,13 +717,13 @@ //! } //! //! impl Serializer for CustomType { -//! fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { +//! fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { //! context.writer.write_i32(self.value); //! context.writer.write_varuint32(self.name.len() as u32); //! context.writer.write_utf8_string(&self.name); //! } //! -//! fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { +//! fn fory_read_data(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { //! let value = context.reader.read_i32(); //! let len = context.reader.read_varuint32() as usize; //! let name = context.reader.read_utf8_string(len); diff --git a/rust/tests/tests/compatible/test_basic_type.rs b/rust/tests/tests/compatible/test_basic_type.rs index 7756dad9be..7efdac6542 100644 --- a/rust/tests/tests/compatible/test_basic_type.rs +++ b/rust/tests/tests/compatible/test_basic_type.rs @@ -439,13 +439,13 @@ fn deserialize_nullable(fory: &Fory, context: &mut ReadContext, auto_conv: bool, fn basic() { let fory = Fory::default().mode(Compatible); // serialize - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); serialize_non_null(&fory, &mut write_context); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); deserialize_non_null(&fory, &mut read_context, false, true); } @@ -454,13 +454,13 @@ fn basic() { fn basic_nullable() { let fory = Fory::default().mode(Compatible); // serialize - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); serialize_nullable(&fory, &mut write_context); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); deserialize_nullable(&fory, &mut read_context, false, true); } @@ -469,21 +469,21 @@ fn basic_nullable() { fn auto_conv() { let fory = Fory::default().mode(Compatible); // serialize_non-null - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); serialize_non_null(&fory, &mut write_context); // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context: ReadContext<'_, '_> = ReadContext::new(&fory, reader, 5); + let mut read_context: ReadContext = ReadContext::new(reader, 5); deserialize_nullable(&fory, &mut read_context, true, true); // serialize_nullable - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); serialize_nullable(&fory, &mut write_context); // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); deserialize_non_null(&fory, &mut read_context, true, true); } diff --git a/rust/tests/tests/compatible/test_container.rs b/rust/tests/tests/compatible/test_container.rs index 78ad8e443f..e6e6426cb4 100644 --- a/rust/tests/tests/compatible/test_container.rs +++ b/rust/tests/tests/compatible/test_container.rs @@ -223,15 +223,15 @@ fn complex_container2() -> Vec, Vec>> { fn container_outer_auto_conv() { let fory = Fory::default().mode(Compatible); // serialize_outer_non-null - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&basic_list(), &mut write_context); fory.serialize_with_context(&basic_set(), &mut write_context); fory.serialize_with_context(&basic_map(), &mut write_context); // deserialize_outer_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( Some(basic_list()), fory.deserialize_with_context::>>(&mut read_context) @@ -249,8 +249,8 @@ fn container_outer_auto_conv() { ); assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_outer_nullable - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&Some(basic_list()), &mut write_context); fory.serialize_with_context(&Some(basic_set()), &mut write_context); fory.serialize_with_context(&Some(basic_map()), &mut write_context); @@ -260,7 +260,7 @@ fn container_outer_auto_conv() { // deserialize_outer_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -302,8 +302,8 @@ fn collection_inner() { fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&basic_list(), &mut write_context); fory.serialize_with_context(&item_list(), &mut write_context); fory.serialize_with_context(&basic_set(), &mut write_context); @@ -315,7 +315,7 @@ fn collection_inner() { // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -368,8 +368,8 @@ fn collection_inner_auto_conv() { fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize_non-null - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&basic_list(), &mut write_context); fory.serialize_with_context(&item_list(), &mut write_context); fory.serialize_with_context(&basic_set(), &mut write_context); @@ -377,7 +377,7 @@ fn collection_inner_auto_conv() { // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( nullable_basic_list(true), fory.deserialize_with_context::>>(&mut read_context) @@ -400,8 +400,8 @@ fn collection_inner_auto_conv() { ); assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_nullable - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&nullable_basic_list(false), &mut write_context); fory.serialize_with_context(&nullable_item_list(false), &mut write_context); fory.serialize_with_context(&nullable_basic_set(false), &mut write_context); @@ -409,7 +409,7 @@ fn collection_inner_auto_conv() { // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -442,8 +442,8 @@ fn map_inner() { fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&basic_map(), &mut write_context); fory.serialize_with_context(&item_map(), &mut write_context); fory.serialize_with_context(&nullable_basic_map(false), &mut write_context); @@ -451,7 +451,7 @@ fn map_inner() { // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( basic_map(), fory.deserialize_with_context::>(&mut read_context) @@ -486,14 +486,14 @@ fn map_inner_auto_conv() { fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize_non-null - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&basic_map(), &mut write_context); fory.serialize_with_context(&item_map(), &mut write_context); // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( nullable_basic_map(true), fory.deserialize_with_context::, Option>>( @@ -508,14 +508,14 @@ fn map_inner_auto_conv() { ); assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_nullable - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&nullable_basic_map(false), &mut write_context); fory.serialize_with_context(&nullable_item_map(false), &mut write_context); // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( basic_map(), fory.deserialize_with_context::>(&mut read_context) @@ -537,14 +537,14 @@ fn complex() { let mut fory2 = Fory::default().mode(Compatible); fory2.register_by_name::("item"); for fory in [fory1, fory2] { - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); fory.serialize_with_context(&nested_collection(), &mut write_context); fory.serialize_with_context(&complex_container1(), &mut write_context); fory.serialize_with_context(&complex_container2(), &mut write_context); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( nested_collection(), fory.deserialize_with_context::>>(&mut read_context) diff --git a/rust/tests/tests/compatible/test_struct_enum.rs b/rust/tests/tests/compatible/test_struct_enum.rs index 6b8791a9f6..22778b32e9 100644 --- a/rust/tests/tests/compatible/test_struct_enum.rs +++ b/rust/tests/tests/compatible/test_struct_enum.rs @@ -91,14 +91,14 @@ fn basic() { fory2.register_by_name::("item"); fory2.register_by_name::("person"); for fory in [fory1, fory2] { - let mut writer = Writer::default(); - let mut write_context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut write_context = WriteContext::new(writer); let person = Person::default(); fory.serialize_with_context(&person, &mut write_context); fory.serialize_with_context(&person, &mut write_context); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(&fory, reader, 5); + let mut read_context = ReadContext::new(reader, 5); assert_eq!( person, fory.deserialize_with_context::(&mut read_context) @@ -395,12 +395,16 @@ fn ext() { } } impl Serializer for ExtItem { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_data(&self.id, fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(fory, context, is_field)?, }) } @@ -441,12 +445,16 @@ fn skip_ext() { id: i32, } impl Serializer for ExtItem { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_data(&self.id, fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(fory, context, is_field)?, }) } @@ -500,12 +508,16 @@ fn compatible_ext() { id: i32, } impl Serializer for ExtItem { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_data(&self.id, fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(fory, context, is_field)?, }) } fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 1d25e960fb..29e95c1df2 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -229,13 +229,13 @@ fn test_string_serializer() { .mode(Compatible) .xlang(true) .compress_string(false); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); let reader_compress = Reader::new(bytes.as_slice()); let fory_compress = Fory::default() .mode(Compatible) .xlang(true) .compress_string(true); - let mut context_compress = ReadContext::new(&fory_compress, reader_compress, 5); + let mut context_compress = ReadContext::new(reader_compress, 5); let test_strings: Vec = vec![ // Latin1 "ab".to_string(), @@ -250,17 +250,20 @@ fn test_string_serializer() { ]; for s in &test_strings { // make is_field=true to skip read/write type_id - assert_eq!(*s, String::fory_read_data(&mut context, true).unwrap()); assert_eq!( *s, - String::fory_read_data(&mut context_compress, true).unwrap() + String::fory_read_data(&fory, &mut context, true).unwrap() + ); + assert_eq!( + *s, + String::fory_read_data(&fory_compress, &mut context_compress, true).unwrap() ); } - let mut writer = Writer::default(); + let writer = Writer::default(); let fory = Fory::default().mode(Compatible).xlang(true); - let mut context = WriteContext::new(&fory, &mut writer); + let mut context = WriteContext::new(writer); for s in &test_strings { - s.fory_write_data(&mut context, true); + s.fory_write_data(&fory, &mut context, true); } fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -291,7 +294,7 @@ fn test_cross_language_serializer() { let reader = Reader::new(bytes.as_slice()); let mut fory = Fory::default().mode(Compatible).xlang(true); fory.register::(101); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); assert_de!(fory, context, bool, true); assert_de!(fory, context, bool, false); assert_de!(fory, context, i32, -1); @@ -319,8 +322,8 @@ fn test_cross_language_serializer() { assert_de!(fory, context, HashMap::, str_map); assert_de!(fory, context, Color, color); - let mut writer = Writer::default(); - let mut context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut context = WriteContext::new(writer); fory.serialize_with_context(&true, &mut context); fory.serialize_with_context(&false, &mut context); fory.serialize_with_context(&-1, &mut context); @@ -421,7 +424,7 @@ fn test_list() { let mut fory = Fory::default().mode(Compatible); fory.register::(102); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); let str_list = vec![Some("a".to_string()), Some("b".to_string())]; let str_list2 = vec![None, Some("b".to_string())]; @@ -447,8 +450,8 @@ fn test_list() { let remote_item_list2: Vec> = fory.deserialize_with_context(&mut context).unwrap(); assert_eq!(remote_item_list2, item_list2); - let mut writer = Writer::default(); - let mut context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut context = WriteContext::new(writer); fory.serialize_with_context(&remote_str_list, &mut context); fory.serialize_with_context(&remote_str_list2, &mut context); fory.serialize_with_context(&remote_item_list, &mut context); @@ -466,7 +469,7 @@ fn test_map() { let mut fory = Fory::default().mode(Compatible); fory.register::(102); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); let str_map = HashMap::from([ (Some("k1".to_string()), Some("v1".to_string())), @@ -536,7 +539,7 @@ fn test_integer() { fory.register::(101); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); let f1 = 1; let f2 = Some(2); let f3 = Some(3); @@ -567,8 +570,8 @@ fn test_integer() { let remote_f6: Option = fory.deserialize_with_context(&mut context).unwrap(); assert_eq!(remote_f6, f6); - let mut writer = Writer::default(); - let mut context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut context = WriteContext::new(writer); fory.serialize_with_context(&remote_item2, &mut context); fory.serialize_with_context(&remote_f1, &mut context); fory.serialize_with_context(&remote_f2, &mut context); @@ -588,15 +591,19 @@ struct MyExt { id: i32, } impl Serializer for MyExt { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, _is_field: bool) { // set is_field=false to write type_id like in java - write_data(&self.id, context, false); + write_data(&self.id, fory, context, false); } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result { Ok(Self { // set is_field=false to write type_id like in java - id: read_data(context, false)?, + id: read_data(fory, context, false)?, }) } @@ -698,7 +705,7 @@ fn test_consistent_named() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(&fory, reader, 5); + let mut context = ReadContext::new(reader, 5); assert_eq!( fory.deserialize_with_context::(&mut context) @@ -732,8 +739,8 @@ fn test_consistent_named() { my_ext ); - let mut writer = Writer::default(); - let mut context = WriteContext::new(&fory, &mut writer); + let writer = Writer::default(); + let mut context = WriteContext::new(writer); fory.serialize_with_context(&color, &mut context); fory.serialize_with_context(&color, &mut context); fory.serialize_with_context(&color, &mut context); diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index 4db19daff1..9f43bce5f3 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -60,13 +60,17 @@ fn test_use() { } impl Serializer for Item { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) { - write_data(&self.f1, context, is_field); + fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + write_data(&self.f1, fory, context, is_field); } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result { Ok(Self { - f1: read_data(context, is_field)?, + f1: read_data(fory, context, is_field)?, f2: 0, }) } diff --git a/rust/tests/tests/test_multi_thread.rs b/rust/tests/tests/test_multi_thread.rs new file mode 100644 index 0000000000..d1055caf6b --- /dev/null +++ b/rust/tests/tests/test_multi_thread.rs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use fory_core::Fory; +use fory_derive::ForyObject; +use std::collections::HashSet; +use std::sync::Arc; +use std::thread; + +#[test] +fn test_simple_multi_thread() { + let fory = Arc::new(Fory::default()); + let src: HashSet<_> = [41, 42, 43, 45, 46, 47].into_iter().collect(); + // serialize + let mut handles = vec![]; + for item in &src { + let fory_clone = Arc::clone(&fory); + let item = *item; + let handle = thread::spawn(move || fory_clone.serialize(&item)); + handles.push(handle); + } + let mut serialized_data = vec![]; + for handle in handles { + let bytes = handle.join().unwrap(); + serialized_data.push(bytes); + } + // deserialize + let mut dest = HashSet::new(); + let mut handles = vec![]; + for bytes in serialized_data { + let fory_clone = Arc::clone(&fory); + let handle = thread::spawn(move || fory_clone.deserialize::(&bytes).unwrap()); + handles.push(handle); + } + for handle in handles { + let value = handle.join().unwrap(); + dest.insert(value); + } + // verify + assert_eq!(dest, src); +} + +#[test] +fn test_struct_multi_thread() { + #[derive(ForyObject, Debug, PartialEq, Eq, Hash, Clone, Copy)] + struct Item1 { + f1: i32, + } + let mut fory = Fory::default(); + fory.register::(101); + let fory = Arc::new(fory); + let src: HashSet<_> = [ + Item1 { f1: 42 }, + Item1 { f1: 43 }, + Item1 { f1: 45 }, + Item1 { f1: 46 }, + Item1 { f1: 47 }, + ] + .into_iter() + .collect(); + // serialize + let mut handles = vec![]; + for item in &src { + let fory_clone = Arc::clone(&fory); + let item = *item; + let handle = thread::spawn(move || fory_clone.serialize(&item)); + handles.push(handle); + } + let mut serialized_data = vec![]; + for handle in handles { + let bytes = handle.join().unwrap(); + serialized_data.push(bytes); + } + // deserialize + let mut dest = HashSet::new(); + let mut handles = vec![]; + for bytes in serialized_data { + let fory_clone = Arc::clone(&fory); + let handle = thread::spawn(move || fory_clone.deserialize::(&bytes).unwrap()); + handles.push(handle); + } + for handle in handles { + let value = handle.join().unwrap(); + dest.insert(value); + } + // verify + assert_eq!(dest, src); +} diff --git a/rust/tests/tests/test_ref_resolver.rs b/rust/tests/tests/test_ref_resolver.rs index 22f6f585ca..5dfca17b54 100644 --- a/rust/tests/tests/test_ref_resolver.rs +++ b/rust/tests/tests/test_ref_resolver.rs @@ -88,7 +88,7 @@ fn test_ref_writer_clear() { assert!(!ref_writer.try_write_rc_ref(&mut writer, &rc)); // Clear the writer - ref_writer.clear(); + ref_writer.reset(); // After clearing, should register as new reference again assert!(!ref_writer.try_write_rc_ref(&mut writer, &rc)); @@ -104,7 +104,7 @@ fn test_ref_reader_clear() { assert!(ref_reader.get_rc_ref::(ref_id).is_some()); // Clear the reader - ref_reader.clear(); + ref_reader.reset(); // After clearing, reference should no longer be found assert!(ref_reader.get_rc_ref::(ref_id).is_none()); From b2bcbaeb2d955dc1a6f5d092aa66f6b468e3e5e3 Mon Sep 17 00:00:00 2001 From: Sanyam Suyal <168440633+SanyamSuyal@users.noreply.github.com> Date: Sun, 12 Oct 2025 20:32:00 +0530 Subject: [PATCH 09/37] feat(ci): cache Bazel binary in Python CI workflow (#2745) ### Problem Fixes #2742 Python CI downloads Bazel binary on every run, which: - Takes 2-5 seconds per run - Can fail due to transient network errors - Wastes bandwidth and CI resources ### Solution Implemented GitHub Actions caching for Bazel binary to avoid repeated downloads. ### Changes - `.github/workflows/ci.yml`: Added actions/cache@v4 step to cache Bazel binary - Caches ~/bin/bazel and ~/.local/bin/bazel - Cache key includes OS, architecture, and Bazel version hash - Invalidates cache when Bazel version changes - `ci/tasks/common.py`: Updated install_bazel() function - Checks if cached Bazel binary exists before downloading - Verifies cached binary works by running bazel --version - Automatically re-downloads if cached binary is corrupted - Skips download entirely when cache is valid ### Testing Tested all scenarios: - Fresh install (no cache) - downloads successfully - Cache hit (valid binary) - skips download, saves time - Corrupted cache - detects corruption and recovers automatically - All Python syntax and YAML validation passed ### Benefits - Faster builds: Saves 2-5 seconds per CI run when cache hits - More reliable: Reduces dependency on network availability - Cost savings: Less bandwidth usage and shorter CI runtime ### Related Follow-up to #2733 (retry logic for Bazel downloads) Fixes #2742 --- .github/workflows/ci.yml | 9 +++++++++ ci/tasks/common.py | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cdf42238f..2bdcc8cc88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,6 +283,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Cache Bazel binary + uses: actions/cache@v4 + with: + path: | + ~/bin/bazel + ~/.local/bin/bazel + key: bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion') }} + restore-keys: | + bazel-${{ runner.os }}-${{ runner.arch }}- - name: Install bazel (Unix) if: runner.os != 'Windows' shell: bash diff --git a/ci/tasks/common.py b/ci/tasks/common.py index c1393ce86e..65ba8b488f 100644 --- a/ci/tasks/common.py +++ b/ci/tasks/common.py @@ -144,6 +144,32 @@ def update_shell_profile(): def install_bazel(): """Download and install bazel.""" + # Check if bazel is already cached (from GitHub Actions cache) + if not is_windows(): + home_bin = os.path.expanduser("~/bin") + bazel_path = os.path.join(home_bin, "bazel") + + # Also check ~/.local/bin for some systems + alt_bin = os.path.expanduser("~/.local/bin") + alt_bazel_path = os.path.join(alt_bin, "bazel") + + for path in [bazel_path, alt_bazel_path]: + if os.path.exists(path) and os.access(path, os.X_OK): + logging.info(f"Bazel already exists at {path}, verifying...") + try: + # Verify it works + result = exec_cmd(f"{path} --version") + logging.info(f"Cached Bazel binary is valid: {result.strip()}") + logging.info("Skipping Bazel download, using cached binary") + return + except Exception as e: + logging.warning(f"Cached Bazel binary at {path} is invalid: {e}") + logging.info("Re-downloading Bazel...") + try: + os.remove(path) + except Exception: + pass + bazel_download_url = get_bazel_download_url() logging.info(f"Downloading bazel from: {bazel_download_url}") From c41389ff5ce6b0d49527fb7db28728d082023799 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 22:25:07 +0530 Subject: [PATCH 10/37] fix(ci): fix sync file (#2750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/scripts/add_doc_headers.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/add_doc_headers.sh b/.github/scripts/add_doc_headers.sh index 588591ae02..7e77729784 100755 --- a/.github/scripts/add_doc_headers.sh +++ b/.github/scripts/add_doc_headers.sh @@ -95,7 +95,12 @@ EOF ) # Process Python guide +rm -rf docs/guide/python_guide.md add_header "python/README.md" "docs/guide/python_guide.md" "$PYTHON_HEADER" # Process Rust guide -add_header "rust/README.md" "docs/guide/rust_guide.md" "$RUST_HEADER" \ No newline at end of file +rm -rf docs/guide/rust_guide.md +add_header "rust/README.md" "docs/guide/rust_guide.md" "$RUST_HEADER" +git add docs/guide/rust_guide.md +git add docs/guide/python_guide.md +git commit -m "Added rust and python docs" \ No newline at end of file From b65eafdd62d68f3644ed31ef8d26cf406491d588 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 22:38:24 +0530 Subject: [PATCH 11/37] feat(rust): rewrite fory derive macro for smaller and faster generated code using compile-time fields sort algorithm (#2749) ## Why? ## What does this PR do? - designed a new fields sort algorithm which is friendly to compile-time languages, the compile-time languages can use this new fields sort algorithm to generate serialize code at compile time - rewrite fory rust derive macro for smaller and faster generated code Given struct: ```rust #[derive(ForyObject, Debug, PartialEq)] struct Person1 { f1: Color1, f2: Color1, // skip f3: Color2, f5: Vec, f6: Option, f7: Option, f8: Color1, last: i8, } ``` For following struct, this PR generates code: ```rust fn fory_write_data( &self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool, ) { fory_core::serializer::write_ref_info_data::< i8, >(&self.last, fory, context, true, true, false); fory_core::serializer::write_ref_info_data::< Vec, >(&self.f5, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Color1, >(&self.f1, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Color1, >(&self.f2, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Color2, >(&self.f3, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Option, >(&self.f6, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Option, >(&self.f7, fory, context, true, false, false); fory_core::serializer::write_ref_info_data::< Color1, >(&self.f8, fory, context, true, false, false); } fn fory_read_data( fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool, ) -> Result { let _last = fory_core::serializer::read_ref_info_data::< i8, >(fory, context, true, true, false)?; let _f5 = fory_core::serializer::read_ref_info_data::< Vec, >(fory, context, true, false, false)?; let _f1 = fory_core::serializer::read_ref_info_data::< Color1, >(fory, context, true, false, false)?; let _f2 = fory_core::serializer::read_ref_info_data::< Color1, >(fory, context, true, false, false)?; let _f3 = fory_core::serializer::read_ref_info_data::< Color2, >(fory, context, true, false, false)?; let _f6 = fory_core::serializer::read_ref_info_data::< Option, >(fory, context, true, false, false)?; let _f7 = fory_core::serializer::read_ref_info_data::< Option, >(fory, context, true, false, false)?; let _f8 = fory_core::serializer::read_ref_info_data::< Color1, >(fory, context, true, false, false)?; Ok(Self { last: _last, f5: _f5, f1: _f1, f2: _f2, f3: _f3, f6: _f6, f7: _f7, f8: _f8, }) } ``` This PR also reverts #2724 since it generats lots of inefficient code and bloat code size ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- ci/run_ci.py | 2 +- .../specification/xlang_serialization_spec.md | 23 +- go/fory/struct.go | 24 +- go/fory/type.go | 20 +- go/fory/type_def.go | 6 +- .../fory/builder/ObjectCodecBuilder.java | 8 +- .../apache/fory/resolver/ClassResolver.java | 15 +- .../apache/fory/resolver/XtypeResolver.java | 57 +- .../serializer/AbstractObjectSerializer.java | 8 +- .../fory/serializer/MetaSharedSerializer.java | 20 +- .../NonexistentClassSerializers.java | 26 +- .../fory/serializer/ObjectSerializer.java | 45 +- .../fory/serializer/SerializationBinding.java | 3 + .../apache/fory/type/DescriptorGrouper.java | 37 +- .../org/apache/fory/type/GenericType.java | 2 +- .../fory/type/DescriptorGrouperTest.java | 30 +- python/pyfory/_struct.py | 50 +- python/pyfory/format/infer.py | 5 + python/pyfory/tests/test_struct.py | 43 +- python/pyfory/type.py | 8 + rust/fory-core/src/buffer.rs | 63 +- rust/fory-core/src/meta/type_meta.rs | 110 ++-- rust/fory-core/src/resolver/context.rs | 13 + rust/fory-core/src/resolver/type_resolver.rs | 17 +- rust/fory-core/src/serializer/bool.rs | 8 + rust/fory-core/src/serializer/mod.rs | 10 +- rust/fory-core/src/serializer/number.rs | 8 + rust/fory-core/src/serializer/option.rs | 11 + rust/fory-core/src/serializer/string.rs | 8 + rust/fory-core/src/types.rs | 24 +- rust/fory-derive/src/object/misc.rs | 13 +- rust/fory-derive/src/object/mod.rs | 2 +- rust/fory-derive/src/object/read.rs | 182 ++---- rust/fory-derive/src/object/serializer.rs | 2 +- rust/fory-derive/src/object/util.rs | 540 +++++++----------- rust/fory-derive/src/object/write.rs | 68 +-- rust/fory-derive/src/util.rs | 20 +- rust/tests/tests/test_simple_struct.rs | 72 +++ 38 files changed, 834 insertions(+), 769 deletions(-) create mode 100644 rust/tests/tests/test_simple_struct.rs diff --git a/ci/run_ci.py b/ci/run_ci.py index 7818a9e606..cdd8f7fbc1 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -293,7 +293,7 @@ def parse_args(): if USE_PYTHON_GO: func() else: - run_shell_script("go") + # run_shell_script("go") pass elif command == "format": if USE_PYTHON_FORMAT: diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index d44590c379..4b562de27b 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -781,24 +781,11 @@ Field will be ordered as following, every group of fields will have its own orde - when same size and type id, sort by snake case field name - types: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64 - nullable primitive fields: same order as primitive fields -- morphic fields: same type together, then sorted by field name lexicographically using snake case style. -- unknown fields: same sort algorithms as morphic fields -- list fields: same sort algorithms as morphic fields -- set fields: same sort algorithms as morphic fields -- map fields: same sort algorithms as morphic fields - -#### Field order - -Fields in a struct are sorted in a ascending order by: - -- primitive fields first: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64, sorted by - type id. -- nullable primitive fields -- morphic types except `list/set/map` -- unknown types -- list types -- set types -- map types +- other internal type fields: sort by type id then snake case field name +- list fields: sort by snake case field name +- set fields: sort by snake case field name +- map fields: sort by snake case field name +- other fields: sort by snake case field name If two fields have same type, then sort by snake_case styled field name. diff --git a/go/fory/struct.go b/go/fory/struct.go index 9833c4b0be..a1a38e449e 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -34,7 +34,7 @@ type structSerializer struct { codegenDelegate Serializer // Optional codegen serializer for performance (like Python's approach) } -var UNKNOWN_TYPE_ID = int16(-1) +var UNKNOWN_TYPE_ID = int16(63) func (s *structSerializer) TypeId() TypeId { return NAMED_STRUCT @@ -378,7 +378,7 @@ func sortFields( } typeTriples = append(typeTriples, triple{ser.TypeId(), ser, name}) } - var boxed, collection, maps, final []triple + var boxed, collection, setFields, maps, otherInternalTypeFields []triple for _, t := range typeTriples { switch { @@ -386,12 +386,14 @@ func sortFields( boxed = append(boxed, t) case isListType(t.typeID): collection = append(collection, t) + case isSetType(t.typeID): + setFields = append(setFields, t) case isMapType(t.typeID): maps = append(maps, t) - case t.typeID == STRING || isPrimitiveArrayType(t.typeID): - final = append(final, t) - default: + case isUserDefinedType(t.typeID) || t.typeID == UNKNOWN_TYPE_ID: others = append(others, t) + default: + otherInternalTypeFields = append(otherInternalTypeFields, t) } } sort.Slice(boxed, func(i, j int) bool { @@ -417,17 +419,23 @@ func sortFields( return s[i].name < s[j].name }) } - sortTuple(final) + sortByTypeIDThenName := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + return s[i].name < s[j].name + }) + } + sortByTypeIDThenName(otherInternalTypeFields) sortTuple(others) sortTuple(collection) sortTuple(maps) all := make([]triple, 0, len(fieldNames)) all = append(all, boxed...) - all = append(all, final...) - all = append(all, others...) + all = append(all, otherInternalTypeFields...) all = append(all, collection...) + all = append(all, setFields...) all = append(all, maps...) + all = append(all, others...) outSer := make([]Serializer, len(all)) outNam := make([]string, len(all)) diff --git a/go/fory/type.go b/go/fory/type.go index 2e49e54fdc..302c907fc7 100644 --- a/go/fory/type.go +++ b/go/fory/type.go @@ -71,8 +71,8 @@ const ( NAMED_STRUCT = 17 // NAMED_COMPATIBLE_STRUCT a compatible_struct whose type mapping will be encoded as a name NAMED_COMPATIBLE_STRUCT = 18 - // EXTENSION a type which will be serialized by a customized serializer - EXTENSION = 19 + // EXT a type which will be serialized by a customized serializer + EXT = 19 // NAMED_EXT an ext type whose type mapping will be encoded as a name NAMED_EXT = 20 // LIST A list of some logical data type @@ -1381,6 +1381,11 @@ func isPrimitiveType(typeID int16) bool { func isListType(typeID int16) bool { return typeID == LIST } + +func isSetType(typeID int16) bool { + return typeID == SET +} + func isMapType(typeID int16) bool { return typeID == MAP } @@ -1418,3 +1423,14 @@ func getPrimitiveTypeSize(typeID int16) int { } return -1 } + +func isUserDefinedType(typeID int16) bool { + return typeID == STRUCT || + typeID == COMPATIBLE_STRUCT || + typeID == NAMED_STRUCT || + typeID == NAMED_COMPATIBLE_STRUCT || + typeID == EXT || + typeID == NAMED_EXT || + typeID == ENUM || + typeID == NAMED_ENUM +} diff --git a/go/fory/type_def.go b/go/fory/type_def.go index d0f9bf5c9f..1603f4f95a 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -252,7 +252,7 @@ func readFieldType(buffer *ByteBuffer) (FieldType, error) { return nil, fmt.Errorf("failed to read value type: %w", err) } return NewMapFieldType(TypeId(typeId), keyType, valueType), nil - case EXTENSION, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + case EXT, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: return NewDynamicFieldType(TypeId(typeId)), nil } return NewSimpleFieldType(TypeId(typeId)), nil @@ -341,7 +341,7 @@ func NewSimpleFieldType(typeId TypeId) *SimpleFieldType { } } -// DynamicFieldType represents a field type that is determined at runtime, like EXTENSION or STRUCT +// DynamicFieldType represents a field type that is determined at runtime, like EXT or STRUCT type DynamicFieldType struct { BaseFieldType } @@ -406,7 +406,7 @@ func buildFieldType(fory *Fory, fieldValue reflect.Value) (FieldType, error) { } typeId = TypeId(typeInfo.TypeID) - if typeId == EXTENSION || typeId == STRUCT || typeId == NAMED_STRUCT || + if typeId == EXT || typeId == STRUCT || typeId == NAMED_STRUCT || typeId == COMPATIBLE_STRUCT || typeId == NAMED_COMPATIBLE_STRUCT { return NewDynamicFieldType(typeId), nil } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 37a85e3ffc..de6fc10cd0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -164,8 +164,6 @@ public Expression buildEncodeExpression() { objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer); addGroupExpressions( objectCodecOptimizer.finalWriteGroups, numGroups, expressions, bean, buffer); - addGroupExpressions( - objectCodecOptimizer.otherWriteGroups, numGroups, expressions, bean, buffer); for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(serializeGroup(Collections.singletonList(descriptor), bean, buffer, false)); @@ -173,6 +171,8 @@ public Expression buildEncodeExpression() { for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getMapDescriptors()) { expressions.add(serializeGroup(Collections.singletonList(d), bean, buffer, false)); } + addGroupExpressions( + objectCodecOptimizer.otherWriteGroups, numGroups, expressions, bean, buffer); return expressions; } @@ -459,14 +459,14 @@ public Expression buildDecodeExpression() { objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer); deserializeReadGroup( objectCodecOptimizer.finalReadGroups, numGroups, expressions, bean, buffer); - deserializeReadGroup( - objectCodecOptimizer.otherReadGroups, numGroups, expressions, bean, buffer); for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false)); } for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getMapDescriptors()) { expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false)); } + deserializeReadGroup( + objectCodecOptimizer.otherReadGroups, numGroups, expressions, bean, buffer); if (isRecord) { if (recordCtrAccessible) { assert bean instanceof FieldsCollector; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 8b9eca8dc6..62fda33225 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1767,13 +1767,14 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - fory.getClassResolver()::isMonomorphic, - descriptors, - descriptorsGroupedOrdered, - descriptorUpdator, - fory.compressInt(), - fory.compressLong(), - DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME); + fory.getClassResolver()::isMonomorphic, + descriptors, + descriptorsGroupedOrdered, + descriptorUpdator, + fory.compressInt(), + fory.compressLong(), + DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME) + .sort(); } /** diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 309e758bd4..9d56c4829f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -38,6 +38,7 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -887,26 +888,43 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - this::isMonomorphic, - descriptors, - descriptorsGroupedOrdered, - descriptorUpdator, - fory.compressInt(), - fory.compressLong(), - (o1, o2) -> { - int xtypeId = getXtypeId(o1.getRawType()); - int xtypeId2 = getXtypeId(o2.getRawType()); - if (xtypeId == xtypeId2) { - return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName()); - } else { - return xtypeId - xtypeId2; - } - }); - } - - private static final int UNKNOWN_TYPE_ID = -1; + clz -> { + ClassInfo classInfo = getClassInfo(clz, false); + if (classInfo == null || clz.isEnum()) { + return false; + } + byte foryTypeId = (byte) (classInfo.xtypeId & 0xff); + if (foryTypeId == 0 + || foryTypeId == Types.UNKNOWN + || Types.isUserDefinedType(foryTypeId)) { + return false; + } + return foryTypeId != Types.LIST && foryTypeId != Types.SET && foryTypeId != Types.MAP; + }, + descriptors, + descriptorsGroupedOrdered, + descriptorUpdator, + fory.compressInt(), + fory.compressLong(), + (o1, o2) -> { + int xtypeId = getXtypeId(o1.getRawType()); + int xtypeId2 = getXtypeId(o2.getRawType()); + if (xtypeId == xtypeId2) { + return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName()); + } else { + return xtypeId - xtypeId2; + } + }) + .setOtherDescriptorComparator(Comparator.comparing(Descriptor::getSnakeCaseName)) + .sort(); + } + + private static final int UNKNOWN_TYPE_ID = Types.UNKNOWN; private int getXtypeId(Class cls) { + if (isSet(cls)) { + return Types.SET; + } if (isCollection(cls)) { return Types.LIST; } @@ -922,6 +940,9 @@ private int getXtypeId(Class cls) { if (cls.isEnum()) { return Types.ENUM; } + if (cls.isArray()) { + return Types.LIST; + } if (ReflectionUtils.isMonomorphic(cls)) { throw new UnsupportedOperationException(cls + " is not supported for xlang serialization"); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 1dc0703a7e..930e8b67ed 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -127,7 +127,13 @@ static Object readOtherFieldValue( SerializationBinding binding, GenericTypeField fieldInfo, MemoryBuffer buffer) { Object fieldValue; boolean nullable = fieldInfo.nullable; - if (fieldInfo.trackingRef) { + if (fieldInfo.genericType.getCls().isEnum()) { + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } else { + return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); + } + } else if (fieldInfo.trackingRef) { fieldValue = binding.readRef(buffer, fieldInfo); } else { binding.preserveRefId(-1); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 7d5272d560..c7f507d722 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -89,11 +89,9 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { "Class version check should be disabled when compatible mode is enabled."); Preconditions.checkArgument( fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); - boolean xlang = fory.isCrossLanguage(); - Collection descriptors = - consolidateFields( - xlang ? fory.getXtypeResolver() : fory.getClassResolver(), type, classDef); - DescriptorGrouper descriptorGrouper = classResolver.createDescriptorGrouper(descriptors, false); + Collection descriptors = consolidateFields(fory._getTypeResolver(), type, classDef); + DescriptorGrouper descriptorGrouper = + fory._getTypeResolver().createDescriptorGrouper(descriptors, false); // d.getField() may be null if not exists in this class when meta share enabled. Tuple3< Tuple2, @@ -212,17 +210,17 @@ public T read(MemoryBuffer buffer) { } } } - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { - Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); + Generics generics = fory.getGenerics(); + for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + Object fieldValue = + AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { fieldAccessor.putObject(obj, fieldValue); } } - Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { - Object fieldValue = - AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); + for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { fieldAccessor.putObject(obj, fieldValue); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 2adfe97dc2..04fe92e771 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -19,6 +19,7 @@ package org.apache.fory.serializer; +import static org.apache.fory.serializer.ObjectSerializer.writeOtherFieldValue; import static org.apache.fory.serializer.SerializationUtils.getTypeResolver; import java.util.ArrayList; @@ -75,6 +76,7 @@ public static final class NonexistentClassSerializer extends Serializer { private final ClassInfoHolder classInfoHolder; private final LongMap fieldsInfoMap; private final SerializationBinding binding; + private final TypeResolver typeResolver; public NonexistentClassSerializer(Fory fory, ClassDef classDef) { super(fory, NonexistentClass.NonexistentMetaShared.class); @@ -82,6 +84,7 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { classInfoHolder = fory.getClassResolver().nilClassInfoHolder(); fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); + typeResolver = fory._getTypeResolver(); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); } @@ -140,21 +143,16 @@ public void write(MemoryBuffer buffer, Object v) { } } } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { - Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - boolean nullable = fieldInfo.nullable; - if (fieldInfo.trackingRef) { - binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } else { - binding.writeNullable(buffer, fieldValue, fieldInfo.classInfoHolder, nullable); - } - } Generics generics = fory.getGenerics(); for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); ObjectSerializer.writeContainerFieldValue( binding, refResolver, classResolver, generics, fieldInfo, buffer, fieldValue); } + for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + Object fieldValue = value.get(fieldInfo.qualifiedFieldName); + writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + } } private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { @@ -214,17 +212,17 @@ public Object read(MemoryBuffer buffer) { } entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { - Object fieldValue = - AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); - } Generics generics = fory.getGenerics(); for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } + for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + Object fieldValue = + AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); + entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); + } obj.setEntries(entries); return obj; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index b26ca8ad78..db1cf2dcc5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -103,6 +103,7 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { } DescriptorGrouper descriptorGrouper = typeResolver.createDescriptorGrouper(descriptors, false); descriptors = descriptorGrouper.getSortedDescriptors(); + System.out.println(descriptors.stream().map(f -> f.getName()).collect(Collectors.toList())); if (isRecord) { List fieldNames = descriptors.stream().map(Descriptor::getName).collect(Collectors.toList()); @@ -132,19 +133,15 @@ public void write(MemoryBuffer buffer, T value) { } // write order: primitive,boxed,final,other,collection,map writeFinalFields(buffer, value, fory, refResolver, typeResolver); - writeOtherFields(buffer, value); writeContainerFields(buffer, value, fory, refResolver, typeResolver); + writeOtherFields(buffer, value); } private void writeOtherFields(MemoryBuffer buffer, T value) { for (GenericTypeField fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - if (fieldInfo.trackingRef) { - binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } else { - binding.writeNullable(buffer, fieldValue, fieldInfo.classInfoHolder, fieldInfo.nullable); - } + writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); } } @@ -239,6 +236,24 @@ static void writeContainerFieldValue( } } + static void writeOtherFieldValue( + SerializationBinding binding, + TypeResolver typeResolver, + MemoryBuffer buffer, + GenericTypeField fieldInfo, + Object fieldValue) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else if (fieldValue.getClass().isEnum()) { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + fieldInfo.genericType.getSerializer(typeResolver).write(buffer, fieldValue); + } else if (fieldInfo.trackingRef) { + binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else { + binding.writeNullable(buffer, fieldValue, fieldInfo.classInfoHolder, fieldInfo.nullable); + } + } + @Override public T read(MemoryBuffer buffer) { if (isRecord) { @@ -286,15 +301,15 @@ public Object[] readFields(MemoryBuffer buffer) { fieldValues[counter++] = fieldValue; } } - for (GenericTypeField fieldInfo : otherFields) { - Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); - fieldValues[counter++] = fieldValue; - } Generics generics = fory.getGenerics(); for (GenericTypeField fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } + for (GenericTypeField fieldInfo : otherFields) { + Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); + fieldValues[counter++] = fieldValue; + } return fieldValues; } @@ -325,17 +340,17 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { fieldAccessor.putObject(obj, fieldValue); } } - for (GenericTypeField fieldInfo : otherFields) { - Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - fieldAccessor.putObject(obj, fieldValue); - } Generics generics = fory.getGenerics(); for (GenericTypeField fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } + for (GenericTypeField fieldInfo : otherFields) { + Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + fieldAccessor.putObject(obj, fieldValue); + } return obj; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 0871890494..3fb1c84179 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -28,6 +28,7 @@ import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; // This polymorphic interface has cost, do not expose it as a public class @@ -37,10 +38,12 @@ abstract class SerializationBinding { protected final Fory fory; protected final RefResolver refResolver; + protected final TypeResolver typeResolver; SerializationBinding(Fory fory) { this.fory = fory; this.refResolver = fory.getRefResolver(); + typeResolver = fory._getTypeResolver(); } abstract void writeRef(MemoryBuffer buffer, T obj); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java index aa1b4cbb59..95213b893b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java @@ -29,6 +29,7 @@ import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; +import org.apache.fory.util.Preconditions; import org.apache.fory.util.record.RecordUtils; /** @@ -55,6 +56,11 @@ public class DescriptorGrouper { } return c; }; + private final Collection descriptors; + private final Predicate> isMonomorphic; + private final Function descriptorUpdater; + private final boolean descriptorsGroupedOrdered; + private boolean sorted = false; /** * When compress disabled, sort primitive descriptors from largest to smallest, if size is the @@ -138,7 +144,7 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole // The key/value type should be final. private final Collection mapDescriptors; private final Collection finalDescriptors; - private final Collection otherDescriptors; + private Collection otherDescriptors; /** * Create a descriptor grouper. @@ -157,6 +163,10 @@ private DescriptorGrouper( Function descriptorUpdater, Comparator primitiveComparator, Comparator comparator) { + this.descriptors = descriptors; + this.isMonomorphic = isMonomorphic; + this.descriptorUpdater = descriptorUpdater; + this.descriptorsGroupedOrdered = descriptorsGroupedOrdered; this.primitiveDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(primitiveComparator); this.boxedDescriptors = @@ -168,6 +178,19 @@ private DescriptorGrouper( descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); this.otherDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); + } + + public DescriptorGrouper setOtherDescriptorComparator(Comparator comparator) { + Preconditions.checkArgument(!sorted); + this.otherDescriptors = + descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); + return this; + } + + public DescriptorGrouper sort() { + if (sorted) { + return this; + } for (Descriptor descriptor : descriptors) { if (TypeUtils.isPrimitive(descriptor.getRawType())) { primitiveDescriptors.add(descriptorUpdater.apply(descriptor)); @@ -183,40 +206,49 @@ private DescriptorGrouper( otherDescriptors.add(descriptorUpdater.apply(descriptor)); } } + sorted = true; + return this; } public List getSortedDescriptors() { + Preconditions.checkArgument(sorted); List descriptors = new ArrayList<>(getNumDescriptors()); descriptors.addAll(getPrimitiveDescriptors()); descriptors.addAll(getBoxedDescriptors()); descriptors.addAll(getFinalDescriptors()); - descriptors.addAll(getOtherDescriptors()); descriptors.addAll(getCollectionDescriptors()); descriptors.addAll(getMapDescriptors()); + descriptors.addAll(getOtherDescriptors()); return descriptors; } public Collection getPrimitiveDescriptors() { + Preconditions.checkArgument(sorted); return primitiveDescriptors; } public Collection getBoxedDescriptors() { + Preconditions.checkArgument(sorted); return boxedDescriptors; } public Collection getCollectionDescriptors() { + Preconditions.checkArgument(sorted); return collectionDescriptors; } public Collection getMapDescriptors() { + Preconditions.checkArgument(sorted); return mapDescriptors; } public Collection getFinalDescriptors() { + Preconditions.checkArgument(sorted); return finalDescriptors; } public Collection getOtherDescriptors() { + Preconditions.checkArgument(sorted); return otherDescriptors; } @@ -250,6 +282,7 @@ public static DescriptorGrouper createDescriptorGrouper( } public int getNumDescriptors() { + Preconditions.checkArgument(sorted); return primitiveDescriptors.size() + boxedDescriptors.size() + collectionDescriptors.size() diff --git a/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java b/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java index 36c9a3eb7e..9e8263da1c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/GenericType.java @@ -194,7 +194,7 @@ public void setSerializer(Serializer serializer) { this.serializer = serializer; } - public Serializer getSerializer(TypeResolver classResolver) { + public Serializer getSerializer(TypeResolver classResolver) { Serializer serializer = this.serializer; if (serializer == null) { serializer = classResolver.getSerializer(cls); diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index 7d6c28eaef..99277da4c0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -150,13 +150,14 @@ public void testGrouper() { new Descriptor(new TypeRef>() {}, "c" + index++, -1, "TestClass")); DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, - descriptors, - false, - null, - false, - false, - DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME); + ReflectionUtils::isMonomorphic, + descriptors, + false, + null, + false, + false, + DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME) + .sort(); { List> classes = grouper.getPrimitiveDescriptors().stream() @@ -232,13 +233,14 @@ public void testGrouper() { public void testCompressedPrimitiveGrouper() { DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, - createDescriptors(), - false, - null, - true, - true, - DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME); + ReflectionUtils::isMonomorphic, + createDescriptors(), + false, + null, + true, + true, + DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME) + .sort(); { List> classes = grouper.getPrimitiveDescriptors().stream() diff --git a/python/pyfory/_struct.py b/python/pyfory/_struct.py index 63da403cec..2b9ae87145 100644 --- a/python/pyfory/_struct.py +++ b/python/pyfory/_struct.py @@ -39,7 +39,7 @@ is_list_type, is_map_type, get_primitive_type_size, - is_primitive_array_type, + is_polymorphic_type, ) from pyfory.type import is_subclass @@ -79,6 +79,13 @@ def visit_list(self, field_name, elem_type, types_path=None): elem_serializer = infer_field("item", elem_type, self, types_path=types_path) return ListSerializer(self.fory, list, elem_serializer) + def visit_set(self, field_name, elem_type, types_path=None): + from pyfory.serializer import SetSerializer # Local import + + # Infer type recursively for type such as Set[Dict[str, str]] + elem_serializer = infer_field("item", elem_type, self, types_path=types_path) + return SetSerializer(self.fory, set, elem_serializer) + def visit_dict(self, field_name, key_type, value_type, types_path=None): from pyfory.serializer import MapSerializer # Local import @@ -117,8 +124,9 @@ def _get_hash(fory, field_names: list, type_hints: dict): def _sort_fields(type_resolver, field_names, serializers): boxed_types = [] collection_types = [] + set_types = [] map_types = [] - final_types = [] + internal_types = [] other_types = [] type_ids = [] for field_name, serializer in zip(field_names, serializers): @@ -127,7 +135,7 @@ def _sort_fields(type_resolver, field_names, serializers): else: type_ids.append( ( - type_resolver.get_typeinfo(serializer.type_).type_id, + type_resolver.get_typeinfo(serializer.type_).type_id & 0xFF, serializer, field_name, ) @@ -135,16 +143,21 @@ def _sort_fields(type_resolver, field_names, serializers): for type_id, serializer, field_name in type_ids: if is_primitive_type(type_id): container = boxed_types + elif type_id == TypeId.SET: + container = set_types elif is_list_type(serializer.type_): container = collection_types elif is_map_type(serializer.type_): container = map_types - elif ( - type_id in {TypeId.STRING} or is_primitive_array_type(type_id) or is_subclass(serializer.type_, enum.Enum) - ) or serializer.type_ in _time_types: - container = final_types - else: + elif is_polymorphic_type(type_id) or type_id in { + TypeId.ENUM, + TypeId.NAMED_ENUM, + }: container = other_types + else: + assert TypeId.LOWER_BOUND < type_id < TypeId.UNKNOWN, (type_id,) + assert type_id != TypeId.UNKNOWN, serializer + container = internal_types container.append((type_id, serializer, field_name)) def sorter(item): @@ -162,10 +175,10 @@ def numeric_sorter(item): boxed_types = sorted(boxed_types, key=numeric_sorter) collection_types = sorted(collection_types, key=sorter) - final_types = sorted(final_types, key=sorter) + internal_types = sorted(internal_types, key=sorter) map_types = sorted(map_types, key=sorter) - other_types = sorted(other_types, key=sorter) - all_types = boxed_types + final_types + other_types + collection_types + map_types + other_types = sorted(other_types, key=lambda item: item[2]) + all_types = boxed_types + internal_types + collection_types + set_types + map_types + other_types return [t[2] for t in all_types], [t[1] for t in all_types] @@ -182,6 +195,11 @@ def visit_list(self, field_name, elem_type, types_path=None): xtype_id = self.fory.type_resolver.get_typeinfo(list).type_id self._hash = self._compute_field_hash(self._hash, abs(xtype_id)) + def visit_set(self, field_name, elem_type, types_path=None): + # TODO add set element type to hash. + xtype_id = self.fory.type_resolver.get_typeinfo(set).type_id + self._hash = self._compute_field_hash(self._hash, abs(xtype_id)) + def visit_dict(self, field_name, key_type, value_type, types_path=None): # TODO add map key/value type to hash. xtype_id = self.fory.type_resolver.get_typeinfo(dict).type_id @@ -237,6 +255,11 @@ def visit_list(self, field_name, elem_type, types_path=None): elem_ids = infer_field("item", elem_type, self, types_path=types_path) return TypeId.LIST, elem_ids + def visit_set(self, field_name, elem_type, types_path=None): + # Infer type recursively for type such as Set[Dict[str, str]] + elem_ids = infer_field("item", elem_type, self, types_path=types_path) + return TypeId.SET, elem_ids + def visit_dict(self, field_name, key_type, value_type, types_path=None): # Infer type recursively for type such as Dict[str, Dict[str, str]] key_ids = infer_field("key", key_type, self, types_path=types_path) @@ -267,6 +290,11 @@ def visit_list(self, field_name, elem_type, types_path=None): elem_types = infer_field("item", elem_type, self, types_path=types_path) return typing.List, elem_types + def visit_set(self, field_name, elem_type, types_path=None): + # Infer type recursively for type such as Set[Dict[str, str]] + elem_types = infer_field("item", elem_type, self, types_path=types_path) + return typing.Set, elem_types + def visit_dict(self, field_name, key_type, value_type, types_path=None): # Infer type recursively for type such as Dict[str, Dict[str, str]] key_types = infer_field("key", key_type, self, types_path=types_path) diff --git a/python/pyfory/format/infer.py b/python/pyfory/format/infer.py index 18410345f9..a9596b5494 100644 --- a/python/pyfory/format/infer.py +++ b/python/pyfory/format/infer.py @@ -109,6 +109,11 @@ def visit_list(self, field_name, elem_type, types_path=None): elem_field = infer_field("item", elem_type, self, types_path=types_path) return pa.field(field_name, pa.list_(elem_field.type)) + def visit_set(self, field_name, elem_type, types_path=None): + # Infer type recursively for type such as Set[Dict[str, str]] + elem_field = infer_field("item", elem_type, self, types_path=types_path) + return pa.field(field_name, pa.list_(elem_field.type)) + def visit_dict(self, field_name, key_type, value_type, types_path=None): # Infer type recursively for type such as Dict[str, Dict[str, str]] key_field = infer_field("key", key_type, self, types_path=types_path) diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index 955c16fccb..2e522964ad 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -16,7 +16,8 @@ # under the License. from dataclasses import dataclass -from typing import Dict, Any, List +import datetime +from typing import Dict, Any, List, Set import os import pytest @@ -137,6 +138,46 @@ def create(cls): ) +def test_sort_fields(): + @dataclass + class TestClass: + f1: pyfory.Int32Type + f2: List[pyfory.Int16Type] + f3: Dict[str, pyfory.Float64Type] + f4: str + f5: pyfory.Float32Type + f6: bytes + f7: bool + f8: Any + f9: Dict[pyfory.Int32Type, pyfory.Float64Type] + f10: List[str] + f11: pyfory.Int8Type + f12: pyfory.Int64Type + f13: pyfory.Float64Type + f14: Set[pyfory.Int32Type] + f15: datetime.datetime + + fory = Fory(xlang=True, ref=True) + serializer = DataClassSerializer(fory, TestClass, xlang=True) + assert serializer._field_names == [ + "f13", + "f5", + "f11", + "f7", + "f12", + "f1", + "f4", + "f15", + "f6", + "f10", + "f2", + "f14", + "f3", + "f9", + "f8", + ] + + def test_data_class_serializer_xlang(): fory = Fory(xlang=True, ref=True) fory.register_type(ComplexObject, typename="example.ComplexObject") diff --git a/python/pyfory/type.py b/python/pyfory/type.py index e596c46784..dc25dd6b65 100644 --- a/python/pyfory/type.py +++ b/python/pyfory/type.py @@ -130,6 +130,7 @@ class TypeId: See `org.apache.fory.types.Type` """ + LOWER_BOUND = 0 # null value NA = 0 # a boolean value (true or false). @@ -397,6 +398,10 @@ class TypeVisitor(ABC): def visit_list(self, field_name, elem_type, types_path=None): pass + @abstractmethod + def visit_set(self, field_name, elem_type, types_path=None): + pass + @abstractmethod def visit_dict(self, field_name, key_type, value_type, types_path=None): pass @@ -428,6 +433,9 @@ def infer_field(field_name, type_, visitor: TypeVisitor, types_path=None): if origin is list or origin == typing.List: elem_type = args[0] return visitor.visit_list(field_name, elem_type, types_path=types_path) + elif origin is set or origin == typing.Set: + elem_type = args[0] + return visitor.visit_set(field_name, elem_type, types_path=types_path) elif origin is dict or origin == typing.Dict: key_type, value_type = args return visitor.visit_dict(field_name, key_type, value_type, types_path=types_path) diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index ec5ea22941..9baf94c659 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -29,23 +29,28 @@ pub struct Writer { } impl Writer { + #[inline(always)] pub fn reset(&mut self) { // keep capacity and reset len to 0 self.bf.clear(); } + #[inline(always)] pub fn dump(&self) -> Vec { self.bf.clone() } + #[inline(always)] pub fn len(&self) -> usize { self.bf.len() } + #[inline(always)] pub fn is_empty(&self) -> bool { self.bf.is_empty() } + #[inline(always)] pub fn reserve(&mut self, additional: usize) { self.reserved += additional; if self.bf.capacity() < self.reserved { @@ -53,10 +58,12 @@ impl Writer { } } + #[inline(always)] pub fn skip(&mut self, len: usize) { self.bf.resize(self.bf.len() + len, 0); } + #[inline(always)] pub fn set_bytes(&mut self, offset: usize, data: &[u8]) { self.bf .get_mut(offset..offset + data.len()) @@ -64,61 +71,75 @@ impl Writer { .copy_from_slice(data); } + #[inline(always)] pub fn write_bytes(&mut self, v: &[u8]) -> usize { self.reserve(v.len()); self.bf.extend_from_slice(v); v.len() } + #[inline(always)] pub fn write_u8(&mut self, value: u8) { self.bf.write_u8(value).unwrap(); } + #[inline(always)] pub fn write_i8(&mut self, value: i8) { self.bf.write_i8(value).unwrap(); } + #[inline(always)] pub fn write_u16(&mut self, value: u16) { self.bf.write_u16::(value).unwrap(); } + #[inline(always)] pub fn write_i16(&mut self, value: i16) { self.bf.write_i16::(value).unwrap(); } + #[inline(always)] pub fn write_u32(&mut self, value: u32) { self.bf.write_u32::(value).unwrap(); } + #[inline(always)] pub fn write_i32(&mut self, value: i32) { self.bf.write_i32::(value).unwrap(); } + #[inline(always)] pub fn write_f32(&mut self, value: f32) { self.bf.write_f32::(value).unwrap(); } + #[inline(always)] pub fn write_i64(&mut self, value: i64) { self.bf.write_i64::(value).unwrap(); } + #[inline(always)] pub fn write_f64(&mut self, value: f64) { self.bf.write_f64::(value).unwrap(); } + #[inline(always)] pub fn write_u64(&mut self, value: u64) { self.bf.write_u64::(value).unwrap(); } + #[inline(always)] pub fn write_varint32(&mut self, value: i32) { let zigzag = ((value as i64) << 1) ^ ((value as i64) >> 31); self._write_varuint32(zigzag as u32) } + #[inline(always)] pub fn write_varuint32(&mut self, value: u32) { self._write_varuint32(value) } + #[inline(always)] fn _write_varuint32(&mut self, value: u32) { if value < 0x80 { self.write_u8(value as u8); @@ -157,15 +178,18 @@ impl Writer { } } + #[inline(always)] pub fn write_varint64(&mut self, value: i64) { let zigzag = ((value << 1) ^ (value >> 63)) as u64; self._write_varuint64(zigzag) } + #[inline(always)] pub fn write_varuint64(&mut self, value: u64) { self._write_varuint64(value) } + #[inline(always)] fn _write_varuint64(&mut self, value: u64) { if value < 0x80 { self.write_u8(value as u8); @@ -264,6 +288,7 @@ impl Writer { } } + #[inline(always)] pub fn write_varuint36_small(&mut self, value: u64) { assert!(value < (1u64 << 36), "value too large for 36-bit varint"); if value < 0x80 { @@ -297,14 +322,17 @@ impl Writer { } } + #[inline(always)] pub fn write_latin1_string(&mut self, s: &str) { write_latin1_simd(self, s); } + #[inline(always)] pub fn write_utf8_string(&mut self, s: &str) { write_utf8_simd(self, s); } + #[inline(always)] pub fn write_utf16_bytes(&mut self, bytes: &[u16]) { write_utf16_simd(self, bytes); } @@ -317,6 +345,7 @@ pub struct Reader { } impl Reader { + #[inline(always)] pub fn new(bf: &[u8]) -> Reader { Reader { bf: bf.as_ptr(), @@ -325,98 +354,114 @@ impl Reader { } } + #[inline(always)] pub fn init(&mut self, bf: &[u8]) { self.bf = bf.as_ptr(); self.len = bf.len(); self.cursor = 0; } + #[inline(always)] pub fn reset(&mut self) { self.bf = std::ptr::null(); self.len = 0; self.cursor = 0; } + #[inline(always)] pub(crate) fn move_next(&mut self, additional: usize) { self.cursor += additional; } - #[inline] + #[inline(always)] unsafe fn ptr_at(&self, offset: usize) -> *const u8 { self.bf.add(offset) } + #[inline(always)] pub fn slice_after_cursor(&self) -> &[u8] { - let remaining = self.len - self.cursor; - if self.bf.is_null() || remaining == 0 { + if self.bf.is_null() || self.cursor >= self.len { &[] } else { + let remaining = self.len - self.cursor; unsafe { std::slice::from_raw_parts(self.bf.add(self.cursor), remaining) } } } + #[inline(always)] pub fn get_cursor(&self) -> usize { self.cursor } + #[inline(always)] pub fn read_u8(&mut self) -> u8 { let result = unsafe { *self.ptr_at(self.cursor) }; self.move_next(1); result } + #[inline(always)] pub fn read_i8(&mut self) -> i8 { self.read_u8() as i8 } + #[inline(always)] pub fn read_u16(&mut self) -> u16 { let result = LittleEndian::read_u16(self.slice_after_cursor()); self.move_next(2); result } + #[inline(always)] pub fn read_i16(&mut self) -> i16 { let result = LittleEndian::read_i16(self.slice_after_cursor()); self.move_next(2); result } + #[inline(always)] pub fn read_u32(&mut self) -> u32 { let result = LittleEndian::read_u32(self.slice_after_cursor()); self.move_next(4); result } + #[inline(always)] pub fn read_i32(&mut self) -> i32 { let result = LittleEndian::read_i32(self.slice_after_cursor()); self.move_next(4); result } + #[inline(always)] pub fn read_u64(&mut self) -> u64 { let result = LittleEndian::read_u64(self.slice_after_cursor()); self.move_next(8); result } + #[inline(always)] pub fn read_i64(&mut self) -> i64 { let result = LittleEndian::read_i64(self.slice_after_cursor()); self.move_next(8); result } + #[inline(always)] pub fn read_f32(&mut self) -> f32 { let result = LittleEndian::read_f32(self.slice_after_cursor()); self.move_next(4); result } + #[inline(always)] pub fn read_f64(&mut self) -> f64 { let result = LittleEndian::read_f64(self.slice_after_cursor()); self.move_next(8); result } + #[inline(always)] pub fn read_varuint32(&mut self) -> u32 { let start = self.cursor; let b0 = unsafe { *self.bf.add(start) as u32 }; @@ -453,11 +498,13 @@ impl Reader { encoded } + #[inline(always)] pub fn read_varint32(&mut self) -> i32 { let encoded = self.read_varuint32(); ((encoded >> 1) as i32) ^ -((encoded & 1) as i32) } + #[inline(always)] pub fn read_varuint64(&mut self) -> u64 { let start = self.cursor; let b0 = unsafe { *self.bf.add(start) } as u64; @@ -522,23 +569,28 @@ impl Reader { var64 } + #[inline(always)] pub fn read_varint64(&mut self) -> i64 { let encoded = self.read_varuint64(); ((encoded >> 1) as i64) ^ -((encoded & 1) as i64) } + #[inline(always)] pub fn read_latin1_string(&mut self, len: usize) -> String { read_latin1_simd(self, len) } + #[inline(always)] pub fn read_utf8_string(&mut self, len: usize) -> String { read_utf8_simd(self, len) } + #[inline(always)] pub fn read_utf16_string(&mut self, len: usize) -> String { read_utf16_simd(self, len) } + #[inline(always)] pub fn read_varuint36small(&mut self) -> u64 { let start = self.cursor; // fast path @@ -580,10 +632,12 @@ impl Reader { result } + #[inline(always)] pub fn skip(&mut self, len: u32) { self.move_next(len as usize); } + #[inline(always)] pub fn get_slice(&self) -> &[u8] { if self.bf.is_null() || self.len == 0 { &[] @@ -592,12 +646,14 @@ impl Reader { } } + #[inline(always)] pub fn read_bytes(&mut self, len: usize) -> &[u8] { let s = unsafe { slice::from_raw_parts(self.bf.add(self.cursor), len) }; self.move_next(len); s } + #[inline(always)] pub fn reset_cursor_to_here(&self) -> impl FnOnce(&mut Self) { let raw_cursor = self.cursor; move |this: &mut Self| { @@ -605,6 +661,7 @@ impl Reader { } } + #[inline(always)] pub fn aligned(&self) -> bool { if self.bf.is_null() { return false; diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 2d56a21926..0ae8e6c5ad 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -21,7 +21,7 @@ use crate::meta::{ murmurhash3_x64_128, Encoding, MetaString, MetaStringDecoder, FIELD_NAME_DECODER, FIELD_NAME_ENCODER, NAMESPACE_DECODER, TYPE_NAME_DECODER, }; -use crate::types::{TypeId, FINAL_TYPES, PRIMITIVE_ARRAY_TYPES, PRIMITIVE_TYPES}; +use crate::types::{TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; use std::clone::Clone; use std::cmp::min; @@ -181,6 +181,7 @@ impl FieldType { #[derive(Debug, PartialEq, Eq, Clone)] pub struct FieldInfo { + pub field_id: i16, pub field_name: String, pub field_type: FieldType, } @@ -188,6 +189,7 @@ pub struct FieldInfo { impl FieldInfo { pub fn new(field_name: &str, field_type: FieldType) -> FieldInfo { FieldInfo { + field_id: -1i16, field_name: field_name.to_string(), field_type, } @@ -223,6 +225,7 @@ impl FieldInfo { .decode(field_name_bytes, encoding) .unwrap(); FieldInfo { + field_id: -1i16, field_name: field_name.original, field_type, } @@ -386,15 +389,16 @@ impl TypeMetaLayer { // group let mut primitive_fields = Vec::new(); let mut nullable_primitive_fields = Vec::new(); - let mut final_fields = Vec::new(); - let mut other_fields = Vec::new(); - let mut unknown_fields = Vec::new(); - let mut collection_fields = Vec::new(); + let mut internal_type_fields = Vec::new(); + let mut list_fields = Vec::new(); + let mut set_fields = Vec::new(); let mut map_fields = Vec::new(); + let mut other_fields = Vec::new(); for field_info in field_infos.into_iter() { let mut type_id = field_info.field_type.type_id; - if type_id == TypeId::ForyNullable as u32 { + let is_nullable = type_id == TypeId::ForyNullable as u32; + if is_nullable { type_id = field_info.field_type.generics.first().unwrap().type_id; if PRIMITIVE_TYPES.contains(&type_id) { nullable_primitive_fields.push(field_info); @@ -402,49 +406,21 @@ impl TypeMetaLayer { } } - let internal_id = type_id & 0xff; if PRIMITIVE_TYPES.contains(&type_id) { primitive_fields.push(field_info); - } else if PRIMITIVE_ARRAY_TYPES.contains(&type_id) - || FINAL_TYPES.contains(&type_id) - || [TypeId::ENUM as u32, TypeId::NAMED_ENUM as u32].contains(&internal_id) - { - final_fields.push(field_info); - } else if [TypeId::LIST as u32, TypeId::SET as u32].contains(&type_id) { - collection_fields.push(field_info); + } else if TypeId::LIST as u32 == type_id { + list_fields.push(field_info); + } else if TypeId::SET as u32 == type_id { + set_fields.push(field_info); } else if TypeId::MAP as u32 == type_id { map_fields.push(field_info); - } else if [ - TypeId::COMPATIBLE_STRUCT as u32, - TypeId::NAMED_COMPATIBLE_STRUCT as u32, - TypeId::EXT as u32, - TypeId::NAMED_EXT as u32, - ] - .contains(&internal_id) - { - other_fields.push(field_info); - } else if internal_id == TypeId::UNKNOWN as u32 { - unknown_fields.push(field_info); + } else if crate::types::is_internal_type(type_id) { + internal_type_fields.push(field_info); } else { - unreachable!("type_id: {type_id}"); + other_fields.push(field_info); } } - fn sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { - let a_id = if a.field_type.type_id == TypeId::ForyNullable as u32 { - a.field_type.generics.first().unwrap().type_id - } else { - a.field_type.type_id - }; - let b_id = if b.field_type.type_id == TypeId::ForyNullable as u32 { - b.field_type.generics.first().unwrap().type_id - } else { - b.field_type.type_id - }; - let a_field_name = &a.field_name; - let b_field_name = &b.field_name; - a_id.cmp(&b_id).then_with(|| a_field_name.cmp(b_field_name)) - } fn get_primitive_type_size(type_id_num: u32) -> i32 { let type_id = TypeId::try_from(type_id_num as i16).unwrap(); match type_id { @@ -491,23 +467,33 @@ impl TypeMetaLayer { compress_a .cmp(&compress_b) .then_with(|| size_b.cmp(&size_a)) + .then_with(|| a_id.cmp(&b_id)) .then_with(|| a_field_name.cmp(b_field_name)) } + fn type_then_name_sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { + a.field_type + .type_id + .cmp(&b.field_type.type_id) + .then_with(|| a.field_name.cmp(&b.field_name)) + } + fn name_sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { + a.field_name.cmp(&b.field_name) + } primitive_fields.sort_by(numeric_sorter); nullable_primitive_fields.sort_by(numeric_sorter); - final_fields.sort_by(sorter); - other_fields.sort_by(sorter); - unknown_fields.sort_by(sorter); - collection_fields.sort_by(sorter); - map_fields.sort_by(sorter); + internal_type_fields.sort_by(type_then_name_sorter); + list_fields.sort_by(name_sorter); + set_fields.sort_by(name_sorter); + map_fields.sort_by(name_sorter); + other_fields.sort_by(name_sorter); let mut sorted_field_infos = Vec::with_capacity(fields_len); sorted_field_infos.extend(primitive_fields); sorted_field_infos.extend(nullable_primitive_fields); - sorted_field_infos.extend(final_fields); - sorted_field_infos.extend(other_fields); - sorted_field_infos.extend(unknown_fields); - sorted_field_infos.extend(collection_fields); + sorted_field_infos.extend(internal_type_fields); + sorted_field_infos.extend(list_fields); + sorted_field_infos.extend(set_fields); sorted_field_infos.extend(map_fields); + sorted_field_infos.extend(other_fields); sorted_field_infos } @@ -550,16 +536,16 @@ impl TypeMetaLayer { pub struct TypeMeta { // assigned valid value and used, only during deserializing hash: i64, - layers: Vec, + layer: TypeMetaLayer, } impl TypeMeta { pub fn get_field_infos(&self) -> &Vec { - self.layers.first().unwrap().get_field_infos() + self.layer.get_field_infos() } pub fn get_type_id(&self) -> u32 { - self.layers.first().unwrap().get_type_id() + self.layer.get_type_id() } pub fn get_hash(&self) -> i64 { @@ -567,11 +553,11 @@ impl TypeMeta { } pub fn get_type_name(&self) -> MetaString { - self.layers.first().unwrap().get_type_name().clone() + self.layer.get_type_name().clone() } pub fn get_namespace(&self) -> MetaString { - self.layers.first().unwrap().get_namespace().clone() + self.layer.get_namespace().clone() } pub fn from_fields( @@ -583,13 +569,7 @@ impl TypeMeta { ) -> TypeMeta { TypeMeta { hash: 0, - layers: vec![TypeMetaLayer::new( - type_id, - namespace, - type_name, - register_by_name, - field_infos, - )], + layer: TypeMetaLayer::new(type_id, namespace, type_name, register_by_name, field_infos), } } #[allow(unused_assignments)] @@ -604,13 +584,11 @@ impl TypeMeta { // let is_compressed: bool = (header & COMPRESS_META_FLAG) != 0; // let meta_hash = header >> (64 - NUM_HASH_BITS); - let mut layers = Vec::new(); // let current_meta_size = 0; // while current_meta_size < meta_size {} let layer = TypeMetaLayer::from_bytes(reader); - layers.push(layer); TypeMeta { - layers, + layer, hash: header, } } @@ -622,7 +600,7 @@ impl TypeMeta { // for layer in self.layers.iter() { // layers_writer.bytes(layer.to_bytes()?.as_slice()); // } - layers_writer.write_bytes(self.layers.first().unwrap().to_bytes()?.as_slice()); + layers_writer.write_bytes(self.layer.to_bytes()?.as_slice()); // global_binary_header:| hash:50bits | is_compressed:1bit | write_fields_meta:1bit | meta_size:12bits | let meta_size = layers_writer.len() as i64; let mut header: i64 = min(META_SIZE_MASK, meta_size); diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index b8e0311797..4b3f81cc9d 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -44,14 +44,17 @@ impl WriteContext { } } + #[inline(always)] pub fn empty(&mut self) -> bool { self.meta_resolver.empty() } + #[inline(always)] pub fn push_meta(&mut self, fory: &Fory, type_id: std::any::TypeId) -> usize { self.meta_resolver.push(type_id, fory) } + #[inline(always)] pub fn write_meta(&mut self, offset: usize) { self.writer.set_bytes( offset, @@ -107,11 +110,13 @@ impl WriteContext { } } + #[inline(always)] pub fn write_meta_string_bytes(&mut self, ms: &MetaString) { self.meta_string_resolver .write_meta_string_bytes(&mut self.writer, ms); } + #[inline(always)] pub fn reset(&mut self) { self.meta_resolver.reset(); self.ref_writer.reset(); @@ -140,16 +145,19 @@ impl ReadContext { } } + #[inline(always)] pub fn init(&mut self, bytes: &[u8], max_dyn_depth: u32) { self.reader.init(bytes); self.max_dyn_depth = max_dyn_depth; self.current_depth = 0; } + #[inline(always)] pub fn get_meta(&self, type_index: usize) -> &Arc { self.meta_resolver.get(type_index) } + #[inline(always)] pub fn load_meta(&mut self, offset: usize) -> usize { self.meta_resolver.load(&mut Reader::new( &self.reader.slice_after_cursor()[offset..], @@ -200,6 +208,7 @@ impl ReadContext { .read_meta_string_bytes(&mut self.reader) } + #[inline(always)] pub fn inc_depth(&mut self) -> Result<(), crate::error::Error> { self.current_depth += 1; if self.current_depth > self.max_dyn_depth { @@ -215,10 +224,12 @@ impl ReadContext { Ok(()) } + #[inline(always)] pub fn dec_depth(&mut self) { self.current_depth = self.current_depth.saturating_sub(1); } + #[inline(always)] pub fn reset(&mut self) { self.reader.reset(); self.meta_resolver.reset(); @@ -239,6 +250,7 @@ impl Pool { } } + #[inline(always)] pub fn get(&self) -> T { let item = self .items @@ -251,6 +263,7 @@ impl Pool { } // put back manually + #[inline(always)] pub fn put(&self, item: T) { self.items.lock().unwrap().push(item); } diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index ab3c57735c..b0f9a5c9d9 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -23,7 +23,7 @@ use crate::meta::{ TYPE_NAME_ENCODINGS, }; use crate::serializer::{ForyDefault, Serializer, StructSerializer}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::{any::Any, collections::HashMap}; type WriteFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool); @@ -182,7 +182,6 @@ pub struct TypeResolver { type_info_cache: HashMap, // Fast lookup by numeric ID for common types type_id_index: Vec, - sorted_field_names_map: RwLock>>>, } const NO_TYPE_ID: u32 = 1000000000; @@ -196,7 +195,6 @@ impl Default for TypeResolver { type_name_map: HashMap::new(), type_info_cache: HashMap::new(), type_id_index: Vec::new(), - sorted_field_names_map: RwLock::new(HashMap::new()), }; resolver.register_builtin_types(); resolver @@ -544,19 +542,6 @@ impl TypeResolver { .expect("named_ext type must be registered in both peers") } - pub fn get_sorted_field_names( - &self, - type_id: std::any::TypeId, - ) -> Option>> { - let map = self.sorted_field_names_map.read().unwrap(); - map.get(&type_id).cloned() - } - - pub fn set_sorted_field_names(&self, field_names: Arc>) { - let mut map = self.sorted_field_names_map.write().unwrap(); - map.insert(std::any::TypeId::of::(), field_names); - } - pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { if let Some(type_info) = self.type_info_cache.get(&rust_type_id) { Some(type_info.get_type_id()) diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index c3a71449d8..a890816d92 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -24,10 +24,12 @@ use crate::types::TypeId; use std::mem; impl Serializer for bool { + #[inline(always)] fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { context.writer.write_u8(if *self { 1 } else { 0 }); } + #[inline(always)] fn fory_read_data( _fory: &Fory, context: &mut ReadContext, @@ -36,10 +38,12 @@ impl Serializer for bool { Ok(context.reader.read_u8() == 1) } + #[inline(always)] fn fory_reserved_space() -> usize { mem::size_of::() } + #[inline(always)] fn fory_get_type_id(_fory: &Fory) -> u32 { TypeId::BOOL as u32 } @@ -48,20 +52,24 @@ impl Serializer for bool { TypeId::BOOL as u32 } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self } + #[inline(always)] fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { write_type_info::(fory, context, is_field); } + #[inline(always)] fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { read_type_info::(fory, context, is_field); } } impl ForyDefault for bool { + #[inline(always)] fn fory_default() -> Self { false } diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 0f3221f7f0..c91eb09ab8 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -22,7 +22,6 @@ use crate::resolver::context::{ReadContext, WriteContext}; use crate::types::{Mode, RefFlag, TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; use std::any::Any; -use std::sync::Arc; pub mod any; mod arc; @@ -47,6 +46,7 @@ pub mod struct_; pub mod trait_object; pub mod weak; +#[inline(always)] pub fn write_ref_info_data( record: &T, fory: &Fory, @@ -68,6 +68,7 @@ pub fn write_ref_info_data( } } +#[inline(always)] pub fn read_ref_info_data( fory: &Fory, context: &mut ReadContext, @@ -105,6 +106,7 @@ pub fn read_ref_info_data( } } +#[inline(always)] fn write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { if is_field { return; @@ -113,6 +115,7 @@ fn write_type_info(fory: &Fory, context: &mut WriteContext, is_fi context.writer.write_varuint32(type_id); } +#[inline(always)] fn read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { if is_field { return; @@ -122,6 +125,7 @@ fn read_type_info(fory: &Fory, context: &mut ReadContext, is_fiel assert_eq!(local_type_id, remote_type_id); } +#[inline(always)] pub fn get_skip_ref_flag(fory: &Fory) -> bool { let elem_type_id = T::fory_get_type_id(fory); !T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id) @@ -325,7 +329,7 @@ pub trait StructSerializer: Serializer + 'static { struct_::actual_type_id(type_id, register_by_name, mode) } - fn fory_get_sorted_field_names(_fory: &Fory) -> Arc> { - unimplemented!() + fn fory_get_sorted_field_names(_fory: &Fory) -> &'static [&'static str] { + &[] } } diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index 429afc4de4..e3646cb118 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -26,10 +26,12 @@ use crate::types::TypeId; macro_rules! impl_num_serializer { ($ty:ty, $writer:expr, $reader:expr, $field_type:expr) => { impl Serializer for $ty { + #[inline] fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { $writer(&mut context.writer, *self); } + #[inline] fn fory_read_data( _fory: &Fory, context: &mut ReadContext, @@ -38,10 +40,12 @@ macro_rules! impl_num_serializer { Ok($reader(&mut context.reader)) } + #[inline] fn fory_reserved_space() -> usize { std::mem::size_of::<$ty>() } + #[inline] fn fory_get_type_id(_fory: &Fory) -> u32 { $field_type as u32 } @@ -50,19 +54,23 @@ macro_rules! impl_num_serializer { $field_type as u32 } + #[inline] fn as_any(&self) -> &dyn std::any::Any { self } + #[inline] fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { write_type_info::(fory, context, is_field); } + #[inline] fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { read_type_info::(fory, context, is_field); } } impl ForyDefault for $ty { + #[inline] fn fory_default() -> Self { 0 as $ty } diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 78211b16a2..168fdc9046 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -22,6 +22,7 @@ use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; impl Serializer for Option { + #[inline(always)] fn fory_read_data( fory: &Fory, context: &mut ReadContext, @@ -30,10 +31,12 @@ impl Serializer for Option { Ok(Some(T::fory_read_data(fory, context, is_field)?)) } + #[inline(always)] fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { T::fory_read_type_info(fory, context, is_field); } + #[inline(always)] fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { if let Some(v) = self { T::fory_write_data(v, fory, context, is_field) @@ -42,18 +45,22 @@ impl Serializer for Option { } } + #[inline(always)] fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { T::fory_write_type_info(fory, context, is_field); } + #[inline(always)] fn fory_reserved_space() -> usize { std::mem::size_of::() } + #[inline(always)] fn fory_get_type_id(fory: &Fory) -> u32 { T::fory_get_type_id(fory) } + #[inline(always)] fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { match self { Some(val) => val.fory_type_id_dyn(fory), @@ -61,20 +68,24 @@ impl Serializer for Option { } } + #[inline(always)] fn fory_is_option() -> bool { true } + #[inline(always)] fn fory_is_none(&self) -> bool { self.is_none() } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self } } impl ForyDefault for Option { + #[inline(always)] fn fory_default() -> Self { None } diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index 9fb68d69b1..e6a7a0b699 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -31,6 +31,7 @@ enum StrEncoding { } impl Serializer for String { + #[inline] fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, _is_field: bool) { let mut len = get_latin1_length(self); if len >= 0 { @@ -51,6 +52,7 @@ impl Serializer for String { } } + #[inline] fn fory_read_data( _fory: &Fory, context: &mut ReadContext, @@ -75,10 +77,12 @@ impl Serializer for String { Ok(s) } + #[inline] fn fory_reserved_space() -> usize { mem::size_of::() } + #[inline(always)] fn fory_get_type_id(_fory: &Fory) -> u32 { TypeId::STRING as u32 } @@ -87,20 +91,24 @@ impl Serializer for String { TypeId::STRING as u32 } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self } + #[inline(always)] fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { write_type_info::(fory, context, is_field); } + #[inline(always)] fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { read_type_info::(fory, context, is_field); } } impl ForyDefault for String { + #[inline(always)] fn fory_default() -> Self { String::new() } diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index d89d257458..9da9cd705d 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -83,7 +83,6 @@ pub enum TypeId { ARROW_RECORD_BATCH = 38, ARROW_TABLE = 39, UNKNOWN = 63, - ForyAny = 256, // only used at receiver peer ForyNullable = 265, } @@ -134,12 +133,6 @@ pub static PRIMITIVE_TYPES: [u32; 7] = [ TypeId::FLOAT64 as u32, ]; -pub static FINAL_TYPES: [u32; 3] = [ - TypeId::STRING as u32, - TypeId::LOCAL_DATE as u32, - TypeId::TIMESTAMP as u32, -]; - pub static PRIMITIVE_ARRAY_TYPES: [u32; 8] = [ TypeId::BOOL_ARRAY as u32, TypeId::BINARY as u32, @@ -179,6 +172,23 @@ pub static PRIMITIVE_ARRAY_TYPE_MAP: &[(&str, u32, &str)] = &[ ("f64", TypeId::FLOAT64_ARRAY as u32, "Vec"), ]; +pub fn is_internal_type(type_id: u32) -> bool { + if type_id == 0 || type_id >= TypeId::UNKNOWN as u32 { + return false; + } + let excluded = [ + TypeId::ENUM as u32, + TypeId::NAMED_ENUM as u32, + TypeId::STRUCT as u32, + TypeId::COMPATIBLE_STRUCT as u32, + TypeId::NAMED_STRUCT as u32, + TypeId::NAMED_COMPATIBLE_STRUCT as u32, + TypeId::EXT as u32, + TypeId::NAMED_EXT as u32, + ]; + !excluded.contains(&type_id) +} + pub fn compute_field_hash(hash: u32, id: i16) -> u32 { let mut new_hash: u64 = (hash as u64) * 31 + (id as u64); while new_hash >= MAX_UNT32 { diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index 80fedeb201..4a2d1374ab 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -65,18 +65,9 @@ pub fn gen_actual_type_id() -> TokenStream { } pub fn gen_get_sorted_field_names(fields: &[&Field]) -> TokenStream { - let create_sorted_field_names = get_sort_fields_ts(fields); + let static_field_names = get_sort_fields_ts(fields); quote! { - let sorted_field_names = match fory.get_type_resolver().get_sorted_field_names::(std::any::TypeId::of::()) { - Some(result) => result, - None => { - #create_sorted_field_names - let arc_sorted_field_names = std::sync::Arc::new(sorted_field_names); - fory.get_type_resolver().set_sorted_field_names::(arc_sorted_field_names.clone()); - arc_sorted_field_names - } - }; - sorted_field_names + #static_field_names } } diff --git a/rust/fory-derive/src/object/mod.rs b/rust/fory-derive/src/object/mod.rs index 6fa6bb0263..5e25ce09fa 100644 --- a/rust/fory-derive/src/object/mod.rs +++ b/rust/fory-derive/src/object/mod.rs @@ -19,7 +19,7 @@ mod derive_enum; mod misc; mod read; mod serializer; -mod util; +pub(crate) mod util; mod write; pub use serializer::derive_serializer; diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index c87e1bfda8..c3ca1c5d76 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -21,7 +21,7 @@ use syn::{Field, Type}; use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, - generic_tree_to_tokens, parse_generic_tree, NullableTypeNode, StructField, + generic_tree_to_tokens, parse_generic_tree, skip_ref_flag, NullableTypeNode, StructField, }; fn create_private_field_name(field: &Field) -> Ident { @@ -83,33 +83,30 @@ fn assign_value(fields: &[&Field]) -> Vec { .collect() } -fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { +fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let ty = &field.ty; - let name_str = field.ident.as_ref().unwrap().to_string(); - match classify_trait_object_field(ty) { StructField::BoxDyn(trait_name) => { let from_any_fn = format_ident!("from_any_internal_{}", trait_name); let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); quote! { - #name_str => { - let ref_flag = context.reader.read_i8(); - if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - panic!("Expected NotNullValue for trait object field"); - } + let ref_flag = context.reader.read_i8(); + if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { + panic!("Expected NotNullValue for trait object field"); + } - let fory_type_id = context.reader.read_varuint32(); + let fory_type_id = context.reader.read_varuint32(); - let harness = fory.get_type_resolver() - .get_harness(fory_type_id) - .expect("Type not registered for trait object field"); + let harness = fory.get_type_resolver() + .get_harness(fory_type_id) + .expect("Type not registered for trait object field"); - let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(fory, context, true, false)?; + let deserializer_fn = harness.get_read_fn(); + let any_box = deserializer_fn(fory, context, true, false)?; + + let base_type_id = fory_type_id >> 8; + let #private_ident = #helper_mod::#from_any_fn(any_box, base_type_id)?; - let base_type_id = fory_type_id >> 8; - #private_ident = #helper_mod::#from_any_fn(any_box, base_type_id)?; - } } } StructField::RcDyn(trait_name) => { @@ -117,10 +114,8 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = std::rc::Rc::::from(wrapper); - } + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = std::rc::Rc::::from(wrapper); } } StructField::ArcDyn(trait_name) => { @@ -128,10 +123,8 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = std::sync::Arc::::from(wrapper); - } + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = std::sync::Arc::::from(wrapper); } } StructField::VecRc(trait_name) => { @@ -139,12 +132,10 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = Some(wrapper_vec.into_iter() - .map(|w| std::rc::Rc::::from(w)) - .collect()); - } + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = wrapper_vec.into_iter() + .map(|w| std::rc::Rc::::from(w)) + .collect(); } } StructField::VecArc(trait_name) => { @@ -152,12 +143,10 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = Some(wrapper_vec.into_iter() - .map(|w| std::sync::Arc::::from(w)) - .collect()); - } + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = wrapper_vec.into_iter() + .map(|w| std::sync::Arc::::from(w)) + .collect(); } } StructField::HashMapRc(key_ty, trait_name) => { @@ -165,12 +154,10 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = Some(wrapper_map.into_iter() - .map(|(k, v)| (k, std::rc::Rc::::from(v))) - .collect()); - } + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = wrapper_map.into_iter() + .map(|(k, v)| (k, std::rc::Rc::::from(v))) + .collect(); } } StructField::HashMapArc(key_ty, trait_name) => { @@ -178,27 +165,21 @@ fn gen_read_match_arm(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; - #private_ident = Some(wrapper_map.into_iter() - .map(|(k, v)| (k, std::sync::Arc::::from(v))) - .collect()); - } + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let #private_ident = wrapper_map.into_iter() + .map(|(k, v)| (k, std::sync::Arc::::from(v))) + .collect(); } } StructField::Forward => { quote! { - #name_str => { - #private_ident = Some(fory_core::serializer::Serializer::fory_read(fory, context, true)?); - } + let #private_ident = fory_core::serializer::Serializer::fory_read(fory, context, true)?; } } _ => { + let skip_ref_flag = skip_ref_flag(ty); quote! { - #name_str => { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); - #private_ident = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); - } + let #private_ident = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, #skip_ref_flag, false)?; } } } @@ -211,94 +192,32 @@ pub fn gen_read_type_info() -> TokenStream { } fn get_fields_loop_ts(fields: &[&Field]) -> TokenStream { - let match_ts: Vec<_> = fields + let read_fields_ts: Vec<_> = fields .iter() .map(|field| { let private_ident = create_private_field_name(field); - gen_read_match_arm(field, &private_ident) + gen_read_field(field, &private_ident) }) .collect(); - #[cfg(not(feature = "fields-loop-unroll"))] - let loop_ts = quote! { - for field_name in field_names { - match field_name.as_str() { - #(#match_ts),* - , _ => unreachable!() - } - } - }; - #[cfg(feature = "fields-loop-unroll")] - let loop_ts = { - let loop_item_ts = fields.iter().enumerate().map(|(i, _field)| { - let idx = syn::Index::from(i); - quote! { - let field_name = field_names.get(#idx).unwrap(); - match field_name.as_str() { - #(#match_ts),* - , _ => { unreachable!() } - } - } - }); - quote! { - #(#loop_item_ts)* - } - }; - loop_ts + quote! { + #(#read_fields_ts)* + } } pub fn gen_read_data(fields: &[&Field]) -> TokenStream { let sorted_read = if fields.is_empty() { quote! {} } else { - let declare_var_ts = - fields - .iter() - .map(|field| { - let private_ident = create_private_field_name(field); - let ty = &field.ty; - match classify_trait_object_field(ty) { - StructField::BoxDyn(_) - | StructField::RcDyn(_) - | StructField::ArcDyn(_) => { - quote! { - let mut #private_ident: #ty = <#ty as fory_core::serializer::ForyDefault>::fory_default(); - } - } - _ => { - quote! { - let mut #private_ident: Option<#ty> = None; - } - } - } - }); let loop_ts = get_fields_loop_ts(fields); quote! { - #(#declare_var_ts)* - let field_names = ::fory_get_sorted_field_names(fory); - let field_names = field_names.as_ref(); #loop_ts } }; let field_idents = fields.iter().map(|field| { let private_ident = create_private_field_name(field); let original_ident = &field.ident; - let ty = &field.ty; - match classify_trait_object_field(ty) { - StructField::BoxDyn(_) | StructField::RcDyn(_) | StructField::ArcDyn(_) => { - quote! { - #original_ident: #private_ident - } - } - StructField::ContainsTraitObject => { - quote! { - #original_ident: #private_ident.unwrap() - } - } - _ => { - quote! { - #original_ident: #private_ident.unwrap_or_default() - } - } + quote! { + #original_ident: #private_ident } }); quote! { @@ -501,7 +420,6 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { let declare_ts: Vec = declare_var(fields); let assign_ts: Vec = assign_value(fields); - let consistent_fields_loop_ts = get_fields_loop_ts(fields); quote! { let remote_type_id = context.reader.read_varuint32(); let meta_index = context.reader.read_varuint32(); @@ -517,9 +435,7 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { let local_type_hash = i64::from_le_bytes(high_bytes.try_into().unwrap()); if meta.get_hash() == local_type_hash { // fast path - let field_names = ::fory_get_sorted_field_names(fory); - let field_names = field_names.as_ref(); - #consistent_fields_loop_ts + ::fory_read_data(fory, context, false) } else { for _field in fields.iter() { #(#pattern_items else)* { @@ -529,10 +445,10 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { fory_core::serializer::skip::skip_field_value(fory, context, &nullable_field_type, read_ref_flag).unwrap(); } } + Ok(Self { + #(#assign_ts),* + }) } - Ok(Self { - #(#assign_ts),* - }) } } diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 20f0421c4a..53b07f20dc 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -135,7 +135,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #actual_type_id_ts } - fn fory_get_sorted_field_names(fory: &fory_core::fory::Fory) -> std::sync::Arc> { + fn fory_get_sorted_field_names(_fory: &fory_core::fory::Fory) -> &'static [&'static str] { #get_sorted_field_names_ts } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 8626bf4629..6841dfa145 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -250,7 +250,7 @@ macro_rules! basic_type_deserialize { }; } -pub fn try_primitive_vec_type(node: &TypeNode) -> Option { +pub(super) fn try_primitive_vec_type(node: &TypeNode) -> Option { if node.name != "Vec" { return None; } @@ -263,7 +263,7 @@ pub fn try_primitive_vec_type(node: &TypeNode) -> Option { None } -pub fn try_vec_of_option_primitive(node: &TypeNode) -> Option { +pub(super) fn try_vec_of_option_primitive(node: &TypeNode) -> Option { if node.name != "Vec" { return None; } @@ -283,7 +283,7 @@ pub fn try_vec_of_option_primitive(node: &TypeNode) -> Option { None } -pub fn try_primitive_vec_type_name(node: &NullableTypeNode) -> Option { +pub(super) fn try_primitive_vec_type_name(node: &NullableTypeNode) -> Option { if node.name != "Vec" { return None; } @@ -782,367 +782,223 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { } type FieldGroup = Vec<(String, String, u32)>; -type FieldGroups = (FieldGroup, FieldGroup, FieldGroup, FieldGroup); -pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { - fn group_fields(fields: &[&Field]) -> FieldGroups { - const PRIMITIVE_TYPE_NAMES: [&str; 7] = ["bool", "i8", "i16", "i32", "i64", "f32", "f64"]; - const FINAL_TYPE_NAMES: [&str; 3] = ["String", "NaiveDate", "NaiveDateTime"]; - const PRIMITIVE_ARRAY_NAMES: [&str; 7] = [ - "Vec", - "Vec", - "Vec", - "Vec", - "Vec", - "Vec", - "Vec", - ]; - - fn extract_option_inner(s: &str) -> Option<&str> { - s.strip_prefix("Option<")?.strip_suffix(">") - } - - macro_rules! match_ty { - ($ty:expr, $(($name:expr, $ret:expr)),+ $(,)?) => { - $( - if $ty == $name { - $ret as u32 - } else - )+ - { - unreachable!("Unknown type: {}", $ty); - } - }; - } - - fn get_primitive_type_id(ty: &str) -> u32 { - match_ty!( - ty, - ("bool", TypeId::BOOL), - ("i8", TypeId::INT8), - ("i16", TypeId::INT16), - ("i32", TypeId::INT32), - ("i64", TypeId::INT64), - ("f32", TypeId::FLOAT32), - ("f64", TypeId::FLOAT64), - ) - } - - let mut primitive_fields = Vec::new(); - let mut nullable_primitive_fields = Vec::new(); - let mut final_fields = Vec::new(); - let mut collection_fields = Vec::new(); - let mut map_fields = Vec::new(); - let mut struct_or_enum_fields = Vec::new(); - - // First handle Forward fields separately to avoid borrow checker issues - for field in fields { - if is_forward_field(&field.ty) { - let ident = field.ident.as_ref().unwrap().to_string(); - collection_fields.push((ident, "Forward".to_string(), TypeId::LIST as u32)); - } - } +type FieldGroups = ( + FieldGroup, + FieldGroup, + FieldGroup, + FieldGroup, + FieldGroup, + FieldGroup, + FieldGroup, +); + +const PRIMITIVE_TYPE_NAMES: [&str; 7] = ["bool", "i8", "i16", "i32", "i64", "f32", "f64"]; + +fn get_primitive_type_id(ty: &str) -> u32 { + match ty { + "bool" => TypeId::BOOL as u32, + "i8" => TypeId::INT8 as u32, + "i16" => TypeId::INT16 as u32, + "i32" => TypeId::INT32 as u32, + "i64" => TypeId::INT64 as u32, + "f32" => TypeId::FLOAT32 as u32, + "f64" => TypeId::FLOAT64 as u32, + _ => unreachable!("Unknown primitive type: {}", ty), + } +} - let mut group_field = |ident: String, ty: &str| { - if PRIMITIVE_TYPE_NAMES.contains(&ty) { - let type_id = get_primitive_type_id(ty); - primitive_fields.push((ident, ty.to_string(), type_id)); - } else if FINAL_TYPE_NAMES.contains(&ty) || PRIMITIVE_ARRAY_NAMES.contains(&ty) { - let type_id = match_ty!( - ty, - ("String", TypeId::STRING), - ("NaiveDate", TypeId::LOCAL_DATE), - ("NaiveDateTime", TypeId::TIMESTAMP), - ("Vec", TypeId::BINARY), - ("Vec", TypeId::BOOL_ARRAY), - ("Vec", TypeId::INT8_ARRAY), - ("Vec", TypeId::INT16_ARRAY), - ("Vec", TypeId::INT32_ARRAY), - ("Vec", TypeId::INT64_ARRAY), - ("Vec", TypeId::FLOAT32_ARRAY), - ("Vec", TypeId::FLOAT64_ARRAY), - ); - final_fields.push((ident, ty.to_string(), type_id)); - } else if ty.starts_with("Vec<") - || ty.starts_with("VecDeque<") - || ty.starts_with("LinkedList<") - || ty.starts_with("BinaryHeap<") - { - collection_fields.push((ident, ty.to_string(), TypeId::LIST as u32)); - } else if ty.starts_with("HashSet<") || ty.starts_with("BTreeSet<") { - collection_fields.push((ident, ty.to_string(), TypeId::SET as u32)); - } else if ty.starts_with("HashMap<") || ty.starts_with("BTreeMap<") { - map_fields.push((ident, ty.to_string(), TypeId::MAP as u32)); - } else { - struct_or_enum_fields.push((ident, ty.to_string(), 0)); - } - }; +fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { + fn extract_option_inner(s: &str) -> Option<&str> { + s.strip_prefix("Option<")?.strip_suffix(">") + } - for field in fields { + let mut primitive_fields = Vec::new(); + let mut nullable_primitive_fields = Vec::new(); + let mut internal_type_fields = Vec::new(); + let mut list_fields = Vec::new(); + let mut set_fields = Vec::new(); + let mut map_fields = Vec::new(); + let mut other_fields = Vec::new(); + + // First handle Forward fields separately to avoid borrow checker issues + for field in fields { + if is_forward_field(&field.ty) { let ident = field.ident.as_ref().unwrap().to_string(); - - // Skip if already handled as Forward field - if is_forward_field(&field.ty) { - continue; - } - - let ty: String = field - .ty - .to_token_stream() - .to_string() - .chars() - .filter(|c| !c.is_whitespace()) - .collect::(); - // handle Option specially - if let Some(inner) = extract_option_inner(&ty) { - if PRIMITIVE_TYPE_NAMES.contains(&inner) { - let type_id = get_primitive_type_id(inner); - nullable_primitive_fields.push((ident, ty.to_string(), type_id)); - } else { - // continue to handle Option - // already avoid Option> at compile-time - group_field(ident, inner); - } - } else { - group_field(ident, &ty); - } - } - - for field in fields { - if is_box_dyn_trait(&field.ty).is_some() { - let ident = field.ident.as_ref().unwrap().to_string(); - if let Some(pos) = struct_or_enum_fields.iter().position(|x| x.0 == ident) { - struct_or_enum_fields[pos].2 = TypeId::UNKNOWN as u32; - } - } + other_fields.push((ident, "Forward".to_string(), TypeId::UNKNOWN as u32)); } + } - fn sorter(a: &(String, String, u32), b: &(String, String, u32)) -> std::cmp::Ordering { - a.2.cmp(&b.2).then_with(|| a.0.cmp(&b.0)) - } - fn get_primitive_type_size(type_id_num: u32) -> i32 { - let type_id = TypeId::try_from(type_id_num as i16).unwrap(); - match type_id { - TypeId::BOOL => 1, - TypeId::INT8 => 1, - TypeId::INT16 => 2, - TypeId::INT32 => 4, - TypeId::VAR_INT32 => 4, - TypeId::INT64 => 8, - TypeId::VAR_INT64 => 8, - TypeId::FLOAT16 => 2, - TypeId::FLOAT32 => 4, - TypeId::FLOAT64 => 8, - _ => unreachable!(), - } + fn get_other_internal_type_id(ty: &str) -> u32 { + match ty { + "String" => TypeId::STRING as u32, + "NaiveDate" => TypeId::LOCAL_DATE as u32, + "NaiveDateTime" => TypeId::TIMESTAMP as u32, + "Duration" => TypeId::DURATION as u32, + "Decimal" => TypeId::DECIMAL as u32, + "Vec" | "bytes" => TypeId::BINARY as u32, + "Vec" => TypeId::BOOL_ARRAY as u32, + "Vec" => TypeId::INT8_ARRAY as u32, + "Vec" => TypeId::INT16_ARRAY as u32, + "Vec" => TypeId::INT32_ARRAY as u32, + "Vec" => TypeId::INT64_ARRAY as u32, + "Vec" => TypeId::FLOAT16_ARRAY as u32, + "Vec" => TypeId::FLOAT32_ARRAY as u32, + "Vec" => TypeId::FLOAT64_ARRAY as u32, + _ => 0, } + } - fn is_compress(type_id: u32) -> bool { - [ - TypeId::INT32 as u32, - TypeId::INT64 as u32, - TypeId::VAR_INT32 as u32, - TypeId::VAR_INT64 as u32, - ] - .contains(&type_id) + let mut group_field = |ident: String, ty: &str| { + if PRIMITIVE_TYPE_NAMES.contains(&ty) { + primitive_fields.push((ident, ty.to_string(), get_primitive_type_id(ty))); + } else if get_other_internal_type_id(ty) > 0 { + let internal_type_id = get_other_internal_type_id(ty); + internal_type_fields.push((ident, ty.to_string(), internal_type_id)); + } else if ty.starts_with("Vec<") + || ty.starts_with("VecDeque<") + || ty.starts_with("LinkedList<") + || ty.starts_with("BinaryHeap<") + { + list_fields.push((ident, ty.to_string(), TypeId::LIST as u32)); + } else if ty.starts_with("HashSet<") || ty.starts_with("BTreeSet<") { + set_fields.push((ident, ty.to_string(), TypeId::SET as u32)); + } else if ty.starts_with("HashMap<") || ty.starts_with("BTreeMap<") { + map_fields.push((ident, ty.to_string(), TypeId::MAP as u32)); + } else { + other_fields.push((ident, ty.to_string(), TypeId::UNKNOWN as u32)); } + }; - fn numeric_sorter( - a: &(String, String, u32), - b: &(String, String, u32), - ) -> std::cmp::Ordering { - let compress_a = is_compress(a.2); - let compress_b = is_compress(b.2); - let size_a = get_primitive_type_size(a.2); - let size_b = get_primitive_type_size(b.2); - compress_a - .cmp(&compress_b) - .then_with(|| size_b.cmp(&size_a)) - .then_with(|| a.0.cmp(&b.0)) + for field in fields { + let ident = field.ident.as_ref().unwrap().to_string(); + + // Skip if already handled as Forward field + if is_forward_field(&field.ty) { + continue; + } + + let ty: String = field + .ty + .to_token_stream() + .to_string() + .chars() + .filter(|c| !c.is_whitespace()) + .collect::(); + // handle Option specially + if let Some(inner) = extract_option_inner(&ty) { + if PRIMITIVE_TYPE_NAMES.contains(&inner) { + let type_id = get_primitive_type_id(inner); + nullable_primitive_fields.push((ident, ty.to_string(), type_id)); + } else { + group_field(ident, inner); + } + } else { + group_field(ident, &ty); } - - primitive_fields.sort_by(numeric_sorter); - nullable_primitive_fields.sort_by(numeric_sorter); - primitive_fields.extend(nullable_primitive_fields); - collection_fields.sort_by(sorter); - map_fields.sort_by(sorter); - let container_fields = { - let mut container_fields = collection_fields; - container_fields.extend(map_fields); - container_fields - }; - ( - primitive_fields, - final_fields, - container_fields, - struct_or_enum_fields, - ) } - fn gen_vec_token_stream(fields: &[(String, String, u32)]) -> TokenStream { - let names = fields.iter().map(|(name, _, _)| { - quote! { #name.to_string() } - }); - quote! { - vec![#(#names),*] + fn get_primitive_type_size(type_id_num: u32) -> i32 { + let type_id = TypeId::try_from(type_id_num as i16).unwrap(); + match type_id { + TypeId::BOOL => 1, + TypeId::INT8 => 1, + TypeId::INT16 => 2, + TypeId::INT32 => 4, + TypeId::VAR_INT32 => 4, + TypeId::INT64 => 8, + TypeId::VAR_INT64 => 8, + TypeId::FLOAT16 => 2, + TypeId::FLOAT32 => 4, + TypeId::FLOAT64 => 8, + _ => unreachable!(), } } - fn gen_vec_tuple_token_stream(fields: &[(String, String, u32)]) -> TokenStream { - let names = fields.iter().map(|(name, _, type_id)| { - quote! { (#type_id, #name.to_string()) } - }); - quote! { - vec![#(#names),*] - } + fn is_compress(type_id: u32) -> bool { + [ + TypeId::INT32 as u32, + TypeId::INT64 as u32, + TypeId::VAR_INT32 as u32, + TypeId::VAR_INT64 as u32, + ] + .contains(&type_id) } - let (all_primitive_fields, final_fields, container_fields, struct_or_enum_fields) = - group_fields(fields); + fn numeric_sorter(a: &(String, String, u32), b: &(String, String, u32)) -> std::cmp::Ordering { + let compress_a = is_compress(a.2); + let compress_b = is_compress(b.2); + let size_a = get_primitive_type_size(a.2); + let size_b = get_primitive_type_size(b.2); + compress_a + .cmp(&compress_b) + .then_with(|| size_b.cmp(&size_a)) + .then_with(|| a.2.cmp(&b.2)) + .then_with(|| a.0.cmp(&b.0)) + } - let all_primitive_field_names_declare_extend_ts = { - if all_primitive_fields.is_empty() { - (quote! {}, quote! {}) - } else { - let all_primitive_field_names_ts = gen_vec_token_stream(&all_primitive_fields); - ( - quote! { - let all_primitive_field_names: Vec = #all_primitive_field_names_ts; - }, - quote! { - sorted_field_names.extend(all_primitive_field_names); - }, - ) - } - }; - let container_field_names_declare_extend_ts = { - if container_fields.is_empty() { - (quote! {}, quote! {}) - } else { - let container_field_names_ts = gen_vec_token_stream(&container_fields); - ( - quote! { - let container_field_names: Vec = #container_field_names_ts; - }, - quote! { - sorted_field_names.extend(container_field_names); - }, - ) - } - }; - let sorter_ts = quote! { - |a: &(u32, String), b: &(u32, String)| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)) - }; - let final_fields_declare_extend_ts = { - if final_fields.is_empty() && struct_or_enum_fields.is_empty() { - (quote! {}, quote! {}) - } else { - let final_fields_ts = gen_vec_tuple_token_stream(&final_fields); - ( - quote! { - let mut final_fields: Vec<(u32, String)> = #final_fields_ts; - }, - quote! { - final_fields.sort_by(#sorter_ts); - for (_, name) in final_fields.drain(..) { sorted_field_names.push(name); } - }, - ) - } - }; - let other_fields_declare_extend_ts = { - if struct_or_enum_fields.is_empty() { - (quote! {}, quote! {}) - } else { - ( - quote! { - let mut other_fields: Vec<(u32, String)> = vec![]; - }, - quote! { - other_fields.sort_by(#sorter_ts); - for (_, name) in other_fields.drain(..) { sorted_field_names.push(name); } - }, - ) - } - }; - let trait_object_fields_ts = { - let trait_obj_fields: Vec<_> = struct_or_enum_fields - .iter() - .filter(|(_, _, type_id)| *type_id == fory_core::types::TypeId::UNKNOWN as u32) - .collect(); - - if trait_obj_fields.is_empty() { - quote! {} - } else { - let names = trait_obj_fields.iter().map(|(name, _, type_id)| { - quote! { - final_fields.push((#type_id, #name.to_string())); - } - }); - quote! { - #(#names)* - } - } - }; + fn type_id_then_name_sorter( + a: &(String, String, u32), + b: &(String, String, u32), + ) -> std::cmp::Ordering { + a.2.cmp(&b.2).then_with(|| a.0.cmp(&b.0)) + } - let group_sort_enum_other_fields = { - if struct_or_enum_fields.is_empty() { - quote! {} - } else { - let ts = struct_or_enum_fields - .iter() - .filter(|(_, _, type_id)| *type_id != fory_core::types::TypeId::UNKNOWN as u32) - .map(|(name, ty, _)| { - let ty_type: Type = syn::parse_str(ty).unwrap(); - quote! { - let field_type_id = <#ty_type as fory_core::serializer::Serializer>::fory_get_type_id(fory); - let internal_id = field_type_id & 0xff; - if internal_id == fory_core::types::TypeId::COMPATIBLE_STRUCT as u32 - || internal_id == fory_core::types::TypeId::NAMED_COMPATIBLE_STRUCT as u32 - || internal_id == fory_core::types::TypeId::STRUCT as u32 - || internal_id == fory_core::types::TypeId::NAMED_STRUCT as u32 - || internal_id == fory_core::types::TypeId::EXT as u32 - || internal_id == fory_core::types::TypeId::NAMED_EXT as u32 - { - other_fields.push((field_type_id, #name.to_string())); - } else if internal_id == fory_core::types::TypeId::ENUM as u32 || internal_id == fory_core::types::TypeId::NAMED_ENUM as u32 { - final_fields.push((field_type_id, #name.to_string())); - } else { - unimplemented!("unknown internal_id when group_sort_enum_other_fields"); - } - } - }) - .collect::>(); - quote! { - { - #(#ts)* - } - } - } - }; + fn name_sorter(a: &(String, String, u32), b: &(String, String, u32)) -> std::cmp::Ordering { + a.0.cmp(&b.0) + } - let (all_primitive_declare, all_primitive_extend) = all_primitive_field_names_declare_extend_ts; - let (container_declare, container_extend) = container_field_names_declare_extend_ts; - let (final_declare, final_extend) = final_fields_declare_extend_ts; - let (other_declare, other_extend) = other_fields_declare_extend_ts; + primitive_fields.sort_by(numeric_sorter); + nullable_primitive_fields.sort_by(numeric_sorter); + internal_type_fields.sort_by(type_id_then_name_sorter); + list_fields.sort_by(name_sorter); + set_fields.sort_by(name_sorter); + map_fields.sort_by(name_sorter); + other_fields.sort_by(name_sorter); + + ( + primitive_fields, + nullable_primitive_fields, + internal_type_fields, + list_fields, + set_fields, + map_fields, + other_fields, + ) +} - let fields_len = fields.len(); +pub(crate) fn get_sorted_field_names(fields: &[&Field]) -> Vec { + let ( + primitive_fields, + nullable_primitive_fields, + internal_type_fields, + list_fields, + set_fields, + map_fields, + other_fields, + ) = group_fields_by_type(fields); + + let mut all_fields = primitive_fields; + all_fields.extend(nullable_primitive_fields); + all_fields.extend(internal_type_fields); + all_fields.extend(list_fields); + all_fields.extend(set_fields); + all_fields.extend(map_fields); + all_fields.extend(other_fields); + + all_fields.into_iter().map(|(name, _, _)| name).collect() +} +pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream { + let sorted_names = get_sorted_field_names(fields); + let names = sorted_names.iter().map(|name| { + quote! { #name } + }); quote! { - let sorted_field_names = { - #all_primitive_declare - #final_declare - #other_declare - #container_declare - - #trait_object_fields_ts - #group_sort_enum_other_fields - - let mut sorted_field_names: Vec = Vec::with_capacity(#fields_len); - #all_primitive_extend - #final_extend - #other_extend - #container_extend - - sorted_field_names - }; + &[#(#names),*] } } + +pub(crate) fn skip_ref_flag(ty: &Type) -> bool { + // !T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id) + PRIMITIVE_TYPE_NAMES.contains(&extract_type_name(ty).as_str()) +} diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 6adba6780c..5736f6aade 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -16,7 +16,8 @@ // under the License. use super::util::{ - classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, StructField, + classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, skip_ref_flag, + StructField, }; use proc_macro2::TokenStream; use quote::quote; @@ -98,15 +99,13 @@ pub fn gen_write_type_info() -> TokenStream { } } -fn gen_write_match_arm(field: &Field) -> TokenStream { +fn gen_write_field(field: &Field) -> TokenStream { let ty = &field.ty; let ident = &field.ident; - let name_str = ident.as_ref().unwrap().to_string(); - match classify_trait_object_field(ty) { StructField::BoxDyn(_) => { quote! { - #name_str => { + { let any_ref = self.#ident.as_any(); let concrete_type_id = any_ref.type_id(); let fory_type_id = fory.get_type_resolver() @@ -131,7 +130,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); } @@ -142,7 +141,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); } @@ -153,7 +152,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) .collect(); @@ -166,7 +165,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) .collect(); @@ -179,7 +178,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) .collect(); @@ -192,7 +191,7 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - #name_str => { + { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) .collect(); @@ -202,63 +201,28 @@ fn gen_write_match_arm(field: &Field) -> TokenStream { } StructField::Forward => { quote! { - #name_str => { + { fory_core::serializer::Serializer::fory_write(&self.#ident, fory, context, true); } } } _ => { + let skip_ref_flag = skip_ref_flag(ty); quote! { - #name_str => { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); - fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, skip_ref_flag, false); - } + fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, #skip_ref_flag, false); } } } } pub fn gen_write_data(fields: &[&Field]) -> TokenStream { - let sorted_serialize = if fields.is_empty() { + if fields.is_empty() { quote! {} } else { - let match_ts: Vec<_> = fields - .iter() - .map(|field| gen_write_match_arm(field)) - .collect(); - #[cfg(not(feature = "fields-loop-unroll"))] - let loop_ts = quote! { - for field_name in sorted_field_names { - match field_name.as_str() { - #(#match_ts),* - , _ => {unreachable!()} - } - } - }; - #[cfg(feature = "fields-loop-unroll")] - let loop_ts = { - let loop_item_ts = fields.iter().enumerate().map(|(i, _field)| { - let idx = syn::Index::from(i); - quote! { - let field_name = sorted_field_names.get(#idx).unwrap(); - match field_name.as_str() { - #(#match_ts),* - , _ => { unreachable!() } - } - } - }); - quote! { - #(#loop_item_ts)* - } - }; + let write_fields_ts: Vec<_> = fields.iter().map(|field| gen_write_field(field)).collect(); quote! { - let sorted_field_names = ::fory_get_sorted_field_names(fory); - let sorted_field_names = sorted_field_names.as_ref(); - #loop_ts + #(#write_fields_ts)* } - }; - quote! { - #sorted_serialize } } diff --git a/rust/fory-derive/src/util.rs b/rust/fory-derive/src/util.rs index edb0121711..6990fc7767 100644 --- a/rust/fory-derive/src/util.rs +++ b/rust/fory-derive/src/util.rs @@ -18,9 +18,23 @@ use syn::{Field, Fields, GenericArgument, PathArguments, Type, TypePath, TypeTraitObject}; pub fn sorted_fields(fields: &Fields) -> Vec<&Field> { - let mut fields = fields.iter().collect::>(); - fields.sort_by(|a, b| a.ident.cmp(&b.ident)); - fields + let fields = fields.iter().collect::>(); + get_sorted_fields(&fields) +} + +pub fn get_sorted_fields<'a>(fields: &[&'a Field]) -> Vec<&'a Field> { + use crate::object::util::get_sorted_field_names; + + let sorted_names = get_sorted_field_names(fields); + let mut sorted_fields = Vec::with_capacity(fields.len()); + + for name in &sorted_names { + if let Some(field) = fields.iter().find(|f| *f.ident.as_ref().unwrap() == name) { + sorted_fields.push(*field); + } + } + + sorted_fields } /// Check if a type is `Box` and return the trait type and trait name if it is diff --git a/rust/tests/tests/test_simple_struct.rs b/rust/tests/tests/test_simple_struct.rs new file mode 100644 index 0000000000..438554a1e8 --- /dev/null +++ b/rust/tests/tests/test_simple_struct.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use fory_core::fory::Fory; +use fory_core::types::Mode::Compatible; +use fory_derive::ForyObject; +use std::collections::HashMap; + +#[test] +fn test_simple() { + // a single test for cargo expand and analysis + // &["f7", "last", "f2", "f5", "f3", "f6", "f1"] + #[derive(ForyObject, Debug)] + struct Animal1 { + f1: HashMap>, + f2: String, + f3: Vec, + f5: String, + f6: Vec, + f7: i8, + last: i8, + } + + // &["f7", "f5", "last", "f4", "f3", "f6", "f1"] + #[derive(ForyObject, Debug)] + struct Animal2 { + f1: HashMap>, + f3: Vec, + f4: String, + f5: i8, + f6: Vec, + f7: i16, + last: i8, + } + let mut fory1 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().mode(Compatible); + fory1.register::(999); + fory2.register::(999); + let animal: Animal1 = Animal1 { + f1: HashMap::from([(1, vec![2])]), + f2: String::from("hello"), + f3: vec![1, 2, 3], + f5: String::from("f5"), + f6: vec![42], + f7: 43, + last: 44, + }; + + let bin = fory1.serialize(&animal); + let obj: Animal2 = fory2.deserialize(&bin).unwrap(); + assert_eq!(animal.f1, obj.f1); + assert_eq!(animal.f3, obj.f3); + assert_eq!(obj.f4, String::default()); + assert_eq!(obj.f5, i8::default()); + assert_eq!(obj.f6, Vec::::default()); + assert_eq!(obj.f7, i16::default()); + assert_eq!(animal.last, obj.last); +} From e0ce954afe3a87d136656a0e947135ec4da242ce Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 22:38:41 +0530 Subject: [PATCH 12/37] fix(ci): fix sync files (#2752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues #2750 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/scripts/add_doc_headers.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/add_doc_headers.sh b/.github/scripts/add_doc_headers.sh index 7e77729784..5e34d4aa00 100755 --- a/.github/scripts/add_doc_headers.sh +++ b/.github/scripts/add_doc_headers.sh @@ -101,6 +101,8 @@ add_header "python/README.md" "docs/guide/python_guide.md" "$PYTHON_HEADER" # Process Rust guide rm -rf docs/guide/rust_guide.md add_header "rust/README.md" "docs/guide/rust_guide.md" "$RUST_HEADER" +git config --global user.email "dev@fory.apache.org" +git config --global user.name "Apache Fory" git add docs/guide/rust_guide.md git add docs/guide/python_guide.md git commit -m "Added rust and python docs" \ No newline at end of file From eb8dc32c94517c0209b550122ffac42ddfb62ced Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 22:56:41 +0530 Subject: [PATCH 13/37] feat(ci): add maven cache to ci for faster build (#2751) ## Why? ## What does this PR do? add maven cache to ci for faster build ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/ci.yml | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bdcc8cc88..406da8cfd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,14 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python3.8 uses: actions/setup-python@v5 with: @@ -89,6 +97,14 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "adopt-openj9" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python3.8 uses: actions/setup-python@v5 with: @@ -117,6 +133,14 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python 3.8 uses: actions/setup-python@v5 with: @@ -139,6 +163,14 @@ jobs: distribution: "graalvm" github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: "true" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python3.8 uses: actions/setup-python@v5 with: @@ -162,6 +194,14 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python 3.8 uses: actions/setup-python@v5 with: @@ -182,6 +222,14 @@ jobs: java-version: 11 distribution: "temurin" cache: sbt + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - uses: sbt/setup-sbt@v1 - name: Install fory java run: cd java && mvn -T10 --no-transfer-progress clean install -DskipTests && cd - @@ -199,6 +247,14 @@ jobs: with: java-version: 11 distribution: "temurin" + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Set up Python 3.8 uses: actions/setup-python@v5 with: @@ -253,6 +309,14 @@ jobs: with: java-version: 11 distribution: temurin + - name: Restore Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Run Rust CI run: python ./ci/run_ci.py rust From 740156cdf9b8bca74e0dd67d2a88caa0b108a938 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 23:04:14 +0530 Subject: [PATCH 14/37] docs(rust): update rust roadmap (#2754) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/README.md b/rust/README.md index 026976d17d..55e1793aed 100644 --- a/rust/README.md +++ b/rust/README.md @@ -968,6 +968,8 @@ cargo clippy --all-targets --all-features -- -D warnings - [x] Trait object serialization with polymorphism - [x] Schema evolution in compatible mode - [x] SIMD optimizations for string encoding +- [ ] Cross-language support for shared and circular reference tracking +- [ ] Cross-language support for trait objects - [ ] Performance optimizations - [ ] More comprehensive benchmarks From c76d80f325910eb252b322f0c607ba61ea526426 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 23:04:24 +0530 Subject: [PATCH 15/37] fix(docs): fix rust doc links build and check (#2753) ## Why? ## What does this PR do? ## Related issues #2752 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/README.md b/rust/README.md index 55e1793aed..acf9d21710 100644 --- a/rust/README.md +++ b/rust/README.md @@ -803,7 +803,7 @@ fory.register::(100); fory.register_by_namespace::("com.example", "MyStruct"); ``` -See [xlang_type_mapping.md](../docs/guide/xlang_type_mapping.md) for type mapping across languages. +See [xlang_type_mapping.md](https://fory.apache.org/docs/specification/fory_xlang_serialization_spec) for type mapping across languages. ## ⚡ Performance @@ -825,9 +825,8 @@ cargo bench --package fory-benchmarks ## 📖 Documentation - **[API Documentation](https://docs.rs/fory)** - Complete API reference -- **[Protocol Specification](../docs/specification/xlang_serialization_spec.md)** - Serialization protocol details -- **[Row Format Specification](../docs/specification/row_format_spec.md)** - Row format details -- **[Type Mapping](../docs/guide/xlang_type_mapping.md)** - Cross-language type mappings +- **[Protocol Specification](https://fory.apache.org/docs/specification/fory_xlang_serialization_spec)** - Serialization protocol details +- **[Type Mapping](https://fory.apache.org/docs/docs/guide/xlang_type_mapping)** - Cross-language type mappings ## 🎯 Use Cases @@ -975,11 +974,11 @@ cargo clippy --all-targets --all-features -- -D warnings ## 📄 License -Licensed under the Apache License, Version 2.0. See [LICENSE](../LICENSE) for details. +Licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/apache/fory/blob/main/LICENSE) for details. ## 🤝 Contributing -We welcome contributions! Please see our [Contributing Guide](../CONTRIBUTING.md) for details. +We welcome contributions! Please see our [Contributing Guide](https://github.com/apache/fory/blob/main/CONTRIBUTING.md) for details. ## 📞 Support From d093d43403180c4361fca71f443668be1721048e Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 23:26:02 +0530 Subject: [PATCH 16/37] fix(java): fix maven cache (#2755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/ci.yml | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 406da8cfd2..7bf93e5996 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,9 +54,8 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -97,9 +96,8 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "adopt-openj9" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -133,9 +131,8 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -163,9 +160,8 @@ jobs: distribution: "graalvm" github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: "true" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -194,9 +190,8 @@ jobs: with: java-version: ${{ matrix.java-version }} distribution: "temurin" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -222,9 +217,8 @@ jobs: java-version: 11 distribution: "temurin" cache: sbt - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -247,9 +241,8 @@ jobs: with: java-version: 11 distribution: "temurin" - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} @@ -309,9 +302,8 @@ jobs: with: java-version: 11 distribution: temurin - - name: Restore Maven Repos - id: restore-maven-cache - uses: actions/cache/restore@v3 + - name: Cache Maven local repository + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} From 3000a1f3780af3e7c7c7a0297af6fd09175efec7 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Sun, 12 Oct 2025 23:26:12 +0530 Subject: [PATCH 17/37] docs(rust): fix rust doc broken link (#2756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/README.md b/rust/README.md index acf9d21710..146b3977ed 100644 --- a/rust/README.md +++ b/rust/README.md @@ -2,7 +2,7 @@ [![Crates.io](https://img.shields.io/crates/v/fory.svg)](https://crates.io/crates/fory) [![Documentation](https://docs.rs/fory/badge.svg)](https://docs.rs/fory) -[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/apache/fory/blob/main/LICENSE) **Apache Fory™** is a blazing fast multi-language serialization framework powered by **JIT compilation** and **zero-copy** techniques, providing up to **ultra-fast performance** while maintaining ease of use and safety. @@ -22,9 +22,9 @@ The Rust implementation provides versatile and high-performance serialization wi | Crate | Description | Version | | ----------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- | -| [`fory`](fory/) | High-level API with derive macros | [![crates.io](https://img.shields.io/crates/v/fory.svg)](https://crates.io/crates/fory) | -| [`fory-core`](fory-core/) | Core serialization engine | [![crates.io](https://img.shields.io/crates/v/fory-core.svg)](https://crates.io/crates/fory-core) | -| [`fory-derive`](fory-derive/) | Procedural macros | [![crates.io](https://img.shields.io/crates/v/fory-derive.svg)](https://crates.io/crates/fory-derive) | +| [`fory`](https://github.com/apache/fory/blob/main/rust/fory) | High-level API with derive macros | [![crates.io](https://img.shields.io/crates/v/fory.svg)](https://crates.io/crates/fory) | +| [`fory-core`](https://github.com/apache/fory/blob/main/rust/fory-core/) | Core serialization engine | [![crates.io](https://img.shields.io/crates/v/fory-core.svg)](https://crates.io/crates/fory-core) | +| [`fory-derive`](https://github.com/apache/fory/blob/main/rust/fory-derive/) | Procedural macros | [![crates.io](https://img.shields.io/crates/v/fory-derive.svg)](https://crates.io/crates/fory-derive) | ## 🏃 Quick Start From 11e233489180b6014ae08dedd51707623df71665 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 13 Oct 2025 08:20:18 +0530 Subject: [PATCH 18/37] docs(rust): fix rust doc lint (#2757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/README.md b/rust/README.md index 146b3977ed..1237b822da 100644 --- a/rust/README.md +++ b/rust/README.md @@ -20,9 +20,9 @@ The Rust implementation provides versatile and high-performance serialization wi ## 📦 Crates -| Crate | Description | Version | -| ----------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- | -| [`fory`](https://github.com/apache/fory/blob/main/rust/fory) | High-level API with derive macros | [![crates.io](https://img.shields.io/crates/v/fory.svg)](https://crates.io/crates/fory) | +| Crate | Description | Version | +| --------------------------------------------------------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- | +| [`fory`](https://github.com/apache/fory/blob/main/rust/fory) | High-level API with derive macros | [![crates.io](https://img.shields.io/crates/v/fory.svg)](https://crates.io/crates/fory) | | [`fory-core`](https://github.com/apache/fory/blob/main/rust/fory-core/) | Core serialization engine | [![crates.io](https://img.shields.io/crates/v/fory-core.svg)](https://crates.io/crates/fory-core) | | [`fory-derive`](https://github.com/apache/fory/blob/main/rust/fory-derive/) | Procedural macros | [![crates.io](https://img.shields.io/crates/v/fory-derive.svg)](https://crates.io/crates/fory-derive) | From db4ea28dac4ab4940a7b4088f9d09abb2684406d Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 13 Oct 2025 17:55:07 +0530 Subject: [PATCH 19/37] feat(rust): fast fory_read_compatible macro to use match by assigned field id (#2758) ## Why? ## What does this PR do? This pr implemented a new fory_read_compatible macro function to use match by assigned field id for better performance. - When read type meta, assign field id based on local field type info. If type info are consistent(excluding nullability), then assign remote field to local field. Otherwise, skip this remote field value when deserializing, use default value instead. - When generating fory_read_compatible code, use match by field id arm, which will compiled to switch. This is much more efficient compared to previous compare field by string - Skip using Option is local field is primitive and not Option. Now the macro generate much more efficient and compact code: ```rust fn fory_read_compatible( fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, ) -> Result { let remote_type_id = context.reader.read_varuint32(); let meta_index = context.reader.read_varuint32(); let meta = context.get_meta(meta_index as usize); let fields = { let meta = context.get_meta(meta_index as usize); meta.get_field_infos().clone() }; let mut _f7: i16 = 0 as i16; let mut _f5: i8 = 0 as i8; let mut _last: i8 = 0 as i8; let mut _f4: Option = None; let mut _f3: Option> = None; let mut _f6: Option> = None; let mut _f1: Option>> = None; let local_type_def = fory .get_type_resolver() .get_type_info(std::any::TypeId::of::()) .get_type_def(); let high_bytes = &local_type_def[..8]; let local_type_hash = i64::from_le_bytes(high_bytes.try_into().unwrap()); if meta.get_hash() == local_type_hash { ::fory_read_data( fory, context, false, ) } else { for _field in fields.iter() { match _field.field_id { 0i16 => { if !&_field.field_type.nullable { _f7 = fory_core::serializer::read_ref_info_data::< i16, >(fory, context, true, true, false)?; } else { if (context.reader.read_bool()) { _f7 = ::fory_default(); } else { _f7 = fory_core::serializer::read_ref_info_data::< i16, >(fory, context, true, true, false)?; } } } 1i16 => { if !&_field.field_type.nullable { _f5 = fory_core::serializer::read_ref_info_data::< i8, >(fory, context, true, true, false)?; } else { if (context.reader.read_bool()) { _f5 = ::fory_default(); } else { _f5 = fory_core::serializer::read_ref_info_data::< i8, >(fory, context, true, true, false)?; } } } 2i16 => { if !&_field.field_type.nullable { _last = fory_core::serializer::read_ref_info_data::< i8, >(fory, context, true, true, false)?; } else { if (context.reader.read_bool()) { _last = ::fory_default(); } else { _last = fory_core::serializer::read_ref_info_data::< i8, >(fory, context, true, true, false)?; } } } 3i16 => { if !&_field.field_type.nullable { _f4 = Some( fory_core::serializer::read_ref_info_data::< String, >(fory, context, true, false, false)?, ); } else { if (context.reader.read_bool()) { _f4 = Some( ::fory_default(), ); } else { _f4 = Some( fory_core::serializer::read_ref_info_data::< String, >(fory, context, true, false, false)?, ); } } } 4i16 => { if !&_field.field_type.nullable { _f3 = Some( fory_core::serializer::read_ref_info_data::< Vec, >(fory, context, true, false, false)?, ); } else { if (context.reader.read_bool()) { _f3 = Some( as fory_core::serializer::ForyDefault>::fory_default(), ); } else { _f3 = Some( fory_core::serializer::read_ref_info_data::< Vec, >(fory, context, true, false, false)?, ); } } } 5i16 => { if !&_field.field_type.nullable { _f6 = Some( fory_core::serializer::read_ref_info_data::< Vec, >(fory, context, true, false, false)?, ); } else { if (context.reader.read_bool()) { _f6 = Some( as fory_core::serializer::ForyDefault>::fory_default(), ); } else { _f6 = Some( fory_core::serializer::read_ref_info_data::< Vec, >(fory, context, true, false, false)?, ); } } } 6i16 => { if !&_field.field_type.nullable { _f1 = Some( fory_core::serializer::read_ref_info_data::< HashMap>, >(fory, context, true, false, false)?, ); } else { if (context.reader.read_bool()) { _f1 = Some( , > as fory_core::serializer::ForyDefault>::fory_default(), ); } else { _f1 = Some( fory_core::serializer::read_ref_info_data::< HashMap>, >(fory, context, true, false, false)?, ); } } } _ => { let field_type = &_field.field_type; let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag( &field_type, ); fory_core::serializer::skip::skip_field_value( fory, context, &field_type, read_ref_flag, ) .unwrap(); } } } Ok(Self { f7: _f7, f5: _f5, last: _last, f4: _f4.unwrap_or_default(), f3: _f3.unwrap_or_default(), f6: _f6.unwrap_or_default(), f1: _f1.unwrap_or_default(), }) } ``` ## Related issues #2492 #2545 Closes #2761 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- AGENTS.md | 6 + CONTRIBUTING.md | 2 + docs/guide/DEVELOPMENT.md | 6 +- rust/fory-core/src/buffer.rs | 7 + rust/fory-core/src/fory.rs | 2 +- rust/fory-core/src/meta/mod.rs | 3 +- rust/fory-core/src/meta/type_meta.rs | 169 +++---- rust/fory-core/src/resolver/context.rs | 11 +- rust/fory-core/src/resolver/meta_resolver.rs | 89 ++-- rust/fory-core/src/resolver/type_resolver.rs | 46 +- rust/fory-core/src/serializer/enum_.rs | 5 +- rust/fory-core/src/serializer/mod.rs | 6 +- rust/fory-core/src/serializer/skip.rs | 16 +- rust/fory-core/src/serializer/struct_.rs | 9 +- rust/fory-derive/src/object/misc.rs | 5 + rust/fory-derive/src/object/read.rs | 243 +++++---- rust/fory-derive/src/object/serializer.rs | 16 +- rust/fory-derive/src/object/util.rs | 488 ++----------------- rust/tests/tests/compatible/test_struct.rs | 1 + rust/tests/tests/test_simple_struct.rs | 2 +- 20 files changed, 377 insertions(+), 755 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 882f20c921..7e2cd126e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,6 +140,12 @@ cargo test --features tests # run specific test cargo test -p fory-tests --test $test_file $test_method +# run specific test under subdirectory +cargo test --test mod $dir$::$test_file::$test_method + +# inspect generated code by fory derive macro +cargo expand --test mod $mod$::$file$ > expanded.rs + # Format code cargo fmt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e84610760..180cd26469 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,8 @@ cd rust cargo test # run test with specific test file and method cargo test -p fory-tests --test $test_file $test_method +# run specific test under subdirectory + cargo test --test mod $dir$::$test_file::$test_method ``` ### JavaScript diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index 3b7105c85b..dc08f8a00c 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -97,7 +97,11 @@ cargo build # run test cargo test # run specific test -cargo test -p fory-tests --test $test_file $test_method +cargo test -p fory-tests --test $test_file $test_method +# run specific test under subdirectory +cargo test --test mod $dir$::$test_file::$test_method +# inspect generated code by fory derive macro +cargo expand --test mod $mod$::$file$ > expanded.rs ``` #### Environment Requirements diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index 9baf94c659..a51f6bdc2b 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -393,6 +393,13 @@ impl Reader { self.cursor } + #[inline(always)] + pub fn read_bool(&mut self) -> bool { + let result = unsafe { *self.ptr_at(self.cursor) }; + self.move_next(1); + result != 0 + } + #[inline(always)] pub fn read_u8(&mut self) -> u8 { let result = unsafe { *self.ptr_at(self.cursor) }; diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index a4c2b940b9..91f4f34740 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -404,7 +404,7 @@ impl Fory { if self.mode == Mode::Compatible { let meta_offset = context.reader.read_i32(); if meta_offset != -1 { - bytes_to_skip = context.load_meta(meta_offset as usize); + bytes_to_skip = context.load_meta(self.get_type_resolver(), meta_offset as usize); } } let result = ::fory_read(self, context, false); diff --git a/rust/fory-core/src/meta/mod.rs b/rust/fory-core/src/meta/mod.rs index d80a13eb84..ce7bd276ad 100644 --- a/rust/fory-core/src/meta/mod.rs +++ b/rust/fory-core/src/meta/mod.rs @@ -25,6 +25,5 @@ pub use meta_string::{ }; pub use string_util::{buffer_rw_string, get_latin1_length, is_latin, murmurhash3_x64_128}; pub use type_meta::{ - FieldInfo, FieldType, NullableFieldType, TypeMeta, TypeMetaLayer, NAMESPACE_ENCODINGS, - TYPE_NAME_ENCODINGS, + FieldInfo, FieldType, TypeMeta, TypeMetaLayer, NAMESPACE_ENCODINGS, TYPE_NAME_ENCODINGS, }; diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 0ae8e6c5ad..2dbbe06a9f 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -21,10 +21,12 @@ use crate::meta::{ murmurhash3_x64_128, Encoding, MetaString, MetaStringDecoder, FIELD_NAME_DECODER, FIELD_NAME_ENCODER, NAMESPACE_DECODER, TYPE_NAME_DECODER, }; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::types::{TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; use std::clone::Clone; use std::cmp::min; +use std::collections::HashMap; const SMALL_NUM_FIELDS_THRESHOLD: usize = 0b11111; const REGISTER_BY_NAME_FLAG: u8 = 0b100000; @@ -59,57 +61,20 @@ static FIELD_NAME_ENCODINGS: &[Encoding] = &[ #[derive(Debug, Eq, Clone)] pub struct FieldType { pub type_id: u32, - pub generics: Vec, -} - -#[derive(Debug, Clone)] -pub struct NullableFieldType { - pub type_id: u32, - pub generics: Vec, pub nullable: bool, + pub generics: Vec, } -impl NullableFieldType { - pub fn from(node: &FieldType) -> Self { - if node.type_id == TypeId::ForyNullable as u32 { - let inner = NullableFieldType::from(&node.generics[0]); - NullableFieldType { - type_id: inner.type_id, - generics: inner.generics, - nullable: true, - } - } else { - let generics = node.generics.iter().map(NullableFieldType::from).collect(); - NullableFieldType { - type_id: node.type_id, - generics, - nullable: false, - } - } - } -} - -impl PartialEq for NullableFieldType { - fn eq(&self, other: &Self) -> bool { - self.type_id == other.type_id && self.generics == other.generics - } -} - -impl Eq for NullableFieldType {} - impl FieldType { - pub fn new(type_id: u32, generics: Vec) -> Self { - FieldType { type_id, generics } + pub fn new(type_id: u32, nullable: bool, generics: Vec) -> Self { + FieldType { + type_id, + nullable, + generics, + } } fn to_bytes(&self, writer: &mut Writer, write_flag: bool, nullable: bool) -> Result<(), Error> { - if self.type_id == TypeId::ForyNullable as u32 { - self.generics - .first() - .unwrap() - .to_bytes(writer, write_flag, true)?; - return Ok(()); - } let mut header = self.type_id; if write_flag { header <<= 2; @@ -147,11 +112,12 @@ impl FieldType { type_id = header; _nullable = nullable.unwrap(); } - let field_type = match type_id { + match type_id { x if x == TypeId::LIST as u32 || x == TypeId::SET as u32 => { let generic = Self::from_bytes(reader, true, None); Self { type_id, + nullable: _nullable, generics: vec![generic], } } @@ -160,21 +126,15 @@ impl FieldType { let val_generic = Self::from_bytes(reader, true, None); Self { type_id, + nullable: _nullable, generics: vec![key_generic, val_generic], } } _ => Self { type_id, + nullable: _nullable, generics: vec![], }, - }; - if _nullable { - Self { - type_id: TypeId::ForyNullable as u32, - generics: vec![field_type], - } - } else { - field_type } } } @@ -241,7 +201,7 @@ impl FieldInfo { let name_size = name_encoded.len() - 1; let mut header: u8 = (min(FIELD_NAME_SIZE_THRESHOLD, name_size) as u8) << 2; // let ref_tracking = false; - let nullable = self.field_type.type_id == TypeId::ForyNullable as u32; + let nullable = self.field_type.nullable; // if ref_tracking { // header |= 1; // } @@ -296,6 +256,16 @@ impl TypeMetaLayer { } } + pub fn empty() -> TypeMetaLayer { + TypeMetaLayer { + type_id: 0, + namespace: MetaString::default(), + type_name: MetaString::default(), + register_by_name: false, + field_infos: vec![], + } + } + pub fn get_type_id(&self) -> u32 { self.type_id } @@ -396,14 +366,11 @@ impl TypeMetaLayer { let mut other_fields = Vec::new(); for field_info in field_infos.into_iter() { - let mut type_id = field_info.field_type.type_id; - let is_nullable = type_id == TypeId::ForyNullable as u32; - if is_nullable { - type_id = field_info.field_type.generics.first().unwrap().type_id; - if PRIMITIVE_TYPES.contains(&type_id) { - nullable_primitive_fields.push(field_info); - continue; - } + let type_id = field_info.field_type.type_id; + let is_nullable = field_info.field_type.nullable; + if is_nullable && PRIMITIVE_TYPES.contains(&type_id) { + nullable_primitive_fields.push(field_info); + continue; } if PRIMITIVE_TYPES.contains(&type_id) { @@ -447,28 +414,21 @@ impl TypeMetaLayer { .contains(&type_id) } fn numeric_sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { - let (a_id, b_id) = { - // for nullable_primitive - if a.field_type.type_id == TypeId::ForyNullable as u32 { - ( - a.field_type.generics.first().unwrap().type_id, - b.field_type.generics.first().unwrap().type_id, - ) - } else { - (a.field_type.type_id, b.field_type.type_id) - } - }; + let (a_id, b_id) = (a.field_type.type_id, b.field_type.type_id); let a_field_name = &a.field_name; let b_field_name = &b.field_name; let compress_a = is_compress(a_id); let compress_b = is_compress(b_id); let size_a = get_primitive_type_size(a_id); let size_b = get_primitive_type_size(b_id); - compress_a - .cmp(&compress_b) - .then_with(|| size_b.cmp(&size_a)) - .then_with(|| a_id.cmp(&b_id)) - .then_with(|| a_field_name.cmp(b_field_name)) + let a_nullable = a.field_type.nullable; + let b_nullable = b.field_type.nullable; + a_nullable + .cmp(&b_nullable) // non-nullable first + .then_with(|| compress_a.cmp(&compress_b)) // fixed-size (false) first, then variable-size (true) last + .then_with(|| size_b.cmp(&size_a)) // when same compress status: larger size first + .then_with(|| a_id.cmp(&b_id)) // when same size: smaller type id first + .then_with(|| a_field_name.cmp(b_field_name)) // when same id: lexicographic name } fn type_then_name_sorter(a: &FieldInfo, b: &FieldInfo) -> std::cmp::Ordering { a.field_type @@ -497,7 +457,7 @@ impl TypeMetaLayer { sorted_field_infos } - fn from_bytes(reader: &mut Reader) -> TypeMetaLayer { + fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) -> TypeMetaLayer { let meta_header = reader.read_u8(); let register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0; let mut num_fields = meta_header as usize & SMALL_NUM_FIELDS_THRESHOLD; @@ -517,11 +477,23 @@ impl TypeMetaLayer { namespace = empty_name.clone(); type_name = empty_name; } + let mut field_infos = Vec::with_capacity(num_fields); for _ in 0..num_fields { field_infos.push(FieldInfo::from_bytes(reader)); } - let sorted_field_infos = Self::sort_field_infos(field_infos); + let mut sorted_field_infos = Self::sort_field_infos(field_infos); + + if register_by_name { + if let Some(type_info_current) = + type_resolver.get_type_info_by_name(&namespace.original, &type_name.original) + { + Self::assign_field_ids(type_info_current, &mut sorted_field_infos); + } + } else if let Some(type_info_current) = type_resolver.get_type_info_by_id(type_id) { + Self::assign_field_ids(type_info_current, &mut sorted_field_infos); + } + // if no type found, keep all fields id as -1 to be skipped. TypeMetaLayer::new( type_id, namespace, @@ -530,6 +502,32 @@ impl TypeMetaLayer { sorted_field_infos, ) } + + fn assign_field_ids(type_info_current: &TypeInfo, field_infos: &mut [FieldInfo]) { + // convert to map: fiend_name -> field_info + let field_info_map = type_info_current + .get_type_meta() + .get_field_infos() + .iter() + .map(|field_info| (field_info.field_name.clone(), field_info.clone())) + .collect::>(); + for field in field_infos.iter_mut() { + match field_info_map.get(&field.field_name.clone()) { + Some(local_field_info) => { + if field.field_type.type_id != local_field_info.field_type.type_id + || field.field_type.generics != local_field_info.field_type.generics + { + field.field_id = -1; + } else { + field.field_id = local_field_info.field_id; + } + } + None => { + field.field_id = -1; + } + } + } + } } #[derive(Debug)] @@ -560,6 +558,13 @@ impl TypeMeta { self.layer.get_namespace().clone() } + pub fn empty() -> TypeMeta { + TypeMeta { + hash: 0, + layer: TypeMetaLayer::empty(), + } + } + pub fn from_fields( type_id: u32, namespace: MetaString, @@ -573,7 +578,7 @@ impl TypeMeta { } } #[allow(unused_assignments)] - pub fn from_bytes(reader: &mut Reader) -> TypeMeta { + pub fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) -> TypeMeta { let header = reader.read_i64(); let mut meta_size = header & META_SIZE_MASK; if meta_size == META_SIZE_MASK { @@ -586,7 +591,7 @@ impl TypeMeta { // let current_meta_size = 0; // while current_meta_size < meta_size {} - let layer = TypeMetaLayer::from_bytes(reader); + let layer = TypeMetaLayer::from_bytes(reader, type_resolver); TypeMeta { layer, hash: header, diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 4b3f81cc9d..f067383418 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -24,7 +24,7 @@ use crate::resolver::metastring_resolver::{ MetaStringBytes, MetaStringReaderResolver, MetaStringWriterResolver, }; use crate::resolver::ref_resolver::{RefReader, RefWriter}; -use crate::resolver::type_resolver::Harness; +use crate::resolver::type_resolver::{Harness, TypeResolver}; use std::sync::{Arc, Mutex}; pub struct WriteContext { @@ -158,10 +158,11 @@ impl ReadContext { } #[inline(always)] - pub fn load_meta(&mut self, offset: usize) -> usize { - self.meta_resolver.load(&mut Reader::new( - &self.reader.slice_after_cursor()[offset..], - )) + pub fn load_meta(&mut self, type_resolver: &TypeResolver, offset: usize) -> usize { + self.meta_resolver.load( + type_resolver, + &mut Reader::new(&self.reader.slice_after_cursor()[offset..]), + ) } pub fn read_any_typeinfo(&mut self, fory: &Fory) -> Arc { diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 6d12d744d9..0e15f653f1 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -19,9 +19,52 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; use crate::fory::Fory; use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER}; +use crate::TypeResolver; use std::collections::HashMap; use std::sync::Arc; +#[derive(Default)] +pub struct MetaWriterResolver { + type_defs: Vec>>, + type_id_index_map: HashMap, +} + +#[allow(dead_code)] +impl MetaWriterResolver { + pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize { + match self.type_id_index_map.get(&type_id) { + None => { + let index = self.type_defs.len(); + self.type_defs.push( + fory.get_type_resolver() + .get_type_info(type_id) + .get_type_def(), + ); + self.type_id_index_map.insert(type_id, index); + index + } + Some(index) => *index, + } + } + + pub fn to_bytes(&self, writer: &mut Writer) -> Result<(), Error> { + writer.write_varuint32(self.type_defs.len() as u32); + for item in &self.type_defs { + writer.write_bytes(item); + } + Ok(()) + } + + pub fn empty(&mut self) -> bool { + self.type_defs.is_empty() + } + + pub fn reset(&mut self) { + self.type_defs.clear(); + self.type_id_index_map.clear(); + } +} + #[derive(Default)] pub struct MetaReaderResolver { pub reading_type_defs: Vec>, @@ -32,11 +75,11 @@ impl MetaReaderResolver { unsafe { self.reading_type_defs.get_unchecked(index) } } - pub fn load(&mut self, reader: &mut Reader) -> usize { + pub fn load(&mut self, type_resolver: &TypeResolver, reader: &mut Reader) -> usize { let meta_size = reader.read_varuint32(); // self.reading_type_defs.reserve(meta_size as usize); for _ in 0..meta_size { - let type_meta = TypeMeta::from_bytes(reader); + let type_meta = TypeMeta::from_bytes(reader, type_resolver); self.reading_type_defs.push(Arc::new(type_meta)); } reader.get_cursor() @@ -71,45 +114,3 @@ impl MetaReaderResolver { self.reading_type_defs.clear(); } } - -#[derive(Default)] -pub struct MetaWriterResolver { - type_defs: Vec>>, - type_id_index_map: HashMap, -} - -#[allow(dead_code)] -impl MetaWriterResolver { - pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize { - match self.type_id_index_map.get(&type_id) { - None => { - let index = self.type_defs.len(); - self.type_defs.push( - fory.get_type_resolver() - .get_type_info(type_id) - .get_type_def(), - ); - self.type_id_index_map.insert(type_id, index); - index - } - Some(index) => *index, - } - } - - pub fn to_bytes(&self, writer: &mut Writer) -> Result<(), Error> { - writer.write_varuint32(self.type_defs.len() as u32); - for item in &self.type_defs { - writer.write_bytes(item); - } - Ok(()) - } - - pub fn empty(&mut self) -> bool { - self.type_defs.is_empty() - } - - pub fn reset(&mut self) { - self.type_defs.clear(); - self.type_id_index_map.clear(); - } -} diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index b0f9a5c9d9..e9b64f345e 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -23,6 +23,7 @@ use crate::meta::{ TYPE_NAME_ENCODINGS, }; use crate::serializer::{ForyDefault, Serializer, StructSerializer}; +use crate::Reader; use std::sync::Arc; use std::{any::Any, collections::HashMap}; @@ -88,6 +89,7 @@ impl Harness { #[derive(Clone, Debug)] pub struct TypeInfo { type_def: Arc>, + type_meta: Arc, type_id: u32, namespace: MetaString, type_name: MetaString, @@ -108,14 +110,16 @@ impl TypeInfo { let type_name_metastring = TYPE_NAME_ENCODER .encode_with_encodings(type_name, TYPE_NAME_ENCODINGS) .unwrap(); + let (type_def_bytes, type_meta) = T::fory_type_def( + fory, + type_id, + namespace_metastring.clone(), + type_name_metastring.clone(), + register_by_name, + ); TypeInfo { - type_def: Arc::from(T::fory_type_def( - fory, - type_id, - namespace_metastring.clone(), - type_name_metastring.clone(), - register_by_name, - )), + type_def: Arc::from(type_def_bytes), + type_meta: Arc::new(type_meta), type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -144,8 +148,11 @@ impl TypeInfo { vec![], ); let type_def = meta.to_bytes().unwrap(); + let type_resolver = _fory.get_type_resolver(); + let meta = TypeMeta::from_bytes(&mut Reader::new(&type_def), type_resolver); TypeInfo { type_def: Arc::from(type_def), + type_meta: Arc::new(meta), type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -169,6 +176,10 @@ impl TypeInfo { self.type_def.clone() } + pub fn get_type_meta(&self) -> Arc { + self.type_meta.clone() + } + pub fn is_registered_by_name(&self) -> bool { self.register_by_name } @@ -180,6 +191,8 @@ pub struct TypeResolver { type_id_map: HashMap, type_name_map: HashMap, type_info_cache: HashMap, + type_info_map_by_id: HashMap, + type_info_map_by_name: HashMap<(String, String), TypeInfo>, // Fast lookup by numeric ID for common types type_id_index: Vec, } @@ -194,6 +207,8 @@ impl Default for TypeResolver { type_id_map: HashMap::new(), type_name_map: HashMap::new(), type_info_cache: HashMap::new(), + type_info_map_by_id: HashMap::new(), + type_info_map_by_name: HashMap::new(), type_id_index: Vec::new(), }; resolver.register_builtin_types(); @@ -209,6 +224,7 @@ impl TypeResolver { ($ty:ty, $type_id:expr) => {{ let type_info = TypeInfo { type_def: Arc::from(vec![]), + type_meta: Arc::new(TypeMeta::empty()), type_id: $type_id as u32, namespace: NAMESPACE_ENCODER .encode_with_encodings("", NAMESPACE_ENCODINGS) @@ -243,12 +259,21 @@ impl TypeResolver { pub fn get_type_info(&self, type_id: std::any::TypeId) -> &TypeInfo { self.type_info_cache.get(&type_id).unwrap_or_else(|| { panic!( - "TypeId {:?} not found in type_info_map, maybe you forgot to register some types", + "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", type_id ) }) } + pub fn get_type_info_by_id(&self, id: u32) -> Option<&TypeInfo> { + self.type_info_map_by_id.get(&id) + } + + pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option<&TypeInfo> { + self.type_info_map_by_name + .get(&(namespace.to_owned(), type_name.to_owned())) + } + /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> u32 { let id_usize = id as usize; @@ -351,6 +376,8 @@ impl TypeResolver { panic!("rs_struct:{:?} already registered", rs_type_id); } self.type_info_cache.insert(rs_type_id, type_info.clone()); + self.type_info_map_by_id + .insert(type_info.type_id, type_info.clone()); let index = T::fory_type_index() as usize; if index >= self.type_id_index.len() { self.type_id_index.resize(index + 1, NO_TYPE_ID); @@ -380,6 +407,9 @@ impl TypeResolver { to_serializer::, )), ); + let string_key = (namespace.original.clone(), type_name.original.clone()); + self.type_info_map_by_name + .insert(string_key, type_info.clone()); } else { let type_id = type_info.type_id; if self.serializer_map.contains_key(&type_id) { diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index fa3c4c245b..da4688955c 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -38,9 +38,10 @@ pub fn type_def( namespace: MetaString, type_name: MetaString, register_by_name: bool, -) -> Vec { +) -> (Vec, TypeMeta) { let meta = TypeMeta::from_fields(type_id, namespace, type_name, register_by_name, vec![]); - meta.to_bytes().unwrap() + let bytes = meta.to_bytes().unwrap(); + (bytes, meta) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index c91eb09ab8..5ce462d5ff 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -17,7 +17,7 @@ use crate::error::Error; use crate::fory::Fory; -use crate::meta::{MetaString, NAMESPACE_DECODER, TYPE_NAME_DECODER}; +use crate::meta::{MetaString, TypeMeta, NAMESPACE_DECODER, TYPE_NAME_DECODER}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::types::{Mode, RefFlag, TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; @@ -318,8 +318,8 @@ pub trait StructSerializer: Serializer + 'static { _namespace: MetaString, _type_name: MetaString, _register_by_name: bool, - ) -> Vec { - Vec::default() + ) -> (Vec, TypeMeta) { + (Vec::default(), TypeMeta::empty()) } fn fory_type_index() -> u32 { diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index dda7ff175a..1dd658a682 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -16,7 +16,7 @@ // under the License. use crate::error::Error; -use crate::meta::NullableFieldType; +use crate::meta::FieldType; use crate::resolver::context::ReadContext; use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE}; use crate::serializer::Serializer; @@ -24,7 +24,7 @@ use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPE use crate::Fory; use chrono::{NaiveDate, NaiveDateTime}; -pub fn get_read_ref_flag(field_type: &NullableFieldType) -> bool { +pub fn get_read_ref_flag(field_type: &FieldType) -> bool { let nullable = field_type.nullable; nullable || !PRIMITIVE_TYPES.contains(&field_type.type_id) } @@ -48,7 +48,7 @@ macro_rules! basic_type_deserialize { pub fn skip_field_value( fory: &Fory, context: &mut ReadContext, - field_type: &NullableFieldType, + field_type: &FieldType, read_ref_flag: bool, ) -> Result<(), Error> { if read_ref_flag { @@ -155,9 +155,8 @@ pub fn skip_field_value( let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let nullable_field_type = NullableFieldType::from(&field_info.field_type); - let read_ref_flag = get_read_ref_flag(&nullable_field_type); - skip_field_value(fory, context, &nullable_field_type, read_ref_flag)?; + let read_ref_flag = get_read_ref_flag(&field_info.field_type); + skip_field_value(fory, context, &field_info.field_type, read_ref_flag)?; } context.dec_depth(); Ok(()) @@ -188,9 +187,8 @@ pub fn skip_field_value( let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let nullable_field_type = NullableFieldType::from(&field_info.field_type); - let read_ref_flag = get_read_ref_flag(&nullable_field_type); - skip_field_value(fory, context, &nullable_field_type, read_ref_flag)?; + let read_ref_flag = get_read_ref_flag(&field_info.field_type); + skip_field_value(fory, context, &field_info.field_type, read_ref_flag)?; } context.dec_depth(); } else if internal_id == ENUM_ID { diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index a171d18596..815bf98840 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -44,7 +44,7 @@ pub fn type_def( type_name: MetaString, register_by_name: bool, mut field_infos: Vec, -) -> Vec { +) -> (Vec, TypeMeta) { let sorted_field_names = T::fory_get_sorted_field_names(fory); let mut sorted_field_infos: Vec = Vec::with_capacity(field_infos.len()); for name in sorted_field_names.iter() { @@ -61,6 +61,10 @@ pub fn type_def( panic!("Field {} not found in field_infos", name); } } + // assign field id in ascending order + for (i, field_info) in sorted_field_infos.iter_mut().enumerate() { + field_info.field_id = i as i16; + } let meta = TypeMeta::from_fields( type_id, namespace, @@ -68,7 +72,8 @@ pub fn type_def( register_by_name, sorted_field_infos, ); - meta.to_bytes().unwrap() + let bytes = meta.to_bytes().unwrap(); + (bytes, meta) } #[inline(always)] diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index 4a2d1374ab..e157505440 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -87,8 +87,10 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new(#name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::LIST as u32, + nullable: false, generics: vec![fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + nullable: false, generics: Vec::new() }] }) @@ -100,10 +102,12 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new(#name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::MAP as u32, + nullable: false, generics: vec![ #key_generic_token, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + nullable: false, generics: Vec::new() } ] @@ -114,6 +118,7 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { quote! { fory_core::meta::FieldInfo::new(#name, fory_core::meta::FieldType { type_id: fory_core::types::TypeId::UNKNOWN as u32, + nullable: false, generics: Vec::new() }) } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index c3ca1c5d76..fc9124638a 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -15,21 +15,23 @@ // specific language governing permissions and limitations // under the License. +use fory_core::types::RefFlag; use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, quote}; use syn::{Field, Type}; use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, - generic_tree_to_tokens, parse_generic_tree, skip_ref_flag, NullableTypeNode, StructField, + extract_type_name, is_primitive_type, parse_generic_tree, skip_ref_flag, StructField, }; fn create_private_field_name(field: &Field) -> Ident { format_ident!("_{}", field.ident.as_ref().expect("")) } -fn create_read_nullable_fn_name(field: &Field) -> Ident { - format_ident!("fory_read_nullable_{}", field.ident.as_ref().expect("")) +fn need_declared_by_option(field: &Field) -> bool { + let type_name = extract_type_name(&field.ty); + type_name == "Option" || !is_primitive_type(type_name.as_str()) } fn declare_var(fields: &[&Field]) -> Vec { @@ -47,8 +49,18 @@ fn declare_var(fields: &[&Field]) -> Vec { } } _ => { - quote! { - let mut #var_name: Option<#ty> = None; + if need_declared_by_option(field) { + quote! { + let mut #var_name: Option<#ty> = None; + } + } else if extract_type_name(&field.ty) == "bool" { + quote! { + let mut #var_name: bool = false; + } + } else { + quote! { + let mut #var_name: #ty = 0 as #ty; + } } } } @@ -74,8 +86,14 @@ fn assign_value(fields: &[&Field]) -> Vec { } } _ => { - quote! { - #name: #var_name.unwrap_or_default() + if need_declared_by_option(field) { + quote! { + #name: #var_name.unwrap_or_default() + } + } else { + quote! { + #name: #var_name + } } } } @@ -228,29 +246,26 @@ pub fn gen_read_data(fields: &[&Field]) -> TokenStream { } } -fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream { +fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenStream { let ty = &field.ty; - let field_name_str = field.ident.as_ref().unwrap().to_string(); match classify_trait_object_field(ty) { StructField::BoxDyn(trait_name) => { let from_any_fn = format_ident!("from_any_internal_{}", trait_name); let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); quote! { - if _field.field_name.as_str() == #field_name_str { - let ref_flag = context.reader.read_i8(); - if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - panic!("Expected NotNullValue for trait object field"); - } - let fory_type_id = context.reader.read_varuint32(); - let harness = fory.get_type_resolver() - .get_harness(fory_type_id) - .expect("Type not registered for trait object field"); - let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(fory, context, true, false).unwrap(); - let base_type_id = fory_type_id >> 8; - #var_name = #helper_mod::#from_any_fn(any_box, base_type_id).unwrap(); + let ref_flag = context.reader.read_i8(); + if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { + panic!("Expected NotNullValue for trait object field"); } + let fory_type_id = context.reader.read_varuint32(); + let harness = fory.get_type_resolver() + .get_harness(fory_type_id) + .expect("Type not registered for trait object field"); + let deserializer_fn = harness.get_read_fn(); + let any_box = deserializer_fn(fory, context, true, false)?; + let base_type_id = fory_type_id >> 8; + #var_name = #helper_mod::#from_any_fn(any_box, base_type_id)?; } } StructField::RcDyn(trait_name) => { @@ -258,10 +273,8 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(std::rc::Rc::::from(wrapper)); - } + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(std::rc::Rc::::from(wrapper)); } } StructField::ArcDyn(trait_name) => { @@ -269,10 +282,8 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(std::sync::Arc::::from(wrapper)); - } + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(std::sync::Arc::::from(wrapper)); } } StructField::VecRc(trait_name) => { @@ -280,12 +291,10 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(wrapper_vec.into_iter() - .map(|w| std::rc::Rc::::from(w)) - .collect()); - } + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(wrapper_vec.into_iter() + .map(|w| std::rc::Rc::::from(w)) + .collect()); } } StructField::VecArc(trait_name) => { @@ -293,12 +302,10 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(wrapper_vec.into_iter() - .map(|w| std::sync::Arc::::from(w)) - .collect()); - } + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(wrapper_vec.into_iter() + .map(|w| std::sync::Arc::::from(w)) + .collect()); } } StructField::HashMapRc(key_ty, trait_name) => { @@ -306,12 +313,10 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(wrapper_map.into_iter() - .map(|(k, v)| (k, std::rc::Rc::::from(v))) - .collect()); - } + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(wrapper_map.into_iter() + .map(|(k, v)| (k, std::rc::Rc::::from(v))) + .collect()); } } StructField::HashMapArc(key_ty, trait_name) => { @@ -319,66 +324,61 @@ fn gen_read_compatible_match_arm(field: &Field, var_name: &Ident) -> TokenStream let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - if _field.field_name.as_str() == #field_name_str { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true).unwrap(); - #var_name = Some(wrapper_map.into_iter() - .map(|(k, v)| (k, std::sync::Arc::::from(v))) - .collect()); - } + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + #var_name = Some(wrapper_map.into_iter() + .map(|(k, v)| (k, std::sync::Arc::::from(v))) + .collect()); } } StructField::ContainsTraitObject => { quote! { - if _field.field_name.as_str() == #field_name_str { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); - } + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); } } StructField::Forward => { quote! { - if _field.field_name.as_str() == #field_name_str { - #var_name = Some(fory_core::serializer::Serializer::fory_read(fory, context, true).unwrap()); - } + #var_name = Some(fory_core::serializer::Serializer::fory_read(fory, context, true)?); } } StructField::None => { let generic_tree = parse_generic_tree(ty); - let generic_token = generic_tree_to_tokens(&generic_tree); - let read_nullable_fn_name = create_read_nullable_fn_name(field); - + let local_nullable = generic_tree.name == "Option"; let _base_ty = match &ty { Type::Path(type_path) => &type_path.path.segments.first().unwrap().ident, _ => panic!("Unsupported type"), }; - quote! { - if _field.field_name.as_str() == #field_name_str { - let local_field_type = #generic_token; - if &_field.field_type == &local_field_type { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false).unwrap_or_else(|_err| { - panic!("Err at deserializing {:?}: {:?}", #field_name_str, _err); - })); + if local_nullable { + quote! { + if _field.field_type.nullable { + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, false, false)?); } else { - let local_nullable_type = fory_core::meta::NullableFieldType::from(&local_field_type); - let remote_nullable_type = fory_core::meta::NullableFieldType::from(&_field.field_type); - if local_nullable_type != remote_nullable_type { - println!("Type not match, just skip: {}", #field_name_str); - let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&remote_nullable_type); - fory_core::serializer::skip::skip_field_value(fory, context, &remote_nullable_type, read_ref_flag).unwrap(); - #var_name = Some(<#ty as fory_core::serializer::ForyDefault>::fory_default()); + #var_name = Some( + fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)? + ); + } + } + } else { + let dec_by_option = need_declared_by_option(field); + if dec_by_option { + quote! { + if !_field.field_type.nullable { + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?); } else { - println!("Try to deserialize_compatible: {}", #field_name_str); - #var_name = Some( - Self::#read_nullable_fn_name( - fory, - context, - &local_nullable_type, - &remote_nullable_type - ).unwrap_or_else(|_err| { - panic!("Err at deserializing {:?}: {:?}", #field_name_str, _err); - }) - ); + #var_name = fory_core::serializer::read_ref_info_data::>(fory, context, true, false, false)? + } + } + } else { + let null_flag = RefFlag::Null as i8; + quote! { + if !_field.field_type.nullable { + #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; + } else { + if context.reader.read_i8() == #null_flag { + #var_name = <#ty as fory_core::serializer::ForyDefault>::fory_default(); + } else { + #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; + } } } } @@ -413,13 +413,24 @@ pub fn gen_read(struct_ident: &Ident) -> TokenStream { } pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { - let pattern_items = fields.iter().map(|field| { - let var_name = create_private_field_name(field); - gen_read_compatible_match_arm(field, &var_name) - }); let declare_ts: Vec = declare_var(fields); let assign_ts: Vec = assign_value(fields); + let match_arms: Vec = fields + .iter() + .enumerate() + .map(|(i, field)| { + let var_name = create_private_field_name(field); + let field_id = i as i16; + let body = gen_read_compatible_match_arm_body(field, &var_name); + quote! { + #field_id => { + #body + } + } + }) + .collect(); + quote! { let remote_type_id = context.reader.read_varuint32(); let meta_index = context.reader.read_varuint32(); @@ -434,15 +445,16 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { let high_bytes = &local_type_def[..8]; let local_type_hash = i64::from_le_bytes(high_bytes.try_into().unwrap()); if meta.get_hash() == local_type_hash { - // fast path ::fory_read_data(fory, context, false) } else { for _field in fields.iter() { - #(#pattern_items else)* { - println!("skip {:?}:{:?}", _field.field_name.as_str(), _field.field_type); - let nullable_field_type = fory_core::meta::NullableFieldType::from(&_field.field_type); - let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&nullable_field_type); - fory_core::serializer::skip::skip_field_value(fory, context, &nullable_field_type, read_ref_flag).unwrap(); + match _field.field_id { + #(#match_arms)* + _ => { + let field_type = &_field.field_type; + let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&field_type); + fory_core::serializer::skip::skip_field_value(fory, context, &field_type, read_ref_flag).unwrap(); + } } } Ok(Self { @@ -451,34 +463,3 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { } } } - -pub fn gen_read_nullable(fields: &[&Field]) -> TokenStream { - let func_tokens: Vec = fields - .iter() - .filter_map(|field| { - let ty = &field.ty; - match classify_trait_object_field(ty) { - StructField::None => { - let fn_name = create_read_nullable_fn_name(field); - let generic_tree = parse_generic_tree(ty); - let nullable_generic_tree = NullableTypeNode::from(generic_tree); - let read_tokens = nullable_generic_tree.to_read_tokens(&vec![], true); - Some(quote! { - fn #fn_name( - fory: &fory_core::fory::Fory, - context: &mut fory_core::resolver::context::ReadContext, - local_nullable_type: &fory_core::meta::NullableFieldType, - remote_nullable_type: &fory_core::meta::NullableFieldType - ) -> Result<#ty, fory_core::error::Error> { - #read_tokens - } - }) - } - _ => None, - } - }) - .collect::>(); - quote! { - #(#func_tokens)* - } -} diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 53b07f20dc..4807846676 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -106,17 +106,6 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { panic!("Union is not supported") } }; - // extra - let (deserialize_nullable_ts,) = match &ast.data { - syn::Data::Struct(s) => { - let fields = sorted_fields(&s.fields); - (read::gen_read_nullable(&fields),) - } - syn::Data::Enum(_s) => (quote! {},), - syn::Data::Union(_) => { - panic!("Union is not supported") - } - }; // Allocate a unique type ID once and share it between both functions let type_idx = misc::allocate_type_id(); @@ -139,7 +128,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #get_sorted_field_names_ts } - fn fory_type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: fory_core::meta::MetaString, type_name: fory_core::meta::MetaString, register_by_name: bool) -> Vec { + fn fory_type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: fory_core::meta::MetaString, type_name: fory_core::meta::MetaString, register_by_name: bool) -> (Vec, fory_core::meta::TypeMeta) { #type_def_ts } } @@ -188,9 +177,6 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #read_compatible_ts } } - impl #name { - #deserialize_nullable_ts - } }; let code = gen.into(); clear_struct_context(); diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 6841dfa145..c0698bf1a6 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -19,12 +19,12 @@ use crate::util::{ detect_collection_with_trait_object, is_arc_dyn_trait, is_box_dyn_trait, is_rc_dyn_trait, CollectionTraitInfo, }; -use fory_core::types::{TypeId, BASIC_TYPE_NAMES, CONTAINER_TYPE_NAMES, PRIMITIVE_ARRAY_TYPE_MAP}; +use fory_core::types::{TypeId, PRIMITIVE_ARRAY_TYPE_MAP}; use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, quote, ToTokens}; use std::cell::RefCell; use std::fmt; -use syn::{parse_str, Field, GenericArgument, PathArguments, Type}; +use syn::{Field, GenericArgument, PathArguments, Type}; thread_local! { static MACRO_CONTEXT: RefCell> = const {RefCell::new(None)}; @@ -212,42 +212,8 @@ pub(super) fn classify_trait_object_field(ty: &Type) -> StructField { #[derive(Debug)] pub(super) struct TypeNode { - name: String, - generics: Vec, -} - -#[derive(Debug)] -pub(super) struct NullableTypeNode { - name: String, - generics: Vec, - nullable: bool, -} - -macro_rules! basic_type_deserialize { - ($name:expr, $nullable:expr; $( ($ty_str:expr, $ty:ty) ),* $(,)?) => { - match $name { - $( - $ty_str => { - if $nullable { - quote! { - <$ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - let res1 = Some(<$ty as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) - .map_err(fory_core::error::Error::from)?); - Ok::, fory_core::error::Error>(res1) - } - } else { - quote! { - <$ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - let res2 = <$ty as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) - .map_err(fory_core::error::Error::from)?; - Ok::<$ty, fory_core::error::Error>(res2) - } - } - } - )* - _ => unreachable!(), - } - }; + pub name: String, + pub generics: Vec, } pub(super) fn try_primitive_vec_type(node: &TypeNode) -> Option { @@ -283,377 +249,6 @@ pub(super) fn try_vec_of_option_primitive(node: &TypeNode) -> Option Option { - if node.name != "Vec" { - return None; - } - let child = node.generics.first()?; - for (generic_name, _, ty_name) in PRIMITIVE_ARRAY_TYPE_MAP { - if child.name == *generic_name { - return Some(ty_name.to_string()); - } - } - None -} - -impl NullableTypeNode { - pub(super) fn to_read_tokens( - &self, - generic_path: &Vec, - read_ref_flag: bool, - ) -> TokenStream { - let tokens = if let Some(primitive_ty_name) = try_primitive_vec_type_name(self) { - let ty_type: Type = parse_str(&primitive_ty_name).expect("Invalid primitive type name"); - let nullable = self.nullable; - if nullable { - quote! { - let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - None - } else { - <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - Some(<#ty_type as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) - .map_err(fory_core::error::Error::from)?) - }; - Ok::, fory_core::error::Error>(res1) - } - } else { - quote! { - let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - Vec::default() - } else { - <#ty_type as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - <#ty_type as fory_core::serializer::Serializer>::fory_read_data(fory, context, true) - .map_err(fory_core::error::Error::from)? - }; - Ok::<#ty_type, fory_core::error::Error>(res2) - } - } - } else if BASIC_TYPE_NAMES.contains(&self.name.as_str()) { - basic_type_deserialize!(self.name.as_str(), self.nullable; - ("bool", bool), - ("i8", i8), - ("i16", i16), - ("i32", i32), - ("i64", i64), - ("f32", f32), - ("f64", f64), - ("String", String), - ("NaiveDate", chrono::NaiveDate), - ("NaiveDateTime", chrono::NaiveDateTime), - ) - } else if CONTAINER_TYPE_NAMES.contains(&self.name.as_str()) { - let ty = parse_str::(&self.to_string()).unwrap(); - let mut new_path = generic_path.clone(); - match self.name.as_str() { - "Vec" => { - new_path.push(0); - let generic_node = self.generics.first().unwrap(); - let element_tokens = generic_node.to_read_tokens(&new_path, false); - let element_ty: Type = parse_str(&generic_node.to_string()).unwrap(); - let vec_ts = quote! { - let length = context.reader.read_varuint32() as usize; - if length == 0 { - Vec::default() - } else { - let mut v = Vec::with_capacity(length); - let header = context.reader.read_u8(); - let has_null = (header & fory_core::serializer::collection::HAS_NULL) != 0; - let is_same_type = (header & fory_core::serializer::collection::IS_SAME_TYPE) != 0; - let read_ref_flag = !(is_same_type && !has_null); - for _ in 0..length { - let ref_flag = if read_ref_flag { - context.reader.read_i8() - } else { - fory_core::types::RefFlag::NotNullValue as i8 - }; - let element = if ref_flag == fory_core::types::RefFlag::Null as i8 { - <#element_ty as fory_core::serializer::ForyDefault>::fory_default() - } else { - #element_tokens? - }; - v.push(element); - } - v - } - }; - if self.nullable { - quote! { - let v = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - None - } else { - Some({#vec_ts}) - }; - Ok::<#ty, fory_core::error::Error>(v) - } - } else { - quote! { - let v = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - Vec::default() - } else { - #vec_ts - }; - Ok::<#ty, fory_core::error::Error>(v) - } - } - } - "HashSet" => { - new_path.push(0); - let generic_node = self.generics.first().unwrap(); - let element_tokens = generic_node.to_read_tokens(&new_path, false); - let element_ty: Type = parse_str(&generic_node.to_string()).unwrap(); - let set_ts = quote! { - let length = context.reader.read_varuint32() as usize; - if length == 0 { - HashSet::default() - } else { - let mut s = HashSet::with_capacity(length); - let header = context.reader.read_u8(); - let has_null = (header & fory_core::serializer::collection::HAS_NULL) != 0; - let is_same_type = (header & fory_core::serializer::collection::IS_SAME_TYPE) != 0; - let read_ref_flag = !(is_same_type && !has_null); - for _ in 0..length { - let ref_flag = if read_ref_flag { - context.reader.read_i8() - } else { - fory_core::types::RefFlag::NotNullValue as i8 - }; - let element = if ref_flag == fory_core::types::RefFlag::Null as i8 { - <#element_ty as fory_core::serializer::ForyDefault>::fory_default() - } else { - #element_tokens? - }; - s.insert(element); - } - s - } - }; - if self.nullable { - quote! { - let s = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - None - } else { - Some({#set_ts}) - }; - Ok::<#ty, fory_core::error::Error>(s) - } - } else { - quote! { - let s = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - HashSet::default() - } else { - #set_ts - }; - Ok::<#ty, fory_core::error::Error>(s) - } - } - } - "HashMap" => { - let key_generic_node = self.generics.first().unwrap(); - let val_generic_node = self.generics.get(1).unwrap(); - - new_path.push(0); - let key_tokens = key_generic_node.to_read_tokens(&new_path, false); - new_path.pop(); - new_path.push(1); - let val_tokens = val_generic_node.to_read_tokens(&new_path, false); - - let key_ty: Type = parse_str(&key_generic_node.to_string()).unwrap(); - let val_ty: Type = parse_str(&val_generic_node.to_string()).unwrap(); - - let map_ts = quote! { - let length = context.reader.read_varuint32(); - let mut map = HashMap::with_capacity(length as usize); - if length == 0 { - map - } else { - let mut len_counter = 0; - loop { - if len_counter == length { - break; - } - let header = context.reader.read_u8(); - if header & fory_core::serializer::map::KEY_NULL != 0 && header & fory_core::serializer::map::VALUE_NULL != 0 { - map.insert(<#key_ty as fory_core::serializer::ForyDefault>::fory_default(), <#val_ty as fory_core::serializer::ForyDefault>::fory_default()); - len_counter += 1; - continue; - } - if header & fory_core::serializer::map::KEY_NULL != 0 { - let value: #val_ty = {#val_tokens}?; - map.insert(<#key_ty as fory_core::serializer::ForyDefault>::fory_default(), value); - len_counter += 1; - continue; - } - if header & fory_core::serializer::map::VALUE_NULL != 0 { - let key: #key_ty = {#key_tokens}?; - map.insert(key, <#val_ty as fory_core::serializer::ForyDefault>::fory_default()); - len_counter += 1; - continue; - } - let chunk_size = context.reader.read_u8(); - <#key_ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - <#val_ty as fory_core::serializer::Serializer>::fory_read_type_info(fory, context, true); - for _ in (0..chunk_size).enumerate() { - let key: #key_ty = {#key_tokens}?; - let value: #val_ty = {#val_tokens}?; - map.insert(key, value); - } - len_counter += chunk_size as u32; - } - map - } - }; - if self.nullable { - quote! { - let m = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - None - } else { - Some({#map_ts}) - }; - Ok::<#ty, fory_core::error::Error>(m) - } - } else { - quote! { - let m = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - HashMap::default() - } else { - #map_ts - }; - Ok::<#ty, fory_core::error::Error>(m) - } - } - } - _ => quote! { compile_error!("Unsupported type for container"); }, - } - } else { - // struct or enum - let nullable_ty = parse_str::(&self.nullable_ty_string()).unwrap(); - let ty = parse_str::(&self.to_string()).unwrap(); - let ts = if self.nullable { - quote! { - let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - None - } else { - let type_id = cur_remote_nullable_type.type_id; - let internal_id = type_id & 0xff; - Some( - if internal_id == COMPATIBLE_STRUCT_ID - || internal_id == NAMED_COMPATIBLE_STRUCT_ID - || internal_id == ENUM_ID - || internal_id == NAMED_ENUM_ID - || internal_id == EXT_ID - || internal_id == NAMED_EXT_ID - { - <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) - .map_err(fory_core::error::Error::from)? - } else { - unimplemented!() - } - ) - }; - Ok::<#ty, fory_core::error::Error>(res1) - } - } else { - quote! { - let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) { - <#ty as fory_core::serializer::ForyDefault>::fory_default() - } else { - let type_id = cur_remote_nullable_type.type_id; - let internal_id = type_id & 0xff; - if internal_id == COMPATIBLE_STRUCT_ID - || internal_id == NAMED_COMPATIBLE_STRUCT_ID - || internal_id == ENUM_ID - || internal_id == NAMED_ENUM_ID - || internal_id == EXT_ID - || internal_id == NAMED_EXT_ID - { - <#nullable_ty as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) - .map_err(fory_core::error::Error::from)? - } else { - unimplemented!() - } - }; - Ok::<#ty, fory_core::error::Error>(res2) - } - }; - quote! { - const COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::COMPATIBLE_STRUCT as u32; - const ENUM_ID: u32 = fory_core::types::TypeId::ENUM as u32; - const NAMED_COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::NAMED_COMPATIBLE_STRUCT as u32; - const NAMED_ENUM_ID: u32 = fory_core::types::TypeId::NAMED_ENUM as u32; - const EXT_ID: u32 = fory_core::types::TypeId::EXT as u32; - const NAMED_EXT_ID: u32 = fory_core::types::TypeId::NAMED_EXT as u32; - #ts - } - }; - let mut cur_remote_nullable_type = quote! { remote_nullable_type }; - for idx in generic_path { - cur_remote_nullable_type = quote! { - #cur_remote_nullable_type.generics.get(#idx as usize).unwrap() - }; - } - let read_ref_flag_ts = if read_ref_flag { - quote! { - let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(cur_remote_nullable_type); - let ref_flag = if read_ref_flag { - context.reader.read_i8() - } else { - fory_core::types::RefFlag::NotNullValue as i8 - }; - if ref_flag == fory_core::types::RefFlag::Null as i8 { - Ok(Default::default()) - } else { - #tokens - } - } - } else { - quote! { #tokens } - }; - quote! { - let cur_remote_nullable_type = &#cur_remote_nullable_type; - #read_ref_flag_ts - } - } - - pub(super) fn from(node: TypeNode) -> Self { - if node.name == "Option" { - let inner = NullableTypeNode::from(node.generics.into_iter().next().unwrap()); - NullableTypeNode { - name: inner.name, - generics: inner.generics, - nullable: true, - } - } else { - let generics = node - .generics - .into_iter() - .map(NullableTypeNode::from) - .collect(); - - NullableTypeNode { - name: node.name, - generics, - nullable: false, - } - } - } - - pub(super) fn nullable_ty_string(&self) -> String { - if self.generics.is_empty() { - self.name.clone() - } else { - format!( - "{}<{}>", - self.name, - self.generics - .iter() - .map(|g| g.to_string()) - .collect::>() - .join(",") - ) - } - } -} - impl fmt::Display for TypeNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.generics.is_empty() { @@ -673,31 +268,7 @@ impl fmt::Display for TypeNode { } } -impl fmt::Display for NullableTypeNode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let inner_type = if self.generics.is_empty() { - self.name.clone() - } else { - format!( - "{}<{}>", - self.name, - self.generics - .iter() - .map(|g| g.to_string()) - .collect::>() - .join(",") - ) - }; - - if self.nullable { - write!(f, "Option<{}>", inner_type) - } else { - write!(f, "{}", inner_type) - } - } -} - -fn extract_type_name(ty: &Type) -> String { +pub(super) fn extract_type_name(ty: &Type) -> String { if let Type::Path(type_path) = ty { type_path.path.segments.last().unwrap().ident.to_string() } else if matches!(ty, Type::TraitObject(_)) { @@ -742,40 +313,55 @@ pub(super) fn parse_generic_tree(ty: &Type) -> TypeNode { } pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { - if node.name == "Option" { - if let Some(first_generic) = node.generics.first() { - if first_generic.name == "Option" { - return quote! { - compile_error!("adjacent Options are not supported"); - }; + // If Option, unwrap it before generating children + let (nullable, base_node) = if node.name == "Option" { + if let Some(inner) = node.generics.first() { + if inner.name == "Option" { + return quote! { compile_error!("Nested adjacent Option is not allowed!"); }; } + // Unwrap Option and propagate parsing + (true, inner) + } else { + return quote! { compile_error!("Missing Option inner type"); }; } - } + } else { + (!PRIMITIVE_TYPE_NAMES.contains(&node.name.as_str()), node) + }; - if let Some(ts) = try_vec_of_option_primitive(node) { + // Vec> rule stays as is + if let Some(ts) = try_vec_of_option_primitive(base_node) { return ts; } - let primitive_vec = try_primitive_vec_type(node); + // Try primitive Vec type + let primitive_vec = try_primitive_vec_type(base_node); + + // Recursively generate children token streams let children_tokens: Vec = if primitive_vec.is_none() { - node.generics.iter().map(generic_tree_to_tokens).collect() + base_node + .generics + .iter() + .map(generic_tree_to_tokens) + .collect() } else { vec![] }; - let ty: syn::Type = syn::parse_str(&node.to_string()).unwrap(); - let get_type_id = if node.name == "Option" { - let option_type_id = TypeId::ForyNullable as u32; - quote! { #option_type_id } - } else if let Some(ts) = primitive_vec { + + // Build the syn::Type from the DISPLAY of base_node, not the original node if Option + let ty: syn::Type = syn::parse_str(&base_node.to_string()).unwrap(); + + let get_type_id = if let Some(ts) = primitive_vec { ts } else { quote! { <#ty as fory_core::serializer::Serializer>::fory_get_type_id(fory) } }; + quote! { fory_core::meta::FieldType::new( #get_type_id, + #nullable, vec![#(#children_tokens),*] as Vec ) } @@ -807,6 +393,10 @@ fn get_primitive_type_id(ty: &str) -> u32 { } } +pub(super) fn is_primitive_type(ty: &str) -> bool { + PRIMITIVE_TYPE_NAMES.contains(&ty) +} + fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { fn extract_option_inner(s: &str) -> Option<&str> { s.strip_prefix("Option<")?.strip_suffix(">") diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 23fd99f2b1..b103aa0712 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -517,6 +517,7 @@ fn named_enum() { #[test] #[allow(clippy::unnecessary_literal_unwrap)] fn boxed() { + // cargo expand --test mod compatible::test_struct > e1.rs #[derive(ForyObject, Debug, PartialEq)] struct Item1 { f1: i32, diff --git a/rust/tests/tests/test_simple_struct.rs b/rust/tests/tests/test_simple_struct.rs index 438554a1e8..cf65d437ec 100644 --- a/rust/tests/tests/test_simple_struct.rs +++ b/rust/tests/tests/test_simple_struct.rs @@ -22,7 +22,7 @@ use std::collections::HashMap; #[test] fn test_simple() { - // a single test for cargo expand and analysis + // a single test for cargo expand and analysis: `cargo expand --test test_simple_struct 2>&1 > expanded.rs` // &["f7", "last", "f2", "f5", "f3", "f6", "f1"] #[derive(ForyObject, Debug)] struct Animal1 { From 6e2212271fdd1957024a35eab051c94c8c46f170 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Tue, 14 Oct 2025 01:58:51 +0530 Subject: [PATCH 20/37] refactor(rust): use compatible bool instead of enum to simplify API (#2763) ## Why? ## What does this PR do? use compatible bool isntead enum to simplify API: ```rust let fory = Fory::default().compatible(true); ``` ## Related issues ## Does this PR introduce any user-facing change? - [x] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 17 ++---- rust/fory-core/src/fory.rs | 50 +++++++-------- rust/fory-core/src/lib.rs | 9 ++- rust/fory-core/src/serializer/enum_.rs | 4 +- rust/fory-core/src/serializer/mod.rs | 6 +- rust/fory-core/src/serializer/struct_.rs | 25 ++++---- rust/fory-core/src/serializer/trait_object.rs | 12 ++-- rust/fory-derive/src/object/derive_enum.rs | 2 +- rust/fory-derive/src/object/misc.rs | 2 +- rust/fory-derive/src/object/read.rs | 14 ++--- rust/fory-derive/src/object/serializer.rs | 2 +- rust/fory/src/lib.rs | 30 ++++----- .../tests/tests/compatible/test_basic_type.rs | 7 +-- rust/tests/tests/compatible/test_container.rs | 23 ++++--- rust/tests/tests/compatible/test_struct.rs | 43 +++++++------ .../tests/compatible/test_struct_enum.rs | 61 +++++++++---------- rust/tests/tests/test_cross_language.rs | 33 +++++----- rust/tests/tests/test_ext.rs | 3 +- rust/tests/tests/test_max_dyn_depth.rs | 5 +- rust/tests/tests/test_rc_arc_trait_object.rs | 3 +- rust/tests/tests/test_simple_struct.rs | 5 +- rust/tests/tests/test_trait_object.rs | 3 +- 22 files changed, 168 insertions(+), 191 deletions(-) diff --git a/rust/README.md b/rust/README.md index 1237b822da..5891cd95ea 100644 --- a/rust/README.md +++ b/rust/README.md @@ -305,7 +305,6 @@ Apache Fory™ supports polymorphic serialization through trait objects, enablin use fory::{Fory, register_trait_type}; use fory::Serializer; use fory::ForyObject; -use fory::Mode; trait Animal: Serializer { fn speak(&self) -> String; @@ -336,7 +335,7 @@ struct Zoo { star_animal: Box, } -let mut fory = Fory::default().mode(Mode::Compatible); +let mut fory = Fory::default().compatible(true); fory.register::(100); fory.register::(101); fory.register::(102); @@ -425,7 +424,7 @@ struct AnimalShelter { registry: HashMap>, } -let mut fory = Fory::default().mode(Mode::Compatible); +let mut fory = Fory::default().compatible(true); fory.register::(100); fory.register::(101); fory.register::(102); @@ -510,7 +509,6 @@ Apache Fory™ supports schema evolution in **Compatible mode**, allowing serial ```rust use fory::Fory; -use fory::Mode; use fory::ForyObject; use std::collections::HashMap; @@ -531,10 +529,10 @@ struct PersonV2 { metadata: HashMap, } -let mut fory1 = Fory::default().mode(Mode::Compatible); +let mut fory1 = Fory::default().compatible(true); fory1.register::(1); -let mut fory2 = Fory::default().mode(Mode::Compatible); +let mut fory2 = Fory::default().compatible(true); fory2.register::(1); let person_v1 = PersonV1 { @@ -789,11 +787,10 @@ Apache Fory™ supports seamless data exchange across multiple languages: ```rust use fory::Fory; -use fory::Mode; // Enable cross-language mode let mut fory = Fory::default() - .mode(Mode::Compatible) + .compatible(true) .xlang(true); // Register types with consistent IDs across languages @@ -887,9 +884,7 @@ let fory = Fory::default(); // SchemaConsistent by default Allows independent schema evolution: ```rust -use fory::Mode; - -let fory = Fory::default().mode(Mode::Compatible); +let fory = Fory::default().compatible(true); ``` ## ⚙️ Configuration diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 91f4f34740..76889b17f9 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -26,7 +26,7 @@ use crate::serializer::{Serializer, StructSerializer}; use crate::types::config_flags::IS_NULL_FLAG; use crate::types::{ config_flags::{IS_CROSS_LANGUAGE_FLAG, IS_LITTLE_ENDIAN_FLAG}, - Language, Mode, MAGIC_NUMBER, SIZE_OF_REF_AND_TYPE, + Language, MAGIC_NUMBER, SIZE_OF_REF_AND_TYPE, }; use crate::util::get_ext_actual_type_id; use anyhow::anyhow; @@ -69,15 +69,15 @@ static EMPTY_STRING: String = String::new(); /// Custom configuration: /// /// ```rust -/// use fory_core::{Fory, Mode}; +/// use fory_core::Fory; /// /// let fory = Fory::default() -/// .mode(Mode::Compatible) +/// .compatible(true) /// .compress_string(true) /// .max_dyn_depth(10); /// ``` pub struct Fory { - mode: Mode, + compatible: bool, xlang: bool, share_meta: bool, type_resolver: TypeResolver, @@ -99,7 +99,7 @@ impl Default for Fory { ReadContext::new(reader, 0) }; Fory { - mode: Mode::SchemaConsistent, + compatible: false, xlang: true, share_meta: false, type_resolver: TypeResolver::default(), @@ -112,14 +112,14 @@ impl Default for Fory { } impl Fory { - /// Sets the serialization mode for this Fory instance. + /// Sets the serialization compatible mode for this Fory instance. /// /// # Arguments /// - /// * `mode` - The serialization mode to use. Options are: - /// - `Mode::SchemaConsistent`: Schema must be consistent between serialization and deserialization. + /// * `compatible` - The serialization compatible mode to use. Options are: + /// - `false`: Schema must be consistent between serialization and deserialization. /// No metadata is shared. This is the fastest mode. - /// - `Mode::Compatible`: Supports schema evolution and type metadata sharing for better + /// - true`: Supports schema evolution and type metadata sharing for better /// cross-version compatibility. /// /// # Returns @@ -128,21 +128,21 @@ impl Fory { /// /// # Note /// - /// Setting the mode also automatically configures the `share_meta` flag: - /// - `Mode::SchemaConsistent` → `share_meta = false` - /// - `Mode::Compatible` → `share_meta = true` + /// Setting the compatible mode also automatically configures the `share_meta` flag: + /// - `false` → `share_meta = false` + /// - true` → `share_meta = true` /// /// # Examples /// /// ```rust - /// use fory_core::{Fory, Mode}; + /// use fory_core::Fory; /// - /// let fory = Fory::default().mode(Mode::Compatible); + /// let fory = Fory::default().compatible(true); /// ``` - pub fn mode(mut self, mode: Mode) -> Self { + pub fn compatible(mut self, compatible: bool) -> Self { // Setting share_meta individually is not supported currently - self.share_meta = mode != Mode::SchemaConsistent; - self.mode = mode; + self.share_meta = compatible; + self.compatible = compatible; self } @@ -253,9 +253,9 @@ impl Fory { /// /// # Returns /// - /// A reference to the current `Mode` (either `SchemaConsistent` or `Compatible`). - pub fn get_mode(&self) -> &Mode { - &self.mode + /// `ture` if the serialization mode is compatible, `false` otherwise`. + pub fn is_compatible(&self) -> bool { + self.compatible } /// Returns whether string compression is enabled. @@ -401,7 +401,7 @@ impl Fory { return Ok(T::fory_default()); } let mut bytes_to_skip = 0; - if self.mode == Mode::Compatible { + if self.compatible { let meta_offset = context.reader.read_i32(); if meta_offset != -1 { bytes_to_skip = context.load_meta(self.get_type_resolver(), meta_offset as usize); @@ -459,11 +459,11 @@ impl Fory { self.write_head::(is_none, &mut context.writer); let meta_start_offset = context.writer.len(); if !is_none { - if self.mode == Mode::Compatible { + if self.compatible { context.writer.write_i32(-1); }; ::fory_write(record, self, context, false); - if self.mode == Mode::Compatible && !context.empty() { + if self.compatible && !context.empty() { context.write_meta(meta_start_offset); } } @@ -498,7 +498,7 @@ impl Fory { /// fory.register::(100); /// ``` pub fn register(&mut self, id: u32) { - let actual_type_id = T::fory_actual_type_id(id, false, &self.mode); + let actual_type_id = T::fory_actual_type_id(id, false, self.compatible); let type_info = TypeInfo::new::(self, actual_type_id, &EMPTY_STRING, &EMPTY_STRING, false); self.type_resolver.register::(&type_info); @@ -539,7 +539,7 @@ impl Fory { namespace: &str, type_name: &str, ) { - let actual_type_id = T::fory_actual_type_id(0, true, &self.mode); + let actual_type_id = T::fory_actual_type_id(0, true, self.compatible); let type_info = TypeInfo::new::(self, actual_type_id, namespace, type_name, true); self.type_resolver.register::(&type_info); } diff --git a/rust/fory-core/src/lib.rs b/rust/fory-core/src/lib.rs index a2ea1fe746..9da7008885 100644 --- a/rust/fory-core/src/lib.rs +++ b/rust/fory-core/src/lib.rs @@ -70,7 +70,7 @@ //! Define custom traits and register implementations: //! //! ```rust,ignore -//! use fory_core::{Fory, register_trait_type, Serializer, Mode}; +//! use fory_core::{Fory, register_trait_type, Serializer}; //! use fory_derive::ForyObject; //! //! trait Animal: Serializer { @@ -99,7 +99,7 @@ //! } //! //! # fn main() { -//! let mut fory = Fory::default().mode(Mode::Compatible); +//! let mut fory = Fory::default().compatible(true); //! fory.register::(100); //! fory.register::(101); //! fory.register::(102); @@ -143,12 +143,11 @@ //! ```rust //! use fory_core::fory::Fory; //! use fory_core::error::Error; -//! use fory_core::types::Mode; //! use fory_core::row::{to_row, from_row}; //! use std::collections::HashMap; //! //! // Create a Fory instance -//! let mut fory = Fory::default().mode(Mode::Compatible); +//! let mut fory = Fory::default().compatible(true); //! //! // Serialize String //! let text = String::from("Hello, Fory!"); @@ -196,4 +195,4 @@ pub use crate::resolver::context::{ReadContext, WriteContext}; pub use crate::resolver::type_resolver::TypeResolver; pub use crate::serializer::weak::{ArcWeak, RcWeak}; pub use crate::serializer::{ForyDefault, Serializer}; -pub use crate::types::{Mode, RefFlag, TypeId}; +pub use crate::types::{RefFlag, TypeId}; diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index da4688955c..a9ccbb4a6e 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -20,10 +20,10 @@ use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; -use crate::types::{Mode, RefFlag, TypeId}; +use crate::types::{RefFlag, TypeId}; #[inline(always)] -pub fn actual_type_id(type_id: u32, register_by_name: bool, _mode: &Mode) -> u32 { +pub fn actual_type_id(type_id: u32, register_by_name: bool, _compatible: bool) -> u32 { if register_by_name { TypeId::NAMED_ENUM as u32 } else { diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 5ce462d5ff..418ae87483 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -19,7 +19,7 @@ use crate::error::Error; use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta, NAMESPACE_DECODER, TYPE_NAME_DECODER}; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::types::{Mode, RefFlag, TypeId, PRIMITIVE_TYPES}; +use crate::types::{RefFlag, TypeId, PRIMITIVE_TYPES}; use anyhow::anyhow; use std::any::Any; @@ -325,8 +325,8 @@ pub trait StructSerializer: Serializer + 'static { fn fory_type_index() -> u32 { unimplemented!() } - fn fory_actual_type_id(type_id: u32, register_by_name: bool, mode: &Mode) -> u32 { - struct_::actual_type_id(type_id, register_by_name, mode) + fn fory_actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { + struct_::actual_type_id(type_id, register_by_name, compatible) } fn fory_get_sorted_field_names(_fory: &Fory) -> &'static [&'static str] { diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 815bf98840..b827627f81 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -19,11 +19,11 @@ use crate::fory::Fory; use crate::meta::{FieldInfo, MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{Serializer, StructSerializer}; -use crate::types::{Mode, RefFlag, TypeId}; +use crate::types::{RefFlag, TypeId}; #[inline(always)] -pub fn actual_type_id(type_id: u32, register_by_name: bool, mode: &Mode) -> u32 { - if mode == &Mode::Compatible { +pub fn actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { + if compatible { if register_by_name { TypeId::NAMED_COMPATIBLE_STRUCT as u32 } else { @@ -123,17 +123,14 @@ pub fn read_type_info(fory: &Fory, context: &mut ReadContext, _is #[inline(always)] pub fn write(this: &T, fory: &Fory, context: &mut WriteContext, _is_field: bool) { - match fory.get_mode() { + if fory.is_compatible() { + context.writer.write_i8(RefFlag::NotNullValue as i8); + T::fory_write_type_info(fory, context, false); + this.fory_write_data(fory, context, true); + } else { // currently same - Mode::SchemaConsistent => { - context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false); - this.fory_write_data(fory, context, true); - } - Mode::Compatible => { - context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false); - this.fory_write_data(fory, context, true); - } + context.writer.write_i8(RefFlag::NotNullValue as i8); + T::fory_write_type_info(fory, context, false); + this.fory_write_data(fory, context, true); } } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 29ea1a7bdc..727f88b1e0 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -29,12 +29,12 @@ pub fn write_trait_object_headers( fory_type_id: u32, concrete_type_id: std::any::TypeId, ) { - use crate::types::{Mode, RefFlag, TypeId}; + use crate::types::{RefFlag, TypeId}; context.writer.write_i8(RefFlag::NotNullValue as i8); context.writer.write_varuint32(fory_type_id); - if fory.get_mode() == &Mode::Compatible + if fory.is_compatible() && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { @@ -45,7 +45,7 @@ pub fn write_trait_object_headers( /// Reads common trait object headers and returns the type ID pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Result { - use crate::types::{Mode, RefFlag, TypeId}; + use crate::types::{RefFlag, TypeId}; let ref_flag = context.reader.read_i8(); if ref_flag != RefFlag::NotNullValue as i8 { @@ -57,7 +57,7 @@ pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Resu let fory_type_id = context.reader.read_varuint32(); - if fory.get_mode() == &Mode::Compatible + if fory.is_compatible() && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { @@ -121,7 +121,7 @@ macro_rules! resolve_and_deserialize { /// # Example /// /// ```rust,ignore -/// use fory_core::{fory::Fory, register_trait_type, serializer::Serializer, types::Mode}; +/// use fory_core::{fory::Fory, register_trait_type, serializer::Serializer}; /// use fory_derive::ForyObject; /// /// trait Animal: Serializer { @@ -148,7 +148,7 @@ macro_rules! resolve_and_deserialize { /// register_trait_type!(Animal, Dog, Cat); /// /// # fn main() { -/// let mut fory = Fory::default().mode(Mode::Compatible); +/// let mut fory = Fory::default().compatible(true); /// fory.register::(100); /// fory.register::(101); /// diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index df35fdd4ac..1524fc9453 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -21,7 +21,7 @@ use syn::DataEnum; pub fn gen_actual_type_id() -> TokenStream { quote! { - fory_core::serializer::enum_::actual_type_id(type_id, register_by_name, mode) + fory_core::serializer::enum_::actual_type_id(type_id, register_by_name, compatible) } } diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index e157505440..40f52502e6 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -60,7 +60,7 @@ fn hash(fields: &[&Field]) -> TokenStream { pub fn gen_actual_type_id() -> TokenStream { quote! { - fory_core::serializer::struct_::actual_type_id(type_id, register_by_name, mode) + fory_core::serializer::struct_::actual_type_id(type_id, register_by_name, compatible) } } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index fc9124638a..3aa53334de 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -391,15 +391,11 @@ pub fn gen_read(struct_ident: &Ident) -> TokenStream { quote! { let ref_flag = context.reader.read_i8(); if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::types::RefFlag::RefValue as i8) { - match fory.get_mode() { - fory_core::types::Mode::SchemaConsistent => { - ::fory_read_type_info(fory, context, false); - ::fory_read_data(fory, context, false) - }, - fory_core::types::Mode::Compatible => { - <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) - }, - _ => unreachable!() + if fory.is_compatible() { + <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) + } else { + ::fory_read_type_info(fory, context, false); + ::fory_read_data(fory, context, false) } } else if ref_flag == (fory_core::types::RefFlag::Null as i8) { Ok(::fory_default()) diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 4807846676..daaf1bdfed 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -120,7 +120,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #type_idx } - fn fory_actual_type_id(type_id: u32, register_by_name: bool, mode: &fory_core::types::Mode) -> u32 { + fn fory_actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { #actual_type_id_ts } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index c5d90d6283..3ba3844f55 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -334,7 +334,7 @@ //! #### Basic Trait Object Serialization //! //! ```rust -//! use fory::{Fory, register_trait_type, Serializer, Mode, Error}; +//! use fory::{Fory, register_trait_type, Serializer, Error}; //! use fory::ForyObject; //! //! trait Animal: Serializer { @@ -366,7 +366,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default().mode(Mode::Compatible); +//! let mut fory = Fory::default().compatible(true); //! fory.register::(100); //! fory.register::(101); //! fory.register::(102); @@ -463,7 +463,7 @@ //! automatically handles the conversion without needing wrappers: //! //! ```rust -//! use fory::{Fory, register_trait_type, Serializer, Mode, Error}; +//! use fory::{Fory, register_trait_type, Serializer, Error}; //! use fory::ForyObject; //! use std::sync::Arc; //! use std::rc::Rc; @@ -493,7 +493,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default().mode(Mode::Compatible); +//! let mut fory = Fory::default().compatible(true); //! fory.register::(100); //! fory.register::(101); //! fory.register::(102); @@ -529,7 +529,7 @@ //! The `register_trait_type!` macro generates `AnimalRc` and `AnimalArc` wrapper types: //! //! ```rust -//! use fory::{Fory, Mode, Error, register_trait_type, Serializer}; +//! use fory::{Fory, Error, register_trait_type, Serializer}; //! use fory::ForyObject; //! use std::sync::Arc; //! use std::rc::Rc; @@ -547,7 +547,7 @@ //! register_trait_type!(Animal, Dog); //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default().mode(Mode::Compatible); +//! let mut fory = Fory::default().compatible(true); //! fory.register::(100); //! //! // For Rc @@ -602,7 +602,7 @@ //! - Nested struct types must be registered on both sides //! //! ```rust -//! use fory::{Fory, Error, Mode}; +//! use fory::{Fory, Error}; //! use fory::ForyObject; //! use std::collections::HashMap; //! @@ -622,10 +622,10 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory1 = Fory::default().mode(Mode::Compatible); +//! let mut fory1 = Fory::default().compatible(true); //! fory1.register::(1); //! -//! let mut fory2 = Fory::default().mode(Mode::Compatible); +//! let mut fory2 = Fory::default().compatible(true); //! fory2.register::(1); //! //! let person_v1 = PersonV1 { @@ -923,9 +923,9 @@ //! - Essential for zero-downtime deployments //! //! ```rust -//! use fory::{Fory, Mode}; +//! use fory::Fory; //! -//! let fory = Fory::default().mode(Mode::Compatible); +//! let fory = Fory::default().compatible(true); //! ``` //! //! ## Cross-Language Serialization @@ -939,11 +939,11 @@ //! **How to enable:** //! //! ```rust -//! use fory::{Fory, Mode}; +//! use fory::Fory; //! use fory::ForyObject; //! //! let mut fory = Fory::default() -//! .mode(Mode::Compatible) +//! .compatible(true) //! .xlang(true); //! //! #[derive(ForyObject)] @@ -1046,7 +1046,7 @@ //! - **[GitHub Repository](https://github.com/apache/fory)** - Source code and issue tracking pub use fory_core::{ - error::Error, fory::Fory, register_trait_type, row::from_row, row::to_row, types::Mode, - types::TypeId, ArcWeak, ForyDefault, RcWeak, ReadContext, Serializer, WriteContext, + error::Error, fory::Fory, register_trait_type, row::from_row, row::to_row, types::TypeId, + ArcWeak, ForyDefault, RcWeak, ReadContext, Serializer, WriteContext, }; pub use fory_derive::{ForyObject, ForyRow}; diff --git a/rust/tests/tests/compatible/test_basic_type.rs b/rust/tests/tests/compatible/test_basic_type.rs index 7efdac6542..1077e913e4 100644 --- a/rust/tests/tests/compatible/test_basic_type.rs +++ b/rust/tests/tests/compatible/test_basic_type.rs @@ -19,7 +19,6 @@ use chrono::{NaiveDate, NaiveDateTime}; use fory_core::buffer::{Reader, Writer}; use fory_core::fory::Fory; use fory_core::resolver::context::{ReadContext, WriteContext}; -use fory_core::types::Mode::Compatible; // primitive_val const BOOL_VAL: bool = true; @@ -437,7 +436,7 @@ fn deserialize_nullable(fory: &Fory, context: &mut ReadContext, auto_conv: bool, // non-null <-> non-null #[test] fn basic() { - let fory = Fory::default().mode(Compatible); + let fory = Fory::default().compatible(true); // serialize let writer = Writer::default(); let mut write_context = WriteContext::new(writer); @@ -452,7 +451,7 @@ fn basic() { // nullable <-> nullable #[test] fn basic_nullable() { - let fory = Fory::default().mode(Compatible); + let fory = Fory::default().compatible(true); // serialize let writer = Writer::default(); let mut write_context = WriteContext::new(writer); @@ -467,7 +466,7 @@ fn basic_nullable() { // non-null -> nullable -> non-null #[test] fn auto_conv() { - let fory = Fory::default().mode(Compatible); + let fory = Fory::default().compatible(true); // serialize_non-null let writer = Writer::default(); let mut write_context = WriteContext::new(writer); diff --git a/rust/tests/tests/compatible/test_container.rs b/rust/tests/tests/compatible/test_container.rs index e6e6426cb4..061a71d1cf 100644 --- a/rust/tests/tests/compatible/test_container.rs +++ b/rust/tests/tests/compatible/test_container.rs @@ -20,7 +20,6 @@ use std::collections::{HashMap, HashSet}; use fory_core::buffer::{Reader, Writer}; use fory_core::fory::Fory; use fory_core::resolver::context::{ReadContext, WriteContext}; -use fory_core::types::Mode::Compatible; use fory_derive::ForyObject; #[derive(ForyObject, PartialEq, Eq, Hash, Debug)] @@ -221,7 +220,7 @@ fn complex_container2() -> Vec, Vec>> { #[test] fn container_outer_auto_conv() { - let fory = Fory::default().mode(Compatible); + let fory = Fory::default().compatible(true); // serialize_outer_non-null let writer = Writer::default(); let mut write_context = WriteContext::new(writer); @@ -296,9 +295,9 @@ fn container_outer_auto_conv() { #[test] fn collection_inner() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize @@ -362,9 +361,9 @@ fn collection_inner() { #[test] fn collection_inner_auto_conv() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize_non-null @@ -436,9 +435,9 @@ fn collection_inner_auto_conv() { #[test] fn map_inner() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize @@ -480,9 +479,9 @@ fn map_inner() { #[test] fn map_inner_auto_conv() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item"); for fory in [fory1, fory2] { // serialize_non-null @@ -532,9 +531,9 @@ fn map_inner_auto_conv() { #[test] fn complex() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item"); for fory in [fory1, fory2] { let writer = Writer::default(); diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index b103aa0712..95f0fbf8d2 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -16,7 +16,6 @@ // under the License. use fory_core::fory::Fory; -use fory_core::types::Mode::Compatible; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; @@ -44,8 +43,8 @@ fn simple() { f7: i16, last: i8, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); let animal: Animal1 = Animal1 { @@ -83,8 +82,8 @@ fn skip_option() { f2: i8, last: i64, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); let item1 = Item1 { @@ -122,8 +121,8 @@ fn nonexistent_struct() { f3: i64, last: String, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(899); fory1.register::(999); fory2.register::(799); @@ -152,7 +151,7 @@ fn option() { f5: Vec>>>, last: i64, } - let mut fory = Fory::default().mode(Compatible); + let mut fory = Fory::default().compatible(true); fory.register::(999); let animal: Animal = Animal { f1: Some(String::from("f1")), @@ -196,8 +195,8 @@ fn nullable() { last: i64, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); @@ -250,8 +249,8 @@ fn nullable_container() { last: i64, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); @@ -300,8 +299,8 @@ fn inner_nullable() { f3: HashMap, last: i64, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); @@ -344,8 +343,8 @@ fn nullable_struct() { f3: Item, last: i64, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(199); fory1.register::(200); fory2.register::(199); @@ -418,11 +417,11 @@ fn enum_without_payload() { last: i8, } - let mut fory1 = Fory::default().mode(Compatible).xlang(true); + let mut fory1 = Fory::default().compatible(true).xlang(true); fory1.register::(101); fory1.register::(102); fory1.register::(103); - let mut fory2 = Fory::default().mode(Compatible).xlang(true); + let mut fory2 = Fory::default().compatible(true).xlang(true); fory2.register::(101); fory2.register::(102); fory2.register::(103); @@ -482,10 +481,10 @@ fn named_enum() { f6: Option, last: i8, } - let mut fory1 = Fory::default().mode(Compatible).xlang(true); + let mut fory1 = Fory::default().compatible(true).xlang(true); fory1.register_by_name::("a"); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible).xlang(true); + let mut fory2 = Fory::default().compatible(true).xlang(true); fory2.register_by_name::("a"); fory2.register::(101); let item1 = Item1 { @@ -538,9 +537,9 @@ fn boxed() { f6: Option, } - let mut fory1 = Fory::default().mode(Compatible).xlang(true); + let mut fory1 = Fory::default().compatible(true).xlang(true); fory1.register::(101); - let mut fory2 = Fory::default().mode(Compatible).xlang(true); + let mut fory2 = Fory::default().compatible(true).xlang(true); fory2.register::(101); let f1 = 1; diff --git a/rust/tests/tests/compatible/test_struct_enum.rs b/rust/tests/tests/compatible/test_struct_enum.rs index 22778b32e9..288c653d22 100644 --- a/rust/tests/tests/compatible/test_struct_enum.rs +++ b/rust/tests/tests/compatible/test_struct_enum.rs @@ -21,7 +21,6 @@ use fory_core::error::Error; use fory_core::fory::{read_data, write_data, Fory}; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; -use fory_core::types::Mode::Compatible; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; @@ -82,11 +81,11 @@ struct Empty {} #[test] fn basic() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); fory1.register::(102); fory1.register::(103); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("color"); fory2.register_by_name::("item"); fory2.register_by_name::("person"); @@ -115,11 +114,11 @@ fn basic() { #[test] fn outer_nullable() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register::(101); fory1.register::(102); fory1.register::(103); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("color"); fory2.register_by_name::("item"); fory2.register_by_name::("person"); @@ -138,18 +137,18 @@ fn skip_basic() { let person_default = Person::default(); let person2_default = Empty::default(); - let mut id_fory1 = Fory::default().mode(Compatible); + let mut id_fory1 = Fory::default().compatible(true); id_fory1.register::(101); id_fory1.register::(102); id_fory1.register::(103); - let mut id_fory2 = Fory::default().mode(Compatible); + let mut id_fory2 = Fory::default().compatible(true); id_fory2.register::(103); - let mut name_fory1 = Fory::default().mode(Compatible); + let mut name_fory1 = Fory::default().compatible(true); name_fory1.register_by_name::("color"); name_fory1.register_by_name::("item"); name_fory1.register_by_name::("person"); - let mut name_fory2 = Fory::default().mode(Compatible); + let mut name_fory2 = Fory::default().compatible(true); name_fory2.register_by_name::("person"); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { @@ -181,20 +180,20 @@ fn nested() { f6: HashMap, f7: Element, } - let mut id_fory1 = Fory::default().mode(Compatible); + let mut id_fory1 = Fory::default().compatible(true); id_fory1.register::(101); id_fory1.register::(102); id_fory1.register::(103); id_fory1.register::(104); - let mut id_fory2 = Fory::default().mode(Compatible); + let mut id_fory2 = Fory::default().compatible(true); id_fory2.register::(104); - let mut name_fory1 = Fory::default().mode(Compatible); + let mut name_fory1 = Fory::default().compatible(true); name_fory1.register_by_name::("item"); name_fory1.register_by_name::("color"); name_fory1.register_by_name::("element"); name_fory1.register_by_name::("nested"); - let mut name_fory2 = Fory::default().mode(Compatible); + let mut name_fory2 = Fory::default().compatible(true); name_fory2.register_by_name::("nested"); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { @@ -288,20 +287,20 @@ fn compatible_nullable() { f28: Some(HashSet::::default()), f29: Some(HashMap::::default()), }; - let mut id_fory1 = Fory::default().mode(Compatible); + let mut id_fory1 = Fory::default().compatible(true); id_fory1.register::(101); id_fory1.register::(102); id_fory1.register::(103); - let mut id_fory2 = Fory::default().mode(Compatible); + let mut id_fory2 = Fory::default().compatible(true); id_fory2.register::(101); id_fory2.register::(102); id_fory2.register::(103); - let mut name_fory1 = Fory::default().mode(Compatible); + let mut name_fory1 = Fory::default().compatible(true); name_fory1.register_by_name::("color"); name_fory1.register_by_name::("item"); name_fory1.register_by_name::("obj"); - let mut name_fory2 = Fory::default().mode(Compatible); + let mut name_fory2 = Fory::default().compatible(true); name_fory2.register_by_name::("color"); name_fory2.register_by_name::("item"); name_fory2.register_by_name::("obj"); @@ -351,20 +350,20 @@ fn name_mismatch() { f29: HashSet, f30: HashMap, } - let mut id_fory1 = Fory::default().mode(Compatible); + let mut id_fory1 = Fory::default().compatible(true); id_fory1.register::(101); id_fory1.register::(102); id_fory1.register::(103); - let mut id_fory2 = Fory::default().mode(Compatible); + let mut id_fory2 = Fory::default().compatible(true); id_fory2.register::(101); id_fory2.register::(102); id_fory2.register::(103); - let mut name_fory1 = Fory::default().mode(Compatible); + let mut name_fory1 = Fory::default().compatible(true); name_fory1.register_by_name::("color"); name_fory1.register_by_name::("item"); name_fory1.register_by_name::("person"); - let mut name_fory2 = Fory::default().mode(Compatible); + let mut name_fory2 = Fory::default().compatible(true); name_fory2.register_by_name::("color"); name_fory2.register_by_name::("item"); name_fory2.register_by_name::("person"); @@ -421,11 +420,11 @@ fn ext() { f1: ExtItem, } - let mut id_fory = Fory::default().mode(Compatible).xlang(true); + let mut id_fory = Fory::default().compatible(true).xlang(true); id_fory.register_serializer::(100); id_fory.register::(101); - let mut name_fory = Fory::default().mode(Compatible).xlang(true); + let mut name_fory = Fory::default().compatible(true).xlang(true); name_fory.register_serializer_by_name::("ext_item"); name_fory.register::(101); @@ -475,17 +474,17 @@ fn skip_ext() { struct ExtWrapper { f1: ExtItem, } - let mut id_fory1 = Fory::default().mode(Compatible).xlang(true); + let mut id_fory1 = Fory::default().compatible(true).xlang(true); id_fory1.register_serializer::(100); id_fory1.register::(101); - let mut id_fory2 = Fory::default().mode(Compatible).xlang(true); + let mut id_fory2 = Fory::default().compatible(true).xlang(true); id_fory2.register_serializer::(100); id_fory2.register::(101); - let mut name_fory1 = Fory::default().mode(Compatible).xlang(true); + let mut name_fory1 = Fory::default().compatible(true).xlang(true); name_fory1.register_serializer_by_name::("ext_item"); name_fory1.register::(101); - let mut name_fory2 = Fory::default().mode(Compatible).xlang(true); + let mut name_fory2 = Fory::default().compatible(true).xlang(true); name_fory2.register_serializer_by_name::("ext_item"); name_fory2.register::(101); @@ -541,17 +540,17 @@ fn compatible_ext() { struct ExtWrapper2 { f1: Option, } - let mut id_fory1 = Fory::default().mode(Compatible).xlang(true); + let mut id_fory1 = Fory::default().compatible(true).xlang(true); id_fory1.register_serializer::(100); id_fory1.register::(101); - let mut id_fory2 = Fory::default().mode(Compatible).xlang(true); + let mut id_fory2 = Fory::default().compatible(true).xlang(true); id_fory2.register_serializer::(100); id_fory2.register::(101); - let mut name_fory1 = Fory::default().mode(Compatible).xlang(true); + let mut name_fory1 = Fory::default().compatible(true).xlang(true); name_fory1.register_serializer_by_name::("ext_item"); name_fory1.register::(101); - let mut name_fory2 = Fory::default().mode(Compatible).xlang(true); + let mut name_fory2 = Fory::default().compatible(true).xlang(true); name_fory2.register_serializer_by_name::("ext_item"); name_fory2.register::(101); diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 29e95c1df2..3b044971c9 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -22,7 +22,6 @@ use fory_core::fory::{read_data, write_data, Fory}; use fory_core::meta::murmurhash3_x64_128; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; -use fory_core::types::Mode::{Compatible, SchemaConsistent}; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; use std::{fs, vec}; @@ -226,13 +225,13 @@ fn test_string_serializer() { let bytes = fs::read(&data_file_path).unwrap(); let reader = Reader::new(bytes.as_slice()); let fory = Fory::default() - .mode(Compatible) + .compatible(true) .xlang(true) .compress_string(false); let mut context = ReadContext::new(reader, 5); let reader_compress = Reader::new(bytes.as_slice()); let fory_compress = Fory::default() - .mode(Compatible) + .compatible(true) .xlang(true) .compress_string(true); let mut context_compress = ReadContext::new(reader_compress, 5); @@ -260,7 +259,7 @@ fn test_string_serializer() { ); } let writer = Writer::default(); - let fory = Fory::default().mode(Compatible).xlang(true); + let fory = Fory::default().compatible(true).xlang(true); let mut context = WriteContext::new(writer); for s in &test_strings { s.fory_write_data(&fory, &mut context, true); @@ -292,7 +291,7 @@ fn test_cross_language_serializer() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let reader = Reader::new(bytes.as_slice()); - let mut fory = Fory::default().mode(Compatible).xlang(true); + let mut fory = Fory::default().compatible(true).xlang(true); fory.register::(101); let mut context = ReadContext::new(reader, 5); assert_de!(fory, context, bool, true); @@ -358,7 +357,7 @@ fn test_cross_language_serializer() { fn test_simple_struct() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().mode(Compatible).xlang(true); + let mut fory = Fory::default().compatible(true).xlang(true); fory.register::(101); fory.register::(102); fory.register::(103); @@ -389,7 +388,7 @@ fn test_simple_struct() { fn test_simple_named_struct() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().mode(Compatible).xlang(true); + let mut fory = Fory::default().compatible(true).xlang(true); fory.register_by_namespace::("demo", "color"); fory.register_by_namespace::("demo", "item"); fory.register_by_namespace::("demo", "simple_struct"); @@ -421,7 +420,7 @@ fn test_list() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().mode(Compatible); + let mut fory = Fory::default().compatible(true); fory.register::(102); let reader = Reader::new(bytes.as_slice()); let mut context = ReadContext::new(reader, 5); @@ -466,7 +465,7 @@ fn test_map() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().mode(Compatible); + let mut fory = Fory::default().compatible(true); fory.register::(102); let reader = Reader::new(bytes.as_slice()); let mut context = ReadContext::new(reader, 5); @@ -535,7 +534,7 @@ fn test_integer() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().mode(Compatible); + let mut fory = Fory::default().compatible(true); fory.register::(101); let reader = Reader::new(bytes.as_slice()); @@ -645,10 +644,10 @@ fn _test_skip_custom(fory1: &Fory, fory2: &Fory) { #[test] fn test_kankankan() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register_serializer::(103); fory1.register::(104); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register::(101); fory2.register::(102); fory2.register_serializer::(103); @@ -665,10 +664,10 @@ fn test_kankankan() { #[test] #[ignore] fn test_skip_id_custom() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register_serializer::(103); fory1.register::(104); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register::(101); fory2.register::(102); fory2.register_serializer::(103); @@ -679,10 +678,10 @@ fn test_skip_id_custom() { #[test] #[ignore] fn test_skip_name_custom() { - let mut fory1 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); fory1.register_serializer_by_name::("my_ext"); fory1.register_by_name::("my_wrapper"); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("color"); fory2.register_by_name::("my_struct"); fory2.register_serializer_by_name::("my_ext"); @@ -693,7 +692,7 @@ fn test_skip_name_custom() { #[test] #[ignore] fn test_consistent_named() { - let mut fory = Fory::default().mode(SchemaConsistent); + let mut fory = Fory::default().compatible(false); fory.register_by_name::("color"); fory.register_by_name::("my_struct"); fory.register_serializer_by_name::("my_ext"); diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index 9f43bce5f3..5faf86c486 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -19,7 +19,6 @@ use fory_core::error::Error; use fory_core::fory::Fory; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; -use fory_core::types::Mode::Compatible; use fory_derive::ForyObject; #[test] @@ -83,7 +82,7 @@ fn test_use() { self } } - let mut fory = Fory::default().mode(Compatible).xlang(true); + let mut fory = Fory::default().compatible(true).xlang(true); let item = Item { f1: 1, f2: 2 }; fory.register_serializer::(100); let bytes = fory.serialize(&item); diff --git a/rust/tests/tests/test_max_dyn_depth.rs b/rust/tests/tests/test_max_dyn_depth.rs index 045f396f29..e4a1bff9fc 100644 --- a/rust/tests/tests/test_max_dyn_depth.rs +++ b/rust/tests/tests/test_max_dyn_depth.rs @@ -27,9 +27,8 @@ struct Container { #[test] fn test_max_dyn_depth_exceeded_box_dyn_any() { - use fory_core::types::Mode; - for mode in [Mode::SchemaConsistent, Mode::Compatible] { - let mut fory = Fory::default().max_dyn_depth(2).mode(mode); + for compatible in [false, true] { + let mut fory = Fory::default().max_dyn_depth(2).compatible(compatible); fory.register::(100); let level3 = Container { diff --git a/rust/tests/tests/test_rc_arc_trait_object.rs b/rust/tests/tests/test_rc_arc_trait_object.rs index 92bfe0a66f..0dd5505cf6 100644 --- a/rust/tests/tests/test_rc_arc_trait_object.rs +++ b/rust/tests/tests/test_rc_arc_trait_object.rs @@ -18,7 +18,6 @@ use fory_core::fory::Fory; use fory_core::register_trait_type; use fory_core::serializer::Serializer; -use fory_core::types::Mode; use fory_core::{unwrap_rc, wrap_rc, wrap_vec_rc}; use fory_derive::ForyObject; use std::collections::HashMap; @@ -26,7 +25,7 @@ use std::rc::Rc; use std::sync::Arc; fn fory_compatible() -> Fory { - Fory::default().mode(Mode::Compatible) + Fory::default().compatible(true) } trait Animal: Serializer + Send + Sync { diff --git a/rust/tests/tests/test_simple_struct.rs b/rust/tests/tests/test_simple_struct.rs index cf65d437ec..9a9bbf53d0 100644 --- a/rust/tests/tests/test_simple_struct.rs +++ b/rust/tests/tests/test_simple_struct.rs @@ -16,7 +16,6 @@ // under the License. use fory_core::fory::Fory; -use fory_core::types::Mode::Compatible; use fory_derive::ForyObject; use std::collections::HashMap; @@ -46,8 +45,8 @@ fn test_simple() { f7: i16, last: i8, } - let mut fory1 = Fory::default().mode(Compatible); - let mut fory2 = Fory::default().mode(Compatible); + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); fory1.register::(999); fory2.register::(999); let animal: Animal1 = Animal1 { diff --git a/rust/tests/tests/test_trait_object.rs b/rust/tests/tests/test_trait_object.rs index bfea6c251e..15bc57b853 100644 --- a/rust/tests/tests/test_trait_object.rs +++ b/rust/tests/tests/test_trait_object.rs @@ -18,12 +18,11 @@ use fory_core::fory::Fory; use fory_core::register_trait_type; use fory_core::serializer::Serializer; -use fory_core::types::Mode; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; fn fory_compatible() -> Fory { - Fory::default().mode(Mode::Compatible) + Fory::default().compatible(true) } #[test] From 0fb16f123a5e1d563695c56957e7ac84f9ef62f4 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Tue, 14 Oct 2025 06:55:21 +0530 Subject: [PATCH 21/37] feat(rust): query type meta from parsed cache to speed up deserialization (#2764) ## Why? ## What does this PR do? ## Related issues Closes #2759 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/fory-core/src/meta/type_meta.rs | 44 +++++++++++++++++--- rust/fory-core/src/resolver/meta_resolver.rs | 21 +++++++++- rust/fory-derive/src/object/read.rs | 4 +- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 2dbbe06a9f..b8f8c5d5f9 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -577,25 +577,59 @@ impl TypeMeta { layer: TypeMetaLayer::new(type_id, namespace, type_name, register_by_name, field_infos), } } - #[allow(unused_assignments)] + pub fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) -> TypeMeta { let header = reader.read_i64(); - let mut meta_size = header & META_SIZE_MASK; + let meta_size = header & META_SIZE_MASK; if meta_size == META_SIZE_MASK { - meta_size += reader.read_varuint32() as i64; + // meta_size += reader.read_varuint32() as i64; + reader.read_varuint32(); } // let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0; // let is_compressed: bool = (header & COMPRESS_META_FLAG) != 0; - // let meta_hash = header >> (64 - NUM_HASH_BITS); + let meta_hash = header >> (64 - NUM_HASH_BITS); // let current_meta_size = 0; // while current_meta_size < meta_size {} let layer = TypeMetaLayer::from_bytes(reader, type_resolver); TypeMeta { layer, - hash: header, + hash: meta_hash, + } + } + + pub fn from_bytes_with_header( + reader: &mut Reader, + type_resolver: &TypeResolver, + header: i64, + ) -> TypeMeta { + let meta_size = header & META_SIZE_MASK; + if meta_size == META_SIZE_MASK { + // meta_size += reader.read_varuint32() as i64; + reader.read_varuint32(); + } + + // let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0; + // let is_compressed: bool = (header & COMPRESS_META_FLAG) != 0; + let meta_hash = header >> (64 - NUM_HASH_BITS); + + // let current_meta_size = 0; + // while current_meta_size < meta_size {} + let layer = TypeMetaLayer::from_bytes(reader, type_resolver); + TypeMeta { + layer, + hash: meta_hash, + } + } + + pub fn skip_bytes(reader: &mut Reader, header: i64) { + let mut meta_size = header & META_SIZE_MASK; + if meta_size == META_SIZE_MASK { + meta_size += reader.read_varuint32() as i64; } + // TODO skio should return result and we need to return it to caller + reader.skip(meta_size as u32); } pub fn to_bytes(&self) -> Result, Error> { diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 0e15f653f1..50a24635fe 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -29,6 +29,8 @@ pub struct MetaWriterResolver { type_id_index_map: HashMap, } +const MAX_PARSED_NUM_TYPE_DEFS: usize = 8192; + #[allow(dead_code)] impl MetaWriterResolver { pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize { @@ -68,6 +70,7 @@ impl MetaWriterResolver { #[derive(Default)] pub struct MetaReaderResolver { pub reading_type_defs: Vec>, + parsed_type_defs: HashMap>, } impl MetaReaderResolver { @@ -79,8 +82,22 @@ impl MetaReaderResolver { let meta_size = reader.read_varuint32(); // self.reading_type_defs.reserve(meta_size as usize); for _ in 0..meta_size { - let type_meta = TypeMeta::from_bytes(reader, type_resolver); - self.reading_type_defs.push(Arc::new(type_meta)); + let meta_header = reader.read_i64(); + if let Some(type_meta) = self.parsed_type_defs.get(&meta_header) { + self.reading_type_defs.push(type_meta.clone()); + TypeMeta::skip_bytes(reader, meta_header); + } else { + let type_meta = Arc::new(TypeMeta::from_bytes_with_header( + reader, + type_resolver, + meta_header, + )); + if self.parsed_type_defs.len() < MAX_PARSED_NUM_TYPE_DEFS { + // avoid malicious type defs to OOM parsed_type_defs + self.parsed_type_defs.insert(meta_header, type_meta.clone()); + } + self.reading_type_defs.push(type_meta); + } } reader.get_cursor() } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 3aa53334de..82a3d8ec77 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -437,9 +437,7 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { }; #(#declare_ts)* - let local_type_def = fory.get_type_resolver().get_type_info(std::any::TypeId::of::()).get_type_def(); - let high_bytes = &local_type_def[..8]; - let local_type_hash = i64::from_le_bytes(high_bytes.try_into().unwrap()); + let local_type_hash = fory.get_type_resolver().get_type_info(std::any::TypeId::of::()).get_type_meta().get_hash(); if meta.get_hash() == local_type_hash { ::fory_read_data(fory, context, false) } else { From 0c35ebb572bf76b1d9b9932e536e2322f99f8f4d Mon Sep 17 00:00:00 2001 From: Steven Schlansker Date: Mon, 13 Oct 2025 19:45:42 -0700 Subject: [PATCH 22/37] feat(java): introduce Compact Row Codec (#2414) ## What does this PR do? Introduce alternate "compact" row encoding that better uses knowledge of fixed-size types and sacrifices alignment to save space. Introduce new Builder pattern to avoid explosion of `Encoders` static methods as more features are added to row format. Optimizations include: * struct stores fixed-size fields (e.g. Int128. FixedSizeBinary) inline in fixed-data area without offset + size * struct of all fixed-sized fields is itself considered fixed-size to store in other struct or array * struct skips null bitmap if all fields are non-nullable * struct sorts fields by fixed-size for best-effort (but not guaranteed) alignment * struct can use less than 8 bytes for small data (int, short, etc) * struct null bitmap stored at end of struct to borrow alignment padding if possible * array stores fixed-size fields inline in fixed-data area without offset+size * array header uses 4 bytes for size (since Collection and array are only int-sized) and leaves remaining 4 bytes for start of null bitmap Fixups include: * toString better handles varbinary / fixed-binary (hex dump of first 256 bytes) * start making Javadoc for row format Compromises: * less alignment could increase access time, but this is opt-in. and I think on modern processors it is not such a big deal. * increased complexity of offset lookup, try to pre-compute in an array when possible and use `StableValue` when it is GA Not compatible with existing row format. ## Related issues Fixes #2337 ## Does this PR introduce any user-facing change? New API for new Compact codec. Existing codec unchanged. --- .../org/apache/fory/builder/CodecBuilder.java | 2 + .../org/apache/fory/memory/MemoryBuffer.java | 2 +- .../org/apache/fory/util/ExceptionUtils.java | 9 + java/fory-format/README.md | 55 +- .../format/encoder/ArrayCodecBuilder.java | 122 +++ .../format/encoder/ArrayEncoderBuilder.java | 9 +- .../encoder/BaseBinaryEncoderBuilder.java | 144 ++-- .../fory/format/encoder/BaseCodecBuilder.java | 75 ++ .../format/encoder/BinaryArrayEncoder.java | 101 +++ .../fory/format/encoder/BinaryMapEncoder.java | 118 +++ .../fory/format/encoder/BinaryRowEncoder.java | 125 ++++ .../encoder/BufferResettingArrayEncoder.java | 80 ++ .../encoder/BufferResettingMapEncoder.java | 93 +++ .../encoder/BufferResettingRowEncoder.java | 78 ++ .../encoder/CompactArrayEncoderBuilder.java | 61 ++ .../format/encoder/CompactCodecFormat.java | 93 +++ .../encoder/CompactMapEncoderBuilder.java | 69 ++ .../encoder/CompactRowEncoderBuilder.java | 127 ++++ .../format/encoder/DefaultCodecFormat.java | 89 +++ .../apache/fory/format/encoder/Encoder.java | 14 +- .../apache/fory/format/encoder/Encoders.java | 515 ++----------- .../apache/fory/format/encoder/Encoding.java | 55 ++ .../format/encoder/GeneratedRowEncoder.java | 3 +- .../fory/format/encoder/MapCodecBuilder.java | 108 +++ .../format/encoder/MapEncoderBuilder.java | 25 +- .../fory/format/encoder/RowCodecBuilder.java | 98 +++ .../fory/format/encoder/RowEncoder.java | 5 +- .../format/encoder/RowEncoderBuilder.java | 28 +- .../org/apache/fory/format/row/Getters.java | 2 +- .../fory/format/row/ToStringValueVisitor.java | 57 ++ .../apache/fory/format/row/ValueVisitor.java | 5 + .../fory/format/row/binary/BinaryArray.java | 50 +- .../fory/format/row/binary/BinaryMap.java | 18 +- .../fory/format/row/binary/BinaryRow.java | 39 +- .../format/row/binary/CompactBinaryArray.java | 138 ++++ .../format/row/binary/CompactBinaryMap.java | 42 ++ .../format/row/binary/CompactBinaryRow.java | 157 ++++ .../fory/format/row/binary/UnsafeTrait.java | 37 +- .../binary/writer/BaseBinaryRowWriter.java | 137 ++++ .../row/binary/writer/BinaryArrayWriter.java | 69 +- .../row/binary/writer/BinaryRowWriter.java | 78 +- .../row/binary/writer/BinaryWriter.java | 52 +- .../writer/CompactBinaryArrayWriter.java | 147 ++++ .../binary/writer/CompactBinaryRowWriter.java | 336 +++++++++ .../writer/FieldAlignmentComparator.java | 42 ++ .../apache/fory/format/type/DataTypes.java | 14 +- .../fory/format/type/TypeInference.java | 4 + .../fory/format/encoder/ArrayEncoderTest.java | 2 +- .../fory/format/encoder/CodecBuilderTest.java | 7 +- .../fory/format/encoder/CompactCodecTest.java | 708 ++++++++++++++++++ .../fory/format/encoder/CustomCodecTest.java | 7 +- .../fory/format/encoder/RowEncoderTest.java | 3 +- 52 files changed, 3775 insertions(+), 679 deletions(-) create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryArrayEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryMapEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingArrayEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingMapEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingRowEncoder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/ToStringValueVisitor.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryArray.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryMap.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryRow.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BaseBinaryRowWriter.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryArrayWriter.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java create mode 100644 java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/FieldAlignmentComparator.java create mode 100644 java/fory-format/src/test/java/org/apache/fory/format/encoder/CompactCodecTest.java diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index 4c3f2498d3..99f827b075 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -124,6 +124,8 @@ public CodecBuilder(CodegenContext ctx, TypeRef beanType) { .forEach(ctx::reserveName); } + public abstract String codecClassName(Class cls); + /** Generate codec class code. */ public abstract String genCode(); diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java index a444e48a96..29331f2691 100644 --- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java @@ -40,7 +40,7 @@ *
  • additional binary compare, swap, and copy methods. *
  • little-endian access. *
  • independent read/write index. - *
  • variant int/long encoding. + *
  • varint int/long encoding. *
  • aligned int/long encoding. * * diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java index 5e08141dcd..e4577f9b60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java @@ -71,4 +71,13 @@ public static RuntimeException handleReadFailed(Fory fory, Throwable t) { } public static void ignore(Object... args) {} + + public static RuntimeException throwAnyway(Throwable t) { + throw ExceptionUtils.throwEvadingChecks(t); + } + + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) + private static E throwEvadingChecks(Throwable throwable) throws E { + throw (E) throwable; + } } diff --git a/java/fory-format/README.md b/java/fory-format/README.md index e867e11cf3..20b8fadcf5 100644 --- a/java/fory-format/README.md +++ b/java/fory-format/README.md @@ -8,12 +8,61 @@ Fory row format is heavily inspired by spark tungsten row format, but with chang - Decimal use arrow decimal format. - Variable-size field can be inline in fixed-size region if small enough. - Allow skip padding by generate Row using aot to put offsets in generated code. -- Support adding fields without breaking compatibility. -The initial fory java row data structure implementation is modified from spark unsafe row/writer. +The initial Fory java row data structure implementation is modified from spark unsafe row/writer. See `Encoders.bean` Javadoc for a list built-in supported types. +## Row Format Java + +To begin using the row format from Java, start with the `Encoders` class: + +``` +// Many built-in types and collections are supported +public record MyRecord(int key, String value) {} + +// The encoder supplier is relatively expensive to create +// It is thread-safe and should be re-used +Supplier> encoderFactory = + Encoders.buildBeanCodec(MyRecord.class) + .build(); + +// Each individual encoder is relatively cheap to create +// It is not thread-safe, but may be reused by the same thread +var encoder = encoderFactory.get(); +byte[] encoded = encoder.encode(new MyRecord(42, "Test")); + +MyRecord deserialized = encoder.decode(encoded); +``` + +## Compact Format + +The default row format is cross-language compatible and alignment-padded for maximum performance. +When data size is a greater concern, the compact format provides an alternate encoding that uses +significantly less space. + +Enable the compact codec on the encoder builder: + +``` +Supplier> encoderFactory = + Encoders.buildBeanCodec(MyRecord.class) + .compactEncoding() + .build(); +``` + +Optimizations include: + +- struct stores fixed-size fields (e.g. Int128. FixedSizeBinary) inline in fixed-data area without offset + size +- struct of all fixed-sized fields is itself considered fixed-size to store in other struct or array +- struct skips null bitmap if all fields are non-nullable +- struct sorts fields by fixed-size for best-effort (but not guaranteed) alignment +- struct can use less than 8 bytes for small data (int, short, etc) +- struct null bitmap stored at end of struct to borrow alignment padding if possible +- array stores fixed-size fields inline in fixed-data area without offset+size +- array header uses 4 bytes for size (since Collection and array are only int-sized) and leaves remaining 4 bytes for start of null bitmap + +## Custom Type Registration + It is possible to register custom type handling and collection factories for the row format - see Encoders.registerCustomCodec and Encoders.registerCustomCollectionFactory. For an interface, Fory can synthesize a simple value implementation, such as the UuidType below. @@ -45,7 +94,7 @@ static class UuidEncoder implements CustomCodec.MemoryBufferCodec { static class SortedSetOfUuidDecoder implements CustomCollectionFactory> { @Override public SortedSet newCollection(final int size) { - return new TreeSet<>(UnsignedUuidComparator.INSTANCE); + return new TreeSet<>(); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java new file mode 100644 index 0000000000..a2faa102ae --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import static org.apache.fory.type.TypeUtils.getRawType; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.ExceptionUtils; + +public class ArrayCodecBuilder> + extends BaseCodecBuilder> { + + private final TypeRef collectionType; + private final Field elementField; + + ArrayCodecBuilder(final TypeRef collectionType) { + super(TypeInference.inferSchema(collectionType, false)); + this.collectionType = collectionType; + elementField = DataTypes.fieldOfSchema(schema, 0); + } + + public Supplier> build() { + final Function> arrayEncoderFactory = buildWithWriter(); + return new Supplier>() { + @Override + public ArrayEncoder get() { + final BinaryArrayWriter writer = codecFormat.newArrayWriter(elementField); + return new BufferResettingArrayEncoder<>( + initialBufferSize, writer, arrayEncoderFactory.apply(writer)); + } + }; + } + + Function> buildWithWriter() { + loadArrayInnerCodecs(); + final Function generatedEncoderFactory = + generatedEncoderFactory(); + return new Function>() { + @Override + public ArrayEncoder apply(final BinaryArrayWriter writer) { + return new BinaryArrayEncoder<>( + writer, generatedEncoderFactory.apply(writer), sizeEmbedded); + } + }; + } + + private void loadArrayInnerCodecs() { + final Set> set = new HashSet<>(); + Encoders.findBeanToken(collectionType, set); + if (set.isEmpty()) { + throw new IllegalArgumentException("can not find bean class."); + } + + for (final TypeRef tt : set) { + Encoders.loadOrGenRowCodecClass(getRawType(tt), codecFormat); + } + } + + Function generatedEncoderFactory() { + final TypeRef elementType = TypeUtils.getElementType(collectionType); + final Class arrayCodecClass = + Encoders.loadOrGenArrayCodecClass(collectionType, elementType, codecFormat); + + final MethodHandle constructorHandle; + try { + final var constructor = + arrayCodecClass.asSubclass(GeneratedArrayEncoder.class).getConstructor(Object[].class); + constructorHandle = + MethodHandles.lookup() + .unreflectConstructor(constructor) + .asType(MethodType.methodType(GeneratedArrayEncoder.class, Object[].class)); + } catch (final NoSuchMethodException | IllegalAccessException e) { + throw new EncoderException( + "Failed to construct array codec for " + + collectionType + + " with element class " + + elementType, + e); + } + return new Function() { + @Override + public GeneratedArrayEncoder apply(final BinaryArrayWriter writer) { + final Object[] references = {writer.getField(), writer, fory}; + try { + return (GeneratedArrayEncoder) constructorHandle.invokeExact(references); + } catch (final Throwable t) { + throw ExceptionUtils.throwAnyway(t); + } + } + }; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java index 2865d62c84..f95473da78 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java @@ -31,7 +31,6 @@ import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.ExpressionUtils; import org.apache.fory.format.row.binary.BinaryArray; -import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.type.TypeInference; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; @@ -104,13 +103,13 @@ public String genCode() { "arrayWriter", ROOT_ARRAY_WRITER_NAME, "arrayWriterType", - ctx.type(BinaryArrayWriter.class), + arrayWriterType(), "fory", FORY_NAME, "foryType", ctx.type(Fory.class)); ctx.addField(ctx.type(Field.class), FIELD_NAME); - ctx.addField(ctx.type(BinaryArrayWriter.class), ROOT_ARRAY_WRITER_NAME); + ctx.addField(ctx.type(arrayWriterType()), ROOT_ARRAY_WRITER_NAME); ctx.addField(ctx.type(Fory.class), FORY_NAME); Expression encodeExpr = buildEncodeExpression(); @@ -136,7 +135,7 @@ public String genCode() { @Override public Expression buildEncodeExpression() { Expression.Reference arrayWriter = - new Expression.Reference(ROOT_ARRAY_WRITER_NAME, arrayWriterTypeToken, false); + new Expression.Reference(ROOT_ARRAY_WRITER_NAME, arrayWriterType(), false); Expression.ListExpression expressions = new Expression.ListExpression(); Expression.Reference inputObject = @@ -148,7 +147,7 @@ public Expression buildEncodeExpression() { Expression.Reference fieldExpr = new Expression.Reference(FIELD_NAME, ARROW_FIELD_TYPE, false); Expression listExpression = - serializeForArrayByWriter(array, arrayWriter, arrayToken, fieldExpr); + serializeForArrayByWriter(array, arrayWriter, arrayToken, null, fieldExpr); expressions.add(listExpression); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java index 102a76f7e2..d208921219 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java @@ -64,6 +64,7 @@ import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.row.binary.BinaryRow; import org.apache.fory.format.row.binary.BinaryUtils; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.row.binary.writer.BinaryRowWriter; import org.apache.fory.format.row.binary.writer.BinaryWriter; @@ -86,9 +87,6 @@ public abstract class BaseBinaryEncoderBuilder extends CodecBuilder { protected static final TypeRef ARROW_FIELD_TYPE = TypeRef.of(Field.class); protected static TypeRef schemaTypeToken = TypeRef.of(Schema.class); protected static TypeRef writerTypeToken = TypeRef.of(BinaryWriter.class); - protected static TypeRef rowWriterTypeToken = TypeRef.of(BinaryRowWriter.class); - protected static TypeRef arrayWriterTypeToken = - TypeRef.of(BinaryArrayWriter.class); protected static TypeRef rowTypeToken = TypeRef.of(Row.class); protected static TypeRef binaryRowTypeToken = TypeRef.of(BinaryRow.class); protected static TypeRef binaryArrayTypeToken = TypeRef.of(BinaryArray.class); @@ -120,6 +118,7 @@ public BaseBinaryEncoderBuilder(CodegenContext context, TypeRef beanType) { this.typeCtx = typeCtx; } + @Override public String codecClassName(Class beanClass) { return codecClassName(beanClass, ""); } @@ -145,6 +144,14 @@ public String codecQualifiedClassName(Class beanClass, String prefix) { return CodeGenerator.getPackage(beanClass) + "." + codecClassName(beanClass, prefix); } + protected TypeRef rowWriterType() { + return TypeRef.of(BinaryRowWriter.class); + } + + protected TypeRef arrayWriterType() { + return TypeRef.of(BinaryArrayWriter.class); + } + /** * Return an expression for serializing an object of given type to row format representation. The * inputObject will be written to position ordinal of row/array using given @@ -155,6 +162,7 @@ protected Expression serializeFor( Expression inputObject, Expression writer, TypeRef typeRef, + Field fieldIfKnown, Expression arrowField, Set> visitedCustomTypes) { Class rawType = getRawType(typeRef); @@ -166,7 +174,13 @@ protected Expression serializeFor( visitedCustomTypes.add(typeRef); Expression doSerialize = serializeFor( - ordinal, newInputObject, writer, rewrittenType, arrowField, visitedCustomTypes); + ordinal, + newInputObject, + writer, + rewrittenType, + fieldIfKnown, + arrowField, + visitedCustomTypes); return new If( new Expression.IsNull(inputObject), new Invoke(writer, "setNullAt", ordinal), @@ -188,6 +202,7 @@ protected Expression serializeFor( new Expression.Cast(unwrapped, elemType), writer, elemType, + fieldIfKnown, arrowField, visitedCustomTypes); } else if (TypeUtils.isPrimitive(rawType)) { @@ -254,24 +269,27 @@ protected Expression serializeFor( // place outer writer operations here, because map key/value arrays need to call // serializeForArray, // but don't setOffsetAndSize for array. - Invoke offset = + Invoke originalWriterIndex = new Invoke(writer, "writerIndex", "writerIndex", TypeUtils.PRIMITIVE_INT_TYPE); - Expression serializeArray = serializeForArray(inputObject, writer, typeRef, arrowField); + Expression serializeArray = + serializeForArray(inputObject, writer, typeRef, fieldIfKnown, arrowField); Arithmetic size = ExpressionUtils.subtract( new Invoke(writer, "writerIndex", "writerIndex", TypeUtils.PRIMITIVE_INT_TYPE), - offset); - Invoke setOffsetAndSize = new Invoke(writer, "setOffsetAndSize", ordinal, offset, size); + originalWriterIndex); + Invoke setOffsetAndSize = + new Invoke(writer, "setOffsetAndSize", ordinal, originalWriterIndex, size); + Expression finishArrayWrite = new ListExpression(size, setOffsetAndSize); ListExpression expression = - new ListExpression(offset, serializeArray, size, setOffsetAndSize); + new ListExpression(originalWriterIndex, serializeArray, finishArrayWrite); return new If( ExpressionUtils.eqNull(inputObject), new Invoke(writer, "setNullAt", ordinal), expression); } else if (TypeUtils.MAP_TYPE.isSupertypeOf(typeRef)) { - return serializeForMap(ordinal, writer, inputObject, typeRef, arrowField); + return serializeForMap(ordinal, writer, inputObject, typeRef, fieldIfKnown, arrowField); } else if (TypeUtils.isBean(rawType, typeCtx)) { - return serializeForBean(ordinal, writer, inputObject, typeRef, arrowField); + return serializeForBean(ordinal, writer, inputObject, fieldIfKnown, typeRef, arrowField); } else if (rawType == BinaryArray.class) { Invoke writeExp = new Invoke( @@ -302,20 +320,28 @@ protected Expression serializeFor( } protected Expression serializeForArray( - Expression inputObject, Expression writer, TypeRef typeRef, Expression arrowField) { + Expression inputObject, + Expression writer, + TypeRef typeRef, + Field fieldIfKnown, + Expression arrowField) { Reference arrayWriter = getOrCreateArrayWriter(typeRef, arrowField, writer); - return serializeForArrayByWriter(inputObject, arrayWriter, typeRef, arrowField); + return serializeForArrayByWriter(inputObject, arrayWriter, typeRef, fieldIfKnown, arrowField); } protected Expression serializeForArrayByWriter( - Expression inputObject, Expression arrayWriter, TypeRef typeRef, Expression arrowField) { + Expression inputObject, + Expression arrayWriter, + TypeRef typeRef, + Field fieldIfKnown, + Expression arrowField) { StaticInvoke arrayElementField = new StaticInvoke( DataTypes.class, "arrayElementField", "elemField", ARROW_FIELD_TYPE, false, arrowField); Class rawType = getRawType(typeRef); if (rawType.isArray()) { FieldValue length = new FieldValue(inputObject, "length", TypeUtils.PRIMITIVE_INT_TYPE); - Invoke reset = new Invoke(arrayWriter, "reset", length); + Expression reset = new Invoke(arrayWriter, "reset", length); if (rawType.getComponentType().isPrimitive()) { return new ListExpression( reset, new Invoke(arrayWriter, "fromPrimitiveArray", inputObject), arrayWriter); @@ -330,30 +356,17 @@ protected Expression serializeForArrayByWriter( value, arrayWriter, Objects.requireNonNull(typeRef.getComponentType()), + null, arrayElementField, new HashSet<>())); return new ListExpression(reset, forEach, arrayWriter); } - } else if (getRawType(typeRef) == Iterable.class) { - ListFromIterable listFromIterable = new ListFromIterable(inputObject); - Invoke size = new Invoke(listFromIterable, "size", TypeUtils.PRIMITIVE_INT_TYPE); - Invoke reset = new Invoke(arrayWriter, "reset", size); - ForEach forEach = - new ForEach( - listFromIterable, - !TypeUtils.getElementType(typeRef).isPrimitive(), - (i, value) -> - serializeFor( - i, - value, - arrayWriter, - TypeUtils.getElementType(typeRef), - arrayElementField, - new HashSet<>())); - return new ListExpression(reset, forEach, arrayWriter); - } else { // collection + } else { + if (getRawType(typeRef) == Iterable.class) { + inputObject = new ListFromIterable(inputObject); + } Invoke size = new Invoke(inputObject, "size", TypeUtils.PRIMITIVE_INT_TYPE); - Invoke reset = new Invoke(arrayWriter, "reset", size); + Expression reset = new Invoke(arrayWriter, "reset", size); ForEach forEach = new ForEach( inputObject, @@ -364,6 +377,7 @@ protected Expression serializeForArrayByWriter( value, arrayWriter, TypeUtils.getElementType(typeRef), + null, arrayElementField, new HashSet<>())); return new ListExpression(reset, forEach, arrayWriter); @@ -377,10 +391,10 @@ protected Reference getOrCreateArrayWriter( t -> { String name = ctx.newName("arrayWriter"); ctx.addField( - ctx.type(BinaryArrayWriter.class), + arrayWriterType().getRawType(), name, - new NewInstance(arrayWriterTypeToken, arrayDataType, writer)); - return new Reference(name, arrayWriterTypeToken, false); + new NewInstance(arrayWriterType(), arrayDataType, writer)); + return new Reference(name, arrayWriterType(), false); }); } @@ -393,6 +407,7 @@ protected Expression serializeForMap( Expression writer, Expression inputObject, TypeRef typeRef, + Field fieldIfKnown, Expression arrowField) { StaticInvoke keyArrayField = new StaticInvoke( @@ -424,7 +439,8 @@ protected Expression serializeForMap( expressions.add(offset, preserve); Invoke keySet = new Invoke(inputObject, "keySet", keySetType); - Expression keySerializationExpr = serializeForArray(keySet, writer, keySetType, keyArrayField); + Expression keySerializationExpr = + serializeForArray(keySet, writer, keySetType, fieldIfKnown, keyArrayField); expressions.add(keySet, keySerializationExpr); expressions.add( @@ -436,7 +452,7 @@ protected Expression serializeForMap( Invoke values = new Invoke(inputObject, "values", valuesType); Expression valueSerializationExpr = - serializeForArray(values, writer, valuesType, valueArrayField); + serializeForArray(values, writer, valuesType, fieldIfKnown, valueArrayField); expressions.add(values, valueSerializationExpr); Arithmetic size = @@ -457,6 +473,7 @@ protected Expression serializeForBean( Expression ordinal, Expression writer, Expression inputObject, + Field fieldIfKnown, TypeRef typeRef, Expression structField) { Class rawType = getRawType(typeRef); @@ -464,13 +481,11 @@ protected Expression serializeForBean( Reference beanEncoder = beanEncoderMap.get(typeRef); if (beanEncoder == null) { // janino generics don't add cast, so this `<${type}>` is only for generated code readability - StaticInvoke schema = - new StaticInvoke( - DataTypes.class, "schemaFromStructField", "schema", SCHEMA_TYPE, false, structField); + Expression schema = createSchemaFromStructField(structField); String rowWriterName = ctx.newName(StringUtils.uncapitalize(rawType.getSimpleName() + "RowWriter")); - NewInstance newRowWriter = new NewInstance(rowWriterTypeToken, schema, writer); - ctx.addField(ctx.type(rowWriterTypeToken), rowWriterName, newRowWriter); + NewInstance newRowWriter = new NewInstance(rowWriterType(), schema, writer); + ctx.addField(ctx.type(rowWriterType()), rowWriterName, newRowWriter); Preconditions.checkArgument(!codecClassName(rawType).contains(".")); String encoderName = ctx.newName(StringUtils.uncapitalize(codecClassName(rawType))); @@ -483,30 +498,49 @@ protected Expression serializeForBean( ExpressionUtils.newObjectArray(schema, newRowWriter, foryRef)); ctx.addField(encoderClass, encoderName, newEncoder); - rowWriter = new Reference(rowWriterName, rowWriterTypeToken); + rowWriter = new Reference(rowWriterName, rowWriterType()); rowWriterMap.put(typeRef, rowWriter); beanEncoder = new Reference(encoderName, codecTypeRef); beanEncoderMap.put(typeRef, beanEncoder); } rowWriter = rowWriterMap.get(typeRef); - Invoke reset = new Invoke(rowWriter, "reset"); + Expression expression = + serializeForNotNullBean(ordinal, writer, inputObject, fieldIfKnown, rowWriter, beanEncoder); + + return new If( + new Expression.IsNull(inputObject), new Invoke(writer, "setNullAt", ordinal), expression); + } + + protected Expression createSchemaFromStructField(Expression structField) { + return new StaticInvoke( + DataTypes.class, "schemaFromStructField", "schema", SCHEMA_TYPE, false, structField); + } + + protected Expression serializeForNotNullBean( + Expression ordinal, + Expression writer, + Expression inputObject, + Field fieldIfKnown, + Reference rowWriter, + Reference beanEncoder) { + Invoke reset = beanWriterReset(writer, rowWriter, ordinal); Invoke offset = new Invoke(writer, "writerIndex", "writerIndex", TypeUtils.PRIMITIVE_INT_TYPE); Invoke toRow = new Invoke(beanEncoder, "toRow", inputObject); Arithmetic size = ExpressionUtils.subtract( new Invoke(writer, "writerIndex", "writerIndex", TypeUtils.PRIMITIVE_INT_TYPE), offset); Invoke setOffsetAndSize = new Invoke(writer, "setOffsetAndSize", ordinal, offset, size); - ListExpression expression = - new ListExpression( - offset, - reset, - toRow, // reset will change writerIndex. must call reset and toRow in pair. - size, - setOffsetAndSize); + return new ListExpression( + offset, + reset, // reset will change writerIndex. must call reset and toRow in pair. + toRow, + size, + setOffsetAndSize); + } - return new If( - new Expression.IsNull(inputObject), new Invoke(writer, "setNullAt", ordinal), expression); + protected Invoke beanWriterReset(Expression writer, Reference rowWriter, Expression ordinal) { + return new Invoke(rowWriter, "reset"); } /** diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java new file mode 100644 index 0000000000..581e102a1e --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.Fory; +import org.apache.fory.format.row.binary.CompactBinaryRow; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; + +public class BaseCodecBuilder> { + protected Schema schema; + protected int initialBufferSize = 16; + protected boolean sizeEmbedded = true; + protected Fory fory; + protected Encoding codecFormat = DefaultCodecFormat.INSTANCE; + + BaseCodecBuilder(final Schema schema) { + this.schema = schema; + } + + /** Configure the Fory instance used for embedded binary serialized objects. */ + public B fory(final Fory fory) { + this.fory = fory; + return castThis(); + } + + /** Configure the initial buffer size used when writing a new row. */ + public B initialBufferSize(final int initialBufferSize) { + this.initialBufferSize = initialBufferSize; + return castThis(); + } + + /** + * Configure whether the encoder expects the size encoded inline in the data, or provided by + * external framing. When true, the default, the encoder will write the size as part of its data, + * to be read by the decoder. When false, the data size must be preserved by external framing you + * provide. + */ + public B withSizeEmbedded(final boolean sizeEmbedded) { + this.sizeEmbedded = sizeEmbedded; + return castThis(); + } + + /** + * Configure compact encoding, which is more space efficient than the default encoding, but is not + * yet stable. See {@link CompactBinaryRow} for details. + */ + public B compactEncoding() { + schema = CompactBinaryRowWriter.sortSchema(schema); + codecFormat = CompactCodecFormat.INSTANCE; + return castThis(); + } + + @SuppressWarnings("unchecked") + protected B castThis() { + return (B) this; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryArrayEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryArrayEncoder.java new file mode 100644 index 0000000000..f956310de7 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryArrayEncoder.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +class BinaryArrayEncoder implements ArrayEncoder { + private final BinaryArrayWriter writer; + private final GeneratedArrayEncoder codec; + private final boolean sizeEmbedded; + + BinaryArrayEncoder( + final BinaryArrayWriter writer, + final GeneratedArrayEncoder codec, + final boolean sizeEmbedded) { + this.writer = writer; + this.codec = codec; + this.sizeEmbedded = sizeEmbedded; + } + + @Override + public Field field() { + return writer.getField(); + } + + @SuppressWarnings("unchecked") + @Override + public T fromArray(final BinaryArray array) { + return (T) codec.fromArray(array); + } + + @Override + public BinaryArray toArray(final T obj) { + return codec.toArray(obj); + } + + @Override + public T decode(final MemoryBuffer buffer) { + return decode(buffer, sizeEmbedded ? buffer.readInt32() : buffer.remaining()); + } + + @Override + public T decode(final byte[] bytes) { + return decode(MemoryUtils.wrap(bytes), bytes.length); + } + + T decode(final MemoryBuffer buffer, final int size) { + final BinaryArray array = writer.newArray(); + final int readerIndex = buffer.readerIndex(); + array.pointTo(buffer, readerIndex, size); + buffer.readerIndex(readerIndex + size); + return fromArray(array); + } + + @Override + public byte[] encode(final T obj) { + final BinaryArray array = toArray(obj); + return writer.getBuffer().getBytes(0, array.getSizeInBytes()); + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + final MemoryBuffer prevBuffer = writer.getBuffer(); + final int writerIndex = buffer.writerIndex(); + if (sizeEmbedded) { + buffer.writeInt32(-1); + } + try { + writer.setBuffer(buffer); + toArray(obj); + final int size = buffer.writerIndex() - writerIndex; + if (sizeEmbedded) { + buffer.putInt32(writerIndex, size - 4); + } + return size; + } finally { + writer.setBuffer(prevBuffer); + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryMapEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryMapEncoder.java new file mode 100644 index 0000000000..b5514d9170 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryMapEncoder.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +class BinaryMapEncoder implements MapEncoder { + private final Encoding format; + private final Field mapField; + private final BinaryArrayWriter valWriter; + private final BinaryArrayWriter keyWriter; + private final GeneratedMapEncoder codec; + private final boolean sizeEmbedded; + + BinaryMapEncoder( + final Encoding format, + final Field mapField, + final BinaryArrayWriter valWriter, + final BinaryArrayWriter keyWriter, + final GeneratedMapEncoder codec, + final boolean sizeEmbedded) { + this.format = format; + this.mapField = mapField; + this.valWriter = valWriter; + this.keyWriter = keyWriter; + this.codec = codec; + this.sizeEmbedded = sizeEmbedded; + } + + @Override + public Field keyField() { + return keyWriter.getField(); + } + + @Override + public Field valueField() { + return valWriter.getField(); + } + + @SuppressWarnings("unchecked") + @Override + public M fromMap(final BinaryArray key, final BinaryArray value) { + return (M) codec.fromMap(key, value); + } + + @Override + public BinaryMap toMap(final M obj) { + return codec.toMap(obj); + } + + @Override + public M decode(final MemoryBuffer buffer) { + return decode(buffer, sizeEmbedded ? buffer.readInt32() : buffer.remaining()); + } + + M decode(final MemoryBuffer buffer, final int size) { + final BinaryMap map = format.newMap(mapField); + final int readerIndex = buffer.readerIndex(); + map.pointTo(buffer, readerIndex, size); + buffer.readerIndex(readerIndex + size); + return fromMap(map); + } + + @Override + public M decode(final byte[] bytes) { + return decode(MemoryUtils.wrap(bytes), bytes.length); + } + + @Override + public byte[] encode(final M obj) { + final BinaryMap map = toMap(obj); + return map.getBuf().getBytes(map.getBaseOffset(), map.getSizeInBytes()); + } + + @Override + public int encode(final MemoryBuffer buffer, final M obj) { + final MemoryBuffer prevBuffer = keyWriter.getBuffer(); + final int writerIndex = buffer.writerIndex(); + if (sizeEmbedded) { + buffer.writeInt32(-1); + } + try { + keyWriter.setBuffer(buffer); + valWriter.setBuffer(buffer); + toMap(obj); + final int size = buffer.writerIndex() - writerIndex; + if (sizeEmbedded) { + buffer.putInt32(writerIndex, size - 4); + } + return size; + } finally { + keyWriter.setBuffer(prevBuffer); + valWriter.setBuffer(prevBuffer); + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java new file mode 100644 index 0000000000..c9266daa0e --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.exception.ClassNotCompatibleException; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +class BinaryRowEncoder implements RowEncoder { + private final Schema schema; + private final Encoding codecFactory; + private final GeneratedRowEncoder codec; + private final BaseBinaryRowWriter writer; + private final boolean sizeEmbedded; + private final long schemaHash; + private final MemoryBuffer buffer = MemoryUtils.buffer(16); + + BinaryRowEncoder( + final Schema schema, + final Encoding codecFactory, + final GeneratedRowEncoder codec, + final BaseBinaryRowWriter writer, + final boolean sizeEmbedded) { + this.schema = schema; + this.codecFactory = codecFactory; + this.codec = codec; + this.writer = writer; + this.sizeEmbedded = sizeEmbedded; + this.schemaHash = DataTypes.computeSchemaHash(schema); + } + + @Override + public Schema schema() { + return schema; + } + + @SuppressWarnings("unchecked") + @Override + public T fromRow(final BinaryRow row) { + return (T) codec.fromRow(row); + } + + @Override + public BinaryRow toRow(final T obj) { + return codec.toRow(obj); + } + + @Override + public T decode(final MemoryBuffer buffer) { + return decode(buffer, sizeEmbedded ? buffer.readInt32() : buffer.remaining()); + } + + T decode(final MemoryBuffer buffer, final int size) { + final long peerSchemaHash = buffer.readInt64(); + if (peerSchemaHash != schemaHash) { + throw new ClassNotCompatibleException( + String.format( + "Schema is not consistent, encoder schema is %s. " + + "self/peer schema hash are %s/%s. " + + "Please check writer schema.", + schema, schemaHash, peerSchemaHash)); + } + final BinaryRow row = codecFactory.newRow(schema); + row.pointTo(buffer, buffer.readerIndex(), size); + buffer.increaseReaderIndex(size - 8); + return fromRow(row); + } + + @Override + public T decode(final byte[] bytes) { + return decode(MemoryUtils.wrap(bytes), bytes.length); + } + + @Override + public byte[] encode(final T obj) { + buffer.writerIndex(0); + buffer.writeInt64(schemaHash); + writer.setBuffer(buffer); + writer.reset(); + final BinaryRow row = toRow(obj); + return buffer.getBytes(0, 8 + row.getSizeInBytes()); + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + final int writerIndex = buffer.writerIndex(); + if (sizeEmbedded) { + buffer.writeInt32(-1); + } + try { + buffer.writeInt64(schemaHash); + writer.setBuffer(buffer); + writer.reset(); + codec.toRow(obj); + final int size = buffer.writerIndex() - writerIndex; + if (sizeEmbedded) { + buffer.putInt32(writerIndex, size - 4); + } + return size; + } finally { + writer.setBuffer(this.buffer); + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingArrayEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingArrayEncoder.java new file mode 100644 index 0000000000..7202a1e456 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingArrayEncoder.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** + * Wrap an ArrayEncoder to provide a fresh buffer for every toArray operation, when not streaming. + */ +class BufferResettingArrayEncoder implements ArrayEncoder { + + private final int initialBufferSize; + private final BinaryArrayWriter writer; + private final ArrayEncoder encoder; + + BufferResettingArrayEncoder( + final int initialBufferSize, final BinaryArrayWriter writer, final ArrayEncoder encoder) { + this.initialBufferSize = initialBufferSize; + this.writer = writer; + this.encoder = encoder; + } + + @Override + public T decode(final MemoryBuffer buffer) { + return encoder.decode(buffer); + } + + @Override + public T decode(final byte[] bytes) { + return encoder.decode(bytes); + } + + @Override + public byte[] encode(final T obj) { + writer.setBuffer(MemoryUtils.buffer(initialBufferSize)); + return encoder.encode(obj); + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + return encoder.encode(buffer, obj); + } + + @Override + public Field field() { + return encoder.field(); + } + + @Override + public T fromArray(final BinaryArray array) { + return encoder.fromArray(array); + } + + @Override + public BinaryArray toArray(final T obj) { + writer.setBuffer(MemoryUtils.buffer(initialBufferSize)); + return encoder.toArray(obj); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingMapEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingMapEncoder.java new file mode 100644 index 0000000000..d76a13ea42 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingMapEncoder.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** Wrap a MapEncoder to provide a fresh buffer for every toMap operation, when not streaming. */ +class BufferResettingMapEncoder implements MapEncoder { + + private final int initialBufferSize; + private final BinaryArrayWriter keyWriter; + private final BinaryArrayWriter valWriter; + private final MapEncoder encoder; + + BufferResettingMapEncoder( + final int initialBufferSize, + final BinaryArrayWriter keyWriter, + final BinaryArrayWriter valWriter, + final MapEncoder encoder) { + this.initialBufferSize = initialBufferSize; + this.keyWriter = keyWriter; + this.valWriter = valWriter; + this.encoder = encoder; + } + + @Override + public Field keyField() { + return encoder.keyField(); + } + + @Override + public Field valueField() { + return encoder.valueField(); + } + + @Override + public T fromMap(final BinaryArray keyArray, final BinaryArray valArray) { + return encoder.fromMap(keyArray, valArray); + } + + @Override + public BinaryMap toMap(final T obj) { + final MemoryBuffer buffer = MemoryUtils.buffer(initialBufferSize); + keyWriter.setBuffer(buffer); + valWriter.setBuffer(buffer); + return encoder.toMap(obj); + } + + @Override + public T decode(final MemoryBuffer buffer) { + return encoder.decode(buffer); + } + + @Override + public T decode(final byte[] bytes) { + return encoder.decode(bytes); + } + + @Override + public byte[] encode(final T obj) { + final MemoryBuffer buffer = MemoryUtils.buffer(initialBufferSize); + keyWriter.setBuffer(buffer); + valWriter.setBuffer(buffer); + return encoder.encode(obj); + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + return encoder.encode(buffer, obj); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingRowEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingRowEncoder.java new file mode 100644 index 0000000000..9106e4ca09 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BufferResettingRowEncoder.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** Wrap a RowEncoder to provide a fresh buffer for every toRow operation, when not streaming. */ +class BufferResettingRowEncoder implements RowEncoder { + + private final int initialBufferSize; + private final BaseBinaryRowWriter writer; + private final RowEncoder encoder; + + BufferResettingRowEncoder( + final int initialBufferSize, final BaseBinaryRowWriter writer, final RowEncoder encoder) { + this.initialBufferSize = initialBufferSize; + this.writer = writer; + this.encoder = encoder; + } + + @Override + public Schema schema() { + return encoder.schema(); + } + + @Override + public T fromRow(final BinaryRow row) { + return encoder.fromRow(row); + } + + @Override + public BinaryRow toRow(final T obj) { + writer.setBuffer(MemoryUtils.buffer(initialBufferSize)); + writer.reset(); + return encoder.toRow(obj); + } + + @Override + public T decode(final MemoryBuffer buffer) { + return encoder.decode(buffer); + } + + @Override + public T decode(final byte[] bytes) { + return encoder.decode(bytes); + } + + @Override + public byte[] encode(final T obj) { + return encoder.encode(obj); + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + return encoder.encode(buffer, obj); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java new file mode 100644 index 0000000000..65f8508e35 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.reflect.TypeRef; + +class CompactArrayEncoderBuilder extends ArrayEncoderBuilder { + public CompactArrayEncoderBuilder(final TypeRef clsType, final TypeRef beanType) { + super(clsType, beanType); + } + + @Override + protected Invoke beanWriterReset( + final Expression writer, final Reference rowWriter, final Expression ordinal) { + return new Invoke(writer, "resetFor", rowWriter, ordinal); + } + + @Override + protected TypeRef rowWriterType() { + return TypeRef.of(CompactBinaryRowWriter.class); + } + + @Override + protected TypeRef arrayWriterType() { + return TypeRef.of(CompactBinaryArrayWriter.class); + } + + @Override + protected String codecSuffix() { + return "CompactCodec"; + } + + @Override + protected Expression createSchemaFromStructField(final Expression schema) { + return CompactRowEncoderBuilder.sortSchema(super.createSchemaFromStructField(schema)); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java new file mode 100644 index 0000000000..359ec8fc4c --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.util.Collection; +import java.util.Map; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.CompactBinaryArray; +import org.apache.fory.format.row.binary.CompactBinaryMap; +import org.apache.fory.format.row.binary.CompactBinaryRow; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; + +enum CompactCodecFormat implements Encoding { + INSTANCE; + + @Override + public BaseBinaryRowWriter newWriter(final Schema schema) { + return new CompactBinaryRowWriter(schema); + } + + @Override + public BaseBinaryRowWriter newWriter(final Schema schema, final MemoryBuffer buffer) { + return new CompactBinaryRowWriter(schema, buffer); + } + + @Override + public BinaryArrayWriter newArrayWriter(final Field field) { + return new CompactBinaryArrayWriter(field); + } + + @Override + public BinaryArrayWriter newArrayWriter(final Field field, final MemoryBuffer buffer) { + return new CompactBinaryArrayWriter(field, buffer); + } + + @Override + public RowEncoderBuilder newRowEncoder(final TypeRef beanType) { + return new CompactRowEncoderBuilder(beanType); + } + + @Override + public ArrayEncoderBuilder newArrayEncoder( + final TypeRef> collectionType, final TypeRef elementType) { + return new CompactArrayEncoderBuilder(collectionType, elementType); + } + + @Override + public MapEncoderBuilder newMapEncoder( + final TypeRef> mapType, final TypeRef beanToken) { + return new CompactMapEncoderBuilder(mapType, beanToken); + } + + @Override + public BinaryRow newRow(final Schema schema) { + return new CompactBinaryRow(schema); + } + + @Override + public BinaryArray newArray(final Field field) { + return new CompactBinaryArray(field); + } + + @Override + public BinaryMap newMap(final Field field) { + return new CompactBinaryMap(field); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java new file mode 100644 index 0000000000..be3d206d59 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.CompactBinaryMap; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.reflect.TypeRef; + +class CompactMapEncoderBuilder extends MapEncoderBuilder { + + public CompactMapEncoderBuilder(final TypeRef clsType, final TypeRef beanType) { + super(clsType, beanType); + } + + @Override + protected Invoke beanWriterReset( + final Expression writer, final Reference rowWriter, final Expression ordinal) { + return new Invoke(writer, "resetFor", rowWriter, ordinal); + } + + @Override + protected TypeRef rowWriterType() { + return TypeRef.of(CompactBinaryRowWriter.class); + } + + @Override + protected TypeRef arrayWriterType() { + return TypeRef.of(CompactBinaryArrayWriter.class); + } + + @Override + protected TypeRef binaryMapType() { + return TypeRef.of(CompactBinaryMap.class); + } + + @Override + protected String codecSuffix() { + return "CompactCodec"; + } + + @Override + protected Expression createSchemaFromStructField(final Expression structField) { + return CompactRowEncoderBuilder.sortSchema(super.createSchemaFromStructField(structField)); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java new file mode 100644 index 0000000000..5d6e389e6e --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.Invoke; +import org.apache.fory.codegen.Expression.ListExpression; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.codegen.Expression.StaticInvoke; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; + +/** Expression builder for building compact row encoder class. */ +class CompactRowEncoderBuilder extends RowEncoderBuilder { + public CompactRowEncoderBuilder(final TypeRef beanType) { + super(beanType); + } + + @Override + protected Schema inferSchema(final TypeRef beanType) { + return CompactBinaryRowWriter.sortSchema(super.inferSchema(beanType)); + } + + @Override + protected String codecSuffix() { + return "CompactCodec"; + } + + @Override + protected TypeRef rowWriterType() { + return TypeRef.of(CompactBinaryRowWriter.class); + } + + @Override + protected TypeRef arrayWriterType() { + return TypeRef.of(CompactBinaryArrayWriter.class); + } + + @Override + protected Expression serializeForNotNullBean( + final Expression ordinal, + final Expression writer, + final Expression inputObject, + final Field fieldIfKnown, + final Reference rowWriter, + final Reference beanEncoder) { + if (fieldIfKnown == null || CompactBinaryRowWriter.fixedWidthFor(fieldIfKnown) == -1) { + return super.serializeForNotNullBean( + ordinal, writer, inputObject, fieldIfKnown, rowWriter, beanEncoder); + } + // resetFor will change writerIndex. must call reset and toRow in pair. + final Invoke reset = beanWriterReset(writer, rowWriter, ordinal); + final Invoke toRow = new Invoke(beanEncoder, "toRow", inputObject); + return new ListExpression(reset, toRow); + } + + @Override + protected Invoke beanWriterReset( + final Expression writer, final Reference rowWriter, final Expression ordinal) { + return new Invoke(writer, "resetFor", rowWriter, ordinal); + } + + @Override + protected Expression serializeForArrayByWriter( + final Expression inputObject, + final Expression arrayWriter, + final TypeRef typeRef, + final Field fieldIfKnown, + final Expression arrowField) { + final Expression result = + super.serializeForArrayByWriter( + inputObject, arrayWriter, typeRef, fieldIfKnown, arrowField); + if (fieldIfKnown == null + || CompactBinaryRowWriter.fixedWidthFor(itemType(fieldIfKnown)) == -1) { + return result; + } + return new ListExpression( + result, + new Invoke( + Invoke.inlineInvoke(arrayWriter, "getBuffer", TypeRef.of(MemoryBuffer.class)), + "writerIndex"), + arrayWriter); + } + + private static Field itemType(final Field fieldIfKnown) { + return fieldIfKnown.getChildren().get(0); + } + + @Override + protected Expression createSchemaFromStructField(final Expression structField) { + return sortSchema(super.createSchemaFromStructField(structField)); + } + + static Expression sortSchema(final Expression schema) { + return new StaticInvoke( + CompactBinaryRowWriter.class, + "sortSchema", + "sortedSchema", + TypeRef.of(Schema.class), + false, + true, + schema); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java new file mode 100644 index 0000000000..3b6bc2e314 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.util.Collection; +import java.util.Map; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.BinaryRowWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; + +enum DefaultCodecFormat implements Encoding { + INSTANCE; + + @Override + public BaseBinaryRowWriter newWriter(final Schema schema) { + return new BinaryRowWriter(schema); + } + + @Override + public BaseBinaryRowWriter newWriter(final Schema schema, final MemoryBuffer buffer) { + return new BinaryRowWriter(schema, buffer); + } + + @Override + public BinaryArrayWriter newArrayWriter(final Field field) { + return new BinaryArrayWriter(field); + } + + @Override + public BinaryArrayWriter newArrayWriter(final Field field, final MemoryBuffer buffer) { + return new BinaryArrayWriter(field, buffer); + } + + @Override + public RowEncoderBuilder newRowEncoder(final TypeRef beanClass) { + return new RowEncoderBuilder(beanClass); + } + + @Override + public ArrayEncoderBuilder newArrayEncoder( + final TypeRef> collectionType, final TypeRef elementType) { + return new ArrayEncoderBuilder(collectionType, elementType); + } + + @Override + public MapEncoderBuilder newMapEncoder( + final TypeRef> mapType, final TypeRef beanToken) { + return new MapEncoderBuilder(mapType, beanToken); + } + + @Override + public BinaryRow newRow(final Schema schema) { + return new BinaryRow(schema); + } + + @Override + public BinaryArray newArray(final Field field) { + return new BinaryArray(field); + } + + @Override + public BinaryMap newMap(final Field field) { + return new BinaryMap(field); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoder.java index 3216b7ca1b..f56b7fd48d 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoder.java @@ -29,12 +29,22 @@ * @param type of value */ public interface Encoder { - + /** Decode a buffer. */ T decode(MemoryBuffer buffer); + /** Decode a byte array. */ T decode(byte[] bytes); + /** Encode to a byte array. */ byte[] encode(T obj); - void encode(MemoryBuffer buffer, T obj); + /** Encode to a buffer. Returns number of bytes written to the buffer. */ + int encode(MemoryBuffer buffer, T obj); + + /** Encode to a buffer. Returns a sliced buffer view of bytes written. */ + default MemoryBuffer encodeSlice(final MemoryBuffer buffer, final T obj) { + final int initialIndex = buffer.writerIndex(); + final int size = encode(buffer, obj); + return buffer.slice(initialIndex, size); + } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java index 9cd2accc9a..b03ea52e6f 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java @@ -28,26 +28,17 @@ import java.util.Map; import java.util.Set; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.Schema; import org.apache.fory.Fory; +import org.apache.fory.builder.CodecBuilder; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.CompileUnit; import org.apache.fory.collection.Tuple2; -import org.apache.fory.exception.ClassNotCompatibleException; -import org.apache.fory.format.row.binary.BinaryArray; -import org.apache.fory.format.row.binary.BinaryMap; -import org.apache.fory.format.row.binary.BinaryRow; -import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.row.binary.writer.BinaryRowWriter; import org.apache.fory.format.type.CustomTypeEncoderRegistry; import org.apache.fory.format.type.CustomTypeRegistration; -import org.apache.fory.format.type.DataTypes; import org.apache.fory.format.type.TypeInference; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.type.TypeResolutionContext; import org.apache.fory.type.TypeUtils; @@ -60,6 +51,22 @@ public class Encoders { private static final Logger LOG = LoggerFactory.getLogger(Encoders.class); + /** Build a row codec with configurable options through a builder. */ + public static RowCodecBuilder buildBeanCodec(Class beanClass) { + return new RowCodecBuilder<>(beanClass); + } + + /** Build an array codec with configurable options through a builder. */ + public static > ArrayCodecBuilder buildArrayCodec( + TypeRef collectionType) { + return new ArrayCodecBuilder<>(collectionType); + } + + /** Build a map codec with configurable options through a builder. */ + public static > MapCodecBuilder buildMapCodec(TypeRef mapType) { + return new MapCodecBuilder<>(mapType); + } + public static RowEncoder bean(Class beanClass) { return bean(beanClass, 16); } @@ -73,48 +80,7 @@ public static RowEncoder bean(Class beanClass, Fory fory) { } public static RowEncoder bean(Class beanClass, Fory fory, int initialBufferSize) { - Schema schema = TypeInference.inferSchema(beanClass); - BinaryRowWriter writer = new BinaryRowWriter(schema); - RowEncoder encoder = bean(beanClass, writer, fory); - return new RowEncoder() { - - @Override - public Schema schema() { - return encoder.schema(); - } - - @Override - public T fromRow(BinaryRow row) { - return encoder.fromRow(row); - } - - @Override - public BinaryRow toRow(T obj) { - writer.setBuffer(MemoryUtils.buffer(initialBufferSize)); - writer.reset(); - return encoder.toRow(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return encoder.decode(buffer); - } - - @Override - public T decode(byte[] bytes) { - return encoder.decode(bytes); - } - - @Override - public byte[] encode(T obj) { - return encoder.encode(obj); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - encoder.encode(buffer, obj); - } - }; + return buildBeanCodec(beanClass).fory(fory).initialBufferSize(initialBufferSize).build().get(); } public static RowEncoder bean(Class beanClass, BinaryRowWriter writer) { @@ -142,92 +108,7 @@ public static RowEncoder bean(Class beanClass, BinaryRowWriter writer) * */ public static RowEncoder bean(Class beanClass, BinaryRowWriter writer, Fory fory) { - Schema schema = writer.getSchema(); - - try { - Class rowCodecClass = loadOrGenRowCodecClass(beanClass); - Object references = new Object[] {schema, writer, fory}; - GeneratedRowEncoder codec = - rowCodecClass - .asSubclass(GeneratedRowEncoder.class) - .getConstructor(Object[].class) - .newInstance(references); - long schemaHash = DataTypes.computeSchemaHash(schema); - - return new RowEncoder() { - private final MemoryBuffer buffer = MemoryUtils.buffer(16); - - @Override - public Schema schema() { - return schema; - } - - @SuppressWarnings("unchecked") - @Override - public T fromRow(BinaryRow row) { - return (T) codec.fromRow(row); - } - - @Override - public BinaryRow toRow(T obj) { - return codec.toRow(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return decode(buffer, buffer.readInt32()); - } - - public T decode(MemoryBuffer buffer, int size) { - long peerSchemaHash = buffer.readInt64(); - if (peerSchemaHash != schemaHash) { - throw new ClassNotCompatibleException( - String.format( - "Schema is not consistent, encoder schema is %s. " - + "self/peer schema hash are %s/%s. " - + "Please check writer schema.", - schema, schemaHash, peerSchemaHash)); - } - BinaryRow row = new BinaryRow(schema); - row.pointTo(buffer, buffer.readerIndex(), size); - buffer.increaseReaderIndex(size - 8); - return fromRow(row); - } - - @Override - public T decode(byte[] bytes) { - return decode(MemoryUtils.wrap(bytes), bytes.length); - } - - @Override - public byte[] encode(T obj) { - buffer.writerIndex(0); - buffer.writeInt64(schemaHash); - writer.setBuffer(buffer); - writer.reset(); - BinaryRow row = toRow(obj); - return buffer.getBytes(0, 8 + row.getSizeInBytes()); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - int writerIndex = buffer.writerIndex(); - buffer.writeInt32(-1); - try { - buffer.writeInt64(schemaHash); - writer.setBuffer(buffer); - writer.reset(); - toRow(obj); - buffer.putInt32(writerIndex, buffer.writerIndex() - writerIndex - 4); - } finally { - writer.setBuffer(this.buffer); - } - } - }; - } catch (Exception e) { - String msg = String.format("Create encoder failed, \nbeanClass: %s", beanClass); - throw new EncoderException(msg, e); - } + return buildBeanCodec(beanClass).fory(fory).buildForWriter().apply(writer); } /** @@ -273,167 +154,13 @@ public static > void registerCustomCollectionFactory( * @param T is a array type, can be a nested list type. * @return */ - public static ArrayEncoder arrayEncoder(TypeRef token) { + public static > ArrayEncoder arrayEncoder(TypeRef token) { return arrayEncoder(token, null); } - public static ArrayEncoder arrayEncoder(TypeRef token, Fory fory) { - Schema schema = TypeInference.inferSchema(token, false); - Field field = DataTypes.fieldOfSchema(schema, 0); - BinaryArrayWriter writer = new BinaryArrayWriter(field); - - Set> set = new HashSet<>(); - findBeanToken(token, set); - if (set.isEmpty()) { - throw new IllegalArgumentException("can not find bean class."); - } - - TypeRef typeRef = null; - for (TypeRef tt : set) { - typeRef = set.iterator().next(); - Encoders.loadOrGenRowCodecClass(getRawType(tt)); - } - ArrayEncoder encoder = arrayEncoder(token, typeRef, writer, fory); - return new ArrayEncoder() { - - @Override - public Field field() { - return encoder.field(); - } - - @Override - public T fromArray(BinaryArray array) { - return encoder.fromArray(array); - } - - @Override - public BinaryArray toArray(T obj) { - return encoder.toArray(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return encoder.decode(buffer); - } - - @Override - public T decode(byte[] bytes) { - return encoder.decode(bytes); - } - - @Override - public byte[] encode(T obj) { - return encoder.encode(obj); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - encoder.encode(buffer, obj); - } - }; - } - - /** - * The underlying implementation uses array, only supported {@link Collection} format, because - * generic type such as List is erased to simply List, so a bean class input param is required. - * - * @return - */ - public static ArrayEncoder arrayEncoder( - Class arrayCls, Class elementType) { - Preconditions.checkNotNull(elementType); - - return (ArrayEncoder) arrayEncoder(TypeUtils.collectionOf(elementType), null); - } - - /** - * Creates an encoder for Java Bean of type T. - * - *

    T must be publicly accessible. - * - *

    supported types for java bean field: - primitive types: boolean, int, double, etc. - boxed - * types: Boolean, Integer, Double, etc. - String - java.math.BigDecimal, java.math.BigInteger - - * time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant - - * collection types: only array and java.util.List currently, map support is in progress - nested - * java bean. - */ - public static ArrayEncoder arrayEncoder( - TypeRef arrayToken, - TypeRef elementType, - BinaryArrayWriter writer, - Fory fory) { - Field field = writer.getField(); - try { - Class rowCodecClass = loadOrGenArrayCodecClass(arrayToken, elementType); - Object references = new Object[] {field, writer, fory}; - GeneratedArrayEncoder codec = - rowCodecClass - .asSubclass(GeneratedArrayEncoder.class) - .getConstructor(Object[].class) - .newInstance(references); - - return new ArrayEncoder() { - - @Override - public Field field() { - return field; - } - - @SuppressWarnings("unchecked") - @Override - public T fromArray(BinaryArray array) { - return (T) codec.fromArray(array); - } - - @Override - public BinaryArray toArray(T obj) { - return codec.toArray(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return decode(buffer, buffer.readInt32()); - } - - public T decode(MemoryBuffer buffer, int size) { - BinaryArray array = new BinaryArray(field); - int readerIndex = buffer.readerIndex(); - array.pointTo(buffer, readerIndex, size); - buffer.readerIndex(readerIndex + size); - return fromArray(array); - } - - @Override - public T decode(byte[] bytes) { - return decode(MemoryUtils.wrap(bytes), bytes.length); - } - - @Override - public byte[] encode(T obj) { - BinaryArray array = toArray(obj); - return writer.getBuffer().getBytes(0, array.getSizeInBytes()); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - MemoryBuffer prevBuffer = writer.getBuffer(); - int writerIndex = buffer.writerIndex(); - buffer.writeInt32(-1); - try { - writer.setBuffer(buffer); - BinaryArray array = toArray(obj); - int size = buffer.writerIndex() - writerIndex - 4; - assert size == array.getSizeInBytes(); - buffer.putInt32(writerIndex, size); - } finally { - writer.setBuffer(prevBuffer); - } - } - }; - } catch (Exception e) { - String msg = String.format("Create encoder failed, \nelementType: %s", elementType); - throw new EncoderException(msg, e); - } + public static > ArrayEncoder arrayEncoder( + TypeRef token, Fory fory) { + return buildArrayCodec(token).fory(fory).build().get(); } /** @@ -455,6 +182,7 @@ public static MapEncoder mapEncoder(TypeRef token) { * * @return */ + @SuppressWarnings("unchecked") public static MapEncoder mapEncoder( Class mapCls, Class keyType, Class valueType) { Preconditions.checkNotNull(keyType); @@ -463,19 +191,21 @@ public static MapEncoder mapEncoder( return (MapEncoder) mapEncoder(TypeUtils.mapOf(keyType, valueType), null); } - public static MapEncoder mapEncoder(TypeRef token, Fory fory) { + @SuppressWarnings("unchecked") + public static , K, V> MapEncoder mapEncoder(TypeRef token, Fory fory) { Preconditions.checkNotNull(token); - Tuple2, TypeRef> tuple2 = TypeUtils.getMapKeyValueType(token); + final Tuple2, TypeRef> tuple2 = TypeUtils.getMapKeyValueType(token); - Set> set1 = beanSet(tuple2.f0); - Set> set2 = beanSet(tuple2.f1); + final Set> set1 = beanSet(tuple2.f0); + final Set> set2 = beanSet(tuple2.f1); LOG.info("Find beans to load: {}, {}", set1, set2); - TypeRef keyToken = token4BeanLoad(set1, tuple2.f0); - TypeRef valToken = token4BeanLoad(set2, tuple2.f1); + final TypeRef keyToken = + (TypeRef) token4BeanLoad(set1, tuple2.f0, DefaultCodecFormat.INSTANCE); + final TypeRef valToken = + (TypeRef) token4BeanLoad(set2, tuple2.f1, DefaultCodecFormat.INSTANCE); - MapEncoder encoder = mapEncoder0(token, keyToken, valToken, fory); - return createMapEncoder(encoder); + return mapEncoder0(token, keyToken, valToken, fory); } /** @@ -489,110 +219,24 @@ public static MapEncoder mapEncoder(TypeRef token, Fory fo * collection types: only array and java.util.List currently, map support is in progress - nested * java bean. */ - public static MapEncoder mapEncoder( - TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { + public static , K, V> MapEncoder mapEncoder( + TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { Preconditions.checkNotNull(mapToken); Preconditions.checkNotNull(keyToken); Preconditions.checkNotNull(valToken); - - Set> set1 = beanSet(keyToken); - Set> set2 = beanSet(valToken); - LOG.info("Find beans to load: {}, {}", set1, set2); - - token4BeanLoad(set1, keyToken); - token4BeanLoad(set2, valToken); - return mapEncoder0(mapToken, keyToken, valToken, fory); } - private static MapEncoder mapEncoder0( - TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { + private static , K, V> MapEncoder mapEncoder0( + TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { Preconditions.checkNotNull(mapToken); Preconditions.checkNotNull(keyToken); Preconditions.checkNotNull(valToken); + return buildMapCodec(mapToken).fory(fory).build().get(); + } - Schema schema = TypeInference.inferSchema(mapToken, false); - Field field = DataTypes.fieldOfSchema(schema, 0); - Field keyField = DataTypes.keyArrayFieldForMap(field); - Field valField = DataTypes.itemArrayFieldForMap(field); - BinaryArrayWriter keyWriter = new BinaryArrayWriter(keyField); - BinaryArrayWriter valWriter = new BinaryArrayWriter(valField, keyWriter.getBuffer()); - try { - Class rowCodecClass = loadOrGenMapCodecClass(mapToken, keyToken, valToken); - Object references = new Object[] {keyField, valField, keyWriter, valWriter, fory, field}; - GeneratedMapEncoder codec = - rowCodecClass - .asSubclass(GeneratedMapEncoder.class) - .getConstructor(Object[].class) - .newInstance(references); - - return new MapEncoder() { - @Override - public Field keyField() { - return keyField; - } - - @Override - public Field valueField() { - return valField; - } - - @SuppressWarnings("unchecked") - @Override - public T fromMap(BinaryArray key, BinaryArray value) { - return (T) codec.fromMap(key, value); - } - - @Override - public BinaryMap toMap(T obj) { - return codec.toMap(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return decode(buffer, buffer.readInt32()); - } - - public T decode(MemoryBuffer buffer, int size) { - BinaryMap map = new BinaryMap(field); - int readerIndex = buffer.readerIndex(); - map.pointTo(buffer, readerIndex, size); - buffer.readerIndex(readerIndex + size); - return fromMap(map); - } - - @Override - public T decode(byte[] bytes) { - return decode(MemoryUtils.wrap(bytes), bytes.length); - } - - @Override - public byte[] encode(T obj) { - BinaryMap map = toMap(obj); - return map.getBuf().getBytes(map.getBaseOffset(), map.getSizeInBytes()); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - MemoryBuffer prevBuffer = keyWriter.getBuffer(); - int writerIndex = buffer.writerIndex(); - buffer.writeInt32(-1); - try { - keyWriter.setBuffer(buffer); - valWriter.setBuffer(buffer); - toMap(obj); - buffer.putInt32(writerIndex, buffer.writerIndex() - writerIndex - 4); - } finally { - keyWriter.setBuffer(prevBuffer); - valWriter.setBuffer(prevBuffer); - } - } - }; - } catch (Exception e) { - String msg = - String.format("Create encoder failed, \nkeyType: %s, valueType: %s", keyToken, valToken); - throw new EncoderException(msg, e); - } + static void loadMapCodecs(TypeRef type, Encoding codecFactory) { + token4BeanLoad(beanSet(type), type, codecFactory); } private static Set> beanSet(TypeRef token) { @@ -606,62 +250,18 @@ token, new TypeResolutionContext(CustomTypeEncoderRegistry.customTypeHandler(), return set; } - private static TypeRef token4BeanLoad(Set> set, TypeRef init) { + private static TypeRef token4BeanLoad( + Set> set, TypeRef init, Encoding codecFactory) { TypeRef keyToken = init; for (TypeRef tt : set) { keyToken = tt; - Encoders.loadOrGenRowCodecClass(getRawType(tt)); + Encoders.loadOrGenRowCodecClass(getRawType(tt), codecFactory); LOG.info("bean {} load finished", getRawType(tt)); } return keyToken; } - private static MapEncoder createMapEncoder(MapEncoder encoder) { - return new MapEncoder() { - - @Override - public Field keyField() { - return encoder.keyField(); - } - - @Override - public Field valueField() { - return encoder.valueField(); - } - - @Override - public T fromMap(BinaryArray key, BinaryArray value) { - return encoder.fromMap(key, value); - } - - @Override - public BinaryMap toMap(T obj) { - return encoder.toMap(obj); - } - - @Override - public T decode(MemoryBuffer buffer) { - return encoder.decode(buffer); - } - - @Override - public T decode(byte[] bytes) { - return encoder.decode(bytes); - } - - @Override - public byte[] encode(T obj) { - return encoder.encode(obj); - } - - @Override - public void encode(MemoryBuffer buffer, T obj) { - encoder.encode(buffer, obj); - } - }; - } - - private static void findBeanToken(TypeRef typeRef, java.util.Set> set) { + static void findBeanToken(TypeRef typeRef, final Set> set) { TypeResolutionContext typeCtx = new TypeResolutionContext(CustomTypeEncoderRegistry.customTypeHandler(), true); Set> visited = new LinkedHashSet<>(); @@ -696,7 +296,7 @@ private static void findBeanToken(TypeRef typeRef, java.util.Set> } } - public static Class loadOrGenRowCodecClass(Class beanClass) { + static Class loadOrGenRowCodecClass(Class beanClass, Encoding codecFactory) { Set> classes = TypeUtils.listBeansRecursiveInclusive( beanClass, @@ -704,12 +304,12 @@ public static Class loadOrGenRowCodecClass(Class beanClass) { if (classes.isEmpty()) { return null; } - LOG.info("Create RowCodec for classes {}", classes); + LOG.info("Create codec for classes {}", classes); CompileUnit[] compileUnits = classes.stream() .map( cls -> { - RowEncoderBuilder codecBuilder = new RowEncoderBuilder(cls); + final CodecBuilder codecBuilder = codecFactory.newRowEncoder(TypeRef.of(cls)); // use genCodeFunc to avoid gen code repeatedly return new CompileUnit( CodeGenerator.getPackage(cls), @@ -720,14 +320,14 @@ public static Class loadOrGenRowCodecClass(Class beanClass) { return loadCls(compileUnits); } - private static Class loadOrGenArrayCodecClass( - TypeRef arrayCls, TypeRef elementType) { + static Class loadOrGenArrayCodecClass( + TypeRef> arrayCls, TypeRef elementType, Encoding codecFactory) { LOG.info("Create ArrayCodec for classes {}", elementType); Class cls = getRawType(elementType); // class name prefix String prefix = TypeInference.inferTypeName(arrayCls); - ArrayEncoderBuilder codecBuilder = new ArrayEncoderBuilder(arrayCls, elementType); + ArrayEncoderBuilder codecBuilder = codecFactory.newArrayEncoder(arrayCls, elementType); CompileUnit compileUnit = new CompileUnit( CodeGenerator.getPackage(cls), @@ -737,8 +337,11 @@ private static Class loadOrGenArrayCodecClass( return loadCls(compileUnit); } - private static Class loadOrGenMapCodecClass( - TypeRef mapCls, TypeRef keyToken, TypeRef valueToken) { + static Class loadOrGenMapCodecClass( + TypeRef> mapCls, + TypeRef keyToken, + TypeRef valueToken, + Encoding codecFactory) { LOG.info("Create MapCodec for classes {}, {}", keyToken, valueToken); boolean keyIsBean = TypeUtils.isBean(keyToken); boolean valIsBean = TypeUtils.isBean(valueToken); @@ -757,7 +360,7 @@ private static Class loadOrGenMapCodecClass( // class name prefix String prefix = TypeInference.inferTypeName(mapCls); - MapEncoderBuilder codecBuilder = new MapEncoderBuilder(mapCls, beanToken); + MapEncoderBuilder codecBuilder = codecFactory.newMapEncoder(mapCls, beanToken); CompileUnit compileUnit = new CompileUnit( CodeGenerator.getPackage(cls), @@ -774,7 +377,7 @@ private static Class loadCls(CompileUnit... compileUnit) { String className = compileUnit[0].getQualifiedClassName(); try { return classLoader.loadClass(className); - } catch (ClassNotFoundException e) { + } catch (final ClassNotFoundException e) { throw new IllegalStateException("Impossible because we just compiled class", e); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java new file mode 100644 index 0000000000..3f73a869f3 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.util.Collection; +import java.util.Map; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; + +interface Encoding { + BaseBinaryRowWriter newWriter(Schema schema); + + BaseBinaryRowWriter newWriter(Schema schema, MemoryBuffer buffer); + + BinaryArrayWriter newArrayWriter(Field field); + + BinaryArrayWriter newArrayWriter(Field field, MemoryBuffer buffer); + + RowEncoderBuilder newRowEncoder(TypeRef beanType); + + ArrayEncoderBuilder newArrayEncoder( + TypeRef> collectionType, TypeRef elementType); + + MapEncoderBuilder newMapEncoder(TypeRef> mapType, TypeRef beanToken); + + BinaryRow newRow(Schema schema); + + BinaryArray newArray(Field field); + + BinaryMap newMap(Field field); +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/GeneratedRowEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/GeneratedRowEncoder.java index 3fd763623e..9af0f20f78 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/GeneratedRowEncoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/GeneratedRowEncoder.java @@ -19,12 +19,13 @@ package org.apache.fory.format.encoder; +import org.apache.fory.annotation.Internal; import org.apache.fory.builder.Generated; import org.apache.fory.format.row.binary.BinaryRow; /** A row format codec for java bean. */ +@Internal public interface GeneratedRowEncoder extends Generated { - BinaryRow toRow(Object obj); Object fromRow(BinaryRow row); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java new file mode 100644 index 0000000000..6b4bdbbf74 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.ExceptionUtils; + +public class MapCodecBuilder> extends BaseCodecBuilder> { + + private final TypeRef mapType; + private final Field field; + private final Field keyField; + private final Field valField; + private final TypeRef keyType; + private final TypeRef valType; + + MapCodecBuilder(final TypeRef mapType) { + super(TypeInference.inferSchema(mapType, false)); + this.mapType = mapType; + field = DataTypes.fieldOfSchema(schema, 0); + keyField = DataTypes.keyArrayFieldForMap(field); + valField = DataTypes.itemArrayFieldForMap(field); + final var kvType = TypeUtils.getMapKeyValueType(mapType); + keyType = kvType.f0; + valType = kvType.f1; + } + + public Supplier> build() { + loadMapInnerCodecs(); + final var mapEncoderFactory = generatedMapEncoder(); + return new Supplier>() { + @Override + public MapEncoder get() { + final BinaryArrayWriter keyWriter = codecFormat.newArrayWriter(keyField); + final BinaryArrayWriter valWriter = + codecFormat.newArrayWriter(valField, keyWriter.getBuffer()); + final var codec = mapEncoderFactory.apply(keyWriter, valWriter); + return new BufferResettingMapEncoder<>( + initialBufferSize, + keyWriter, + valWriter, + new BinaryMapEncoder(codecFormat, field, valWriter, keyWriter, codec, sizeEmbedded)); + } + }; + } + + private void loadMapInnerCodecs() { + Encoders.loadMapCodecs(keyType, codecFormat); + Encoders.loadMapCodecs(valType, codecFormat); + } + + BiFunction generatedMapEncoder() { + final Class arrayCodecClass = + Encoders.loadOrGenMapCodecClass(mapType, keyType, valType, codecFormat); + + final MethodHandle constructorHandle; + try { + final var constructor = + arrayCodecClass.asSubclass(GeneratedMapEncoder.class).getConstructor(Object[].class); + constructorHandle = + MethodHandles.lookup() + .unreflectConstructor(constructor) + .asType(MethodType.methodType(GeneratedMapEncoder.class, Object[].class)); + } catch (final NoSuchMethodException | IllegalAccessException e) { + throw new EncoderException("Failed to construct array codec for " + mapType, e); + } + return new BiFunction() { + @Override + public GeneratedMapEncoder apply( + final BinaryArrayWriter keyWriter, final BinaryArrayWriter valWriter) { + final Object[] references = {keyField, valField, keyWriter, valWriter, fory, field}; + try { + return (GeneratedMapEncoder) constructorHandle.invokeExact(references); + } catch (final Throwable t) { + throw ExceptionUtils.throwAnyway(t); + } + } + }; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java index 94468406fc..974a59ad89 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java @@ -32,7 +32,6 @@ import org.apache.fory.codegen.ExpressionUtils; import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.row.binary.BinaryMap; -import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.type.TypeInference; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; @@ -52,7 +51,6 @@ public class MapEncoderBuilder extends BaseBinaryEncoderBuilder { private static final String ROOT_KEY_WRITER_NAME = "keyArrayWriter"; private static final String ROOT_VALUE_WRITER_NAME = "valueArrayWriter"; - private static final TypeRef ARROW_FIELD_TYPE = TypeRef.of(Field.class); private final TypeRef mapToken; public MapEncoderBuilder(Class mapCls, Class keyClass) { @@ -97,7 +95,7 @@ public String genCode() { "keyArrayWriter", ROOT_KEY_WRITER_NAME, "arrayWriterType", - ctx.type(BinaryArrayWriter.class), + ctx.type(arrayWriterType()), "valueField", VALUE_FIELD_NAME, "fieldType", @@ -105,7 +103,7 @@ public String genCode() { "valueArrayWriter", ROOT_VALUE_WRITER_NAME, "arrayWriterType", - ctx.type(BinaryArrayWriter.class), + ctx.type(arrayWriterType()), "fory", FORY_NAME, "foryType", @@ -116,8 +114,8 @@ public String genCode() { ctx.type(Field.class)); ctx.addField(ctx.type(Field.class), KEY_FIELD_NAME); ctx.addField(ctx.type(Field.class), VALUE_FIELD_NAME); - ctx.addField(ctx.type(BinaryArrayWriter.class), ROOT_KEY_WRITER_NAME); - ctx.addField(ctx.type(BinaryArrayWriter.class), ROOT_VALUE_WRITER_NAME); + ctx.addField(ctx.type(arrayWriterType()), ROOT_KEY_WRITER_NAME); + ctx.addField(ctx.type(arrayWriterType()), ROOT_VALUE_WRITER_NAME); ctx.addField(ctx.type(Fory.class), FORY_NAME); ctx.addField(ctx.type(Field.class), FIELD_NAME); @@ -158,9 +156,9 @@ public Expression buildEncodeExpression() { new Expression.Cast(inputObject, mapToken, ctx.newName(getRawType(mapToken)), false, false); Expression.Reference keyArrayWriter = - new Expression.Reference(ROOT_KEY_WRITER_NAME, arrayWriterTypeToken, false); + new Expression.Reference(ROOT_KEY_WRITER_NAME, arrayWriterType(), false); Expression.Reference valArrayWriter = - new Expression.Reference(ROOT_VALUE_WRITER_NAME, arrayWriterTypeToken, false); + new Expression.Reference(ROOT_VALUE_WRITER_NAME, arrayWriterType(), false); Expression.Reference fieldExpr = new Expression.Reference(FIELD_NAME, ARROW_FIELD_TYPE, false); Expression.Reference keyFieldExpr = @@ -180,7 +178,7 @@ public Expression buildEncodeExpression() { expressions.add( new Expression.Invoke(keyArrayWriter, "writeDirectly", Expression.Literal.ofInt(-1))); Expression keySerializationExpr = - serializeForArrayByWriter(keySet, keyArrayWriter, keySetType, keyFieldExpr); + serializeForArrayByWriter(keySet, keyArrayWriter, keySetType, null, keyFieldExpr); Expression.Invoke keyArray = new Expression.Invoke(keyArrayWriter, "toArray", TypeRef.of(BinaryArray.class)); expressions.add(map); @@ -195,7 +193,7 @@ public Expression buildEncodeExpression() { Expression.Invoke values = new Expression.Invoke(map, "values", valuesType); Expression valueSerializationExpr = - serializeForArrayByWriter(values, valArrayWriter, valuesType, valFieldExpr); + serializeForArrayByWriter(values, valArrayWriter, valuesType, null, valFieldExpr); Expression.Invoke valArray = new Expression.Invoke(valArrayWriter, "toArray", TypeRef.of(BinaryArray.class)); @@ -203,11 +201,14 @@ public Expression buildEncodeExpression() { expressions.add(valArray); expressions.add( new Expression.Return( - new Expression.NewInstance( - TypeRef.of(BinaryMap.class), keyArray, valArray, fieldExpr))); + new Expression.NewInstance(binaryMapType(), keyArray, valArray, fieldExpr))); return expressions; } + protected TypeRef binaryMapType() { + return TypeRef.of(BinaryMap.class); + } + /** * Returns an expression that deserialize row as a java bean of type {@link * MapEncoderBuilder#mapToken}. diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java new file mode 100644 index 0000000000..adbd2922dd --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.util.ExceptionUtils; + +public class RowCodecBuilder extends BaseCodecBuilder> { + + private final Class beanClass; + + RowCodecBuilder(final Class beanClass) { + super(TypeInference.inferSchema(beanClass)); + this.beanClass = beanClass; + } + + /** + * Create a codec factory with the configuration settings from this builder. The resulting factory + * should be re-used if possible for creating subsequent encoder instances. Encoders are not + * thread-safe, so create a separate encoder for each thread that uses it. For platform threads + * consider {@link ThreadLocal#withInitial(Supplier)}, or get a new encoder instance for each + * virtual thread. + */ + public Supplier> build() { + final Function> rowEncoderFactory = buildForWriter(); + return new Supplier>() { + @Override + public RowEncoder get() { + final BaseBinaryRowWriter writer = codecFormat.newWriter(schema); + return new BufferResettingRowEncoder( + initialBufferSize, writer, rowEncoderFactory.apply(writer)); + } + }; + } + + Function> buildForWriter() { + final Function rowEncoderFactory = + rowEncoderFactory(); + return new Function>() { + @Override + public RowEncoder apply(final BaseBinaryRowWriter writer) { + return new BinaryRowEncoder( + schema, codecFormat, rowEncoderFactory.apply(writer), writer, sizeEmbedded); + } + }; + } + + Function rowEncoderFactory() { + final Class rowCodecClass = Encoders.loadOrGenRowCodecClass(beanClass, codecFormat); + MethodHandle constructorHandle; + try { + final var constructor = + rowCodecClass.asSubclass(GeneratedRowEncoder.class).getConstructor(Object[].class); + constructorHandle = + MethodHandles.lookup() + .unreflectConstructor(constructor) + .asType(MethodType.methodType(GeneratedRowEncoder.class, Object[].class)); + } catch (final NoSuchMethodException | IllegalAccessException e) { + throw new EncoderException("Failed to construct codec for " + beanClass, e); + } + return new Function() { + @Override + public GeneratedRowEncoder apply(final BaseBinaryRowWriter writer) { + try { + final Object[] references = {schema, writer, fory}; + return (GeneratedRowEncoder) constructorHandle.invokeExact(references); + } catch (final ReflectiveOperationException e) { + throw new EncoderException("Failed to construct codec for " + beanClass, e); + } catch (final Throwable e) { + throw ExceptionUtils.throwAnyway(e); + } + } + }; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoder.java index 43a7487cb1..5eb9515186 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoder.java @@ -22,7 +22,10 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.fory.format.row.binary.BinaryRow; -/** Encoder to encode/decode object to/from row. */ +/** + * Encoder to encode/decode object to/from row. A RowEncoder instance is reusable but not + * thread-safe. + */ public interface RowEncoder extends Encoder { Schema schema(); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java index 6a3f599223..bcb044e9a9 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java @@ -49,7 +49,6 @@ import org.apache.fory.format.row.binary.BinaryMap; import org.apache.fory.format.row.binary.BinaryRow; import org.apache.fory.format.row.binary.BinaryUtils; -import org.apache.fory.format.row.binary.writer.BinaryRowWriter; import org.apache.fory.format.type.DataTypes; import org.apache.fory.format.type.TypeInference; import org.apache.fory.logging.Logger; @@ -64,7 +63,7 @@ /** Expression builder for building jit row encoder class. */ @SuppressWarnings("UnstableApiUsage") -public class RowEncoderBuilder extends BaseBinaryEncoderBuilder { +class RowEncoderBuilder extends BaseBinaryEncoderBuilder { private static final Logger LOG = LoggerFactory.getLogger(RowEncoderBuilder.class); static final String SCHEMA_NAME = "schema"; static final String ROOT_ROW_NAME = "row"; @@ -72,7 +71,7 @@ public class RowEncoderBuilder extends BaseBinaryEncoderBuilder { private final String className; private final SortedMap descriptorsMap; - private final Schema schema; + protected final Schema schema; protected static final String BEAN_CLASS_NAME = "beanClass"; protected Reference beanClassRef = new Reference(BEAN_CLASS_NAME, CLASS_TYPE); private final CodegenContext generatedBeanImpl; @@ -86,7 +85,7 @@ public RowEncoderBuilder(TypeRef beanType) { super(new CodegenContext(), beanType); Preconditions.checkArgument(beanClass.isInterface() || TypeUtils.isBean(beanType, typeCtx)); className = codecClassName(beanClass); - this.schema = TypeInference.inferSchema(getRawType(beanType)); + this.schema = inferSchema(beanType); this.descriptorsMap = Descriptor.getDescriptorsMap(beanClass); ctx.reserveName(ROOT_ROW_WRITER_NAME); ctx.reserveName(SCHEMA_NAME); @@ -114,6 +113,10 @@ public RowEncoderBuilder(TypeRef beanType) { } } + protected Schema inferSchema(TypeRef beanType) { + return TypeInference.inferSchema(getRawType(beanType)); + } + @Override protected String codecSuffix() { return "RowCodec"; @@ -131,6 +134,7 @@ public String genCode() { // don't addImport(beanClass), because user class may name collide. // janino don't support generics, so GeneratedCodec has no generics ctx.implementsInterfaces(ctx.type(GeneratedRowEncoder.class)); + String rowWriterType = ctx.type(rowWriterType()); String constructorCode = StringUtils.format( "${schema} = (${schemaType})${references}[0];\n" @@ -145,13 +149,13 @@ public String genCode() { "rowWriter", ROOT_ROW_WRITER_NAME, "rowWriterType", - ctx.type(BinaryRowWriter.class), + rowWriterType, "fory", FORY_NAME, "foryType", ctx.type(Fory.class)); ctx.addField(ctx.type(Schema.class), SCHEMA_NAME); - ctx.addField(ctx.type(BinaryRowWriter.class), ROOT_ROW_WRITER_NAME); + ctx.addField(rowWriterType, ROOT_ROW_WRITER_NAME); ctx.addField(ctx.type(Fory.class), FORY_NAME); Expression encodeExpr = buildEncodeExpression(); @@ -186,7 +190,7 @@ public String genCode() { public Expression buildEncodeExpression() { Reference inputObject = new Reference(ROOT_OBJECT_NAME, TypeUtils.OBJECT_TYPE, false); Expression bean = tryCastIfPublic(inputObject, beanType); - Reference writer = new Reference(ROOT_ROW_WRITER_NAME, rowWriterTypeToken, false); + Reference writer = new Reference(ROOT_ROW_WRITER_NAME, rowWriterType(), false); Reference schemaExpr = new Reference(SCHEMA_NAME, schemaTypeToken, false); CustomCodec customCodec = customTypeHandler.findCodec(beanClass, beanClass); @@ -198,16 +202,17 @@ public Expression buildEncodeExpression() { Expression.ListExpression expressions = new Expression.ListExpression(); // schema field's name must correspond to descriptor's name. for (int i = 0; i < numFields; i++) { - Descriptor d = getDescriptorByFieldName(schema.getFields().get(i).getName()); + Field field = schema.getFields().get(i); + Descriptor d = getDescriptorByFieldName(field.getName()); Preconditions.checkNotNull(d); TypeRef fieldType = d.getTypeRef(); Expression fieldValue = getFieldValue(bean, d); Literal ordinal = Literal.ofInt(i); - Expression.StaticInvoke field = + Expression.StaticInvoke arrowField = new Expression.StaticInvoke( DataTypes.class, "fieldOfSchema", ARROW_FIELD_TYPE, false, schemaExpr, ordinal); Expression fieldExpr = - serializeFor(ordinal, fieldValue, writer, fieldType, field, new HashSet<>()); + serializeFor(ordinal, fieldValue, writer, fieldType, field, arrowField, new HashSet<>()); expressions.add(fieldExpr); } expressions.add( @@ -242,7 +247,8 @@ public Expression buildDecodeExpression() { fieldNames.add(d.getName()); descriptors[i] = d; TypeRef fieldType = d.getTypeRef(); - Expression.Variable value = new Expression.Variable(d.getName(), nullValue(fieldType)); + Expression.Variable value = + new Expression.Variable("value_" + d.getName(), nullValue(fieldType)); values[i] = value; expressions.add(value); Expression.Invoke isNullAt = diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/Getters.java b/java/fory-format/src/main/java/org/apache/fory/format/row/Getters.java index 6be6c4b80d..65d86edd70 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/Getters.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/Getters.java @@ -64,6 +64,6 @@ public interface Getters { MapData getMap(int ordinal); default Object get(int ordinal, Field field) { - return field.getType().accept(new ValueVisitor(this)).apply(ordinal); + return field.getType().accept(new ToStringValueVisitor(this)).apply(ordinal); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/ToStringValueVisitor.java b/java/fory-format/src/main/java/org/apache/fory/format/row/ToStringValueVisitor.java new file mode 100644 index 0000000000..f1cbffff68 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/ToStringValueVisitor.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row; + +import java.util.function.Function; +import org.apache.arrow.vector.types.pojo.ArrowType; + +class ToStringValueVisitor extends ValueVisitor { + ToStringValueVisitor(final Getters getters) { + super(getters); + } + + @Override + public Function visit(final ArrowType.Binary type) { + return super.visit(type).andThen(this::binaryToString); + } + + @Override + public Function visit(final ArrowType.FixedSizeBinary type) { + return super.visit(type).andThen(this::binaryToString); + } + + private String binaryToString(final Object obj) { + final byte[] bytes = (byte[]) obj; + final int clampedLen = Math.min(bytes.length, 256); + final StringBuilder result = new StringBuilder(clampedLen * 2 + 5); + result.append("0x"); + for (int i = 0; i < clampedLen; i++) { + final String hexStr = Integer.toHexString(bytes[i] & 0xff); + if (hexStr.length() == 1) { + result.append('0'); + } + result.append(hexStr); + } + if (bytes.length > clampedLen) { + result.append("..."); + } + return result.toString(); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/ValueVisitor.java b/java/fory-format/src/main/java/org/apache/fory/format/row/ValueVisitor.java index 61001cfa86..487f281003 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/ValueVisitor.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/ValueVisitor.java @@ -88,6 +88,11 @@ public Function visit(ArrowType.Binary type) { return getters::getBinary; } + @Override + public Function visit(ArrowType.FixedSizeBinary type) { + return getters::getBinary; + } + @Override public Function visit(ArrowType.Decimal type) { return getters::getDecimal; diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java index 076f2a1b40..f08bf77e06 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryArray.java @@ -47,34 +47,56 @@ */ public class BinaryArray extends UnsafeTrait implements ArrayData { private final Field field; - private final int elementSize; + protected final int elementSize; private MemoryBuffer buffer; private int numElements; + private int bitmapOffset; private int elementOffset; private int baseOffset; private int sizeInBytes; public BinaryArray(Field field) { + this(field, elementSize(field)); + } + + protected BinaryArray(Field field, int elementSize) { this.field = field; + this.elementSize = elementSize; + initializeExtData(1); // Only require at most one slot to cache the schema for array type. + } + + private static int elementSize(Field field) { int width = DataTypes.getTypeWidth(field.getChildren().get(0).getType()); // variable-length element type if (width < 0) { - this.elementSize = 8; + return 8; } else { - this.elementSize = width; + return width; } - initializeExtData(1); // Only require at most one slot to cache the schema for array type. } public void pointTo(MemoryBuffer buffer, int offset, int sizeInBytes) { + this.buffer = buffer; + this.baseOffset = offset; // Read the numElements of key array from the aligned first 8 bytes as int. - final int numElements = (int) buffer.getInt64(offset); + final int numElements = readNumElements(); assert numElements >= 0 : "numElements (" + numElements + ") should >= 0"; this.numElements = numElements; - this.buffer = buffer; - this.baseOffset = offset; this.sizeInBytes = sizeInBytes; - this.elementOffset = offset + calculateHeaderInBytes(this.numElements); + this.bitmapOffset = bitmapOffset(); + this.elementOffset = elementOffset(); + } + + protected int readNumElements() { + return (int) buffer.getInt64(baseOffset); + } + + protected int elementOffset() { + return baseOffset + calculateHeaderInBytes(this.numElements); + } + + protected int bitmapOffset() { + return baseOffset + 8; } public Field getField() { @@ -114,19 +136,19 @@ int getOffset(int ordinal) { @Override public void setNotNullAt(int ordinal) { assertIndexIsValid(ordinal); - BitUtils.unset(buffer, baseOffset + 8, ordinal); + BitUtils.unset(buffer, bitmapOffset, ordinal); } @Override public void setNullAt(int ordinal) { - BitUtils.set(buffer, baseOffset + 8, ordinal); + BitUtils.set(buffer, bitmapOffset, ordinal); // we assume the corresponding column was already 0 // or will be set to 0 later by the caller side } @Override public boolean isNullAt(int ordinal) { - return BitUtils.isSet(buffer, baseOffset + 8, ordinal); + return BitUtils.isSet(buffer, bitmapOffset, ordinal); } @Override @@ -154,6 +176,10 @@ public void setDecimal(int ordinal, BigDecimal value) { throw new UnsupportedOperationException(); } + public byte[] toBytes() { + return buffer.getBytes(baseOffset, sizeInBytes); + } + public boolean[] toBooleanArray() { boolean[] values = new boolean[numElements]; buffer.copyToUnsafe(elementOffset, values, Platform.BOOLEAN_ARRAY_OFFSET, numElements); @@ -200,7 +226,7 @@ public double[] toDoubleArray() { public ArrayData copy() { MemoryBuffer copyBuf = MemoryUtils.buffer(sizeInBytes); buffer.copyTo(baseOffset, copyBuf, 0, sizeInBytes); - BinaryArray arrayCopy = new BinaryArray(field); + BinaryArray arrayCopy = newArray(field); arrayCopy.pointTo(copyBuf, 0, sizeInBytes); return arrayCopy; } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java index 4ff14f4122..e23f9abf9e 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryMap.java @@ -45,8 +45,16 @@ public class BinaryMap implements MapData { public BinaryMap(Field field) { this.field = field; - this.keys = new BinaryArray(DataTypes.keyArrayFieldForMap(field)); - this.values = new BinaryArray(DataTypes.itemArrayFieldForMap(field)); + this.keys = newArray(DataTypes.keyArrayFieldForMap(field)); + this.values = newArray(DataTypes.itemArrayFieldForMap(field)); + } + + protected BinaryArray newArray(Field field) { + return new BinaryArray(field); + } + + protected BinaryMap newMap(Field field) { + return new BinaryMap(field); } public BinaryMap(BinaryArray keys, BinaryArray values, Field field) { @@ -111,7 +119,7 @@ public BinaryArray valueArray() { public MapData copy() { MemoryBuffer copyBuf = MemoryUtils.buffer(sizeInBytes); buf.copyTo(baseOffset, copyBuf, 0, sizeInBytes); - BinaryMap mapCopy = new BinaryMap(field); + BinaryMap mapCopy = newMap(field); mapCopy.pointTo(copyBuf, 0, sizeInBytes); return mapCopy; } @@ -129,6 +137,10 @@ public void writeTo(ByteBuffer buffer) { buffer.position(pos + sizeInBytes); } + public byte[] toBytes() { + return buf.getBytes(baseOffset, sizeInBytes); + } + @Override public String toString() { return "BinaryMap{" diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryRow.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryRow.java index ad9f1384ff..d2d4d0cbae 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryRow.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/BinaryRow.java @@ -61,21 +61,25 @@ * */ public class BinaryRow extends UnsafeTrait implements Row { - private final Schema schema; - private final int numFields; - private final int bitmapWidthInBytes; - private MemoryBuffer buffer; - private int baseOffset; - private int sizeInBytes; + final Schema schema; + final int numFields; + final int bitmapWidthInBytes; + MemoryBuffer buffer; + int baseOffset; + int sizeInBytes; public BinaryRow(Schema schema) { this.schema = schema; this.numFields = schema.getFields().size(); Preconditions.checkArgument(numFields > 0); - this.bitmapWidthInBytes = BitUtils.calculateBitmapWidthInBytes(numFields); + this.bitmapWidthInBytes = computeBitmapWidthInBytes(); initializeExtData(numFields); } + protected int computeBitmapWidthInBytes() { + return BitUtils.calculateBitmapWidthInBytes(numFields); + } + public void pointTo(MemoryBuffer buffer, int offset, int sizeInBytes) { this.buffer = buffer; this.baseOffset = offset; @@ -117,20 +121,24 @@ public void assertIndexIsValid(int index) { checkArgument(index < numFields, "index (%d) should < %d", index, numFields); } + protected int nullBitmapOffset() { + return baseOffset; + } + @Override public boolean isNullAt(int ordinal) { - return BitUtils.isSet(buffer, baseOffset, ordinal); + return BitUtils.isSet(buffer, nullBitmapOffset(), ordinal); } @Override public boolean anyNull() { - return BitUtils.anySet(buffer, baseOffset, bitmapWidthInBytes); + return BitUtils.anySet(buffer, nullBitmapOffset(), bitmapWidthInBytes); } @Override public void setNullAt(int ordinal) { assertIndexIsValid(ordinal); - BitUtils.set(buffer, baseOffset, ordinal); + BitUtils.set(buffer, nullBitmapOffset(), ordinal); assert DataTypes.getTypeWidth(schema.getFields().get(ordinal).getType()) > 0 : "field[ " + ordinal @@ -144,9 +152,10 @@ public void setNullAt(int ordinal) { buffer.putInt64(getOffset(ordinal), 0); } + @Override public void setNotNullAt(int ordinal) { assertIndexIsValid(ordinal); - BitUtils.unset(buffer, baseOffset, ordinal); + BitUtils.unset(buffer, nullBitmapOffset(), ordinal); } @Override @@ -173,11 +182,15 @@ public BinaryMap getMap(int ordinal) { public Row copy() { MemoryBuffer copyBuf = MemoryUtils.buffer(sizeInBytes); buffer.copyTo(baseOffset, copyBuf, 0, sizeInBytes); - BinaryRow copyRow = new BinaryRow(schema); + BinaryRow copyRow = rowForCopy(); copyRow.pointTo(copyBuf, 0, sizeInBytes); return copyRow; } + protected BinaryRow rowForCopy() { + return new BinaryRow(schema); + } + @Override public String toString() { if (buffer == null) { @@ -211,7 +224,7 @@ public String toDebugString() { if (i != 0) { build.append(','); } - build.append(Long.toHexString(buffer.getInt64(baseOffset + i))); + build.append(Long.toHexString(buffer.getInt64(nullBitmapOffset() + i))); } return build.toString(); } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryArray.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryArray.java new file mode 100644 index 0000000000..5dc8f258ec --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryArray.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.writer.BinaryWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; + +public class CompactBinaryArray extends BinaryArray { + private final Field elementField; + private final int fixedWidth; + private final boolean elementNullable; + private int headerInBytes; + + public CompactBinaryArray(final Field field) { + super(field, CompactBinaryArrayWriter.elementWidth(field)); + elementField = field.getChildren().get(0); + fixedWidth = CompactBinaryRowWriter.fixedWidthFor(elementField); + elementNullable = elementField.isNullable(); + } + + @Override + protected int elementOffset() { + return getBaseOffset() + headerInBytes; + } + + public static int calculateHeaderInBytes( + final int fixedWidth, final int numElements, final boolean elementNullable) { + int headerInBytes = 4 + (elementNullable ? (numElements + 7) / 8 : 0); + if (fixedWidth % 8 == 0) { + headerInBytes = BinaryWriter.roundNumberOfBytesToNearestWord(headerInBytes); + } + return headerInBytes; + } + + @Override + protected int bitmapOffset() { + return getBaseOffset() + 4; + } + + @Override + protected int readNumElements() { + final var numElements = getBuffer().getInt32(getBaseOffset()); + headerInBytes = calculateHeaderInBytes(fixedWidth, numElements, elementNullable); + return numElements; + } + + @Override + public boolean isNullAt(final int ordinal) { + if (!elementNullable) { + return false; + } + return super.isNullAt(ordinal); + } + + @Override + public MemoryBuffer getBuffer(final int ordinal) { + if (fixedWidth == -1) { + return super.getBuffer(ordinal); + } + if (isNullAt(ordinal)) { + return null; + } + return getBuffer().slice(getOffset(ordinal), elementSize); + } + + @Override + public byte[] getBinary(final int ordinal) { + if (fixedWidth == -1) { + return super.getBinary(ordinal); + } + if (isNullAt(ordinal)) { + return null; + } + final byte[] bytes = new byte[elementSize]; + getBuffer().get(getOffset(ordinal), bytes, 0, elementSize); + return bytes; + } + + @Override + protected BinaryRow getStruct(final int ordinal, final Field field, final int extDataSlot) { + if (isNullAt(ordinal)) { + return null; + } + assert field == elementField; + if (fixedWidth == -1) { + return super.getStruct(ordinal, field, extDataSlot); + } + if (extData[extDataSlot] == null) { + extData[extDataSlot] = DataTypes.createSchema(field); + } + final BinaryRow row = newRow((Schema) extData[extDataSlot]); + row.pointTo(getBuffer(), getOffset(ordinal), fixedWidth); + return row; + } + + @Override + protected Schema newSchema(final Field field) { + return CompactBinaryRowWriter.sortSchema(super.newSchema(field)); + } + + @Override + protected BinaryRow newRow(final Schema schema) { + // TODO: don't re-compute fixed offsets + return new CompactBinaryRow(schema, CompactBinaryRowWriter.fixedOffsets(schema)); + } + + @Override + protected BinaryArray newArray(final Field field) { + return new CompactBinaryArray(field); + } + + @Override + protected BinaryMap newMap(final Field field) { + return new CompactBinaryMap(field); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryMap.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryMap.java new file mode 100644 index 0000000000..49446d3f2b --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryMap.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary; + +import org.apache.arrow.vector.types.pojo.Field; + +public class CompactBinaryMap extends BinaryMap { + public CompactBinaryMap(final Field field) { + super(field); + } + + public CompactBinaryMap(final BinaryArray keys, final BinaryArray values, final Field field) { + super(keys, values, field); + } + + @Override + protected BinaryArray newArray(final Field field) { + return new CompactBinaryArray(field); + } + + @Override + protected BinaryMap newMap(final Field field) { + return new CompactBinaryMap(field); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryRow.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryRow.java new file mode 100644 index 0000000000..8cf28e0232 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/CompactBinaryRow.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; + +/** + * A compact version of {@link BinaryRow}. The compact encoding includes additional optimizations: + * + *

      + *
    • fixed size binary objects are stored in the fixed size section with no pointer needed + *
    • small values can take up fewer than 8 bytes + *
    • null bitmap is skipped if all fields are primitive / not-nullable + *
    • the header is packed better, with the null-bitmap allowed to borrow alignment padding at + * end of fixed section + *
    • data alignment is relaxed, which could lead to less performance in very intensive memory + * operations + *
    + * + * The compact format is still under development and may not be stable yet. + */ +public class CompactBinaryRow extends BinaryRow { + private final boolean allFieldsNotNullable; + private final int[] fixedOffsets; + private final int bitmapOffset; + + public CompactBinaryRow(final Schema schema) { + this(schema, CompactBinaryRowWriter.fixedOffsets(schema)); + } + + public CompactBinaryRow(final Schema schema, final int[] fixedOffsets) { + super(schema); + this.fixedOffsets = fixedOffsets; + bitmapOffset = fixedOffsets[fixedOffsets.length - 1]; + allFieldsNotNullable = CompactBinaryRowWriter.allNotNullable(schema.getFields()); + } + + @Override + protected int computeBitmapWidthInBytes() { + // cannot use field due to initialization order + if (CompactBinaryRowWriter.allNotNullable(schema.getFields())) { + return 0; + } + return CompactBinaryRowWriter.headerBytes(schema); + } + + @Override + public boolean isNullAt(final int ordinal) { + if (allFieldsNotNullable) { + return false; + } + return super.isNullAt(ordinal); + } + + // TODO: this should use StableValue once it's available + @Override + public int getOffset(final int ordinal) { + return baseOffset + fixedOffsets[ordinal]; + } + + @Override + public MemoryBuffer getBuffer(final int ordinal) { + final int fixedWidthBinary = CompactBinaryRowWriter.fixedWidthFor(schema, ordinal); + if (fixedWidthBinary >= 0) { + if (isNullAt(ordinal)) { + return null; + } + return getBuffer().slice(getOffset(ordinal), fixedWidthBinary); + } else { + return super.getBuffer(ordinal); + } + } + + @Override + public byte[] getBinary(final int ordinal) { + final int fixedWidthBinary = CompactBinaryRowWriter.fixedWidthFor(schema, ordinal); + if (fixedWidthBinary >= 0) { + if (isNullAt(ordinal)) { + return null; + } + final byte[] bytes = new byte[fixedWidthBinary]; + getBuffer().get(getOffset(ordinal), bytes, 0, fixedWidthBinary); + return bytes; + } else { + return super.getBinary(ordinal); + } + } + + @Override + protected BinaryRow getStruct(final int ordinal, final Field field, final int extDataSlot) { + if (isNullAt(ordinal)) { + return null; + } + final int fixedWidthBinary = CompactBinaryRowWriter.fixedWidthFor(schema, ordinal); + if (fixedWidthBinary == -1) { + return super.getStruct(ordinal, field, extDataSlot); + } + if (extData[extDataSlot] == null) { + extData[extDataSlot] = DataTypes.createSchema(field); + } + final BinaryRow row = newRow((Schema) extData[extDataSlot]); + row.pointTo(getBuffer().slice(getOffset(ordinal), fixedWidthBinary), 0, fixedWidthBinary); + return row; + } + + @Override + protected Schema newSchema(final Field field) { + return CompactBinaryRowWriter.sortSchema(super.newSchema(field)); + } + + @Override + protected BinaryRow newRow(final Schema schema) { + // TODO: avoid re-computing these offsets + return new CompactBinaryRow(schema, CompactBinaryRowWriter.fixedOffsets(schema)); + } + + @Override + protected BinaryArray newArray(final Field field) { + return new CompactBinaryArray(field); + } + + @Override + protected BinaryMap newMap(final Field field) { + return new CompactBinaryMap(field); + } + + @Override + protected int nullBitmapOffset() { + return baseOffset + bitmapOffset; + } + + @Override + protected BinaryRow rowForCopy() { + return new CompactBinaryRow(schema, fixedOffsets); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/UnsafeTrait.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/UnsafeTrait.java index ad7447e8b3..a2b2393b70 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/UnsafeTrait.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/UnsafeTrait.java @@ -36,7 +36,7 @@ /** Internal to binary row format to reuse code, don't use it in anywhere else. */ abstract class UnsafeTrait implements Getters, Setters { - private Object[] extData; + protected Object[] extData; abstract MemoryBuffer getBuffer(); @@ -65,46 +65,55 @@ void initializeExtData(int numSlots) { // ####################### getters ####################### // ########################################################### + @Override public boolean getBoolean(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getBoolean(getOffset(ordinal)); } + @Override public byte getByte(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getByte(getOffset(ordinal)); } + @Override public short getInt16(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getInt16(getOffset(ordinal)); } + @Override public int getInt32(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getInt32(getOffset(ordinal)); } + @Override public long getInt64(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getInt64(getOffset(ordinal)); } + @Override public float getFloat32(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getFloat32(getOffset(ordinal)); } + @Override public double getFloat64(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getFloat64(getOffset(ordinal)); } + @Override public int getDate(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getInt32(getOffset(ordinal)); } + @Override public long getTimestamp(int ordinal) { assertIndexIsValid(ordinal); return getBuffer().getInt64(getOffset(ordinal)); @@ -157,7 +166,7 @@ BigDecimal getDecimal(int ordinal, ArrowType.Decimal decimalType) { * @param extDataSlot the ext data slot used to cache the schema for the struct. * @return the binary row representation of the struct. */ - BinaryRow getStruct(int ordinal, Field field, int extDataSlot) { + protected BinaryRow getStruct(int ordinal, Field field, int extDataSlot) { if (isNullAt(ordinal)) { return null; } @@ -165,13 +174,21 @@ BinaryRow getStruct(int ordinal, Field field, int extDataSlot) { final int relativeOffset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; if (extData[extDataSlot] == null) { - extData[extDataSlot] = DataTypes.createSchema(field); + extData[extDataSlot] = newSchema(field); } - BinaryRow row = new BinaryRow((Schema) extData[extDataSlot]); + BinaryRow row = newRow((Schema) extData[extDataSlot]); row.pointTo(getBuffer(), getBaseOffset() + relativeOffset, size); return row; } + protected Schema newSchema(Field field) { + return DataTypes.createSchema(field); + } + + protected BinaryRow newRow(Schema schema) { + return new BinaryRow(schema); + } + BinaryArray getArray(int ordinal, Field field) { if (isNullAt(ordinal)) { return null; @@ -179,11 +196,15 @@ BinaryArray getArray(int ordinal, Field field) { final long offsetAndSize = getInt64(ordinal); final int relativeOffset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; - BinaryArray array = new BinaryArray(field); + BinaryArray array = newArray(field); array.pointTo(getBuffer(), getBaseOffset() + relativeOffset, size); return array; } + protected BinaryArray newArray(Field field) { + return new BinaryArray(field); + } + BinaryMap getMap(int ordinal, Field field) { if (isNullAt(ordinal)) { return null; @@ -191,11 +212,15 @@ BinaryMap getMap(int ordinal, Field field) { final long offsetAndSize = getInt64(ordinal); final int relativeOffset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; - BinaryMap map = new BinaryMap(field); + BinaryMap map = newMap(field); map.pointTo(getBuffer(), getBaseOffset() + relativeOffset, size); return map; } + protected BinaryMap newMap(Field field) { + return new BinaryMap(field); + } + // ########################################################### // ####################### setters ####################### // ########################################################### diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BaseBinaryRowWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BaseBinaryRowWriter.java new file mode 100644 index 0000000000..6e5ede8bd6 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BaseBinaryRowWriter.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary.writer; + +import java.math.BigDecimal; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** + * Writer to write data into buffer using row format, see {@link BinaryRow}. + * + *

    Must call {@code reset()} before use this writer to write a nested row. + * + *

    Must call {@code reset(buffer)}/{@code reset(buffer, offset)} before use this writer to write + * a new row. + */ +public abstract class BaseBinaryRowWriter extends BinaryWriter { + private final Schema schema; + + public BaseBinaryRowWriter(Schema schema) { + super(MemoryUtils.buffer(schema.getFields().size() * 32), 0); + super.startIndex = 0; + this.schema = schema; + } + + protected BaseBinaryRowWriter(Schema schema, int bitMapOffset) { + super(MemoryUtils.buffer(schema.getFields().size() * 32), bitMapOffset); + super.startIndex = 0; + this.schema = schema; + } + + public BaseBinaryRowWriter(Schema schema, BinaryWriter writer) { + super(writer.getBuffer(), 0); + writer.children.add(this); + // Since we must call reset before use this writer, + // there's no need to set `super.startIndex = writer.writerIndex();` + this.schema = schema; + } + + protected BaseBinaryRowWriter(Schema schema, MemoryBuffer buffer, int bitMapOffset) { + super(buffer, bitMapOffset); + this.schema = schema; + } + + protected BaseBinaryRowWriter(Schema schema, BinaryWriter writer, int bitMapOffset) { + super(writer.getBuffer(), bitMapOffset); + writer.children.add(this); + // Since we must call reset before use this writer, + // there's no need to set `super.startIndex = writer.writerIndex();` + this.schema = schema; + } + + protected abstract int fixedSize(); + + protected abstract int headerInBytes(); + + public Schema getSchema() { + return schema; + } + + /** + * Call {@code reset()} before write nested row to buffer + * + *

    reset BinaryRowWriter(schema, writer) increase writerIndex, which increase writer's + * writerIndex, so we need to record writer's writerIndex before call reset, so we can call + * writer's {@code setOffsetAndSize(int ordinal, int absoluteOffset, int size)}. Reset will + * change writerIndex, please use it very carefully + */ + public void reset() { + super.startIndex = buffer.writerIndex(); + int fixedSize = fixedSize(); + grow(fixedSize); + buffer._increaseWriterIndexUnsafe(fixedSize); + resetHeader(); + } + + protected void resetHeader() { + int bitmapStart = startIndex + bytesBeforeBitMap; + int end = bitmapStart + headerInBytes(); + for (int i = bitmapStart; i < end; i += 1) { + buffer.putByte(i, 0); + } + } + + @Override + public void write(int ordinal, BigDecimal value) { + writeDecimal(ordinal, value, (ArrowType.Decimal) schema.getFields().get(ordinal).getType()); + } + + @Override + public void setNullAt(int ordinal) { + super.setNullAt(ordinal); + clearValue(ordinal); + } + + protected void clearValue(int ordinal) { + write(ordinal, 0L); + } + + public BinaryRow getRow() { + BinaryRow row = newRow(); + int size = size(); + row.pointTo(buffer, startIndex, size); + return row; + } + + public BinaryRow copyToRow() { + BinaryRow row = newRow(); + int size = size(); + MemoryBuffer buffer = MemoryUtils.buffer(size); + this.buffer.copyTo(startIndex, buffer, 0, size); + row.pointTo(buffer, startIndex, size); + return row; + } + + protected abstract BinaryRow newRow(); +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java index 8627121724..2c1bb7f436 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryArrayWriter.java @@ -48,10 +48,10 @@ public class BinaryArrayWriter extends BinaryWriter { public static int MAX_ROUNDED_ARRAY_LENGTH = Integer.MAX_VALUE - 15; - private final Field field; - private final int elementSize; - private int numElements; - private int headerInBytes; + protected final Field field; + protected final int elementSize; + protected int numElements; + protected int headerInBytes; /** Must call reset before using writer constructed by this constructor. */ public BinaryArrayWriter(Field field) { @@ -73,17 +73,26 @@ public BinaryArrayWriter(Field field, BinaryWriter writer) { } public BinaryArrayWriter(Field field, MemoryBuffer buffer) { - super(buffer, 8); - this.field = field; + this(field, buffer, 8, elementWidth(field)); + } + + private static int elementWidth(Field field) { int width = DataTypes.getTypeWidth(field.getChildren().get(0).getType()); // variable-length element type if (width < 0) { - this.elementSize = 8; + return 8; } else { - this.elementSize = width; + return width; } } + protected BinaryArrayWriter( + Field field, MemoryBuffer buffer, int bytesBeforeBitMap, int elementSize) { + super(buffer, bytesBeforeBitMap); + this.field = field; + this.elementSize = elementSize; + } + /** * reset BinaryArrayWriter(ArrayType type, BinaryWriter writer) increase writerIndex, which * increase writer's writerIndex, we need to record writer's writerIndex before call reset, so we @@ -91,10 +100,9 @@ public BinaryArrayWriter(Field field, MemoryBuffer buffer) { * will change writerIndex, please use it very carefully. */ public void reset(int numElements) { - super.startIndex = writerIndex(); + startIndex = writerIndex(); this.numElements = numElements; - // numElements use 8 byte, nullBitsSizeInBytes use multiple of 8 byte - this.headerInBytes = BinaryArray.calculateHeaderInBytes(numElements); + this.headerInBytes = calculateHeaderInBytes(); long dataSize = numElements * (long) elementSize; if (dataSize > MAX_ROUNDED_ARRAY_LENGTH) { throw new UnsupportedOperationException("Can't alloc binary array, it's too big"); @@ -103,23 +111,32 @@ public void reset(int numElements) { buffer.grow(headerInBytes + fixedPartInBytes); // Write numElements and clear out null bits to header - // store numElements in header in aligned 8 byte, though numElements is 4 byte int - buffer.putInt64(startIndex, numElements); + int numElementsSize = writeNumElements(); int end = startIndex + headerInBytes; - for (int i = startIndex + 8; i < end; i += 8) { - buffer.putInt64(i, 0L); + for (int i = startIndex + numElementsSize; i < end; i += 1) { + buffer.putByte(i, 0); } // fill 0 into reminder part of 8-bytes alignment for (int i = elementSize * numElements; i < fixedPartInBytes; i++) { buffer.putByte(startIndex + headerInBytes + i, (byte) 0); } + resetAdvanceWriter(fixedPartInBytes); + } + + protected void resetAdvanceWriter(int fixedPartInBytes) { buffer._increaseWriterIndexUnsafe(headerInBytes + fixedPartInBytes); } - private void assertIndexIsValid(int index) { - assert index >= 0 : "index (" + index + ") should >= 0"; - assert index < numElements : "index (" + index + ") should < " + numElements; + protected int writeNumElements() { + // store numElements in header in aligned 8 byte, though numElements is 4 byte int + buffer.putInt64(startIndex, numElements); + return 8; + } + + protected int calculateHeaderInBytes() { + // numElements use 8 byte, nullBitsSizeInBytes use multiple of 8 byte + return BinaryArray.calculateHeaderInBytes(numElements); } @Override @@ -162,6 +179,10 @@ public void write(int ordinal, BigDecimal value) { writeDecimal(ordinal, value, (ArrowType.Decimal) field.getChildren().get(0).getType()); } + protected void primitiveArrayAdvance(int size) { + // no need to increasewriterIndex, because reset has already increased writerIndex + } + private void fromPrimitiveArray(Object arr, int offset, int numElements, Field type) { if (DataTypes.getTypeId(type.getChildren().get(0).getType()) != DataTypes.getTypeId(this.field.getChildren().get(0).getType())) { @@ -171,9 +192,9 @@ private void fromPrimitiveArray(Object arr, int offset, int numElements, Field t type.getChildren().get(0).getType(), this.field.getChildren().get(0).getType()); throw new IllegalArgumentException(msg); } - buffer.copyFromUnsafe( - startIndex + headerInBytes, arr, offset, numElements * (long) elementSize); - // no need to increasewriterIndex, because reset has already increased writerIndex + int size = numElements * elementSize; + buffer.copyFromUnsafe(startIndex + headerInBytes, arr, offset, size); + primitiveArrayAdvance(size); } public void fromPrimitiveArray(byte[] arr) { @@ -206,12 +227,16 @@ public void fromPrimitiveArray(double[] arr) { } public BinaryArray toArray() { - BinaryArray array = new BinaryArray(field); + BinaryArray array = newArray(); int size = size(); array.pointTo(buffer, startIndex, size); return array; } + public BinaryArray newArray() { + return new BinaryArray(field); + } + public Field getField() { return field; } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryRowWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryRowWriter.java index c46538881b..4e48b167ab 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryRowWriter.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryRowWriter.java @@ -21,12 +21,9 @@ import static org.apache.fory.memory.BitUtils.calculateBitmapWidthInBytes; -import java.math.BigDecimal; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.fory.format.row.binary.BinaryRow; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.memory.MemoryUtils; /** * Writer to write data into buffer using row format, see {@link BinaryRow}. @@ -36,51 +33,41 @@ *

    Must call {@code reset(buffer)}/{@code reset(buffer, offset)} before use this writer to write * a new row. */ -public class BinaryRowWriter extends BinaryWriter { - private final Schema schema; +public class BinaryRowWriter extends BaseBinaryRowWriter { private final int headerInBytes; // fixed width size: bitmap + fixed width region // variable-length region follows startOffset + fixedSize private final int fixedSize; public BinaryRowWriter(Schema schema) { - super(MemoryUtils.buffer(schema.getFields().size() * 32), 0); - super.startIndex = 0; - this.schema = schema; - this.headerInBytes = calculateBitmapWidthInBytes(schema.getFields().size()); - this.fixedSize = headerInBytes + schema.getFields().size() * 8; + super(schema); + headerInBytes = headerBytes(schema); + fixedSize = fixedAreaSize(); + } + + public BinaryRowWriter(Schema schema, MemoryBuffer buffer) { + super(schema, buffer, 0); + headerInBytes = headerBytes(schema); + fixedSize = fixedAreaSize(); } public BinaryRowWriter(Schema schema, BinaryWriter writer) { - super(writer.getBuffer(), 0); - writer.children.add(this); - // Since we must call reset before use this writer, - // there's no need to set `super.startIndex = writer.writerIndex();` - this.schema = schema; - this.headerInBytes = calculateBitmapWidthInBytes(schema.getFields().size()); - this.fixedSize = headerInBytes + schema.getFields().size() * 8; + super(schema, writer); + this.headerInBytes = headerBytes(schema); + this.fixedSize = fixedAreaSize(); + } + + private int fixedAreaSize() { + return headerInBytes + getSchema().getFields().size() * 8; } - public Schema getSchema() { - return schema; + static int headerBytes(Schema schema) { + return calculateBitmapWidthInBytes(schema.getFields().size()); } - /** - * Call {@code reset()} before write nested row to buffer - * - *

    reset BinaryRowWriter(schema, writer) increase writerIndex, which increase writer's - * writerIndex, so we need to record writer's writerIndex before call reset, so we can call - * writer's {@code setOffsetAndSize(int ordinal, int absoluteOffset, int size)}. Reset will - * change writerIndex, please use it very carefully - */ - public void reset() { - super.startIndex = buffer.writerIndex(); - grow(fixedSize); - buffer._increaseWriterIndexUnsafe(fixedSize); - int end = startIndex + headerInBytes; - for (int i = startIndex; i < end; i += 8) { - buffer.putInt64(i, 0L); - } + @Override + protected int headerInBytes() { + return headerInBytes; } @Override @@ -124,8 +111,8 @@ public void write(int ordinal, float value) { } @Override - public void write(int ordinal, BigDecimal value) { - writeDecimal(ordinal, value, (ArrowType.Decimal) schema.getFields().get(ordinal).getType()); + protected int fixedSize() { + return fixedSize; } @Override @@ -134,19 +121,8 @@ public void setNullAt(int ordinal) { write(ordinal, 0L); } - public BinaryRow getRow() { - BinaryRow row = new BinaryRow(schema); - int size = size(); - row.pointTo(buffer, startIndex, size); - return row; - } - - public BinaryRow copyToRow() { - BinaryRow row = new BinaryRow(schema); - int size = size(); - MemoryBuffer buffer = MemoryUtils.buffer(size); - this.buffer.copyTo(startIndex, buffer, 0, size); - row.pointTo(buffer, startIndex, size); - return row; + @Override + protected BinaryRow newRow() { + return new BinaryRow(getSchema()); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryWriter.java index 8e08b054ca..c3ecf6ab38 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryWriter.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/BinaryWriter.java @@ -53,7 +53,7 @@ public static int roundNumberOfBytesToNearestWord(int numBytes) { // avoid polymorphic setNullAt/setNotNullAt to inline for performance. // array use 8 byte for numElements - private final int bytesBeforeBitMap; + protected final int bytesBeforeBitMap; protected final List children; protected BinaryWriter(MemoryBuffer buffer, int bytesBeforeBitMap) { @@ -103,11 +103,11 @@ protected final void grow(int neededSize) { buffer.grow(neededSize); } - public final void setOffsetAndSize(int ordinal, int size) { + public void setOffsetAndSize(int ordinal, int size) { setOffsetAndSize(ordinal, buffer.writerIndex(), size); } - public final void setOffsetAndSize(int ordinal, int absoluteOffset, int size) { + public void setOffsetAndSize(int ordinal, int absoluteOffset, int size) { final long relativeOffset = absoluteOffset - startIndex; final long offsetAndSize = (relativeOffset << 32) | (long) size; write(ordinal, offsetAndSize); @@ -128,11 +128,11 @@ public void setNullAt(int ordinal) { BitUtils.set(buffer, startIndex + bytesBeforeBitMap, ordinal); } - public final void setNotNullAt(int ordinal) { + public void setNotNullAt(int ordinal) { BitUtils.unset(buffer, startIndex + bytesBeforeBitMap, ordinal); } - public final boolean isNullAt(int ordinal) { + public boolean isNullAt(int ordinal) { return BitUtils.isSet(buffer, startIndex + bytesBeforeBitMap, ordinal); } @@ -184,31 +184,36 @@ public final void write(int ordinal, BinaryArray array) { } /** This operation will increase writerIndex by aligned 8-byte. */ - public final void writeUnaligned(int ordinal, byte[] input, int offset, int numBytes) { + public void writeUnaligned(int ordinal, byte[] input, int offset, int numBytes) { final int roundedSize = roundNumberOfBytesToNearestWord(numBytes); buffer.grow(roundedSize); zeroOutPaddingBytes(numBytes); - buffer.put(buffer.writerIndex(), input, offset, numBytes); + buffer.put(bufferWriteIndexFor(ordinal), input, offset, numBytes); setOffsetAndSize(ordinal, numBytes); - buffer._increaseWriterIndexUnsafe(roundedSize); + if (copyShouldIncreaseWriterIndex(ordinal)) { + buffer._increaseWriterIndexUnsafe(roundedSize); + } } /** This operation will increase writerIndex by aligned 8-byte. */ - public final void writeUnaligned(int ordinal, MemoryBuffer input, int offset, int numBytes) { + public void writeUnaligned(int ordinal, MemoryBuffer input, int offset, int numBytes) { final int roundedSize = roundNumberOfBytesToNearestWord(numBytes); buffer.grow(roundedSize); zeroOutPaddingBytes(numBytes); - buffer.copyFrom(buffer.writerIndex(), input, offset, numBytes); + buffer.copyFrom(bufferWriteIndexFor(ordinal), input, offset, numBytes); setOffsetAndSize(ordinal, numBytes); - buffer._increaseWriterIndexUnsafe(roundedSize); + if (copyShouldIncreaseWriterIndex(ordinal)) { + buffer._increaseWriterIndexUnsafe(roundedSize); + } } - public final void writeAlignedBytes( - int ordinal, MemoryBuffer input, int baseOffset, int numBytes) { + public void writeAlignedBytes(int ordinal, MemoryBuffer input, int baseOffset, int numBytes) { buffer.grow(numBytes); - buffer.copyFrom(buffer.writerIndex(), input, baseOffset, numBytes); + buffer.copyFrom(bufferWriteIndexFor(ordinal), input, baseOffset, numBytes); setOffsetAndSize(ordinal, numBytes); - buffer.increaseWriterIndex(numBytes); + if (copyShouldIncreaseWriterIndex(ordinal)) { + buffer.increaseWriterIndex(numBytes); + } } protected final void writeDecimal(int ordinal, BigDecimal value, ArrowType.Decimal type) { @@ -219,15 +224,28 @@ protected final void writeDecimal(int ordinal, BigDecimal value, ArrowType.Decim DecimalUtility.writeBigDecimalToArrowBuf( value, arrowBuf, 0, DecimalUtils.DECIMAL_BYTE_LENGTH); buffer.copyFromUnsafe( - writerIndex(), null, arrowBuf.memoryAddress(), DecimalUtils.DECIMAL_BYTE_LENGTH); + bufferWriteIndexFor(ordinal), + null, + arrowBuf.memoryAddress(), + DecimalUtils.DECIMAL_BYTE_LENGTH); arrowBuf.getReferenceManager().release(); setOffsetAndSize(ordinal, writerIndex(), DecimalUtils.DECIMAL_BYTE_LENGTH); - increaseWriterIndex(DecimalUtils.DECIMAL_BYTE_LENGTH); + if (copyShouldIncreaseWriterIndex(ordinal)) { + increaseWriterIndex(DecimalUtils.DECIMAL_BYTE_LENGTH); + } } else { setNullAt(ordinal); } } + protected int bufferWriteIndexFor(int ordinal) { + return buffer.writerIndex(); + } + + protected boolean copyShouldIncreaseWriterIndex(int ordinal) { + return true; + } + /** write long value to position pointed by current writerIndex. */ public final void writeDirectly(long value) { buffer.grow(8); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryArrayWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryArrayWriter.java new file mode 100644 index 0000000000..0f6b827cab --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryArrayWriter.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary.writer; + +import static org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter.fixedWidthFor; + +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.CompactBinaryArray; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +public class CompactBinaryArrayWriter extends BinaryArrayWriter { + + private final int fixedWidth; + private final boolean elementNullable; + + /** Must call reset before using writer constructed by this constructor. */ + public CompactBinaryArrayWriter(final Field field) { + // buffer size can grow + this(field, MemoryUtils.buffer(64)); + super.startIndex = 0; + } + + /** + * Write data to writer's buffer. + * + *

    Must call reset before using writer constructed by this constructor + */ + public CompactBinaryArrayWriter(final Field field, final BinaryWriter writer) { + this(field, writer.buffer); + writer.children.add(this); + // Since we must call reset before use this writer, + // there's no need to set `super.startIndex = writer.writerIndex();` + } + + public CompactBinaryArrayWriter(final Field field, final MemoryBuffer buffer) { + super(CompactBinaryRowWriter.sortField(field), buffer, 4, elementWidth(field)); + final Field elementField = this.field.getChildren().get(0); + fixedWidth = fixedWidthFor(elementField); + elementNullable = elementField.isNullable(); + } + + public static int elementWidth(final Field field) { + final int width = fixedWidthFor(field.getChildren().get(0)); + if (width < 0) { + return 8; + } else { + return width; + } + } + + @Override + protected int writeNumElements() { + buffer.putInt32(startIndex, numElements); + return 4; + } + + @Override + protected int calculateHeaderInBytes() { + return CompactBinaryArray.calculateHeaderInBytes(fixedWidth, numElements, elementNullable); + } + + @Override + protected void primitiveArrayAdvance(final int size) { + buffer._increaseWriterIndexUnsafe(size); + } + + @Override + protected int bufferWriteIndexFor(final int ordinal) { + if (fixedWidth > 0) { + return getOffset(ordinal); + } else { + return super.bufferWriteIndexFor(ordinal); + } + } + + @Override + protected boolean copyShouldIncreaseWriterIndex(final int ordinal) { + if (fixedWidth > 0) { + return false; + } else { + return super.copyShouldIncreaseWriterIndex(ordinal); + } + } + + @Override + public void setOffsetAndSize(final int ordinal, final int absoluteOffset, final int size) { + if (fixedWidth > 0) { + return; + } + super.setOffsetAndSize(ordinal, absoluteOffset, size); + } + + @Override + public void setNotNullAt(final int ordinal) { + if (!elementNullable) { + return; + } + super.setNotNullAt(ordinal); + } + + @Override + public void setNullAt(final int ordinal) { + if (!elementNullable) { + throw new NullPointerException("unexpected null element at ordinal " + ordinal); + } + super.setNullAt(ordinal); + if (fixedWidth > 0) { + final int off = getOffset(ordinal); + for (int i = off; i < off + fixedWidth; i++) { + getBuffer().putByte(i, 0); + } + } + } + + public void resetFor(final CompactBinaryRowWriter nestedWriter, final int ordinal) { + if (fixedWidth > 0) { + nestedWriter.startIndex = getOffset(ordinal); + nestedWriter.resetHeader(); + } else { + nestedWriter.reset(); + } + } + + @Override + public BinaryArray newArray() { + return new CompactBinaryArray(field); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java new file mode 100644 index 0000000000..aa92450e0e --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary.writer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.CompactBinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; + +/** Writer class to produce {@link CompactBinaryRow}-formatted rows. */ +public class CompactBinaryRowWriter extends BaseBinaryRowWriter { + + private final int headerSize; + private final boolean allFieldsFixedSize; + private final boolean allFieldsNotNullable; + private final int fixedSize; + private final int[] fixedOffsets; + + public CompactBinaryRowWriter(final Schema schema) { + super(sortSchema(schema), computeFixedRegionSize(schema)); + headerSize = headerBytes(schema); + allFieldsFixedSize = allFieldsFixedSize(schema); + fixedSize = computeAlignedFixedRegionSize(); + fixedOffsets = fixedOffsets(schema); + allFieldsNotNullable = allNotNullable(schema.getFields()); + } + + public CompactBinaryRowWriter(final Schema schema, final BinaryWriter writer) { + super(sortSchema(schema), writer, computeFixedRegionSize(schema)); + headerSize = headerBytes(schema); + allFieldsFixedSize = allFieldsFixedSize(schema); + fixedSize = computeAlignedFixedRegionSize(); + fixedOffsets = fixedOffsets(schema); + allFieldsNotNullable = allNotNullable(schema.getFields()); + } + + public CompactBinaryRowWriter(final Schema schema, final MemoryBuffer buffer) { + super(sortSchema(schema), buffer, computeFixedRegionSize(schema)); + headerSize = headerBytes(schema); + allFieldsFixedSize = allFieldsFixedSize(schema); + fixedSize = computeAlignedFixedRegionSize(); + fixedOffsets = fixedOffsets(schema); + allFieldsNotNullable = allNotNullable(schema.getFields()); + } + + private static boolean allFieldsFixedSize(final Schema schema) { + for (final Field f : schema.getFields()) { + if (fixedWidthFor(f) == -1) { + return false; + } + } + return true; + } + + private int computeAlignedFixedRegionSize() { + final int totalUnalignedSize = headerSize + bytesBeforeBitMap; + if (allFieldsFixedSize) { + return totalUnalignedSize; + } + return roundNumberOfBytesToNearestWord(totalUnalignedSize); + } + + // TODO: this should use StableValue once it's available + public static boolean allNotNullable(final List fields) { + for (final Field f : fields) { + if (f.isNullable()) { + return false; + } + } + return true; + } + + public static int headerBytes(final Schema schema) { + return headerBytes(schema.getFields()); + } + + private static int headerBytes(final List fields) { + if (allNotNullable(fields)) { + return 0; + } + return (fields.size() + 7) / 8; + } + + public static Schema sortSchema(final Schema schema) { + return new Schema(sortFields(schema.getFields()), schema.getCustomMetadata()); + } + + @Override + protected int fixedSize() { + return fixedSize; + } + + @Override + protected int headerInBytes() { + return headerSize; + } + + @Override + public int getOffset(final int ordinal) { + return startIndex + fixedOffsets[ordinal]; + } + + private static int fixedBinarySize(final Field field) { + final ArrowType type = field.getType(); + if (type.getTypeID() == ArrowTypeID.FixedSizeBinary) { + return ((ArrowType.FixedSizeBinary) type).getByteWidth(); + } + return -1; + } + + /** Total size of fixed region: fixed size inline values plus variable sized values' pointers. */ + static int computeFixedRegionSize(final Schema schema) { + int fixedSize = 0; + for (final Field f : schema.getFields()) { + fixedSize += fixedRegionSpaceFor(f); + } + return fixedSize; + } + + /** Number of bytes used for a field if fixed, -1 if variable sized. */ + public static int fixedWidthFor(final Schema schema, final int ordinal) { + return fixedWidthFor(schema.getFields().get(ordinal)); + } + + /** Number of bytes used for a field if fixed, -1 if variable sized. */ + public static int fixedWidthFor(final Field f) { + int fixedWidth = DataTypes.getTypeWidth(f.getType()); + if (fixedWidth == -1) { + if (f.getType().getTypeID() == ArrowTypeID.Struct) { + int nestedWidth = headerBytes(f.getChildren()); + for (final Field child : f.getChildren()) { + final int childWidth = fixedWidthFor(child); + if (childWidth == -1) { + return -1; + } + nestedWidth += childWidth; + } + return nestedWidth; + } + fixedWidth = fixedBinarySize(f); + } + return fixedWidth; + } + + /** + * Number of bytes used in fixed-data area for a field - 8 (combined offset + size) if + * variable-sized. + */ + static int fixedRegionSpaceFor(final Field f) { + final int fixedWidth = fixedWidthFor(f); + if (fixedWidth == -1) { + return 8; + } + return fixedWidth; + } + + // TODO: this should use StableValue once it's available + public static int[] fixedOffsets(final Schema schema) { + final List fields = schema.getFields(); + final int[] result = new int[fields.size() + 1]; + int off = 0; + for (int i = 0; i < fields.size(); i++) { + result[i] = off; + off += fixedRegionSpaceFor(fields.get(i)); + } + result[fields.size()] = off; + return result; + } + + static List sortFields(final List fields) { + final ArrayList sortedFields = new ArrayList<>(); + for (final Field f : fields) { + sortedFields.add(sortField(f)); + } + Collections.sort(sortedFields, new FieldAlignmentComparator()); + return sortedFields; + } + + static Field sortField(final Field field) { + final List children = field.getChildren(); + final List sortedChildren = sortFields(children); + if (children.equals(sortedChildren)) { + return field; + } + return new Field(field.getName(), field.getFieldType(), sortedChildren); + } + + public void resetFor(final CompactBinaryRowWriter nestedWriter, final int ordinal) { + if (fixedWidthFor(getSchema(), ordinal) == -1) { + nestedWriter.reset(); + } else { + nestedWriter.startIndex = getOffset(ordinal); + nestedWriter.resetHeader(); + } + } + + @Override + protected void resetHeader() { + final int bitmapStart = startIndex + bytesBeforeBitMap; + final int end = startIndex + computeAlignedFixedRegionSize(); + for (int i = bitmapStart; i < end; i += 1) { + buffer.putByte(i, 0); + } + } + + @Override + public void setNullAt(final int ordinal) { + if (allFieldsNotNullable) { + throw new IllegalStateException( + "Field " + getSchema().getFields().get(ordinal) + " has null value for not-null field"); + } + super.setNullAt(ordinal); + } + + @Override + protected void clearValue(final int ordinal) { + final int fixedWidth = fixedWidthFor(getSchema(), ordinal); + if (fixedWidth > 0) { + final int off = getOffset(ordinal); + for (int i = off; i < off + fixedWidth; i++) { + getBuffer().putByte(i, 0); + } + } else { + super.clearValue(ordinal); + } + } + + @Override + public void setNotNullAt(final int ordinal) { + if (allFieldsNotNullable) { + return; + } + super.setNotNullAt(ordinal); + } + + @Override + public boolean isNullAt(final int ordinal) { + if (allFieldsNotNullable) { + return false; + } + return super.isNullAt(ordinal); + } + + @Override + public void write(final int ordinal, final byte value) { + final int offset = getOffset(ordinal); + buffer.putByte(offset, value); + } + + @Override + public void write(final int ordinal, final boolean value) { + final int offset = getOffset(ordinal); + buffer.putBoolean(offset, value); + } + + @Override + public void write(final int ordinal, final short value) { + final int offset = getOffset(ordinal); + buffer.putInt16(offset, value); + } + + @Override + public void write(final int ordinal, final int value) { + final int offset = getOffset(ordinal); + buffer.putInt32(offset, value); + } + + @Override + public void write(final int ordinal, final float value) { + final int offset = getOffset(ordinal); + buffer.putFloat32(offset, value); + } + + @Override + public void writeUnaligned( + final int ordinal, final byte[] input, final int offset, final int numBytes) { + final int inlineWidth = fixedWidthFor(getSchema(), ordinal); + if (inlineWidth > 0) { + buffer.put(getOffset(ordinal), input, 0, numBytes); + } else { + super.writeUnaligned(ordinal, input, offset, numBytes); + } + } + + @Override + public void writeUnaligned( + final int ordinal, final MemoryBuffer input, final int offset, final int numBytes) { + final int inlineWidth = fixedWidthFor(getSchema(), ordinal); + if (inlineWidth > 0) { + assert inlineWidth == numBytes; + buffer.copyFrom(getOffset(ordinal), input, 0, numBytes); + } else { + super.writeUnaligned(ordinal, input, offset, numBytes); + } + } + + @Override + public void writeAlignedBytes( + final int ordinal, final MemoryBuffer input, final int baseOffset, final int numBytes) { + final int inlineWidth = fixedWidthFor(getSchema(), ordinal); + if (inlineWidth > 0) { + buffer.copyFrom(getOffset(ordinal), input, 0, numBytes); + } else { + super.writeAlignedBytes(ordinal, input, baseOffset, numBytes); + } + } + + @Override + protected BinaryRow newRow() { + return new CompactBinaryRow(getSchema(), fixedOffsets); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/FieldAlignmentComparator.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/FieldAlignmentComparator.java new file mode 100644 index 0000000000..651a05e937 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/FieldAlignmentComparator.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.row.binary.writer; + +import java.util.Comparator; +import org.apache.arrow.vector.types.pojo.Field; + +class FieldAlignmentComparator implements Comparator { + @Override + public int compare(final Field f1, final Field f2) { + final int f1Size = CompactBinaryRowWriter.fixedRegionSpaceFor(f1); + final int f2Size = CompactBinaryRowWriter.fixedRegionSpaceFor(f2); + final int f1Align = tryAlign(f1Size); + final int f2Align = tryAlign(f2Size); + final int alignCmp = Integer.compare(f1Align, f2Align); + if (alignCmp != 0) { + return alignCmp; + } + return -Integer.compare(f1Size, f2Size); + } + + protected int tryAlign(final int size) { + return size == 4 || size == 2 || (size & 7) == 0 ? 0 : 1; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/type/DataTypes.java b/java/fory-format/src/main/java/org/apache/fory/format/type/DataTypes.java index 216e2868ec..a918d260c7 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/type/DataTypes.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/type/DataTypes.java @@ -74,6 +74,12 @@ public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { return -1; } + @Override + public Integer visit( + org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary type) { + return type.getByteWidth(); + } + @Override public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { return -1; @@ -91,7 +97,7 @@ public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { @Override public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { - return type.getBitWidth() / 8; + return (type.getBitWidth() + 7) / 8; } @Override @@ -211,6 +217,12 @@ public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) return ArrowType.STRUCT; } + @Override + public ArrowType visit( + org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary type) { + return ArrowType.FIXED_SIZE_BINARY; + } + @Override public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { return ArrowType.LIST; diff --git a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java index 7c52942313..031aae8817 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java @@ -163,6 +163,10 @@ private static Field inferField(String name, TypeRef typeRef, TypeResolutionC if (replacementField != null) { return replacementField; } + TypeRef replacementType = customEncoder.encodedType(); + if (replacementType != null && !typeRef.equals(replacementType)) { + return inferField(name, replacementType, ctx); + } } if (rawType == boolean.class) { return field(name, DataTypes.notNullFieldType(ArrowType.Bool.INSTANCE)); diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ArrayEncoderTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ArrayEncoderTest.java index c5f795c40a..52d35ef38f 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ArrayEncoderTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ArrayEncoderTest.java @@ -44,7 +44,7 @@ public void testListEncoder() { } ArrayEncoder> encoder = - Encoders.arrayEncoder(bars.getClass(), RowEncoderTest.Bar.class); + Encoders.arrayEncoder(new TypeRef>() {}); BinaryArray array = encoder.toArray(bars); List newBars = encoder.fromArray(array); diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CodecBuilderTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CodecBuilderTest.java index 3bb5be7cc1..9c9dfb5254 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CodecBuilderTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CodecBuilderTest.java @@ -39,13 +39,14 @@ public void genCode() { @Test public void loadOrGenRowCodecClass() { - Class codecClass = Encoders.loadOrGenRowCodecClass(BeanA.class); + Class codecClass = Encoders.loadOrGenRowCodecClass(BeanA.class, DefaultCodecFormat.INSTANCE); assertTrue(GeneratedRowEncoder.class.isAssignableFrom(codecClass)); assertTrue( - GeneratedRowEncoder.class.isAssignableFrom(Encoders.loadOrGenRowCodecClass(BeanB.class))); + GeneratedRowEncoder.class.isAssignableFrom( + Encoders.loadOrGenRowCodecClass(BeanB.class, DefaultCodecFormat.INSTANCE))); assertTrue( GeneratedRowEncoder.class.isAssignableFrom( - Encoders.loadOrGenRowCodecClass(AtomicLong.class))); + Encoders.loadOrGenRowCodecClass(AtomicLong.class, DefaultCodecFormat.INSTANCE))); } static void testStreamingEncode(Encoder encoder, Object object) { diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CompactCodecTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CompactCodecTest.java new file mode 100644 index 0000000000..fd58499049 --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CompactCodecTest.java @@ -0,0 +1,708 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; + +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.UUID; +import lombok.Data; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.fory.format.encoder.RowEncoderTest.Bar; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.row.binary.CompactBinaryRow; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.reflect.TypeRef; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CompactCodecTest { + + static { + Encoders.registerCustomCodec(UUID.class, new CompactUUIDCodec()); + Encoders.registerCustomCodec(NotNullByte.class, new NotNullByteCodec()); + } + + @Data + public static class CompactType { + public float f1; + public double f2; + public byte f3; + public short f4; + public int f5; + public long f6; + public String f7; + } + + @Test + public void testCompactType() { + final CompactType bean1 = new CompactType(); + bean1.f1 = 1; + bean1.f2 = 2; + bean1.f3 = 3; + bean1.f4 = 4; + bean1.f5 = 5; + bean1.f6 = 6; + bean1.f7 = "7"; + final RowEncoder encoder = + Encoders.buildBeanCodec(CompactType.class).compactEncoding().build().get(); + final List fields = encoder.schema().getFields(); + assertEquals(fields.get(0).getName(), "f2"); + assertEquals(fields.get(1).getName(), "f6"); + assertEquals(fields.get(2).getName(), "f7"); + assertEquals(fields.get(3).getName(), "f1"); + assertEquals(fields.get(4).getName(), "f5"); + assertEquals(fields.get(5).getName(), "f4"); + assertEquals(fields.get(6).getName(), "f3"); + + final BinaryRow row = encoder.toRow(bean1); + assertEquals(row.getClass(), CompactBinaryRow.class); + assertEquals(row.getOffset(0), 0); + assertEquals(row.getFloat64(0), 2); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final CompactType deserializedBean = encoder.fromRow(row); + assertEquals(bean1, deserializedBean); + assertEquals(buffer.size(), 4 + 7 + 1 + 4 + 8 + 1 + 2 + 4 + 8 + 8 + 1); + } + + @Test + public void testEncodeResetsNullBits() { + final CompactType bean1 = new CompactType(); + bean1.f1 = 1; + bean1.f2 = 2; + bean1.f3 = 3; + bean1.f4 = 4; + bean1.f5 = 5; + bean1.f6 = 6; + final RowEncoder encoder = + Encoders.buildBeanCodec(CompactType.class).compactEncoding().build().get(); + + byte[] encoded = encoder.encode(bean1); + bean1.f7 = "Not null!"; + encoded = encoder.encode(bean1); + final CompactType deserializedBean = encoder.decode(encoded); + assertEquals(deserializedBean, bean1); + } + + @Data + public static class CompactListType { + public List f1; + } + + @Test + public void testCompactListType() { + final CompactType bean1 = new CompactType(); + bean1.f1 = 1; + bean1.f2 = 2; + bean1.f3 = 3; + bean1.f4 = 4; + bean1.f5 = 5; + bean1.f6 = 6; + bean1.f7 = "7"; + + final CompactType bean2 = new CompactType(); + bean2.f1 = 7; + bean2.f2 = 6; + bean2.f3 = 5; + bean2.f4 = 4; + bean2.f5 = 3; + bean2.f6 = 2; + bean2.f7 = "1"; + + final CompactListType list = new CompactListType(); + list.f1 = Arrays.asList(bean1, bean2); + + final RowEncoder encoder = + Encoders.buildBeanCodec(CompactListType.class).compactEncoding().build().get(); + + final BinaryRow row = encoder.toRow(list); + assertEquals(row.getClass(), CompactBinaryRow.class); + assertEquals(row.getArray(0).getStruct(0).getClass(), CompactBinaryRow.class); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final CompactListType deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, list); + assertEquals(buffer.size(), 37 + 48 + 48); + } + + @Data + public static class CompactUuidType { + public UUID f1; + + public CompactUuidType() {} + + public CompactUuidType(final UUID f1) { + this.f1 = f1; + } + } + + @Test + public void testCompactUuidType() { + final CompactUuidType bean1 = new CompactUuidType(); + bean1.f1 = new UUID(42, 24); + final RowEncoder encoder = + Encoders.buildBeanCodec(CompactUuidType.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final CompactUuidType deserializedBean = encoder.fromRow(row); + assertEquals(bean1, deserializedBean); + assertEquals(buffer.size(), 16 + 1); + } + + static class CompactUUIDCodec implements CustomCodec.MemoryBufferCodec { + @Override + public Field getField(final String fieldName) { + return Field.nullable(fieldName, new ArrowType.FixedSizeBinary(16)); + } + + @Override + public MemoryBuffer encode(final UUID value) { + final MemoryBuffer result = MemoryBuffer.newHeapBuffer(16); + result.putInt64(0, value.getMostSignificantBits()); + result.putInt64(8, value.getLeastSignificantBits()); + return result; + } + + @Override + public UUID decode(final MemoryBuffer value) { + return new UUID(value.readInt64(), value.readInt64()); + } + } + + @Data + public static class Nested1 { + public short f1; + } + + @Data + public static class Nested2 { + public int f1; + } + + @Test + public void testAllNonnullElideBitmap() { + final Nested1 bean1 = new Nested1(); + bean1.f1 = 42; + final Nested2 bean2 = new Nested2(); + bean2.f1 = 75; + final RowEncoder encoder = + Encoders.buildBeanCodec(Nested1.class).compactEncoding().build().get(); + BinaryRow row = encoder.toRow(bean1); + MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final Nested1 deserializedBean = encoder.fromRow(row); + assertEquals(bean1, deserializedBean); + assertEquals(buffer.size(), 2); + + final RowEncoder encoder2 = + Encoders.buildBeanCodec(Nested2.class).compactEncoding().build().get(); + row = encoder2.toRow(bean2); + buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final Nested2 deserializedBean2 = encoder2.fromRow(row); + assertEquals(bean2, deserializedBean2); + assertEquals(buffer.size(), 4); + } + + @Data + public static class InlineNestedType { + public Nested1 f1; + public Nested2 f2; + } + + @Test + public void testInlineNestedType() { + final InlineNestedType bean1 = new InlineNestedType(); + bean1.f1 = new Nested1(); + bean1.f1.f1 = 42; + bean1.f2 = new Nested2(); + bean1.f2.f1 = 75; + final RowEncoder encoder = + Encoders.buildBeanCodec(InlineNestedType.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + assertEquals(row.getSchema().getFields().get(0).getName(), "f2"); + assertEquals(row.getSchema().getFields().get(1).getName(), "f1"); + assertEquals(row.getOffset(0), 0); + assertEquals(row.getOffset(1), 4); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final InlineNestedType deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + assertEquals(buffer.size(), 7); + } + + @Data + public static class InlineNestedArrayType { + public InlineNestedType f1; + public String f2; + public UUID[] f3; + } + + @Test + public void testInlineNestedArrayType() { + final InlineNestedArrayType bean1 = new InlineNestedArrayType(); + bean1.f1 = new InlineNestedType(); + bean1.f1.f1 = new Nested1(); + bean1.f1.f1.f1 = 42; + bean1.f1.f2 = new Nested2(); + bean1.f1.f2.f1 = 75; + bean1.f2 = "luna"; + bean1.f3 = new UUID[] {new UUID(1, 2), new UUID(3, 4), new UUID(5, 6)}; + final RowEncoder encoder = + Encoders.buildBeanCodec(InlineNestedArrayType.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final InlineNestedArrayType deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + assertEquals(buffer.size(), 88); + } + + @Data + public static class InlinePrimitiveNestedArrayType { + public InlineNestedType f1; + public String f2; + public short[] f3; + } + + @Test + public void testInlinePrimitiveNestedArrayType() { + final InlinePrimitiveNestedArrayType bean1 = new InlinePrimitiveNestedArrayType(); + bean1.f1 = new InlineNestedType(); + bean1.f1.f1 = new Nested1(); + bean1.f1.f1.f1 = 42; + bean1.f1.f2 = new Nested2(); + bean1.f1.f2.f1 = 75; + bean1.f2 = "luna"; + bean1.f3 = new short[] {1, 2, 3}; + final RowEncoder encoder = + Encoders.buildBeanCodec(InlinePrimitiveNestedArrayType.class) + .compactEncoding() + .build() + .get(); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final InlinePrimitiveNestedArrayType deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + assertEquals(buffer.size(), 50); + } + + @Data + public static class CompactMapType { + public Map map; + } + + @Test + public void testCompactMapType() { + final CompactMapType bean1 = new CompactMapType(); + bean1.map = new HashMap<>(); + final UUID u1 = new UUID(42, 24); + bean1.map.put(u1, new CompactUuidType(u1)); + final UUID u2 = new UUID(55, 66); + bean1.map.put(u2, new CompactUuidType(u2)); + final RowEncoder encoder = + Encoders.buildBeanCodec(CompactMapType.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + assertEquals( + row.getMap(0).keyArray().toString(), + "[0x2a000000000000001800000000000000,0x37000000000000004200000000000000]"); + assertEquals( + row.getMap(0).valueArray().toString(), + "[{f1=0x2a000000000000001800000000000000},{f1=0x37000000000000004200000000000000}]"); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final CompactMapType deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + assertEquals(buffer.size(), 109); + } + + @Test + public void testTwoFieldsAndASet() { + final TwoFieldsAndASet bean1 = new TwoFieldsAndASet(); + bean1.f1 = new UUID(42, 24); + bean1.f2 = new UUID(55, 66); + bean1.f3 = Set.of(LocalDate.of(2112, 1, 1)); + final RowEncoder encoder = + Encoders.buildBeanCodec(TwoFieldsAndASet.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final TwoFieldsAndASet deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + } + + @Data + public static class TwoFieldsAndASet { + public UUID f1; + public UUID f2; + public Set f3; + } + + @Test + public void testNestedConfigObject() { + final InnerConfigObject inner = new InnerConfigObject(); + inner.f1 = new UUID(1, 2); + inner.f2 = new UUID(3, 4); + inner.f3 = new UUID(5, 6); + inner.f4 = true; + inner.f5 = "Indubitably"; + inner.f6 = EnumSet.of(ConfigEnum.B, ConfigEnum.C); + final OuterConfigObject bean1 = new OuterConfigObject(); + bean1.f1 = inner; + bean1.f2 = LocalDate.of(2112, 1, 1); + final RowEncoder encoder = + Encoders.buildBeanCodec(OuterConfigObject.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final OuterConfigObject deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, bean1); + } + + @Data + public static class OuterConfigObject { + public InnerConfigObject f1; + public LocalDate f2; + } + + @Data + public static class InnerConfigObject { + public UUID f1; + public UUID f2; + public UUID f3; + public boolean f4; + public String f5; + public Set f6; + } + + public enum ConfigEnum { + A, + B, + C + } + + @Test + public void testBiglyBean() { + final BiglyBean big = new BiglyBean(); + big.f1 = new UUID(1, 2); + big.f2 = ConfigEnum.B; + big.f3 = ConfigEnum.C; + big.f4 = new UUID(3, 4); + big.f5 = new UUID(5, 6); + big.f6 = Optional.of("Indubitably"); + big.f7 = LocalDate.of(2112, 2, 2); + big.f8 = Optional.of(LocalDate.of(1221, 3, 4)); + big.f9 = OptionalLong.of(-42); + big.f10 = OptionalLong.of(-24); + big.f11 = 1234; + big.f12 = 4321; + big.f13 = Instant.ofEpochMilli(12345678); + big.f14 = Instant.ofEpochMilli(87654321); + big.f15 = Optional.of(ConfigEnum.B); + final RowEncoder encoder = + Encoders.buildBeanCodec(BiglyBean.class).compactEncoding().build().get(); + final BinaryRow row = encoder.toRow(big); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final BiglyBean deserializedBean = encoder.fromRow(row); + assertEquals(deserializedBean, big); + } + + @Data + public static class BiglyBean { + public UUID f1; + public ConfigEnum f2; + public ConfigEnum f3; + public UUID f4; + public UUID f5; + public Optional f6; + public LocalDate f7; + public Optional f8; + public OptionalLong f9; + public OptionalLong f10; + public long f11; + public long f12; + public Instant f13; + public Instant f14; + public Optional f15; + } + + @Test + public void testArrayOfStruct() { + final ArrayNestedBean bean = new ArrayNestedBean(); + bean.f1 = new UUID(1, 2); + bean.f2 = new UUID(3, 4); + bean.f3 = Instant.ofEpochMilli(12345678); + bean.f4 = true; + bean.f5 = Instant.ofEpochMilli(87654321); + bean.f6 = ConfigEnum.B; + bean.f7 = new UUID(5, 6); + bean.f8 = false; + bean.f9 = new ArrayNestedNestedBean(); + bean.f9.f1 = Instant.ofEpochMilli(43218765); + bean.f9.f2 = "Indubitably"; + bean.f9.f3 = EnumSet.of(ConfigEnum.B, ConfigEnum.C); + bean.f9.f4 = new UUID(7, 8); + bean.f9.f5 = "luna"; + bean.f9.f6 = List.of(); + final ArrayEncoder> encoder = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .build() + .get(); + final List expected = List.of(bean, bean); + final BinaryArray arr = encoder.toArray(expected); + final MemoryBuffer buffer = MemoryUtils.wrap(arr.toBytes()); + arr.pointTo(buffer, 0, buffer.size()); + final List deserializedBean = encoder.fromArray(arr); + assertEquals(deserializedBean, expected); + } + + @Data + public static class ArrayNestedBean { + public UUID f1; + public UUID f2; + public Instant f3; + public boolean f4; + public Instant f5; + public ConfigEnum f6; + public UUID f7; + public boolean f8; + public ArrayNestedNestedBean f9; + } + + @Data + public static class ArrayNestedNestedBean { + public Instant f1; + public String f2; + public Set f3; + public UUID f4; + public String f5; + public List f6; + } + + @Data + public static class ArrayNestedNestedNestedBean { + public Instant f1; + public String f2; + public String f3; + public String f4; + } + + @Test + public void testNotNullByteArray() { + final List expected = new ArrayList<>(); + for (int i = 0; i < 64; i++) { + expected.add(new NotNullByte((byte) i)); + } + final ArrayEncoder> encoder = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .build() + .get(); + final BinaryArray arr = encoder.toArray(expected); + final MemoryBuffer buffer = MemoryUtils.wrap(arr.toBytes()); + arr.pointTo(buffer, 0, buffer.size()); + final List deserializedBean = encoder.fromArray(arr); + assertEquals(deserializedBean, expected); + assertEquals(arr.getSizeInBytes(), 68); + } + + @Test + public void testArrayBufferReuse() { + final List expected = new ArrayList<>(); + final List expectedRev = new ArrayList<>(); + for (int i = 0; i < 128; i++) { + final NotNullByte val = new NotNullByte((byte) i); + expected.add(val); + if (i < 16) { + expectedRev.add(0, val); + } + } + final ArrayEncoder> encoder = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .build() + .get(); + final BinaryArray arr = encoder.toArray(expected); + final BinaryArray subRev = encoder.toArray(expectedRev); + assertNotSame(arr.getBuffer(), subRev.getBuffer()); + } + + @Data + public static class NotNullByte { + public byte b; + + public NotNullByte(final byte b) { + this.b = b; + } + } + + public static class NotNullByteCodec implements CustomCodec { + @Override + public Field getField(final String fieldName) { + return Field.notNullable(fieldName, new ArrowType.Int(8, true)); + } + + @Override + public TypeRef encodedType() { + return new TypeRef() {}; + } + + @Override + public Byte encode(final NotNullByte value) { + return value.b; + } + + @Override + public NotNullByte decode(final Byte value) { + return new NotNullByte(value); + } + } + + @Test + public void testCompactMapEncoder() { + final InnerConfigObject inner = new InnerConfigObject(); + inner.f1 = new UUID(1, 2); + inner.f2 = new UUID(3, 4); + inner.f3 = new UUID(5, 6); + inner.f4 = true; + inner.f5 = "Indubitably"; + inner.f6 = EnumSet.of(ConfigEnum.B, ConfigEnum.C); + final MapEncoder> encoder = + Encoders.buildMapCodec(new TypeRef>() {}) + .compactEncoding() + .build() + .get(); + final var expected = Map.of(inner.f1, inner); + final BinaryMap map = encoder.toMap(expected); + final MemoryBuffer buffer = MemoryUtils.wrap(map.toBytes()); + map.pointTo(buffer, 0, buffer.size()); + final var deserializedMap = encoder.fromMap(map); + assertEquals(deserializedMap, expected); + } + + @Test + public void testNullClearOffset() { + final RowEncoder encoder = + Encoders.buildBeanCodec(Bar.class).compactEncoding().build().get(); + final Bar bar = new Bar(); + bar.f1 = 42; + bar.f2 = null; + final byte[] nullBefore = encoder.encode(bar); + + bar.f2 = "not null"; + // write offset and size + encoder.encode(bar); + + bar.f2 = null; + final byte[] nullAfter = encoder.encode(bar); + Assert.assertEquals(nullAfter, nullBefore); + } + + @Data + public static class MediumBean { + public UUID f1; + public LocalDate f2; + public UUID f3; + } + + @Test + public void testOffsetBuffer() { + final MediumBean bean1 = new MediumBean(); + bean1.f1 = new UUID(1, 2); + bean1.f2 = LocalDate.ofEpochDay(56); + bean1.f3 = new UUID(3, 4); + + final MediumBean bean2 = new MediumBean(); + bean2.f1 = new UUID(7, 8); + bean2.f2 = LocalDate.ofEpochDay(1112); + bean2.f3 = new UUID(9, 10); + + final MediumBean bean3 = new MediumBean(); + bean3.f1 = new UUID(13, 14); + bean3.f2 = LocalDate.ofEpochDay(1718); + bean3.f3 = new UUID(15, 16); + + final ArrayEncoder> encoder = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .withSizeEmbedded(false) + .build() + .get(); + final List expected = Arrays.asList(bean1, bean2, bean3, null); + final byte[] bytes = encoder.encode(expected); + final int offset = 17; + final var buf = ByteBuffer.allocate(bytes.length + offset); + buf.put(new byte[offset]); + buf.put(bytes); + buf.position(offset); + buf.limit(offset + bytes.length); + final List deserializedBean = encoder.decode(MemoryBuffer.fromByteBuffer(buf)); + assertEquals(deserializedBean, expected); + } + + @Data + public static class HeaderClearObject { + public UUID f1; + public InnerHeaderClearObject f2; + } + + @Data + public static class InnerHeaderClearObject { + public UUID f1; + public UUID f2; + public UUID f3; + } + + @Test + public void testHeaderClear() { + final InnerHeaderClearObject inner = new InnerHeaderClearObject(); + inner.f1 = UUID.randomUUID(); + + final HeaderClearObject outer = new HeaderClearObject(); + outer.f2 = inner; + + final RowEncoder encoder = + Encoders.buildBeanCodec(HeaderClearObject.class).compactEncoding().build().get(); + final byte[] bytes = encoder.encode(outer); + final HeaderClearObject deserializedBean = encoder.decode(bytes); + assertEquals(deserializedBean, outer); + } +} diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java index 33b0faa4b2..c59b63f52f 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/CustomCodecTest.java @@ -196,7 +196,7 @@ public CustomByteBuf3 decode(final BinaryArray value) { } } - static class UuidEncoder implements CustomCodec.MemoryBufferCodec { + public static class UuidEncoder implements CustomCodec.MemoryBufferCodec { @Override public MemoryBuffer encode(final UUID value) { final MemoryBuffer result = MemoryBuffer.newHeapBuffer(16); @@ -209,6 +209,11 @@ public MemoryBuffer encode(final UUID value) { public UUID decode(final MemoryBuffer value) { return new UUID(value.readInt64(), value.readInt64()); } + + @Override + public Field getField(final String fieldName) { + return Field.nullable(fieldName, new ArrowType.FixedSizeBinary(16)); + } } static class SortedSetOfUuidDecoder implements CustomCollectionFactory> { diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/RowEncoderTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/RowEncoderTest.java index c80febb64a..83774ea4fb 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/RowEncoderTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/RowEncoderTest.java @@ -114,7 +114,8 @@ public void testPrivateBean() { PrivateStruct s = new PrivateStruct(); s.f1 = ofHashMap(10L, 100L); s.f2 = ofHashMap("k", "v"); - PrivateStruct s1 = encoder.decode(encoder.encode(s)); + byte[] encoded = encoder.encode(s); + PrivateStruct s1 = encoder.decode(encoded); Assert.assertEquals(s1.f1, s.f1); Assert.assertEquals(s1.f2, s.f2); } From db4f7b32fa258a56d46a5112cb10f4d8a0cbb0c2 Mon Sep 17 00:00:00 2001 From: Zhong Junjie Date: Tue, 14 Oct 2025 23:13:14 +0800 Subject: [PATCH 23/37] feat(go): Add pointer field test for meta share mode (#2674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? There is no test of pointer field for meta share mode before, so I created some and fixed bugs regard that ## What does this PR do? Add more test to refine meta share mode. - PointerFields: test serialization and deserialization of pointer fields - PointerFieldsInconsistent: test serialization and deserialization of Inconsistent pointer fields. - ComplexRoundTrip: more complex object to test metashare mode, contains interface type field. There are some bug fix: - field order problem introduce by #2749, re-enable the go ci - add a runtime value param to readTypeInfo function, so we can choose pointer or value to read with based on the runtime-value. ## Related issues #2192 ## Does this PR introduce any user-facing change? - [x] Does this PR introduce any public API change? no - [x] Does this PR introduce any binary protocol compatibility change? no ## Benchmark --- ci/run_ci.py | 2 +- ci/run_ci.sh | 10 +-- go/fory/fory.go | 3 +- go/fory/fory_metashare_test.go | 128 +++++++++++++++++++++++++++++++++ go/fory/map.go | 4 +- go/fory/serializer.go | 1 + go/fory/slice.go | 4 +- go/fory/struct.go | 11 ++- go/fory/type.go | 26 ++++++- go/fory/type_def.go | 31 ++++---- 10 files changed, 191 insertions(+), 29 deletions(-) diff --git a/ci/run_ci.py b/ci/run_ci.py index cdd8f7fbc1..7818a9e606 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -293,7 +293,7 @@ def parse_args(): if USE_PYTHON_GO: func() else: - # run_shell_script("go") + run_shell_script("go") pass elif command == "format": if USE_PYTHON_FORMAT: diff --git a/ci/run_ci.sh b/ci/run_ci.sh index f27835341f..5a3e01ed1f 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -372,11 +372,11 @@ case $1 in ;; go) echo "Executing fory go tests for go" - cd "$ROOT/go/fory" - go install ./cmd/fory - cd "$ROOT/go/fory/tests" - go generate - go test -v +# cd "$ROOT/go/fory" +# go install ./cmd/fory +# cd "$ROOT/go/fory/tests" +# go generate +# go test -v cd "$ROOT/go/fory" go test -v echo "Executing fory go tests succeeds" diff --git a/go/fory/fory.go b/go/fory/fory.go index 3e77eda686..5d1eaf0250 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -539,11 +539,12 @@ func (f *Fory) readReferencableBySerializer(buf *ByteBuffer, value reflect.Value func (f *Fory) readData(buffer *ByteBuffer, value reflect.Value, serializer Serializer) (err error) { if serializer == nil { - typeInfo, err := f.typeResolver.readTypeInfo(buffer) + typeInfo, err := f.typeResolver.readTypeInfo(buffer, value) if err != nil { return fmt.Errorf("read typeinfo failed: %w", err) } serializer = typeInfo.Serializer + var concrete reflect.Value var type_ reflect.Type /* diff --git a/go/fory/fory_metashare_test.go b/go/fory/fory_metashare_test.go index e8e32ba278..500a0ffef8 100644 --- a/go/fory/fory_metashare_test.go +++ b/go/fory/fory_metashare_test.go @@ -61,6 +61,26 @@ type MapDataClass struct { Counters map[string]int32 } +type ComplexObject1 struct { + F1 interface{} + F2 string + F3 []string + F4 map[int8]int32 + F5 int8 + F6 int16 + F7 int32 + F8 int64 + F9 float32 + F10 float64 + F11 [2]int16 + F12 []int16 +} + +type ComplexObject2 struct { + F1 interface{} + F2 map[int8]int32 +} + type UnsortedStruct struct { StringField string FloatField float64 @@ -81,6 +101,14 @@ type InconsistentMapDataClass struct { Counters map[int32]int32 // Different key type } +type PointerDataClass struct { + Inner *SimpleDataClass +} + +type PointerInconsistentDataClass struct { + Inner *InconsistentDataClass +} + type NestedOuter struct { Name string Inner SimpleDataClass @@ -122,6 +150,48 @@ func TestCompatibleSerializationScenarios(t *testing.T) { assert.Equal(t, in.Name, out.Name) }, }, + { + name: "ComplexRoundTrip", + tag: "ComplexObject1", + writeType: ComplexObject1{}, + readType: ComplexObject1{}, + input: func() ComplexObject1 { + nested := ComplexObject2{ + F1: true, + F2: map[int8]int32{-1: 2}, + } + return ComplexObject1{ + F1: nested, + F2: "abc", + F3: []string{"abc", "abc"}, + F4: map[int8]int32{1: 2}, + F5: MaxInt8, + F6: MaxInt16, + F7: MaxInt32, + F8: MaxInt64, + F9: float32(0.5), + F10: 1 / 3.0, + F11: [2]int16{1, 2}, + F12: []int16{-1, 4}, + } + }(), + writerSetup: func(f *Fory) error { + return f.RegisterNamedType(ComplexObject2{}, "test.ComplexObject2") + }, + readerSetup: func(f *Fory) error { + return f.RegisterNamedType(ComplexObject2{}, "test.ComplexObject2") + }, + assertFunc: func(t *testing.T, input interface{}, output interface{}) { + in := input.(ComplexObject1) + out := output.(ComplexObject1) + assert.Equal(t, in, out) + inNested := in.F1.(ComplexObject2) + outNested, ok := out.F1.(ComplexObject2) + if assert.True(t, ok, "expected nested ComplexObject2 type") { + assert.Equal(t, inNested, outNested) + } + }, + }, { name: "InconsistentTypeFallsBackToZeroValue", tag: "TestStruct", @@ -250,6 +320,64 @@ func TestCompatibleSerializationScenarios(t *testing.T) { assert.Equal(t, in.Counters, out.Counters) }, }, + { + name: "PointerFields", + tag: "PointerDataClass", + writeType: PointerDataClass{}, + readType: PointerDataClass{}, + input: func() PointerDataClass { + return PointerDataClass{ + Inner: &SimpleDataClass{ + Name: "inner", + Age: 18, + Active: true, + }, + } + }(), + writerSetup: func(f *Fory) error { + return f.RegisterNamedType(SimpleDataClass{}, "SimpleDataClass") + }, + readerSetup: func(f *Fory) error { + return f.RegisterNamedType(SimpleDataClass{}, "SimpleDataClass") + }, + assertFunc: func(t *testing.T, input interface{}, output interface{}) { + in := input.(PointerDataClass) + out := output.(PointerDataClass) + if assert.NotNil(t, out.Inner) { + assert.Equal(t, *in.Inner, *out.Inner) + } + }, + }, + { + name: "PointerFieldsInconsistent", + tag: "PointerDataClass", + writeType: PointerDataClass{}, + readType: PointerInconsistentDataClass{}, + input: func() PointerDataClass { + return PointerDataClass{ + Inner: &SimpleDataClass{ + Name: "inner", + Age: 18, + Active: true, + }, + } + }(), + writerSetup: func(f *Fory) error { + return f.RegisterNamedType(SimpleDataClass{}, "SimpleDataClass") + }, + readerSetup: func(f *Fory) error { + return f.RegisterNamedType(InconsistentDataClass{}, "SimpleDataClass") + }, + assertFunc: func(t *testing.T, input interface{}, output interface{}) { + in := input.(PointerDataClass) + out := output.(PointerInconsistentDataClass) + if assert.NotNil(t, out.Inner) { + assert.Zero(t, out.Inner.Name) + assert.Equal(t, in.Inner.Age, out.Inner.Age) + assert.Equal(t, in.Inner.Active, out.Inner.Active) + } + }, + }, { name: "InconsistentMapValues", tag: "MapDataClass", diff --git a/go/fory/map.go b/go/fory/map.go index 21e53ea36e..f3c9d40c55 100644 --- a/go/fory/map.go +++ b/go/fory/map.go @@ -397,7 +397,7 @@ func (s mapSerializer) Read(f *Fory, buf *ByteBuffer, type_ reflect.Type, value valDeclType := (chunkHeader & VALUE_DECL_TYPE) != 0 chunkSize := int(buf.ReadUint8()) if !keyDeclType { - ti, err := resolver.readTypeInfo(buf) + ti, err := resolver.readTypeInfo(buf, value) if err != nil { return err } @@ -405,7 +405,7 @@ func (s mapSerializer) Read(f *Fory, buf *ByteBuffer, type_ reflect.Type, value keyType = ti.Type } if !valDeclType { - ti, err := resolver.readTypeInfo(buf) + ti, err := resolver.readTypeInfo(buf, value) if err != nil { return err } diff --git a/go/fory/serializer.go b/go/fory/serializer.go index 47afdb9b13..533c07cdca 100644 --- a/go/fory/serializer.go +++ b/go/fory/serializer.go @@ -462,6 +462,7 @@ func (s *ptrToValueSerializer) Write(f *Fory, buf *ByteBuffer, value reflect.Val } func (s *ptrToValueSerializer) Read(f *Fory, buf *ByteBuffer, type_ reflect.Type, value reflect.Value) error { + fmt.Printf("type_: %v\n", type_) newValue := reflect.New(type_.Elem()) value.Set(newValue) return s.valueSerializer.Read(f, buf, type_.Elem(), newValue.Elem()) diff --git a/go/fory/slice.go b/go/fory/slice.go index 3aae29fb32..513c738917 100644 --- a/go/fory/slice.go +++ b/go/fory/slice.go @@ -310,7 +310,7 @@ func (s sliceSerializer) readDifferentTypes(f *Fory, buf *ByteBuffer, value refl continue } - typeInfo, _ := f.typeResolver.readTypeInfo(buf) + typeInfo, _ := f.typeResolver.readTypeInfo(buf, value) // Create new element and deserialize from buffer elem := reflect.New(typeInfo.Type).Elem() @@ -399,6 +399,8 @@ func (s *sliceConcreteValueSerializer) Read(f *Fory, buf *ByteBuffer, type_ refl if err := readBySerializer(f, buf, value.Index(i), elemSerializer, s.referencable); err != nil { return err } + + prevType = elemType } return nil } diff --git a/go/fory/struct.go b/go/fory/struct.go index a1a38e449e..53ccebc7b4 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -338,6 +338,11 @@ func typesCompatible(actual, expected reflect.Type) bool { return elementTypesCompatible(actual.Key(), expected.Key()) && elementTypesCompatible(actual.Elem(), expected.Elem()) } } + // we can safely treat slice and array as same + if (actual.Kind() == reflect.Array && expected.Kind() == reflect.Slice) || + (actual.Kind() == reflect.Slice && expected.Kind() == reflect.Array) { + return true + } return false } @@ -384,7 +389,7 @@ func sortFields( switch { case isPrimitiveType(t.typeID): boxed = append(boxed, t) - case isListType(t.typeID): + case isListType(t.typeID), isPrimitiveArrayType(t.typeID): collection = append(collection, t) case isSetType(t.typeID): setFields = append(setFields, t) @@ -411,7 +416,7 @@ func sortFields( } return ai.name < aj.name }) - sortTuple := func(s []triple) { + sortByTypeIDThenName := func(s []triple) { sort.Slice(s, func(i, j int) bool { if s[i].typeID != s[j].typeID { return s[i].typeID < s[j].typeID @@ -419,7 +424,7 @@ func sortFields( return s[i].name < s[j].name }) } - sortByTypeIDThenName := func(s []triple) { + sortTuple := func(s []triple) { sort.Slice(s, func(i, j int) bool { return s[i].name < s[j].name }) diff --git a/go/fory/type.go b/go/fory/type.go index 302c907fc7..8834dcc5e2 100644 --- a/go/fory/type.go +++ b/go/fory/type.go @@ -113,6 +113,8 @@ const ( ARROW_RECORD_BATCH = 38 // ARROW_TABLE an arrow table object ARROW_TABLE = 39 + // UNKNOWN an unknown type + UNKNOWN = 63 // UINT8 Unsigned 8-bit little-endian integer UINT8 = 100 // Not in mapping table, assign a higher value @@ -863,6 +865,10 @@ func (r *typeResolver) getTypeDef(typ reflect.Type, create bool) (*TypeDef, erro return nil, fmt.Errorf("TypeDef not found for type %s", typ) } + // don't create TypeDef for pointer types, we create TypeDef for its element type instead. + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } zero := reflect.Zero(typ) typeDef, err := buildTypeDef(r.fory, zero) if err != nil { @@ -872,13 +878,27 @@ func (r *typeResolver) getTypeDef(typ reflect.Type, create bool) (*TypeDef, erro return typeDef, nil } -func (r *typeResolver) readSharedTypeMeta(buffer *ByteBuffer) (TypeInfo, error) { +func (r *typeResolver) readSharedTypeMeta(buffer *ByteBuffer, value reflect.Value) (TypeInfo, error) { context := r.fory.metaContext index := buffer.ReadVarInt32() // shared meta index id if index < 0 || index >= int32(len(context.readTypeInfos)) { return TypeInfo{}, fmt.Errorf("TypeInfo not found for index %d", index) } info := context.readTypeInfos[index] + /* + todo: fix this logic for more elegant handle + There are two corner case: + 1. value is pointer but read info not + 2. value is not pointer but read info is pointer + */ + if value.Kind() == reflect.Ptr && IsNamespacedType(info.Serializer.TypeId()) { + info.Serializer = &ptrToStructSerializer{ + type_: value.Type().Elem(), + structSerializer: *info.Serializer.(*structSerializer), + } + } else if info.Type.Kind() == reflect.Ptr && value.Kind() != reflect.Ptr { + info.Type = info.Type.Elem() + } return info, nil } @@ -1183,7 +1203,7 @@ func (r *typeResolver) readTypeByReadTag(buffer *ByteBuffer) (reflect.Type, erro return r.typeTagToSerializers[metaString].(*ptrToStructSerializer).type_, err } -func (r *typeResolver) readTypeInfo(buffer *ByteBuffer) (TypeInfo, error) { +func (r *typeResolver) readTypeInfo(buffer *ByteBuffer, value reflect.Value) (TypeInfo, error) { // Read variable-length type ID typeID := buffer.ReadVarInt32() internalTypeID := typeID // Extract lower 8 bits for internal type ID @@ -1192,7 +1212,7 @@ func (r *typeResolver) readTypeInfo(buffer *ByteBuffer) (TypeInfo, error) { } if IsNamespacedType(TypeId(internalTypeID)) { if r.metaShareEnabled() { - return r.readSharedTypeMeta(buffer) + return r.readSharedTypeMeta(buffer, value) } // Read namespace and type name metadata bytes nsBytes, err := r.metaStringResolver.ReadMetaStringBytes(buffer) diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 1603f4f95a..a449b96c7d 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -252,7 +252,7 @@ func readFieldType(buffer *ByteBuffer) (FieldType, error) { return nil, fmt.Errorf("failed to read value type: %w", err) } return NewMapFieldType(TypeId(typeId), keyType, valueType), nil - case EXT, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: + case UNKNOWN, EXT, STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT: return NewDynamicFieldType(TypeId(typeId)), nil } return NewSimpleFieldType(TypeId(typeId)), nil @@ -362,9 +362,23 @@ func (d *DynamicFieldType) getTypeInfo(fory *Fory) (TypeInfo, error) { // buildFieldType builds field type from reflect.Type, handling collection, map recursively func buildFieldType(fory *Fory, fieldValue reflect.Value) (FieldType, error) { fieldType := fieldValue.Type() + // Handle Interface type, we can't determine the actual type here, so leave it as dynamic type + if fieldType.Kind() == reflect.Interface { + return NewDynamicFieldType(UNKNOWN), nil + } + + var typeId TypeId + typeInfo, err := fory.typeResolver.getTypeInfo(fieldValue, true) + if err != nil { + return nil, err + } + typeId = TypeId(typeInfo.TypeID) + if typeId < 0 { + typeId = -typeId // restore pointer type id + } // Handle slice and array types - if fieldType.Kind() == reflect.Slice || fieldType.Kind() == reflect.Array { + if typeId == LIST || typeId == SET { // Create a zero value of the element type for recursive processing elemType := fieldType.Elem() elemValue := reflect.Zero(elemType) @@ -378,7 +392,7 @@ func buildFieldType(fory *Fory, fieldValue reflect.Value) (FieldType, error) { } // Handle map types - if fieldType.Kind() == reflect.Map { + if typeId == MAP { // Create zero values for key and value types keyType := fieldType.Key() valueType := fieldType.Elem() @@ -398,16 +412,7 @@ func buildFieldType(fory *Fory, fieldValue reflect.Value) (FieldType, error) { return NewMapFieldType(MAP, keyFieldType, valueFieldType), nil } - // For all other types, get the type ID and treat as ObjectFieldType - var typeId TypeId - typeInfo, err := fory.typeResolver.getTypeInfo(fieldValue, true) - if err != nil { - return nil, err - } - typeId = TypeId(typeInfo.TypeID) - - if typeId == EXT || typeId == STRUCT || typeId == NAMED_STRUCT || - typeId == COMPATIBLE_STRUCT || typeId == NAMED_COMPATIBLE_STRUCT { + if isUserDefinedType(typeId) { return NewDynamicFieldType(typeId), nil } From 487bcfd96dd6a87640bee352e8409b1c29a86bbd Mon Sep 17 00:00:00 2001 From: urlyy Date: Wed, 15 Oct 2025 00:59:24 +0800 Subject: [PATCH 24/37] feat(rust): make Serializer api to return Result && replace panic/expect/assert/unwrap with Result (#2765) ## What does this PR do? - simply refactor `error.rs` - replace `panic/expect/unwrap` with `Result` - replace `assert` with `ensure` - change Serialize api to return `Result` ## Related issues close #2706 ## Does this PR introduce any user-facing change? 1. user should use `unwrap/?` when se/de. 2. when they customize serializer, will return `Result`. - [x] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? --------- Co-authored-by: chaokunyang --- rust/benches/src/serializers/fory.rs | 44 +-- rust/fory-core/Cargo.toml | 1 - rust/fory-core/benches/simd_bench.rs | 12 +- rust/fory-core/src/buffer.rs | 287 ++++++++++-------- rust/fory-core/src/error.rs | 102 ++++++- rust/fory-core/src/fory.rs | 85 +++--- rust/fory-core/src/lib.rs | 6 +- rust/fory-core/src/meta/meta_string.rs | 47 +-- rust/fory-core/src/meta/string_util.rs | 58 ++-- rust/fory-core/src/meta/type_meta.rs | 112 +++---- rust/fory-core/src/resolver/context.rs | 62 ++-- rust/fory-core/src/resolver/meta_resolver.rs | 39 +-- .../src/resolver/metastring_resolver.rs | 115 ++++--- rust/fory-core/src/resolver/ref_resolver.rs | 15 +- rust/fory-core/src/resolver/type_resolver.rs | 192 ++++++------ rust/fory-core/src/row/row.rs | 69 +++-- rust/fory-core/src/row/writer.rs | 14 +- rust/fory-core/src/serializer/any.rs | 176 +++++++---- rust/fory-core/src/serializer/arc.rs | 60 ++-- rust/fory-core/src/serializer/bool.rs | 34 ++- rust/fory-core/src/serializer/box_.rs | 27 +- rust/fory-core/src/serializer/collection.rs | 50 +-- rust/fory-core/src/serializer/datetime.rs | 81 +++-- rust/fory-core/src/serializer/enum_.rs | 65 ++-- rust/fory-core/src/serializer/heap.rs | 31 +- rust/fory-core/src/serializer/list.rs | 113 ++++--- rust/fory-core/src/serializer/map.rs | 146 +++++---- rust/fory-core/src/serializer/mod.rs | 153 ++++++---- rust/fory-core/src/serializer/mutex.rs | 38 ++- rust/fory-core/src/serializer/number.rs | 34 ++- rust/fory-core/src/serializer/option.rs | 27 +- .../src/serializer/primitive_list.rs | 43 ++- rust/fory-core/src/serializer/rc.rs | 56 ++-- rust/fory-core/src/serializer/refcell.rs | 36 ++- rust/fory-core/src/serializer/set.rs | 62 ++-- rust/fory-core/src/serializer/skip.rs | 58 ++-- rust/fory-core/src/serializer/string.rs | 40 ++- rust/fory-core/src/serializer/struct_.rs | 61 ++-- rust/fory-core/src/serializer/trait_object.rs | 195 ++++++------ rust/fory-core/src/serializer/weak.rs | 100 ++++-- rust/fory-core/src/types.rs | 5 +- rust/fory-derive/src/fory_row.rs | 5 +- rust/fory-derive/src/lib.rs | 2 +- rust/fory-derive/src/object/derive_enum.rs | 10 +- rust/fory-derive/src/object/misc.rs | 2 +- rust/fory-derive/src/object/read.rs | 38 +-- rust/fory-derive/src/object/serializer.rs | 14 +- rust/fory-derive/src/object/util.rs | 2 +- rust/fory-derive/src/object/write.rs | 35 +-- rust/fory/src/lib.rs | 45 +-- .../tests/tests/compatible/test_basic_type.rs | 145 +++++---- rust/tests/tests/compatible/test_container.rs | 128 +++++--- rust/tests/tests/compatible/test_struct.rs | 96 +++--- .../tests/compatible/test_struct_enum.rs | 223 ++++++++------ rust/tests/tests/test_any.rs | 45 +-- rust/tests/tests/test_box.rs | 22 +- rust/tests/tests/test_buffer.rs | 6 +- rust/tests/tests/test_collection.rs | 12 +- rust/tests/tests/test_complex_refs.rs | 10 +- rust/tests/tests/test_complex_struct.rs | 44 +-- rust/tests/tests/test_cross_language.rs | 238 +++++++++------ rust/tests/tests/test_ext.rs | 15 +- rust/tests/tests/test_list.rs | 21 +- rust/tests/tests/test_map.rs | 9 +- rust/tests/tests/test_max_dyn_depth.rs | 16 +- rust/tests/tests/test_multi_thread.rs | 6 +- rust/tests/tests/test_mutex.rs | 6 +- rust/tests/tests/test_rc_arc.rs | 42 +-- rust/tests/tests/test_rc_arc_trait_object.rs | 38 +-- rust/tests/tests/test_refcell.rs | 10 +- rust/tests/tests/test_row.rs | 3 +- rust/tests/tests/test_simple_struct.rs | 6 +- rust/tests/tests/test_trait_object.rs | 123 ++++---- rust/tests/tests/test_weak.rs | 24 +- 74 files changed, 2649 insertions(+), 1743 deletions(-) diff --git a/rust/benches/src/serializers/fory.rs b/rust/benches/src/serializers/fory.rs index dee13c50f0..f451f64f02 100644 --- a/rust/benches/src/serializers/fory.rs +++ b/rust/benches/src/serializers/fory.rs @@ -32,27 +32,27 @@ impl ForySerializer { let mut fory = Fory::default(); // Register simple types - fory.register::(100); - fory.register::(101); - fory.register::(102); + fory.register::(100).unwrap(); + fory.register::(101).unwrap(); + fory.register::(102).unwrap(); // Register medium types - fory.register::(200); - fory.register::(201); - fory.register::(202); + fory.register::(200).unwrap(); + fory.register::(201).unwrap(); + fory.register::(202).unwrap(); // Register complex types - fory.register::(300); - fory.register::(301); - fory.register::(302); - fory.register::(303); - fory.register::(304); + fory.register::(300).unwrap(); + fory.register::(301).unwrap(); + fory.register::(302).unwrap(); + fory.register::(303).unwrap(); + fory.register::(304).unwrap(); // Register realworld types - fory.register::(400); - fory.register::(401); - fory.register::(402); - fory.register::(403); + fory.register::(400).unwrap(); + fory.register::(401).unwrap(); + fory.register::(402).unwrap(); + fory.register::(403).unwrap(); Self { fory } } @@ -60,7 +60,7 @@ impl ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &SimpleStruct) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -70,7 +70,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &SimpleList) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -80,7 +80,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &SimpleMap) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -90,7 +90,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &Person) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -100,7 +100,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &Company) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -110,7 +110,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &ECommerceData) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { @@ -120,7 +120,7 @@ impl Serializer for ForySerializer { impl Serializer for ForySerializer { fn serialize(&self, data: &SystemData) -> Result, Box> { - Ok(self.fory.serialize(data)) + Ok(self.fory.serialize(data)?) } fn deserialize(&self, data: &[u8]) -> Result> { diff --git a/rust/fory-core/Cargo.toml b/rust/fory-core/Cargo.toml index 91c91a4195..10ff46726a 100644 --- a/rust/fory-core/Cargo.toml +++ b/rust/fory-core/Cargo.toml @@ -28,7 +28,6 @@ quote = { default-features = false, version = "1.0" } byteorder = { version = "1.4" } chrono = "0.4" thiserror = { default-features = false, version = "1.0" } -anyhow = "1" num_enum = "0.5.1" paste = "1.0" diff --git a/rust/fory-core/benches/simd_bench.rs b/rust/fory-core/benches/simd_bench.rs index 7fc0e422a8..3d49aa5278 100644 --- a/rust/fory-core/benches/simd_bench.rs +++ b/rust/fory-core/benches/simd_bench.rs @@ -171,7 +171,7 @@ fn benchmark_read_latin1(c: &mut Criterion) { c.bench_function(&name_simd, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_latin1_simd(black_box(&mut reader), black_box(s_ascii.len())); + read_latin1_simd(black_box(&mut reader), black_box(s_ascii.len())).unwrap(); }) }); @@ -179,7 +179,7 @@ fn benchmark_read_latin1(c: &mut Criterion) { c.bench_function(&name_scalar, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_latin1_standard(black_box(&mut reader), black_box(s_ascii.len())); + read_latin1_standard(black_box(&mut reader), black_box(s_ascii.len())).unwrap(); }) }); } @@ -199,7 +199,7 @@ fn benchmark_read_utf8(c: &mut Criterion) { c.bench_function(&name_simd, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_utf8_simd(black_box(&mut reader), black_box(s.len())); + read_utf8_simd(black_box(&mut reader), black_box(s.len())).unwrap(); }) }); @@ -207,7 +207,7 @@ fn benchmark_read_utf8(c: &mut Criterion) { c.bench_function(&name_scalar, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_utf8_standard(black_box(&mut reader), black_box(s.len())); + read_utf8_standard(black_box(&mut reader), black_box(s.len())).unwrap(); }) }); } @@ -231,7 +231,7 @@ fn benchmark_read_utf16(c: &mut Criterion) { c.bench_function(&name_simd, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_utf16_simd(black_box(&mut reader), black_box(data.len())); + read_utf16_simd(black_box(&mut reader), black_box(data.len())).unwrap(); }) }); @@ -239,7 +239,7 @@ fn benchmark_read_utf16(c: &mut Criterion) { c.bench_function(&name_scalar, |b| { b.iter(|| { let mut reader = Reader::new(black_box(&data)); - read_utf16_standard(black_box(&mut reader), black_box(data.len())); + read_utf16_standard(black_box(&mut reader), black_box(data.len())).unwrap(); }) }); } diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index a51f6bdc2b..5769377388 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::error::Error; use crate::meta::buffer_rw_string::{ read_latin1_simd, read_utf16_simd, read_utf8_simd, write_latin1_simd, write_utf16_simd, write_utf8_simd, @@ -67,7 +68,7 @@ impl Writer { pub fn set_bytes(&mut self, offset: usize, data: &[u8]) { self.bf .get_mut(offset..offset + data.len()) - .expect("//todo") + .unwrap() .copy_from_slice(data); } @@ -181,12 +182,12 @@ impl Writer { #[inline(always)] pub fn write_varint64(&mut self, value: i64) { let zigzag = ((value << 1) ^ (value >> 63)) as u64; - self._write_varuint64(zigzag) + self._write_varuint64(zigzag); } #[inline(always)] pub fn write_varuint64(&mut self, value: u64) { - self._write_varuint64(value) + self._write_varuint64(value); } #[inline(always)] @@ -374,18 +375,15 @@ impl Reader { } #[inline(always)] - unsafe fn ptr_at(&self, offset: usize) -> *const u8 { - self.bf.add(offset) + pub(crate) fn ptr_at(&self, offset: usize) -> *const u8 { + unsafe { self.bf.add(offset) } } #[inline(always)] pub fn slice_after_cursor(&self) -> &[u8] { - if self.bf.is_null() || self.cursor >= self.len { - &[] - } else { - let remaining = self.len - self.cursor; - unsafe { std::slice::from_raw_parts(self.bf.add(self.cursor), remaining) } - } + let remaining = self.len - self.cursor; + let ptr = unsafe { self.bf.add(self.cursor) }; + unsafe { std::slice::from_raw_parts(ptr, remaining) } } #[inline(always)] @@ -394,215 +392,249 @@ impl Reader { } #[inline(always)] - pub fn read_bool(&mut self) -> bool { - let result = unsafe { *self.ptr_at(self.cursor) }; - self.move_next(1); - result != 0 + fn check_bound(&self, n: usize) -> Result<(), Error> { + // The upper layer guarantees it is non-null + // if self.bf.is_null() { + // return Err(Error::InvalidData("buffer pointer is null".into())); + // } + if self.cursor + n > self.len { + Err(Error::BufferOutOfBound(self.cursor, n, self.len)) + } else { + Ok(()) + } } #[inline(always)] - pub fn read_u8(&mut self) -> u8 { - let result = unsafe { *self.ptr_at(self.cursor) }; + pub fn read_bool(&mut self) -> Result { + Ok(self.read_u8()? != 0) + } + + #[inline(always)] + pub fn read_u8(&mut self) -> Result { + self.check_bound(1)?; + let ptr = self.ptr_at(self.cursor); self.move_next(1); - result + let result = unsafe { *ptr }; + Ok(result) } #[inline(always)] - pub fn read_i8(&mut self) -> i8 { - self.read_u8() as i8 + pub fn read_i8(&mut self) -> Result { + Ok(self.read_u8()? as i8) } #[inline(always)] - pub fn read_u16(&mut self) -> u16 { - let result = LittleEndian::read_u16(self.slice_after_cursor()); + pub fn read_u16(&mut self) -> Result { + self.check_bound(2)?; + let slice = self.slice_after_cursor(); + let result = LittleEndian::read_u16(slice); self.move_next(2); - result + Ok(result) } #[inline(always)] - pub fn read_i16(&mut self) -> i16 { - let result = LittleEndian::read_i16(self.slice_after_cursor()); - self.move_next(2); - result + pub fn read_i16(&mut self) -> Result { + Ok(self.read_u16()? as i16) } #[inline(always)] - pub fn read_u32(&mut self) -> u32 { - let result = LittleEndian::read_u32(self.slice_after_cursor()); + pub fn read_u32(&mut self) -> Result { + self.check_bound(4)?; + let slice = self.slice_after_cursor(); + let result = LittleEndian::read_u32(slice); self.move_next(4); - result + Ok(result) } #[inline(always)] - pub fn read_i32(&mut self) -> i32 { - let result = LittleEndian::read_i32(self.slice_after_cursor()); - self.move_next(4); - result + pub fn read_i32(&mut self) -> Result { + Ok(self.read_u32()? as i32) } #[inline(always)] - pub fn read_u64(&mut self) -> u64 { - let result = LittleEndian::read_u64(self.slice_after_cursor()); + pub fn read_u64(&mut self) -> Result { + self.check_bound(8)?; + let slice = self.slice_after_cursor(); + let result = LittleEndian::read_u64(slice); self.move_next(8); - result + Ok(result) } #[inline(always)] - pub fn read_i64(&mut self) -> i64 { - let result = LittleEndian::read_i64(self.slice_after_cursor()); - self.move_next(8); - result + pub fn read_i64(&mut self) -> Result { + Ok(self.read_u64()? as i64) } #[inline(always)] - pub fn read_f32(&mut self) -> f32 { - let result = LittleEndian::read_f32(self.slice_after_cursor()); + pub fn read_f32(&mut self) -> Result { + self.check_bound(4)?; + let slice = self.slice_after_cursor(); + let result = LittleEndian::read_f32(slice); self.move_next(4); - result + Ok(result) } #[inline(always)] - pub fn read_f64(&mut self) -> f64 { - let result = LittleEndian::read_f64(self.slice_after_cursor()); + pub fn read_f64(&mut self) -> Result { + self.check_bound(8)?; + let slice = self.slice_after_cursor(); + let result = LittleEndian::read_f64(slice); self.move_next(8); - result + Ok(result) } #[inline(always)] - pub fn read_varuint32(&mut self) -> u32 { - let start = self.cursor; - let b0 = unsafe { *self.bf.add(start) as u32 }; + pub fn read_varuint32(&mut self) -> Result { + self.check_bound(1)?; + let slice = self.slice_after_cursor(); + let b0 = slice[0] as u32; if b0 < 0x80 { - self.cursor += 1; - return b0; + self.move_next(1); + return Ok(b0); } - let mut encoded = b0 & 0x7F; - let b1 = unsafe { *self.bf.add(start + 1) as u32 }; - encoded |= (b1 & 0x7F) << 7; + self.check_bound(2)?; + let b1 = slice[1] as u32; + let mut encoded = (b0 & 0x7F) | ((b1 & 0x7F) << 7); if b1 < 0x80 { - self.cursor += 2; - return encoded; + self.move_next(2); + return Ok(encoded); } - let b2 = unsafe { *self.bf.add(start + 2) as u32 }; + self.check_bound(3)?; + let b2 = slice[2] as u32; encoded |= (b2 & 0x7F) << 14; if b2 < 0x80 { - self.cursor += 3; - return encoded; + self.move_next(3); + return Ok(encoded); } - let b3 = unsafe { *self.bf.add(start + 3) as u32 }; + self.check_bound(4)?; + let b3 = slice[3] as u32; encoded |= (b3 & 0x7F) << 21; if b3 < 0x80 { - self.cursor += 4; - return encoded; + self.move_next(4); + return Ok(encoded); } - let b4 = unsafe { *self.bf.add(start + 4) as u32 }; + self.check_bound(5)?; + let b4 = slice[4] as u32; encoded |= b4 << 28; - self.cursor += 5; - encoded + self.move_next(5); + Ok(encoded) } #[inline(always)] - pub fn read_varint32(&mut self) -> i32 { - let encoded = self.read_varuint32(); - ((encoded >> 1) as i32) ^ -((encoded & 1) as i32) + pub fn read_varint32(&mut self) -> Result { + let encoded = self.read_varuint32()?; + Ok(((encoded >> 1) as i32) ^ -((encoded & 1) as i32)) } #[inline(always)] - pub fn read_varuint64(&mut self) -> u64 { - let start = self.cursor; - let b0 = unsafe { *self.bf.add(start) } as u64; + pub fn read_varuint64(&mut self) -> Result { + self.check_bound(1)?; + let slice = self.slice_after_cursor(); + let b0 = slice[0] as u64; if b0 < 0x80 { - self.cursor += 1; - return b0; + self.move_next(1); + return Ok(b0); } - let mut var64 = b0 & 0x7F; - let b1 = unsafe { *self.bf.add(start + 1) } as u64; - var64 |= (b1 & 0x7F) << 7; + self.check_bound(2)?; + let b1 = slice[1] as u64; + let mut var64 = (b0 & 0x7F) | ((b1 & 0x7F) << 7); if b1 < 0x80 { - self.cursor += 2; - return var64; + self.move_next(2); + return Ok(var64); } - let b2 = unsafe { *self.bf.add(start + 2) } as u64; + self.check_bound(3)?; + let b2 = slice[2] as u64; var64 |= (b2 & 0x7F) << 14; if b2 < 0x80 { - self.cursor += 3; - return var64; + self.move_next(3); + return Ok(var64); } - let b3 = unsafe { *self.bf.add(start + 3) } as u64; + self.check_bound(4)?; + let b3 = slice[3] as u64; var64 |= (b3 & 0x7F) << 21; if b3 < 0x80 { - self.cursor += 4; - return var64; + self.move_next(4); + return Ok(var64); } - let b4 = unsafe { *self.bf.add(start + 4) } as u64; + self.check_bound(5)?; + let b4 = slice[4] as u64; var64 |= (b4 & 0x7F) << 28; if b4 < 0x80 { - self.cursor += 5; - return var64; + self.move_next(5); + return Ok(var64); } - let b5 = unsafe { *self.bf.add(start + 5) } as u64; + self.check_bound(6)?; + let b5 = slice[5] as u64; var64 |= (b5 & 0x7F) << 35; if b5 < 0x80 { - self.cursor += 6; - return var64; + self.move_next(6); + return Ok(var64); } - let b6 = unsafe { *self.bf.add(start + 6) } as u64; + self.check_bound(7)?; + let b6 = slice[6] as u64; var64 |= (b6 & 0x7F) << 42; if b6 < 0x80 { - self.cursor += 7; - return var64; + self.move_next(7); + return Ok(var64); } - let b7 = unsafe { *self.bf.add(start + 7) } as u64; + self.check_bound(8)?; + let b7 = slice[7] as u64; var64 |= (b7 & 0x7F) << 49; if b7 < 0x80 { - self.cursor += 8; - return var64; + self.move_next(8); + return Ok(var64); } - let b8 = unsafe { *self.bf.add(start + 8) } as u64; + self.check_bound(9)?; + let b8 = slice[8] as u64; var64 |= (b8 & 0xFF) << 56; - self.cursor += 9; - var64 + self.move_next(9); + Ok(var64) } #[inline(always)] - pub fn read_varint64(&mut self) -> i64 { - let encoded = self.read_varuint64(); - ((encoded >> 1) as i64) ^ -((encoded & 1) as i64) + pub fn read_varint64(&mut self) -> Result { + let encoded = self.read_varuint64()?; + Ok(((encoded >> 1) as i64) ^ -((encoded & 1) as i64)) } #[inline(always)] - pub fn read_latin1_string(&mut self, len: usize) -> String { + pub fn read_latin1_string(&mut self, len: usize) -> Result { + self.check_bound(len)?; read_latin1_simd(self, len) } #[inline(always)] - pub fn read_utf8_string(&mut self, len: usize) -> String { + pub fn read_utf8_string(&mut self, len: usize) -> Result { + self.check_bound(len)?; read_utf8_simd(self, len) } #[inline(always)] - pub fn read_utf16_string(&mut self, len: usize) -> String { + pub fn read_utf16_string(&mut self, len: usize) -> Result { + self.check_bound(len)?; read_utf16_simd(self, len) } #[inline(always)] - pub fn read_varuint36small(&mut self) -> u64 { + pub fn read_varuint36small(&mut self) -> Result { let start = self.cursor; - // fast path - if self.slice_after_cursor().len() >= 8 { - let bulk = self.read_u64(); + let slice = self.slice_after_cursor(); + + if slice.len() >= 8 { + // here already check bound + let bulk = self.read_u64()?; let mut result = bulk & 0x7F; let mut read_idx = start; @@ -623,41 +655,52 @@ impl Reader { } } self.cursor = read_idx + 1; - return result; + return Ok(result); } - // slow path + let mut result = 0u64; let mut shift = 0; while self.cursor < self.len { - let b = self.read_u8(); + let b = { + // read_u8 but no need to check bound + let ptr = self.ptr_at(self.cursor); + self.move_next(1); + unsafe { *ptr } + }; result |= ((b & 0x7F) as u64) << shift; if (b & 0x80) == 0 { break; } shift += 7; + if shift >= 36 { + return Err(Error::EncodeError("varuint36small overflow".into())); + } } - result + Ok(result) } #[inline(always)] - pub fn skip(&mut self, len: u32) { - self.move_next(len as usize); + pub fn skip(&mut self, len: usize) -> Result<(), Error> { + self.check_bound(len)?; + self.move_next(len); + Ok(()) } #[inline(always)] - pub fn get_slice(&self) -> &[u8] { + pub fn get_slice(&self) -> Result<&[u8], Error> { if self.bf.is_null() || self.len == 0 { - &[] + Ok(&[]) } else { - unsafe { slice::from_raw_parts(self.bf, self.len) } + Ok(unsafe { slice::from_raw_parts(self.bf, self.len) }) } } #[inline(always)] - pub fn read_bytes(&mut self, len: usize) -> &[u8] { + pub fn read_bytes(&mut self, len: usize) -> Result<&[u8], Error> { + self.check_bound(len)?; let s = unsafe { slice::from_raw_parts(self.bf.add(self.cursor), len) }; self.move_next(len); - s + Ok(s) } #[inline(always)] @@ -673,7 +716,7 @@ impl Reader { if self.bf.is_null() { return false; } - unsafe { (self.bf.add(self.cursor) as usize) % std::mem::align_of::() == 0 } + unsafe { (self.bf.add(self.cursor) as usize) % align_of::() == 0 } } } diff --git a/rust/fory-core/src/error.rs b/rust/fory-core/src/error.rs index 3d51dbf2d0..4cbee205c0 100644 --- a/rust/fory-core/src/error.rs +++ b/rust/fory-core/src/error.rs @@ -15,38 +15,118 @@ // specific language governing permissions and limitations // under the License. +use std::borrow::Cow; +use std::io; + use thiserror::Error; #[derive(Error, Debug)] #[non_exhaustive] pub enum Error { - #[error("Fory on Rust not support Ref type")] - Ref, + #[error("Type mismatch: type_a = {0}, type_b = {1}")] + TypeMismatch(u32, u32), + + #[error("Buffer out of bound: {0} + {1} > {2}")] + BufferOutOfBound(usize, usize, usize), + + #[error("IO error: {0}")] + Io(#[from] io::Error), + + #[error("{0}")] + EncodeError(Cow<'static, str>), + + #[error("{0}")] + InvalidData(Cow<'static, str>), + + #[error("{0}")] + InvalidRef(Cow<'static, str>), + + #[error("{0}")] + UnknownEnum(Cow<'static, str>), + + #[error("{0}")] + TypeError(Cow<'static, str>), - #[error(transparent)] - Other(#[from] anyhow::Error), + #[error("{0}")] + EncodingError(Cow<'static, str>), + + #[error("{0}")] + DepthExceed(Cow<'static, str>), + + /// Do not construct this variant directly; use [`Error::unknown`] instead. + #[error("{0}")] + Unknown(Cow<'static, str>), +} + +impl Error { + /// Creates a new [`Error::Unknown`] from a string or static message. + /// + /// This function is a convenient way to produce an error message + /// from a literal, `String`, or any type that can be converted into + /// a [`Cow<'static, str>`]. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::unknown("Something went wrong"); + /// let err = Error::unknown(format!("ID:{} not found", 1)); + /// ``` + #[inline(always)] + pub fn unknown>>(s: S) -> Self { + Error::Unknown(s.into()) + } } -/// Works like anyhow's [ensure](https://docs.rs/anyhow/latest/anyhow/macro.ensure.html) -/// But return `Return` +/// Ensures a condition is true; otherwise returns an [`Error`]. +/// +/// # Examples +/// ``` +/// use fory_core::ensure; +/// use fory_core::error::Error; +/// +/// fn check_value(n: i32) -> Result<(), Error> { +/// ensure!(n > 0, "value must be positive"); +/// ensure!(n < 10, "value {} too large", n); +/// Ok(()) +/// } +/// ``` #[macro_export] macro_rules! ensure { ($cond:expr, $msg:literal) => { if !$cond { - return Err(anyhow::anyhow!($msg).into()); + return Err($crate::error::Error::unknown($msg)); } }; ($cond:expr, $err:expr) => { if !$cond { - return Err($err.into()); + return Err($err); } }; ($cond:expr, $fmt:expr, $($arg:tt)*) => { if !$cond { - return Err(anyhow::anyhow!($fmt, $($arg)*).into()); + return Err($crate::error::Error::unknown(format!($fmt, $($arg)*))); } }; } -// Re-export anyhow::Error since it may appear in the public API. -pub use anyhow::Error as AnyhowError; +/// Returns early with an [`Error`]. +/// +/// # Examples +/// ``` +/// use fory_core::bail; +/// use fory_core::error::Error; +/// +/// fn fail_fast() -> Result<(), Error> { +/// bail!("something went wrong"); +/// } +/// ``` +#[macro_export] +macro_rules! bail { + ($err:expr) => { + return Err($crate::error::Error::unknown($err)) + }; + ($fmt:expr, $($arg:tt)*) => { + return Err($crate::error::Error::unknown(format!($fmt, $($arg)*))) + }; +} diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 76889b17f9..2a6322ff42 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -29,7 +29,6 @@ use crate::types::{ Language, MAGIC_NUMBER, SIZE_OF_REF_AND_TYPE, }; use crate::util::get_ext_actual_type_id; -use anyhow::anyhow; static EMPTY_STRING: String = String::new(); @@ -312,28 +311,32 @@ impl Fory { fn read_head(&self, reader: &mut Reader) -> Result { if self.xlang { - let magic_numer = reader.read_u16(); + let magic_numer = reader.read_u16()?; ensure!( magic_numer == MAGIC_NUMBER, - anyhow!( - "The fory xlang serialization must start with magic number {:X}. \ + Error::InvalidData( + format!( + "The fory xlang serialization must start with magic number {:X}. \ Please check whether the serialization is based on the xlang protocol \ and the data didn't corrupt.", - MAGIC_NUMBER + MAGIC_NUMBER + ) + .into() ) ) } - let bitmap = reader.read_u8(); + let bitmap = reader.read_u8()?; let peer_is_xlang = (bitmap & IS_CROSS_LANGUAGE_FLAG) != 0; ensure!( self.xlang == peer_is_xlang, - anyhow!("header bitmap mismatch at xlang bit") + Error::InvalidData("header bitmap mismatch at xlang bit".into()) ); let is_little_endian = (bitmap & IS_LITTLE_ENDIAN_FLAG) != 0; ensure!( is_little_endian, - anyhow!( + Error::InvalidData( "Big endian is not supported for now, please ensure peer machine is little endian." + .into() ) ); let is_none = (bitmap & IS_NULL_FLAG) != 0; @@ -341,7 +344,7 @@ impl Fory { return Ok(true); } if peer_is_xlang { - let _peer_lang = reader.read_u8(); + let _peer_lang = reader.read_u8()?; } Ok(false) } @@ -402,14 +405,15 @@ impl Fory { } let mut bytes_to_skip = 0; if self.compatible { - let meta_offset = context.reader.read_i32(); + let meta_offset = context.reader.read_i32()?; if meta_offset != -1 { - bytes_to_skip = context.load_meta(self.get_type_resolver(), meta_offset as usize); + bytes_to_skip = + context.load_meta(self.get_type_resolver(), meta_offset as usize)?; } } let result = ::fory_read(self, context, false); if bytes_to_skip > 0 { - context.reader.skip(bytes_to_skip as u32); + context.reader.skip(bytes_to_skip)?; } context.ref_reader.resolve_callbacks(); result @@ -442,19 +446,19 @@ impl Fory { /// let point = Point { x: 10, y: 20 }; /// let bytes = fory.serialize(&point); /// ``` - pub fn serialize(&self, record: &T) -> Vec { + pub fn serialize(&self, record: &T) -> Result, Error> { let mut context = self.write_context_pool.get(); - let result = self.serialize_with_context(record, &mut context); + let result = self.serialize_with_context(record, &mut context)?; context.reset(); self.write_context_pool.put(context); - result + Ok(result) } pub fn serialize_with_context( &self, record: &T, context: &mut WriteContext, - ) -> Vec { + ) -> Result, Error> { let is_none = record.fory_is_none(); self.write_head::(is_none, &mut context.writer); let meta_start_offset = context.writer.len(); @@ -462,12 +466,12 @@ impl Fory { if self.compatible { context.writer.write_i32(-1); }; - ::fory_write(record, self, context, false); + ::fory_write(record, self, context, false)?; if self.compatible && !context.empty() { context.write_meta(meta_start_offset); } } - context.writer.dump() + Ok(context.writer.dump()) } /// Registers a struct type with a numeric type ID for serialization. @@ -497,11 +501,14 @@ impl Fory { /// let mut fory = Fory::default(); /// fory.register::(100); /// ``` - pub fn register(&mut self, id: u32) { + pub fn register( + &mut self, + id: u32, + ) -> Result<(), Error> { let actual_type_id = T::fory_actual_type_id(id, false, self.compatible); let type_info = - TypeInfo::new::(self, actual_type_id, &EMPTY_STRING, &EMPTY_STRING, false); - self.type_resolver.register::(&type_info); + TypeInfo::new::(self, actual_type_id, &EMPTY_STRING, &EMPTY_STRING, false)?; + self.type_resolver.register::(&type_info) } /// Registers a struct type with a namespace and type name for cross-language serialization. @@ -538,10 +545,10 @@ impl Fory { &mut self, namespace: &str, type_name: &str, - ) { + ) -> Result<(), Error> { let actual_type_id = T::fory_actual_type_id(0, true, self.compatible); - let type_info = TypeInfo::new::(self, actual_type_id, namespace, type_name, true); - self.type_resolver.register::(&type_info); + let type_info = TypeInfo::new::(self, actual_type_id, namespace, type_name, true)?; + self.type_resolver.register::(&type_info) } /// Registers a struct type with a type name (using the default namespace). @@ -573,8 +580,8 @@ impl Fory { pub fn register_by_name( &mut self, type_name: &str, - ) { - self.register_by_namespace::("", type_name); + ) -> Result<(), Error> { + self.register_by_namespace::("", type_name) } /// Registers a custom serializer type with a numeric type ID. @@ -604,7 +611,10 @@ impl Fory { /// let mut fory = Fory::default(); /// fory.register_serializer::(200); /// ``` - pub fn register_serializer(&mut self, id: u32) { + pub fn register_serializer( + &mut self, + id: u32, + ) -> Result<(), Error> { let actual_type_id = get_ext_actual_type_id(id, false); let type_info = TypeInfo::new_with_empty_fields::( self, @@ -612,8 +622,8 @@ impl Fory { &EMPTY_STRING, &EMPTY_STRING, false, - ); - self.type_resolver.register_serializer::(&type_info); + )?; + self.type_resolver.register_serializer::(&type_info) } /// Registers a custom serializer type with a namespace and type name. @@ -636,11 +646,11 @@ impl Fory { &mut self, namespace: &str, type_name: &str, - ) { + ) -> Result<(), Error> { let actual_type_id = get_ext_actual_type_id(0, true); let type_info = - TypeInfo::new_with_empty_fields::(self, actual_type_id, namespace, type_name, true); - self.type_resolver.register_serializer::(&type_info); + TypeInfo::new_with_empty_fields::(self, actual_type_id, namespace, type_name, true)?; + self.type_resolver.register_serializer::(&type_info) } /// Registers a custom serializer type with a type name (using the default namespace). @@ -656,8 +666,11 @@ impl Fory { /// # Notes /// /// This is a convenience method that calls `register_serializer_by_namespace` with an empty namespace. - pub fn register_serializer_by_name(&mut self, type_name: &str) { - self.register_serializer_by_namespace::("", type_name); + pub fn register_serializer_by_name( + &mut self, + type_name: &str, + ) -> Result<(), Error> { + self.register_serializer_by_namespace::("", type_name) } } @@ -666,8 +679,8 @@ pub fn write_data( fory: &Fory, context: &mut WriteContext, is_field: bool, -) { - T::fory_write_data(this, fory, context, is_field); +) -> Result<(), Error> { + T::fory_write_data(this, fory, context, is_field) } pub fn read_data( diff --git a/rust/fory-core/src/lib.rs b/rust/fory-core/src/lib.rs index 9da7008885..f4f9f5d6f0 100644 --- a/rust/fory-core/src/lib.rs +++ b/rust/fory-core/src/lib.rs @@ -151,13 +151,13 @@ //! //! // Serialize String //! let text = String::from("Hello, Fory!"); -//! let serialized_str = fory.serialize(&text); +//! let serialized_str = fory.serialize(&text).unwrap(); //! let deserialized_str: String = fory.deserialize(&serialized_str).unwrap(); //! assert_eq!(text, deserialized_str); //! //! // Serialize Vec //! let vec_data = vec![1, 2, 3, 4, 5]; -//! let serialized_vec = fory.serialize(&vec_data); +//! let serialized_vec = fory.serialize(&vec_data).unwrap(); //! let deserialized_vec: Vec = fory.deserialize(&serialized_vec).unwrap(); //! assert_eq!(vec_data, deserialized_vec); //! @@ -165,7 +165,7 @@ //! let mut map = HashMap::new(); //! map.insert("key1".to_string(), 100); //! map.insert("key2".to_string(), 200); -//! let serialized_map = fory.serialize(&map); +//! let serialized_map = fory.serialize(&map).unwrap(); //! let deserialized_map: HashMap = fory.deserialize(&serialized_map).unwrap(); //! assert_eq!(map, deserialized_map); //! // Register types for object serialization diff --git a/rust/fory-core/src/meta/meta_string.rs b/rust/fory-core/src/meta/meta_string.rs index 9fb3d1df2f..357849361a 100644 --- a/rust/fory-core/src/meta/meta_string.rs +++ b/rust/fory-core/src/meta/meta_string.rs @@ -18,7 +18,6 @@ use crate::ensure; use crate::error::Error; use crate::meta::string_util; -use anyhow::anyhow; // equal to "std::i16::MAX" const SHORT_MAX_VALUE: usize = 32767; @@ -77,8 +76,9 @@ impl MetaString { ) -> Result { let mut strip_last_char = false; if encoding != Encoding::Utf8 { - ensure!(!bytes.is_empty(), anyhow!("Encoded data cannot be empty")); - + if bytes.is_empty() { + return Err(Error::EncodeError("Encoded data cannot be empty".into())); + } strip_last_char = (bytes[0] & 0x80) != 0; } Ok(MetaString { @@ -142,9 +142,12 @@ impl MetaStringEncoder { ensure!( input.len() < SHORT_MAX_VALUE, - anyhow!( - "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", - input.len() + Error::EncodeError( + format!( + "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", + input.len() + ) + .into() ) ); @@ -260,14 +263,17 @@ impl MetaStringEncoder { } ensure!( input.len() < SHORT_MAX_VALUE, - anyhow!( - "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", - input.len() + Error::EncodeError( + format!( + "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", + input.len() + ) + .into() ) ); ensure!( encoding == Encoding::Utf8 || self.is_latin(input), - anyhow!("Non-ASCII characters in meta string are not allowed") + Error::EncodeError("Non-ASCII characters in meta string are not allowed".into()) ); if input.is_empty() { @@ -415,8 +421,9 @@ impl MetaStringEncoder { '_' => Ok(27), '$' => Ok(28), '|' => Ok(29), - _ => Err(anyhow!( - "Unsupported character for LOWER_UPPER_DIGIT_SPECIAL encoding: {c}" + _ => Err(Error::EncodeError( + format!("Unsupported character for LOWER_UPPER_DIGIT_SPECIAL encoding: {c}",) + .into(), ))?, }, 6 => match c { @@ -429,8 +436,9 @@ impl MetaStringEncoder { } else if c == self.special_char2 { Ok(63) } else { - Err(anyhow!( - "Invalid character value for LOWER_SPECIAL decoding: {c:?}" + Err(Error::EncodeError( + format!("Invalid character value for LOWER_SPECIAL decoding: {c:?}",) + .into(), ))? } } @@ -539,8 +547,8 @@ impl MetaStringDecoder { 27 => Ok('_'), 28 => Ok('$'), 29 => Ok('|'), - _ => Err(anyhow!( - "Invalid character value for LOWER_SPECIAL decoding: {char_value}" + _ => Err(Error::EncodeError( + format!("Invalid character value for LOWER_SPECIAL decoding: {char_value}",).into(), ))?, } } @@ -552,8 +560,11 @@ impl MetaStringDecoder { 52..=61 => Ok((b'0' + char_value - 52) as char), 62 => Ok(self.special_char1), 63 => Ok(self.special_char2), - _ => Err(anyhow!( - "Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {char_value}" + _ => Err(Error::EncodeError( + format!( + "Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {char_value}", + ) + .into(), ))?, } } diff --git a/rust/fory-core/src/meta/string_util.rs b/rust/fory-core/src/meta/string_util.rs index b7ffcdd777..4b4692d629 100644 --- a/rust/fory-core/src/meta/string_util.rs +++ b/rust/fory-core/src/meta/string_util.rs @@ -534,6 +534,7 @@ pub mod buffer_rw_string { const SIMD_CHUNK_SIZE: usize = 32; use crate::buffer::{Reader, Writer}; + use crate::error::Error; #[inline] pub fn write_latin1_standard(writer: &mut Writer, s: &str) { @@ -567,23 +568,23 @@ pub mod buffer_rw_string { } #[inline] - pub fn read_latin1_standard(reader: &mut Reader, len: usize) -> String { + pub fn read_latin1_standard(reader: &mut Reader, len: usize) -> Result { let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let result: String = slice.iter().map(|&b| b as char).collect(); reader.move_next(len); - result + Ok(result) } #[inline] - pub fn read_utf8_standard(reader: &mut Reader, len: usize) -> String { + pub fn read_utf8_standard(reader: &mut Reader, len: usize) -> Result { let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let result = String::from_utf8_lossy(slice).to_string(); reader.move_next(len); - result + Ok(result) } #[inline] - pub fn read_utf16_standard(reader: &mut Reader, len: usize) -> String { + pub fn read_utf16_standard(reader: &mut Reader, len: usize) -> Result { assert!(len % 2 == 0, "UTF-16 length must be even"); let slice = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; let units: Vec = slice @@ -595,7 +596,7 @@ pub mod buffer_rw_string { // lossy .unwrap_or_else(|_| String::from("�")); reader.move_next(len); - result + Ok(result) } #[inline] @@ -769,9 +770,9 @@ pub mod buffer_rw_string { } #[inline] - pub fn read_latin1_simd(reader: &mut Reader, len: usize) -> String { + pub fn read_latin1_simd(reader: &mut Reader, len: usize) -> Result { if len == 0 { - return String::new(); + return Ok(String::new()); } let src = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; @@ -862,15 +863,14 @@ pub mod buffer_rw_string { i += 1; } } - reader.move_next(len); - unsafe { String::from_utf8_unchecked(out) } + Ok(unsafe { String::from_utf8_unchecked(out) }) } #[inline] - pub fn read_utf8_simd(reader: &mut Reader, len: usize) -> String { + pub fn read_utf8_simd(reader: &mut Reader, len: usize) -> Result { if len == 0 { - return String::new(); + return Ok(String::new()); } let src = unsafe { std::slice::from_raw_parts(reader.bf.add(reader.cursor), len) }; @@ -925,11 +925,11 @@ pub mod buffer_rw_string { } reader.move_next(len); - result + Ok(result) } #[inline] - pub fn read_utf16_simd(reader: &mut Reader, len: usize) -> String { + pub fn read_utf16_simd(reader: &mut Reader, len: usize) -> Result { assert_eq!(len % 2, 0, "UTF-16 length must be even"); unsafe fn simd_impl(bytes: &[u8]) -> String { let len = bytes.len(); @@ -1005,7 +1005,7 @@ pub mod buffer_rw_string { if std::arch::is_x86_feature_detected!("avx2") { let s = unsafe { simd_impl(slice) }; reader.move_next(len); - return s; + return Ok(s); } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] @@ -1013,7 +1013,7 @@ pub mod buffer_rw_string { if std::arch::is_x86_feature_detected!("sse2") { let s = unsafe { simd_impl(slice) }; reader.move_next(len); - return s; + return Ok(s); } } #[cfg(target_arch = "aarch64")] @@ -1021,7 +1021,7 @@ pub mod buffer_rw_string { if std::arch::is_aarch64_feature_detected!("neon") { let s = unsafe { simd_impl(slice) }; reader.move_next(len); - return s; + return Ok(s); } } @@ -1050,8 +1050,8 @@ pub mod buffer_rw_string { let bytes = &*writer.dump(); let bytes_len = bytes.len() / 2; let mut reader = Reader::new(bytes); - assert_eq!(read_latin1_standard(&mut reader, bytes_len), s); - assert_eq!(read_latin1_standard(&mut reader, bytes_len), s); + assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s); let mut writer = Writer::default(); write_latin1_standard(&mut writer, s); @@ -1059,8 +1059,8 @@ pub mod buffer_rw_string { let bytes = &*writer.dump(); let bytes_len = bytes.len() / 2; let mut reader = Reader::new(bytes); - assert_eq!(read_latin1_simd(&mut reader, bytes_len), s); - assert_eq!(read_latin1_simd(&mut reader, bytes_len), s); + assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s); } } @@ -1082,16 +1082,16 @@ pub mod buffer_rw_string { write_utf8_standard(&mut writer, s); let bytes = &*writer.dump(); let mut reader = Reader::new(bytes); - assert_eq!(read_utf8_simd(&mut reader, bytes_len), s); - assert_eq!(read_utf8_simd(&mut reader, bytes_len), s); + assert_eq!(read_utf8_simd(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_utf8_simd(&mut reader, bytes_len).unwrap(), s); let mut writer = Writer::default(); write_utf8_simd(&mut writer, s); write_utf8_simd(&mut writer, s); let bytes = &*writer.dump(); let mut reader = Reader::new(bytes); - assert_eq!(read_utf8_standard(&mut reader, bytes_len), s); - assert_eq!(read_utf8_standard(&mut reader, bytes_len), s); + assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s); } } @@ -1113,16 +1113,16 @@ pub mod buffer_rw_string { write_utf16_standard(&mut writer, &utf16); let bytes = &*writer.dump(); let mut reader = Reader::new(bytes); - assert_eq!(read_utf16_simd(&mut reader, bytes_len), s); - assert_eq!(read_utf16_simd(&mut reader, bytes_len), s); + assert_eq!(read_utf16_simd(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_utf16_simd(&mut reader, bytes_len).unwrap(), s); let mut writer = Writer::default(); write_utf16_simd(&mut writer, &utf16); write_utf16_simd(&mut writer, &utf16); let bytes = &*writer.dump(); let mut reader = Reader::new(bytes); - assert_eq!(read_utf16_standard(&mut reader, bytes_len), s); - assert_eq!(read_utf16_standard(&mut reader, bytes_len), s); + assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s); + assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s); } } } diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index b8f8c5d5f9..8e0d3e9e1d 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -23,7 +23,6 @@ use crate::meta::{ }; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::types::{TypeId, PRIMITIVE_TYPES}; -use anyhow::anyhow; use std::clone::Clone; use std::cmp::min; use std::collections::HashMap; @@ -100,8 +99,12 @@ impl FieldType { Ok(()) } - fn from_bytes(reader: &mut Reader, read_flag: bool, nullable: Option) -> Self { - let header = reader.read_varuint32(); + fn from_bytes( + reader: &mut Reader, + read_flag: bool, + nullable: Option, + ) -> Result { + let header = reader.read_varuint32()?; let type_id; let _nullable; if read_flag { @@ -112,9 +115,9 @@ impl FieldType { type_id = header; _nullable = nullable.unwrap(); } - match type_id { + Ok(match type_id { x if x == TypeId::LIST as u32 || x == TypeId::SET as u32 => { - let generic = Self::from_bytes(reader, true, None); + let generic = Self::from_bytes(reader, true, None)?; Self { type_id, nullable: _nullable, @@ -122,8 +125,8 @@ impl FieldType { } } x if x == TypeId::MAP as u32 => { - let key_generic = Self::from_bytes(reader, true, None); - let val_generic = Self::from_bytes(reader, true, None); + let key_generic = Self::from_bytes(reader, true, None)?; + let val_generic = Self::from_bytes(reader, true, None)?; Self { type_id, nullable: _nullable, @@ -135,7 +138,7 @@ impl FieldType { nullable: _nullable, generics: vec![], }, - } + }) } } @@ -160,35 +163,35 @@ impl FieldInfo { 0x00 => Ok(Encoding::Utf8), 0x01 => Ok(Encoding::AllToLowerSpecial), 0x02 => Ok(Encoding::LowerUpperDigitSpecial), - _ => Err(anyhow!( - "Unsupported encoding of field name in type meta, value:{value}" + _ => Err(Error::EncodingError( + format!("Unsupported encoding of field name in type meta, value:{value}").into(), ))?, } } - pub fn from_bytes(reader: &mut Reader) -> FieldInfo { - let header = reader.read_u8(); + pub fn from_bytes(reader: &mut Reader) -> Result { + let header = reader.read_u8()?; let nullable = (header & 2) != 0; // let ref_tracking = (header & 1) != 0; - let encoding = Self::u8_to_encoding((header >> 6) & 0b11).unwrap(); + let encoding = Self::u8_to_encoding((header >> 6) & 0b11)?; let mut name_size = ((header >> 2) & FIELD_NAME_SIZE_THRESHOLD as u8) as usize; if name_size == FIELD_NAME_SIZE_THRESHOLD { - name_size += reader.read_varuint32() as usize; + name_size += reader.read_varuint32()? as usize; } name_size += 1; - let field_type = FieldType::from_bytes(reader, false, Option::from(nullable)); + let field_type = FieldType::from_bytes(reader, false, Option::from(nullable))?; - let field_name_bytes = reader.read_bytes(name_size); + let field_name_bytes = reader.read_bytes(name_size)?; let field_name = FIELD_NAME_DECODER .decode(field_name_bytes, encoding) .unwrap(); - FieldInfo { + Ok(FieldInfo { field_id: -1i16, field_name: field_name.original, field_type, - } + }) } fn to_bytes(&self) -> Result, Error> { @@ -295,36 +298,36 @@ impl TypeMetaLayer { } pub fn write_namespace(&self, writer: &mut Writer) { - Self::write_name(writer, &self.namespace, NAMESPACE_ENCODINGS); + Self::write_name(writer, &self.namespace, NAMESPACE_ENCODINGS) } pub fn write_type_name(&self, writer: &mut Writer) { - Self::write_name(writer, &self.type_name, TYPE_NAME_ENCODINGS); + Self::write_name(writer, &self.type_name, TYPE_NAME_ENCODINGS) } fn read_name( reader: &mut Reader, decoder: &MetaStringDecoder, encodings: &[Encoding], - ) -> MetaString { - let header = reader.read_u8(); + ) -> Result { + let header = reader.read_u8()?; let encoding_idx = header & 0b11; let length = header >> 2; let length = if length >= BIG_NAME_THRESHOLD as u8 { - BIG_NAME_THRESHOLD + reader.read_varuint32() as usize + BIG_NAME_THRESHOLD + reader.read_varuint32()? as usize } else { length as usize }; - let bytes = reader.read_bytes(length); + let bytes = reader.read_bytes(length)?; let encoding = encodings[encoding_idx as usize]; - decoder.decode(bytes, encoding).unwrap() + decoder.decode(bytes, encoding) } - pub fn read_namespace(reader: &mut Reader) -> MetaString { + pub fn read_namespace(reader: &mut Reader) -> Result { Self::read_name(reader, &NAMESPACE_DECODER, NAMESPACE_ENCODINGS) } - pub fn read_type_name(reader: &mut Reader) -> MetaString { + pub fn read_type_name(reader: &mut Reader) -> Result { Self::read_name(reader, &TYPE_NAME_DECODER, TYPE_NAME_ENCODINGS) } @@ -457,22 +460,25 @@ impl TypeMetaLayer { sorted_field_infos } - fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) -> TypeMetaLayer { - let meta_header = reader.read_u8(); + fn from_bytes( + reader: &mut Reader, + type_resolver: &TypeResolver, + ) -> Result { + let meta_header = reader.read_u8()?; let register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0; let mut num_fields = meta_header as usize & SMALL_NUM_FIELDS_THRESHOLD; if num_fields == SMALL_NUM_FIELDS_THRESHOLD { - num_fields += reader.read_varuint32() as usize; + num_fields += reader.read_varuint32()? as usize; } let type_id; let namespace; let type_name; if register_by_name { - namespace = Self::read_namespace(reader); - type_name = Self::read_type_name(reader); + namespace = Self::read_namespace(reader)?; + type_name = Self::read_type_name(reader)?; type_id = 0; } else { - type_id = reader.read_varuint32(); + type_id = reader.read_varuint32()?; let empty_name = MetaString::default(); namespace = empty_name.clone(); type_name = empty_name; @@ -480,7 +486,7 @@ impl TypeMetaLayer { let mut field_infos = Vec::with_capacity(num_fields); for _ in 0..num_fields { - field_infos.push(FieldInfo::from_bytes(reader)); + field_infos.push(FieldInfo::from_bytes(reader)?); } let mut sorted_field_infos = Self::sort_field_infos(field_infos); @@ -494,13 +500,13 @@ impl TypeMetaLayer { Self::assign_field_ids(type_info_current, &mut sorted_field_infos); } // if no type found, keep all fields id as -1 to be skipped. - TypeMetaLayer::new( + Ok(TypeMetaLayer::new( type_id, namespace, type_name, register_by_name, sorted_field_infos, - ) + )) } fn assign_field_ids(type_info_current: &TypeInfo, field_infos: &mut [FieldInfo]) { @@ -578,12 +584,15 @@ impl TypeMeta { } } - pub fn from_bytes(reader: &mut Reader, type_resolver: &TypeResolver) -> TypeMeta { - let header = reader.read_i64(); + pub fn from_bytes( + reader: &mut Reader, + type_resolver: &TypeResolver, + ) -> Result { + let header = reader.read_i64()?; let meta_size = header & META_SIZE_MASK; if meta_size == META_SIZE_MASK { // meta_size += reader.read_varuint32() as i64; - reader.read_varuint32(); + reader.read_varuint32()?; } // let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0; @@ -592,22 +601,22 @@ impl TypeMeta { // let current_meta_size = 0; // while current_meta_size < meta_size {} - let layer = TypeMetaLayer::from_bytes(reader, type_resolver); - TypeMeta { + let layer = TypeMetaLayer::from_bytes(reader, type_resolver)?; + Ok(TypeMeta { layer, hash: meta_hash, - } + }) } pub fn from_bytes_with_header( reader: &mut Reader, type_resolver: &TypeResolver, header: i64, - ) -> TypeMeta { + ) -> Result { let meta_size = header & META_SIZE_MASK; if meta_size == META_SIZE_MASK { - // meta_size += reader.read_varuint32() as i64; - reader.read_varuint32(); + // meta_size += reader.read_varuint32()? as i64; + reader.read_varuint32()?; } // let write_fields_meta = (header & HAS_FIELDS_META_FLAG) != 0; @@ -616,20 +625,19 @@ impl TypeMeta { // let current_meta_size = 0; // while current_meta_size < meta_size {} - let layer = TypeMetaLayer::from_bytes(reader, type_resolver); - TypeMeta { + let layer = TypeMetaLayer::from_bytes(reader, type_resolver)?; + Ok(TypeMeta { layer, hash: meta_hash, - } + }) } - pub fn skip_bytes(reader: &mut Reader, header: i64) { + pub fn skip_bytes(reader: &mut Reader, header: i64) -> Result<(), Error> { let mut meta_size = header & META_SIZE_MASK; if meta_size == META_SIZE_MASK { - meta_size += reader.read_varuint32() as i64; + meta_size += reader.read_varuint32()? as i64; } - // TODO skio should return result and we need to return it to caller - reader.skip(meta_size as u32); + reader.skip(meta_size as usize) } pub fn to_bytes(&self) -> Result, Error> { diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index f067383418..e95ad95f32 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -18,6 +18,7 @@ use crate::buffer::{Reader, Writer}; use crate::fory::Fory; +use crate::error::Error; use crate::meta::{MetaString, TypeMeta}; use crate::resolver::meta_resolver::{MetaReaderResolver, MetaWriterResolver}; use crate::resolver::metastring_resolver::{ @@ -50,7 +51,7 @@ impl WriteContext { } #[inline(always)] - pub fn push_meta(&mut self, fory: &Fory, type_id: std::any::TypeId) -> usize { + pub fn push_meta(&mut self, fory: &Fory, type_id: std::any::TypeId) -> Result { self.meta_resolver.push(type_id, fory) } @@ -60,25 +61,25 @@ impl WriteContext { offset, &((self.writer.len() - offset - 4) as u32).to_le_bytes(), ); - self.meta_resolver.to_bytes(&mut self.writer).unwrap() + self.meta_resolver.to_bytes(&mut self.writer); } pub fn write_any_typeinfo( &mut self, fory: &Fory, concrete_type_id: std::any::TypeId, - ) -> Arc { + ) -> Result, Error> { use crate::types::TypeId as ForyTypeId; let type_resolver = fory.get_type_resolver(); - let type_info = type_resolver.get_type_info(concrete_type_id); + let type_info = type_resolver.get_type_info(concrete_type_id)?; let fory_type_id = type_info.get_type_id(); if type_info.is_registered_by_name() { if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); if fory.is_share_meta() { - let meta_index = self.push_meta(fory, concrete_type_id) as u32; + let meta_index = self.push_meta(fory, concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { type_info.get_namespace().write_to(&mut self.writer); @@ -86,7 +87,7 @@ impl WriteContext { } } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(fory, concrete_type_id) as u32; + let meta_index = self.push_meta(fory, concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(u32::MAX); @@ -95,25 +96,25 @@ impl WriteContext { } type_resolver .get_name_harness(type_info.get_namespace(), type_info.get_type_name()) - .expect("Name harness not found") + .ok_or_else(|| Error::TypeError("Name harness not found".into())) } else { if fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(fory, concrete_type_id) as u32; + let meta_index = self.push_meta(fory, concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(fory_type_id); } type_resolver .get_harness(fory_type_id) - .expect("ID harness not found") + .ok_or_else(|| Error::TypeError("ID harness not found".into())) } } #[inline(always)] - pub fn write_meta_string_bytes(&mut self, ms: &MetaString) { + pub fn write_meta_string_bytes(&mut self, ms: &MetaString) -> Result<(), Error> { self.meta_string_resolver - .write_meta_string_bytes(&mut self.writer, ms); + .write_meta_string_bytes(&mut self.writer, ms) } #[inline(always)] @@ -158,69 +159,74 @@ impl ReadContext { } #[inline(always)] - pub fn load_meta(&mut self, type_resolver: &TypeResolver, offset: usize) -> usize { + pub fn load_meta( + &mut self, + type_resolver: &TypeResolver, + offset: usize, + ) -> Result { self.meta_resolver.load( type_resolver, &mut Reader::new(&self.reader.slice_after_cursor()[offset..]), ) } - pub fn read_any_typeinfo(&mut self, fory: &Fory) -> Arc { + pub fn read_any_typeinfo(&mut self, fory: &Fory) -> Result, Error> { use crate::types::TypeId as ForyTypeId; - let fory_type_id = self.reader.read_varuint32(); + let fory_type_id = self.reader.read_varuint32()?; let type_resolver = fory.get_type_resolver(); if fory_type_id == u32::MAX { - let namespace = self.meta_resolver.read_metastring(&mut self.reader); - let type_name = self.meta_resolver.read_metastring(&mut self.reader); + let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; + let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; type_resolver .get_name_harness(&namespace, &type_name) - .expect("Name harness not found") + .ok_or_else(|| Error::TypeError("Name harness not found".into())) } else if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { if fory.is_share_meta() { let _meta_index = self.reader.read_varuint32(); } else { - let namespace = self.meta_resolver.read_metastring(&mut self.reader); - let type_name = self.meta_resolver.read_metastring(&mut self.reader); + let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; + let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; return type_resolver .get_name_harness(&namespace, &type_name) - .expect("Name harness not found"); + .ok_or_else(|| Error::TypeError("Name harness not found".into())); } type_resolver .get_harness(fory_type_id) - .expect("ID harness not found") + .ok_or_else(|| Error::TypeError("ID harness not found".into())) } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { let _meta_index = self.reader.read_varuint32(); type_resolver .get_harness(fory_type_id) - .expect("ID harness not found") + .ok_or_else(|| Error::TypeError("ID harness not found".into())) } else { type_resolver .get_harness(fory_type_id) - .expect("ID harness not found") + .ok_or_else(|| Error::TypeError("ID harness not found".into())) } } - pub fn read_meta_string_bytes(&mut self) -> MetaStringBytes { + pub fn read_meta_string_bytes(&mut self) -> Result { self.meta_string_resolver .read_meta_string_bytes(&mut self.reader) } #[inline(always)] - pub fn inc_depth(&mut self) -> Result<(), crate::error::Error> { + pub fn inc_depth(&mut self) -> Result<(), Error> { self.current_depth += 1; if self.current_depth > self.max_dyn_depth { - return Err(crate::error::Error::Other(crate::error::AnyhowError::msg( + return Err(Error::DepthExceed( format!( "Maximum dynamic object nesting depth ({}) exceeded. Current depth: {}. \ This may indicate a circular reference or overly deep object graph. \ Consider increasing max_dyn_depth if this is expected.", self.max_dyn_depth, self.current_depth - ), - ))); + ) + .into(), + )); } Ok(()) } diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 50a24635fe..2fb0b5b084 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -33,28 +33,27 @@ const MAX_PARSED_NUM_TYPE_DEFS: usize = 8192; #[allow(dead_code)] impl MetaWriterResolver { - pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> usize { + pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> Result { match self.type_id_index_map.get(&type_id) { None => { let index = self.type_defs.len(); self.type_defs.push( fory.get_type_resolver() - .get_type_info(type_id) + .get_type_info(type_id)? .get_type_def(), ); self.type_id_index_map.insert(type_id, index); - index + Ok(index) } - Some(index) => *index, + Some(index) => Ok(*index), } } - pub fn to_bytes(&self, writer: &mut Writer) -> Result<(), Error> { + pub fn to_bytes(&self, writer: &mut Writer) { writer.write_varuint32(self.type_defs.len() as u32); for item in &self.type_defs { writer.write_bytes(item); } - Ok(()) } pub fn empty(&mut self) -> bool { @@ -78,20 +77,24 @@ impl MetaReaderResolver { unsafe { self.reading_type_defs.get_unchecked(index) } } - pub fn load(&mut self, type_resolver: &TypeResolver, reader: &mut Reader) -> usize { - let meta_size = reader.read_varuint32(); + pub fn load( + &mut self, + type_resolver: &TypeResolver, + reader: &mut Reader, + ) -> Result { + let meta_size = reader.read_varuint32()?; // self.reading_type_defs.reserve(meta_size as usize); for _ in 0..meta_size { - let meta_header = reader.read_i64(); + let meta_header = reader.read_i64()?; if let Some(type_meta) = self.parsed_type_defs.get(&meta_header) { self.reading_type_defs.push(type_meta.clone()); - TypeMeta::skip_bytes(reader, meta_header); + TypeMeta::skip_bytes(reader, meta_header)?; } else { let type_meta = Arc::new(TypeMeta::from_bytes_with_header( reader, type_resolver, meta_header, - )); + )?); if self.parsed_type_defs.len() < MAX_PARSED_NUM_TYPE_DEFS { // avoid malicious type defs to OOM parsed_type_defs self.parsed_type_defs.insert(meta_header, type_meta.clone()); @@ -99,22 +102,22 @@ impl MetaReaderResolver { self.reading_type_defs.push(type_meta); } } - reader.get_cursor() + Ok(reader.get_cursor()) } - pub fn read_metastring(&self, reader: &mut Reader) -> MetaString { - let len = reader.read_varuint32() as usize; + pub fn read_metastring(&self, reader: &mut Reader) -> Result { + let len = reader.read_varuint32()? as usize; if len == 0 { - return MetaString { + return Ok(MetaString { bytes: vec![], encoding: Encoding::Utf8, original: String::new(), strip_last_char: false, special_char1: '\0', special_char2: '\0', - }; + }); } - let bytes = reader.read_bytes(len); + let bytes = reader.read_bytes(len)?; let encoding_byte = bytes[0] & 0x07; let encoding = match encoding_byte { 0x00 => Encoding::Utf8, @@ -124,7 +127,7 @@ impl MetaReaderResolver { 0x04 => Encoding::AllToLowerSpecial, _ => Encoding::Utf8, }; - NAMESPACE_DECODER.decode(bytes, encoding).unwrap() + NAMESPACE_DECODER.decode(bytes, encoding) } pub fn reset(&mut self) { diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs b/rust/fory-core/src/resolver/metastring_resolver.rs index 6e5f63cfd2..ddc206927f 100644 --- a/rust/fory-core/src/resolver/metastring_resolver.rs +++ b/rust/fory-core/src/resolver/metastring_resolver.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::convert::TryInto; use crate::buffer::Writer; +use crate::error::Error; use crate::meta::murmurhash3_x64_128; use crate::meta::{Encoding, MetaString}; use crate::Reader; @@ -78,7 +79,7 @@ impl MetaStringBytes { String::from_utf8_lossy(&self.bytes).into_owned() } - pub(crate) fn from_metastring(meta_string: &MetaString) -> Self { + pub(crate) fn from_metastring(meta_string: &MetaString) -> Result { let mut bytes = meta_string.bytes.to_vec(); let mut hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64; hash_code = hash_code.abs(); @@ -95,10 +96,18 @@ impl MetaStringBytes { if bytes.len() < 16 { bytes.resize(16, 0); } - let first8 = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); - let second8 = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); - Self::new(bytes, hash_code, encoding, first8, second8) + let first8: [u8; 8] = bytes[0..8].try_into().map_err(|_| { + Error::InvalidData(format!("expected at least 8 bytes, got {}", bytes.len()).into()) + })?; + let first8 = u64::from_le_bytes(first8); + + let second8: [u8; 8] = bytes[8..16].try_into().map_err(|_| { + Error::InvalidData(format!("expected at least 16 bytes, got {}", bytes.len()).into()) + })?; + let second8 = u64::from_le_bytes(second8); + + Ok(Self::new(bytes, hash_code, encoding, first8, second8)) } } @@ -169,8 +178,12 @@ impl MetaStringWriterResolver { } } - pub fn write_meta_string_bytes(&mut self, writer: &mut Writer, ms: &MetaString) { - let mut mb = MetaStringBytes::from_metastring(ms); + pub fn write_meta_string_bytes( + &mut self, + writer: &mut Writer, + ms: &MetaString, + ) -> Result<(), Error> { + let mut mb = MetaStringBytes::from_metastring(ms)?; let id = mb.dynamic_write_id; if id == MetaStringBytes::DEFAULT_DYNAMIC_WRITE_STRING_ID { let id_usize = self.dynamic_write_id; @@ -193,6 +206,7 @@ impl MetaStringWriterResolver { let header = ((id as u32 + 1) << 1) | 1; writer.write_varuint32(header); } + Ok(()) } pub fn reset_write(&mut self) { @@ -235,48 +249,51 @@ impl MetaStringReaderResolver { &mut self, reader: &mut Reader, header: u32, - ) -> MetaStringBytes { + ) -> Result { let len = (header >> 2) as usize; if (header & 0b10) == 0 { if len <= Self::SMALL_STRING_THRESHOLD { - let mb = self.read_small_meta_string_bytes(reader, len); + let mb = self.read_small_meta_string_bytes(reader, len)?; self.update_dynamic_string(mb.clone()); - mb + Ok(mb) } else { - let hash_code = reader.read_i64(); - let mb = self.read_big_meta_string_bytes(reader, len, hash_code); + let hash_code = reader.read_i64()?; + let mb = self.read_big_meta_string_bytes(reader, len, hash_code)?; self.update_dynamic_string(mb.clone()); - mb + Ok(mb) } } else { let idx = len - 1; - self.dynamic_read[idx] - .as_ref() - .expect("dynamic id not found") - .clone() + self.dynamic_read + .get(idx) + .and_then(|opt| opt.clone()) + .ok_or_else(|| Error::InvalidData("dynamic id not found".into())) } } - pub fn read_meta_string_bytes(&mut self, reader: &mut Reader) -> MetaStringBytes { - let header = reader.read_varuint32(); + pub fn read_meta_string_bytes( + &mut self, + reader: &mut Reader, + ) -> Result { + let header = reader.read_varuint32()?; let len = (header >> 1) as usize; if (header & 0b1) == 0 { if len > Self::SMALL_STRING_THRESHOLD { - let hash_code = reader.read_i64(); - let mb = self.read_big_meta_string_bytes(reader, len, hash_code); + let hash_code = reader.read_i64()?; + let mb = self.read_big_meta_string_bytes(reader, len, hash_code)?; self.update_dynamic_string(mb.clone()); - mb + Ok(mb) } else { - let mb = self.read_small_meta_string_bytes(reader, len); + let mb = self.read_small_meta_string_bytes(reader, len)?; self.update_dynamic_string(mb.clone()); - mb + Ok(mb) } } else { let idx = len - 1; - self.dynamic_read[idx] - .as_ref() - .expect("dynamic id not found") - .clone() + self.dynamic_read + .get(idx) + .and_then(|opt| opt.clone()) + .ok_or_else(|| Error::InvalidData("dynamic id not found".into())) } } @@ -285,12 +302,12 @@ impl MetaStringReaderResolver { reader: &mut Reader, len: usize, hash_code: i64, - ) -> MetaStringBytes { + ) -> Result { if let Some(existing) = self.hash_to_meta.get(&hash_code) { - reader.skip(len as u32); - existing.clone() + reader.skip(len)?; + Ok(existing.clone()) } else { - let bytes = reader.read_bytes(len).to_vec(); + let bytes = reader.read_bytes(len)?.to_vec(); let mut first8 = 0; let mut second8 = 0; for (i, b) in bytes.iter().enumerate().take(8) { @@ -303,27 +320,31 @@ impl MetaStringReaderResolver { } let mb = MetaStringBytes::new(bytes, hash_code, Encoding::Utf8, first8, second8); self.hash_to_meta.insert(hash_code, mb.clone()); - mb + Ok(mb) } } - fn read_small_meta_string_bytes(&mut self, reader: &mut Reader, len: usize) -> MetaStringBytes { - let encoding_val = reader.read_u8(); + fn read_small_meta_string_bytes( + &mut self, + reader: &mut Reader, + len: usize, + ) -> Result { + let encoding_val = reader.read_u8()?; if len == 0 { debug_assert_eq!(encoding_val, Encoding::Utf8 as i16 as u8); - return MetaStringBytes::EMPTY.clone(); + return Ok(MetaStringBytes::EMPTY.clone()); } let (v1, v2) = if len <= 8 { - let v1 = Self::read_bytes_as_u64(reader, len); + let v1 = Self::read_bytes_as_u64(reader, len)?; (v1, 0) } else { - let v1 = reader.read_i64() as u64; - let v2 = Self::read_bytes_as_u64(reader, len - 8); + let v1 = reader.read_i64()? as u64; + let v2 = Self::read_bytes_as_u64(reader, len - 8)?; (v1, v2) }; let key = (v1, v2, encoding_val); if let Some(existing) = self.small_map.get(&key) { - existing.clone() + Ok(existing.clone()) } else { let mut data = Vec::with_capacity(len); for i in 0..usize::min(8, len) { @@ -346,17 +367,17 @@ impl MetaStringReaderResolver { v2, ); self.small_map.insert(key, mb.clone()); - mb + Ok(mb) } } - fn read_bytes_as_u64(reader: &mut Reader, len: usize) -> u64 { + fn read_bytes_as_u64(reader: &mut Reader, len: usize) -> Result { let mut v = 0; - let slice = reader.read_bytes(len); + let slice = reader.read_bytes(len)?; for (i, b) in slice.iter().take(len).enumerate() { v |= (*b as u64) << (8 * i); } - v + Ok(v) } fn update_dynamic_string(&mut self, mb: MetaStringBytes) { @@ -377,14 +398,14 @@ impl MetaStringReaderResolver { } } - pub fn read_meta_string(&mut self, reader: &mut Reader) -> String { - let mb = self.read_meta_string_bytes(reader); - if let Some(s) = self.meta_string_bytes_to_string.get(&mb) { + pub fn read_meta_string(&mut self, reader: &mut Reader) -> Result { + let mb = self.read_meta_string_bytes(reader)?; + Ok(if let Some(s) = self.meta_string_bytes_to_string.get(&mb) { s.clone() } else { let s = mb.decode_lossy(); self.meta_string_bytes_to_string.insert(mb, s.clone()); s - } + }) } } diff --git a/rust/fory-core/src/resolver/ref_resolver.rs b/rust/fory-core/src/resolver/ref_resolver.rs index 77433d48b5..3ea8ad6c05 100644 --- a/rust/fory-core/src/resolver/ref_resolver.rs +++ b/rust/fory-core/src/resolver/ref_resolver.rs @@ -16,6 +16,7 @@ // under the License. use crate::buffer::{Reader, Writer}; +use crate::error::Error; use crate::types::RefFlag; use std::any::Any; use std::collections::HashMap; @@ -288,15 +289,17 @@ impl RefReader { /// # Panics /// /// Panics if an invalid reference flag value is encountered - pub fn read_ref_flag(&self, reader: &mut Reader) -> RefFlag { - let flag_value = reader.read_i8(); - match flag_value { + pub fn read_ref_flag(&self, reader: &mut Reader) -> Result { + let flag_value = reader.read_i8()?; + Ok(match flag_value { -3 => RefFlag::Null, -2 => RefFlag::Ref, -1 => RefFlag::NotNullValue, 0 => RefFlag::RefValue, - _ => panic!("Invalid reference flag: {}", flag_value), - } + _ => Err(Error::InvalidRef( + format!("Invalid reference flag: {}", flag_value).into(), + ))?, + }) } /// Read a reference ID from the reader. @@ -308,7 +311,7 @@ impl RefReader { /// # Returns /// /// The reference ID as a u32 - pub fn read_ref_id(&self, reader: &mut Reader) -> u32 { + pub fn read_ref_id(&self, reader: &mut Reader) -> Result { reader.read_u32() } diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index e9b64f345e..7cd89cd66e 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -27,7 +27,7 @@ use crate::Reader; use std::sync::Arc; use std::{any::Any, collections::HashMap}; -type WriteFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool); +type WriteFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool) -> Result<(), Error>; type ReadFn = fn( fory: &Fory, &mut ReadContext, @@ -35,7 +35,8 @@ type ReadFn = fn( skip_ref_flag: bool, ) -> Result, Error>; -type WriteDataFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool); +type WriteDataFn = + fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool) -> Result<(), Error>; type ReadDataFn = fn(fory: &Fory, &mut ReadContext, is_field: bool) -> Result, Error>; type ToSerializerFn = fn(Box) -> Result, Error>; @@ -103,28 +104,26 @@ impl TypeInfo { namespace: &str, type_name: &str, register_by_name: bool, - ) -> TypeInfo { - let namespace_metastring = NAMESPACE_ENCODER - .encode_with_encodings(namespace, NAMESPACE_ENCODINGS) - .unwrap(); - let type_name_metastring = TYPE_NAME_ENCODER - .encode_with_encodings(type_name, TYPE_NAME_ENCODINGS) - .unwrap(); + ) -> Result { + let namespace_metastring = + NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; + let type_name_metastring = + TYPE_NAME_ENCODER.encode_with_encodings(type_name, TYPE_NAME_ENCODINGS)?; let (type_def_bytes, type_meta) = T::fory_type_def( fory, type_id, namespace_metastring.clone(), type_name_metastring.clone(), register_by_name, - ); - TypeInfo { + )?; + Ok(TypeInfo { type_def: Arc::from(type_def_bytes), type_meta: Arc::new(type_meta), type_id, namespace: namespace_metastring, type_name: type_name_metastring, register_by_name, - } + }) } pub fn new_with_empty_fields( @@ -133,13 +132,11 @@ impl TypeInfo { namespace: &str, type_name: &str, register_by_name: bool, - ) -> TypeInfo { - let namespace_metastring = NAMESPACE_ENCODER - .encode_with_encodings(namespace, NAMESPACE_ENCODINGS) - .unwrap(); - let type_name_metastring = TYPE_NAME_ENCODER - .encode_with_encodings(type_name, TYPE_NAME_ENCODINGS) - .unwrap(); + ) -> Result { + let namespace_metastring = + NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; + let type_name_metastring = + TYPE_NAME_ENCODER.encode_with_encodings(type_name, TYPE_NAME_ENCODINGS)?; let meta = TypeMeta::from_fields( type_id, namespace_metastring.clone(), @@ -147,17 +144,17 @@ impl TypeInfo { register_by_name, vec![], ); - let type_def = meta.to_bytes().unwrap(); + let type_def = meta.to_bytes()?; let type_resolver = _fory.get_type_resolver(); - let meta = TypeMeta::from_bytes(&mut Reader::new(&type_def), type_resolver); - TypeInfo { + let meta = TypeMeta::from_bytes(&mut Reader::new(&type_def), type_resolver)?; + Ok(TypeInfo { type_def: Arc::from(type_def), type_meta: Arc::new(meta), type_id, namespace: namespace_metastring, type_name: type_name_metastring, register_by_name, - } + }) } pub fn get_type_id(&self) -> u32 { @@ -211,14 +208,16 @@ impl Default for TypeResolver { type_info_map_by_name: HashMap::new(), type_id_index: Vec::new(), }; - resolver.register_builtin_types(); + resolver.register_builtin_types().unwrap(); resolver } } impl TypeResolver { - fn register_builtin_types(&mut self) { + fn register_builtin_types(&mut self) -> Result<(), Error> { use crate::types::TypeId; + let namespace = NAMESPACE_ENCODER.encode_with_encodings("", NAMESPACE_ENCODINGS)?; + let type_name = TYPE_NAME_ENCODER.encode_with_encodings("", TYPE_NAME_ENCODINGS)?; macro_rules! register_basic_type { ($ty:ty, $type_id:expr) => {{ @@ -226,15 +225,11 @@ impl TypeResolver { type_def: Arc::from(vec![]), type_meta: Arc::new(TypeMeta::empty()), type_id: $type_id as u32, - namespace: NAMESPACE_ENCODER - .encode_with_encodings("", NAMESPACE_ENCODINGS) - .unwrap(), - type_name: TYPE_NAME_ENCODER - .encode_with_encodings("", TYPE_NAME_ENCODINGS) - .unwrap(), + namespace: namespace.clone(), + type_name: type_name.clone(), register_by_name: false, }; - self.register_serializer::<$ty>(&type_info); + self.register_serializer::<$ty>(&type_info)?; }}; } @@ -254,15 +249,16 @@ impl TypeResolver { register_basic_type!(Vec, TypeId::INT64_ARRAY); register_basic_type!(Vec, TypeId::FLOAT32_ARRAY); register_basic_type!(Vec, TypeId::FLOAT64_ARRAY); + + Ok(()) } - pub fn get_type_info(&self, type_id: std::any::TypeId) -> &TypeInfo { - self.type_info_cache.get(&type_id).unwrap_or_else(|| { - panic!( + pub fn get_type_info(&self, type_id: std::any::TypeId) -> Result<&TypeInfo, Error> { + self.type_info_cache.get(&type_id) + .ok_or_else(|| Error::TypeError(format!( "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", type_id - ) - }) + ).into())) } pub fn get_type_info_by_id(&self, id: u32) -> Option<&TypeInfo> { @@ -275,34 +271,37 @@ impl TypeResolver { } /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) - pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> u32 { + pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> Result { let id_usize = id as usize; if id_usize < self.type_id_index.len() { let type_id = self.type_id_index[id_usize]; if type_id != NO_TYPE_ID { - return type_id; + return Ok(type_id); } } - panic!( - "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", - type_id - ) + Err(Error::TypeError( + format!( + "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", + type_id + ) + .into(), + )) } pub fn register( &mut self, type_info: &TypeInfo, - ) { + ) -> Result<(), Error> { fn write( this: &dyn Any, fory: &Fory, context: &mut WriteContext, is_field: bool, - ) { + ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { Some(v) => { - let skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory); + let skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory)?; crate::serializer::write_ref_info_data( v, fory, @@ -310,7 +309,8 @@ impl TypeResolver { is_field, skip_ref_flag, true, - ); + )?; + Ok(()) } None => todo!(), } @@ -339,12 +339,10 @@ impl TypeResolver { fory: &Fory, context: &mut WriteContext, is_field: bool, - ) { + ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => { - T2::fory_write_data(v, fory, context, is_field); - } + Some(v) => T2::fory_write_data(v, fory, context, is_field), None => todo!(), } } @@ -365,15 +363,17 @@ impl TypeResolver { ) -> Result, Error> { match boxed_any.downcast::() { Ok(concrete) => Ok(Box::new(*concrete) as Box), - Err(_) => Err(Error::Other(anyhow::anyhow!( - "Failed to downcast to concrete type" - ))), + Err(_) => Err(Error::TypeError( + "Failed to downcast to concrete type".into(), + )), } } let rs_type_id = std::any::TypeId::of::(); if self.type_info_cache.contains_key(&rs_type_id) { - panic!("rs_struct:{:?} already registered", rs_type_id); + return Err(Error::TypeError( + format!("rs_struct:{:?} already registered", rs_type_id).into(), + )); } self.type_info_cache.insert(rs_type_id, type_info.clone()); self.type_info_map_by_id @@ -381,8 +381,10 @@ impl TypeResolver { let index = T::fory_type_index() as usize; if index >= self.type_id_index.len() { self.type_id_index.resize(index + 1, NO_TYPE_ID); - } else if self.type_id_index.get(index).unwrap() != &NO_TYPE_ID { - panic!("please:{:?} already registered", type_info.type_id); + } else if self.type_id_index[index] != NO_TYPE_ID { + return Err(Error::TypeError( + format!("please:{:?} already registered", type_info.type_id).into(), + )); } self.type_id_index[index] = type_info.type_id; @@ -391,10 +393,13 @@ impl TypeResolver { let type_name = &type_info.type_name; let key = (namespace.clone(), type_name.clone()); if self.name_serializer_map.contains_key(&key) { - panic!( - "Namespace:{:?} Name:{:?} already registered_by_name", - namespace, type_name - ); + return Err(Error::InvalidData( + format!( + "Namespace:{:?} Name:{:?} already registered_by_name", + namespace, type_name + ) + .into(), + )); } self.type_name_map.insert(rs_type_id, key.clone()); self.name_serializer_map.insert( @@ -413,7 +418,9 @@ impl TypeResolver { } else { let type_id = type_info.type_id; if self.serializer_map.contains_key(&type_id) { - panic!("TypeId {:?} already registered_by_id", type_id); + return Err(Error::TypeError( + format!("TypeId {:?} already registered_by_id", type_id).into(), + )); } self.type_id_map.insert(rs_type_id, type_id); self.serializer_map.insert( @@ -427,20 +434,22 @@ impl TypeResolver { )), ); } + Ok(()) } - pub fn register_serializer(&mut self, type_info: &TypeInfo) { + pub fn register_serializer( + &mut self, + type_info: &TypeInfo, + ) -> Result<(), Error> { fn write( this: &dyn Any, fory: &Fory, context: &mut WriteContext, is_field: bool, - ) { + ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => { - v.fory_write(fory, context, is_field); - } + Some(v) => Ok(v.fory_write(fory, context, is_field)?), None => todo!(), } } @@ -469,12 +478,10 @@ impl TypeResolver { fory: &Fory, context: &mut WriteContext, is_field: bool, - ) { + ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => { - T2::fory_write_data(v, fory, context, is_field); - } + Some(v) => T2::fory_write_data(v, fory, context, is_field), None => todo!(), } } @@ -495,15 +502,17 @@ impl TypeResolver { ) -> Result, Error> { match boxed_any.downcast::() { Ok(concrete) => Ok(Box::new(*concrete) as Box), - Err(_) => Err(Error::Other(anyhow::anyhow!( - "Failed to downcast to concrete type" - ))), + Err(_) => Err(Error::TypeError( + "Failed to downcast to concrete type".into(), + )), } } let rs_type_id = std::any::TypeId::of::(); if self.type_info_cache.contains_key(&rs_type_id) { - panic!("rs_struct:{:?} already registered", rs_type_id); + return Err(Error::TypeError( + format!("rs_struct:{:?} already registered", rs_type_id).into(), + )); } self.type_info_cache.insert(rs_type_id, type_info.clone()); if type_info.register_by_name { @@ -511,10 +520,13 @@ impl TypeResolver { let type_name = &type_info.type_name; let key = (namespace.clone(), type_name.clone()); if self.name_serializer_map.contains_key(&key) { - panic!( - "Namespace:{:?} Name:{:?} already registered_by_name", - namespace, type_name - ); + return Err(Error::InvalidData( + format!( + "Namespace:{:?} Name:{:?} already registered_by_name", + namespace, type_name + ) + .into(), + )); } self.type_name_map.insert(rs_type_id, key.clone()); self.name_serializer_map.insert( @@ -530,7 +542,9 @@ impl TypeResolver { } else { let type_id = type_info.type_id; if self.serializer_map.contains_key(&type_id) { - panic!("TypeId {:?} already registered_by_id", type_id); + return Err(Error::TypeError( + format!("TypeId {:?} already registered_by_id", type_id).into(), + )); } self.type_id_map.insert(rs_type_id, type_id); self.serializer_map.insert( @@ -544,6 +558,7 @@ impl TypeResolver { )), ); } + Ok(()) } pub fn get_harness(&self, id: u32) -> Option> { @@ -559,17 +574,22 @@ impl TypeResolver { self.name_serializer_map.get(&key).cloned() } - pub fn get_ext_harness(&self, id: u32) -> &Harness { + pub fn get_ext_harness(&self, id: u32) -> Result, Error> { self.serializer_map .get(&id) - .unwrap_or_else(|| panic!("ext type must be registered in both peers")) + .cloned() + .ok_or_else(|| Error::TypeError("ext type must be registered in both peers".into())) } - pub fn get_ext_name_harness(&self, namespace: &MetaString, type_name: &MetaString) -> &Harness { + pub fn get_ext_name_harness( + &self, + namespace: &MetaString, + type_name: &MetaString, + ) -> Result, Error> { let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map - .get(&key) - .expect("named_ext type must be registered in both peers") + self.name_serializer_map.get(&key).cloned().ok_or_else(|| { + Error::TypeError("named_ext type must be registered in both peers".into()) + }) } pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { diff --git a/rust/fory-core/src/row/row.rs b/rust/fory-core/src/row/row.rs index 587b202e5e..69795ce52d 100644 --- a/rust/fory-core/src/row/row.rs +++ b/rust/fory-core/src/row/row.rs @@ -17,7 +17,6 @@ use crate::util::EPOCH; use crate::{buffer::Writer, error::Error}; -use anyhow::anyhow; use byteorder::{ByteOrder, LittleEndian}; use chrono::{DateTime, Days, NaiveDate, NaiveDateTime}; use std::collections::BTreeMap; @@ -31,7 +30,7 @@ use super::{ pub trait Row<'a> { type ReadResult; - fn write(v: &Self, writer: &mut Writer); + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error>; fn cast(bytes: &'a [u8]) -> Self::ReadResult; } @@ -45,8 +44,9 @@ macro_rules! impl_row_for_number { impl<'a> Row<'a> for $tt { type ReadResult = Self; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { $writer(writer, *v); + Ok(()) } fn cast(bytes: &[u8]) -> Self::ReadResult { @@ -65,8 +65,9 @@ impl_row_for_number!(f64, Writer::write_f64, LittleEndian::read_f64); impl<'a> Row<'a> for String { type ReadResult = &'a str; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { writer.write_bytes(v.as_bytes()); + Ok(()) } fn cast(bytes: &'a [u8]) -> Self::ReadResult { @@ -77,8 +78,9 @@ impl<'a> Row<'a> for String { impl Row<'_> for bool { type ReadResult = Self; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { writer.write_u8(if *v { 1 } else { 0 }); + Ok(()) } fn cast(bytes: &[u8]) -> Self::ReadResult { @@ -89,43 +91,46 @@ impl Row<'_> for bool { impl Row<'_> for NaiveDate { type ReadResult = Result; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { let days_since_epoch = v.signed_duration_since(EPOCH).num_days(); writer.write_u32(days_since_epoch as u32); + Ok(()) } fn cast(bytes: &[u8]) -> Self::ReadResult { let days = LittleEndian::read_u32(bytes); EPOCH .checked_add_days(Days::new(days.into())) - .ok_or(Error::from(anyhow!( - "Date out of range, {days} days since epoch" - ))) + .ok_or(Error::InvalidData( + format!("Date out of range, {days} days since epoch").into(), + )) } } impl Row<'_> for NaiveDateTime { type ReadResult = Result; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { writer.write_i64(v.and_utc().timestamp_millis()); + Ok(()) } fn cast(bytes: &[u8]) -> Self::ReadResult { let timestamp = LittleEndian::read_u64(bytes); DateTime::from_timestamp_millis(timestamp as i64) .map(|dt| dt.naive_utc()) - .ok_or(Error::from(anyhow!( - "Date out of range, timestamp:{timestamp}" - ))) + .ok_or(Error::InvalidData( + format!("Date out of range, timestamp:{timestamp}").into(), + )) } } impl<'a> Row<'a> for Vec { type ReadResult = &'a [u8]; - fn write(v: &Self, writer: &mut Writer) { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { writer.write_bytes(v); + Ok(()) } fn cast(bytes: &'a [u8]) -> Self::ReadResult { @@ -155,13 +160,14 @@ impl<'a, T: Row<'a>> ArrayGetter<'a, T> { impl<'a, T: Row<'a>> Row<'a> for Vec { type ReadResult = ArrayGetter<'a, T>; - fn write(v: &Self, writer: &mut Writer) { - let mut array_writer = ArrayWriter::new(v.len(), writer); - v.iter().enumerate().for_each(|(idx, item)| { + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { + let mut array_writer = ArrayWriter::new(v.len(), writer)?; + for (idx, item) in v.iter().enumerate() { let callback_info = array_writer.write_start(idx); - ::write(item, array_writer.get_writer()); + ::write(item, array_writer.get_writer())?; array_writer.write_end(callback_info); - }); + } + Ok(()) } fn cast(row: &'a [u8]) -> Self::ReadResult { @@ -215,26 +221,27 @@ impl<'a, T1: Row<'a> + Ord, T2: Row<'a> + Ord> MapGetter<'a, T1, T2> { impl<'a, T1: Row<'a> + Ord, T2: Row<'a> + Ord> Row<'a> for BTreeMap { type ReadResult = MapGetter<'a, T1, T2>; - fn write(v: &Self, writer: &mut Writer) { - let mut map_writter = MapWriter::new(writer); + fn write(v: &Self, writer: &mut Writer) -> Result<(), Error> { + let mut map_writer = MapWriter::new(writer); { - let callback_info = map_writter.write_start(0); - let mut array_writer = ArrayWriter::new(v.len(), map_writter.get_writer()); - v.keys().enumerate().for_each(|(idx, item)| { + let callback_info = map_writer.write_start(0); + let mut array_writer = ArrayWriter::new(v.len(), map_writer.get_writer())?; + for (idx, item) in v.keys().enumerate() { let callback_info = array_writer.write_start(idx); - ::write(item, array_writer.get_writer()); + ::write(item, array_writer.get_writer())?; array_writer.write_end(callback_info); - }); - map_writter.write_end(callback_info); + } + map_writer.write_end(callback_info); } { - let mut array_writer = ArrayWriter::new(v.len(), map_writter.get_writer()); - v.values().enumerate().for_each(|(idx, item)| { + let mut array_writer = ArrayWriter::new(v.len(), map_writer.get_writer())?; + for (idx, item) in v.values().enumerate() { let callback_info = array_writer.write_start(idx); - ::write(item, array_writer.get_writer()); + ::write(item, array_writer.get_writer())?; array_writer.write_end(callback_info); - }); + } } + Ok(()) } fn cast(row: &'a [u8]) -> Self::ReadResult { diff --git a/rust/fory-core/src/row/writer.rs b/rust/fory-core/src/row/writer.rs index 58e71c6feb..1a5de4757e 100644 --- a/rust/fory-core/src/row/writer.rs +++ b/rust/fory-core/src/row/writer.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::buffer::Writer; - use super::{bit_util::calculate_bitmap_width_in_bytes, row::Row}; +use crate::buffer::Writer; +use crate::error::Error; pub struct WriteCallbackInfo { field_offset: usize, @@ -110,7 +110,7 @@ impl ArrayWriter<'_> { 8 + bit_map_width_in_bytes + num_fields * 8 } - pub fn new(num_fields: usize, writer: &mut Writer) -> ArrayWriter<'_> { + pub fn new(num_fields: usize, writer: &mut Writer) -> Result, Error> { let base_offset = writer.len(); let bit_map_width_in_bytes = calculate_bitmap_width_in_bytes(num_fields); let array_writer = ArrayWriter { @@ -127,7 +127,7 @@ impl ArrayWriter<'_> { .writer .write_u64(num_fields as u64); array_writer.field_writer_helper.writer.skip(fixed_size - 8); - array_writer + Ok(array_writer) } pub fn get_writer(&mut self) -> &mut Writer { @@ -181,8 +181,8 @@ impl MapWriter<'_> { } } -pub fn to_row<'a, T: Row<'a>>(v: &T) -> Vec { +pub fn to_row<'a, T: Row<'a>>(v: &T) -> Result, Error> { let mut writer = Writer::default(); - T::write(v, &mut writer); - writer.dump() + T::write(v, &mut writer)?; + Ok(writer.dump()) } diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index 745c687b3d..52e63675f9 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -30,25 +30,25 @@ pub fn serialize_any_box( fory: &Fory, context: &mut WriteContext, is_field: bool, -) { +) -> Result<(), Error> { context.writer.write_i8(RefFlag::NotNullValue as i8); let concrete_type_id = (**any_box).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**any_box, fory, context, is_field); + serializer_fn(&**any_box, fory, context, is_field) } /// Helper function to deserialize to `Box` pub fn deserialize_any_box(fory: &Fory, context: &mut ReadContext) -> Result, Error> { context.inc_depth()?; - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag != RefFlag::NotNullValue as i8 { - return Err(Error::Other(anyhow::anyhow!( - "Expected NotNullValue for Box" - ))); + return Err(Error::InvalidRef( + "Expected NotNullValue for Box".into(), + )); } - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let result = deserializer_fn(fory, context, true); context.dec_depth(); @@ -62,12 +62,22 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - serialize_any_box(self, fory, context, is_field); + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + serialize_any_box(self, fory, context, is_field) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - serialize_any_box(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + serialize_any_box(self, fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { @@ -82,27 +92,37 @@ impl Serializer for Box { deserialize_any_box(fory, context) } - fn fory_get_type_id(_fory: &Fory) -> u32 { - panic!("Box has no static type ID - use fory_type_id_dyn") + fn fory_get_type_id(_fory: &Fory) -> Result { + unreachable!("Box has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { let concrete_type_id = (**self).type_id(); fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered") + .ok_or_else(|| Error::TypeError("Type not registered".into())) } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { - // Box is polymorphic - type info is written per element + fn fory_write_type_info( + _fory: &Fory, + _context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { + // Rc is polymorphic - type info is written per element + Ok(()) } - fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { - // Box is polymorphic - type info is read per element + fn fory_read_type_info( + _fory: &Fory, + _context: &mut ReadContext, + _is_field: bool, + ) -> Result<(), Error> { + // Rc is polymorphic - type info is read per element + Ok(()) } fn as_any(&self) -> &dyn Any { @@ -117,39 +137,52 @@ impl ForyDefault for Rc { } impl Serializer for Rc { - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if !context .ref_writer .try_write_rc_ref(&mut context.writer, self) { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, fory, context, is_field); - } + serializer_fn(&**self, fory, context, is_field)? + }; + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - self.fory_write(fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + self.fory_write(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { - RefFlag::Null => Err(anyhow::anyhow!("Rc cannot be null").into()), + RefFlag::Null => Err(Error::InvalidRef("Rc cannot be null".into())), RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; context .ref_reader .get_rc_ref::(ref_id) .ok_or_else(|| { - anyhow::anyhow!("Rc reference {} not found", ref_id).into() + Error::InvalidData( + format!("Rc reference {} not found", ref_id).into(), + ) }) } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); @@ -157,7 +190,7 @@ impl Serializer for Rc { } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); @@ -176,27 +209,37 @@ impl Serializer for Rc { Self::fory_read(fory, context, is_field) } - fn fory_get_type_id(_fory: &Fory) -> u32 { - panic!("Rc has no static type ID - use fory_type_id_dyn") + fn fory_get_type_id(_fory: &Fory) -> Result { + unreachable!("Rc has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { let concrete_type_id = (**self).type_id(); fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered") + .ok_or_else(|| Error::TypeError("Type not registered".into())) } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info( + _fory: &Fory, + _context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { // Rc is polymorphic - type info is written per element + Ok(()) } - fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + _context: &mut ReadContext, + _is_field: bool, + ) -> Result<(), Error> { // Rc is polymorphic - type info is read per element + Ok(()) } fn as_any(&self) -> &dyn Any { @@ -211,39 +254,52 @@ impl ForyDefault for Arc { } impl Serializer for Arc { - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if !context .ref_writer .try_write_arc_ref(&mut context.writer, self) { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, fory, context, is_field); + serializer_fn(&**self, fory, context, is_field)?; } + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - self.fory_write(fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + self.fory_write(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { - RefFlag::Null => Err(anyhow::anyhow!("Arc cannot be null").into()), + RefFlag::Null => Err(Error::InvalidRef("Arc cannot be null".into())), RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; context .ref_reader .get_arc_ref::(ref_id) .ok_or_else(|| { - anyhow::anyhow!("Arc reference {} not found", ref_id).into() + Error::InvalidData( + format!("Arc reference {} not found", ref_id).into(), + ) }) } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); @@ -251,7 +307,7 @@ impl Serializer for Arc { } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed = deserializer_fn(fory, context, true)?; context.dec_depth(); @@ -270,27 +326,37 @@ impl Serializer for Arc { Self::fory_read(fory, context, is_field) } - fn fory_get_type_id(_fory: &Fory) -> u32 { - panic!("Arc has no static type ID - use fory_type_id_dyn") + fn fory_get_type_id(_fory: &Fory) -> Result { + unreachable!("Arc has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { let concrete_type_id = (**self).type_id(); fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered") + .ok_or_else(|| Error::TypeError("Type not registered".into())) } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info( + _fory: &Fory, + _context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { // Arc is polymorphic - type info is written per element + Ok(()) } - fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + _context: &mut ReadContext, + _is_field: bool, + ) -> Result<(), Error> { // Arc is polymorphic - type info is read per element + Ok(()) } fn as_any(&self) -> &dyn Any { diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index cc0a0bc875..41377a98ac 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -20,7 +20,6 @@ use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; -use anyhow::anyhow; use std::sync::Arc; impl Serializer for Arc { @@ -28,50 +27,67 @@ impl Serializer for Arc true } - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if !context .ref_writer .try_write_arc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), fory, context, is_field); - } + T::fory_write_data(self.as_ref(), fory, context, is_field)? + }; + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // When Arc is nested inside another shared ref (like Rc>), // the outer ref calls fory_write_data on the inner Arc. // We still need to track the Arc's own references here. - self.fory_write(fory, context, is_field); + self.fory_write(fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; - match ref_flag { - RefFlag::Null => Err(anyhow!("Arc cannot be null").into()), + Ok(match ref_flag { + RefFlag::Null => Err(Error::InvalidRef("Arc cannot be null".into()))?, RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; context .ref_reader .get_arc_ref::(ref_id) - .ok_or_else(|| anyhow!("Arc reference {} not found", ref_id).into()) + .ok_or(Error::InvalidData( + format!("Arc reference {ref_id} not found").into(), + ))? } RefFlag::NotNullValue => { let inner = T::fory_read_data(fory, context, is_field)?; - Ok(Arc::new(inner)) + Arc::new(inner) } RefFlag::RefValue => { let ref_id = context.ref_reader.reserve_ref_id(); let inner = T::fory_read_data(fory, context, is_field)?; let arc = Arc::new(inner); context.ref_reader.store_arc_ref_at(ref_id, arc.clone()); - Ok(arc) + arc } - } + }) } fn fory_read_data( @@ -84,8 +100,12 @@ impl Serializer for Arc Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -94,11 +114,11 @@ impl Serializer for Arc 4 } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { (**self).fory_type_id_dyn(fory) } diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index a890816d92..7095038427 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -25,8 +25,14 @@ use std::mem; impl Serializer for bool { #[inline(always)] - fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + _fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { context.writer.write_u8(if *self { 1 } else { 0 }); + Ok(()) } #[inline(always)] @@ -35,7 +41,7 @@ impl Serializer for bool { context: &mut ReadContext, _is_field: bool, ) -> Result { - Ok(context.reader.read_u8() == 1) + Ok(context.reader.read_u8()? == 1) } #[inline(always)] @@ -44,12 +50,12 @@ impl Serializer for bool { } #[inline(always)] - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::BOOL as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::BOOL as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::BOOL as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::BOOL as u32) } #[inline(always)] @@ -58,13 +64,21 @@ impl Serializer for bool { } #[inline(always)] - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } #[inline(always)] - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } diff --git a/rust/fory-core/src/serializer/box_.rs b/rust/fory-core/src/serializer/box_.rs index 88c4c1893a..63063649d0 100644 --- a/rust/fory-core/src/serializer/box_.rs +++ b/rust/fory-core/src/serializer/box_.rs @@ -30,27 +30,40 @@ impl Serializer for Box { Ok(Box::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { T::fory_write_data(self.as_ref(), fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { T::fory_reserved_space() } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { (**self).fory_type_id_dyn(fory) } diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 488bc84ee0..45938f4beb 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -20,7 +20,7 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; use crate::types::PRIMITIVE_ARRAY_TYPES; -use crate::Fory; +use crate::{ensure, Fory}; // const TRACKING_REF: u8 = 0b1; @@ -36,14 +36,20 @@ pub fn write_collection_type_info( context: &mut WriteContext, is_field: bool, collection_type_id: u32, -) { +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } context.writer.write_varuint32(collection_type_id); + Ok(()) } -pub fn write_collection<'a, T, I>(iter: I, fory: &Fory, context: &mut WriteContext, is_field: bool) +pub fn write_collection<'a, T, I>( + iter: I, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) -> Result<(), Error> where T: Serializer + 'a, I: IntoIterator, @@ -53,7 +59,7 @@ where let len = iter.len(); context.writer.write_varuint32(len as u32); if len == 0 { - return; + return Ok(()); } let mut header = 0; let mut has_null = false; @@ -77,13 +83,14 @@ where header |= IS_SAME_TYPE; } context.writer.write_u8(header); - T::fory_write_type_info(fory, context, is_field); + T::fory_write_type_info(fory, context, is_field)?; // context.writer.reserve((T::reserved_space() + SIZE_OF_REF_AND_TYPE) * len); if T::fory_is_polymorphic() || T::fory_is_shared_ref() { // TOTO: make it xlang compatible for item in iter { - item.fory_write(fory, context, is_field); + item.fory_write(fory, context, is_field)?; } + Ok(()) } else { // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); let skip_ref_flag = is_same_type && !has_null; @@ -95,8 +102,9 @@ where is_field, skip_ref_flag, true, - ); + )?; } + Ok(()) } } @@ -104,16 +112,24 @@ pub fn read_collection_type_info( context: &mut ReadContext, is_field: bool, collection_type_id: u32, -) { +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let remote_collection_type_id = context.reader.read_varuint32(); - assert_eq!(collection_type_id, remote_collection_type_id); + let remote_collection_type_id = context.reader.read_varuint32()?; if PRIMITIVE_ARRAY_TYPES.contains(&remote_collection_type_id) { - panic!("Vec belongs to the `number_array` type, and Vec> belongs to the `list` type. You should not read data of type `number_array` as data of type `list`"); + return Err(Error::TypeError( + "Vec belongs to the `number_array` type, \ + and Vec> belongs to the `list` type. \ + You should not read data of type `number_array` as data of type `list`." + .into(), + )); } - assert_eq!(remote_collection_type_id, collection_type_id); + ensure!( + collection_type_id == remote_collection_type_id, + Error::TypeMismatch(collection_type_id, remote_collection_type_id) + ); + Ok(()) } pub fn read_collection(fory: &Fory, context: &mut ReadContext) -> Result @@ -121,13 +137,13 @@ where T: Serializer + ForyDefault, C: FromIterator, { - let len = context.reader.read_varuint32(); + let len = context.reader.read_varuint32()?; if len == 0 { return Ok(C::from_iter(std::iter::empty())); } - let header = context.reader.read_u8(); + let header = context.reader.read_u8()?; let declared = (header & DECL_ELEMENT_TYPE) != 0; - T::fory_read_type_info(fory, context, declared); + T::fory_read_type_info(fory, context, declared)?; let has_null = (header & HAS_NULL) != 0; let is_same_type = (header & IS_SAME_TYPE) != 0; if T::fory_is_polymorphic() || T::fory_is_shared_ref() { diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs index d709227e30..4c49e26cc0 100644 --- a/rust/fory-core/src/serializer/datetime.rs +++ b/rust/fory-core/src/serializer/datetime.rs @@ -23,15 +23,20 @@ use crate::serializer::Serializer; use crate::serializer::{read_type_info, write_type_info, ForyDefault}; use crate::types::TypeId; use crate::util::EPOCH; -use anyhow::anyhow; use chrono::{DateTime, Days, NaiveDate, NaiveDateTime}; use std::mem; impl Serializer for NaiveDateTime { - fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + _fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { let dt = self.and_utc(); let micros = dt.timestamp() * 1_000_000 + dt.timestamp_subsec_micros() as i64; context.writer.write_i64(micros); + Ok(()) } fn fory_read_data( @@ -39,46 +44,60 @@ impl Serializer for NaiveDateTime { context: &mut ReadContext, _is_field: bool, ) -> Result { - let micros = context.reader.read_i64(); + let micros = context.reader.read_i64()?; let seconds = micros / 1_000_000; let subsec_micros = (micros % 1_000_000) as u32; let nanos = subsec_micros * 1_000; DateTime::from_timestamp(seconds, nanos) .map(|dt| dt.naive_utc()) - .ok_or(Error::from(anyhow!( - "Date out of range, timestamp micros: {micros}" - ))) + .ok_or(Error::InvalidData( + format!("Date out of range, timestamp micros: {micros}").into(), + )) } fn fory_reserved_space() -> usize { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::TIMESTAMP as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::TIMESTAMP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::TIMESTAMP as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::TIMESTAMP as u32) } fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } impl Serializer for NaiveDate { - fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + _fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { let days_since_epoch = self.signed_duration_since(EPOCH).num_days(); context.writer.write_i32(days_since_epoch as i32); + Ok(()) } fn fory_read_data( @@ -86,36 +105,44 @@ impl Serializer for NaiveDate { context: &mut ReadContext, _is_field: bool, ) -> Result { - let days = context.reader.read_i32(); + let days = context.reader.read_i32()?; EPOCH .checked_add_days(Days::new(days as u64)) - .ok_or(Error::from(anyhow!( - "Date out of range, {days} days since epoch" - ))) + .ok_or(Error::InvalidData( + format!("Date out of range, {days} days since epoch").into(), + )) } fn fory_reserved_space() -> usize { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::LOCAL_DATE as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::LOCAL_DATE as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::LOCAL_DATE as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::LOCAL_DATE as u32) } fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index a9ccbb4a6e..60d6930fb9 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta}; @@ -45,47 +46,60 @@ pub fn type_def( } #[inline(always)] -pub fn write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { +pub fn write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let type_id = T::fory_get_type_id(fory); + let type_id = T::fory_get_type_id(fory)?; context.writer.write_varuint32(type_id); let is_named_enum = type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { - return; + return Ok(()); } let rs_type_id = std::any::TypeId::of::(); if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id) as u32; + let meta_index = context.push_meta(fory, rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = fory.get_type_resolver().get_type_info(rs_type_id); + let type_info = fory.get_type_resolver().get_type_info(rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - context.write_meta_string_bytes(&namespace); - context.write_meta_string_bytes(&type_name); + context.write_meta_string_bytes(&namespace)?; + context.write_meta_string_bytes(&type_name)?; } + Ok(()) } #[inline(always)] -pub fn read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { +pub fn read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let local_type_id = T::fory_get_type_id(fory); - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(local_type_id, remote_type_id); + let local_type_id = T::fory_get_type_id(fory)?; + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); let is_named_enum = local_type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { - return; + return Ok(()); } if fory.is_share_meta() { - let _meta_index = context.reader.read_varuint32(); + let _meta_index = context.reader.read_varuint32()?; } else { - let _namespace_msb = context.read_meta_string_bytes(); - let _type_name_msb = context.read_meta_string_bytes(); + let _namespace_msb = context.read_meta_string_bytes()?; + let _type_name_msb = context.read_meta_string_bytes()?; } + Ok(()) } #[inline(always)] @@ -93,15 +107,20 @@ pub fn read_compatible( fory: &Fory, context: &mut ReadContext, ) -> Result { - T::fory_read_type_info(fory, context, true); + T::fory_read_type_info(fory, context, true)?; T::fory_read_data(fory, context, true) } #[inline(always)] -pub fn write(this: &T, fory: &Fory, context: &mut WriteContext, is_field: bool) { +pub fn write( + this: &T, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) -> Result<(), Error> { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, is_field); - this.fory_write_data(fory, context, is_field); + T::fory_write_type_info(fory, context, is_field)?; + this.fory_write_data(fory, context, is_field) } #[inline(always)] @@ -110,11 +129,11 @@ pub fn read( context: &mut ReadContext, is_field: bool, ) -> Result { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag == RefFlag::Null as i8 { Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { - T::fory_read_type_info(fory, context, false); + T::fory_read_type_info(fory, context, false)?; T::fory_read_data(fory, context, is_field) } else { unimplemented!() diff --git a/rust/fory-core/src/serializer/heap.rs b/rust/fory-core/src/serializer/heap.rs index f61b6095c2..94532ec07b 100644 --- a/rust/fory-core/src/serializer/heap.rs +++ b/rust/fory-core/src/serializer/heap.rs @@ -29,12 +29,21 @@ use std::collections::BinaryHeap; use std::mem; impl Serializer for BinaryHeap { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection(self, fory, context, is_field) } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection_type_info(context, is_field, TypeId::SET as u32); + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection_type_info(context, is_field, TypeId::SET as u32) } fn fory_read_data( @@ -45,7 +54,11 @@ impl Serializer for BinaryHeap { read_collection(fory, context) } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -53,12 +66,12 @@ impl Serializer for BinaryHeap { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs index 61b6d99042..399ad8c511 100644 --- a/rust/fory-core/src/serializer/list.rs +++ b/rust/fory-core/src/serializer/list.rs @@ -44,25 +44,26 @@ fn check_primitive() -> Option { } impl Serializer for Vec { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { match check_primitive::() { - Some(_) => { - primitive_list::fory_write_data(self, context); - } - None => { - write_collection(self, fory, context, is_field); - } + Some(_) => primitive_list::fory_write_data(self, context), + None => write_collection(self, fory, context, is_field), } } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { match check_primitive::() { - Some(type_id) => { - primitive_list::fory_write_type_info(context, is_field, type_id); - } - None => { - write_collection_type_info(context, is_field, TypeId::LIST as u32); - } + Some(type_id) => primitive_list::fory_write_type_info(context, is_field, type_id), + None => write_collection_type_info(context, is_field, TypeId::LIST as u32), } } @@ -77,7 +78,11 @@ impl Serializer for Vec { } } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { match check_primitive::() { Some(type_id) => primitive_list::fory_read_type_info(context, is_field, type_id), None => read_collection_type_info(context, is_field, TypeId::LIST as u32), @@ -94,18 +99,18 @@ impl Serializer for Vec { } } - fn fory_get_type_id(_fory: &Fory) -> u32 { - match check_primitive::() { + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(match check_primitive::() { Some(type_id) => type_id as u32, None => TypeId::LIST as u32, - } + }) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - match check_primitive::() { + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(match check_primitive::() { Some(type_id) => type_id as u32, None => TypeId::LIST as u32, - } + }) } fn as_any(&self) -> &dyn std::any::Any { @@ -120,12 +125,21 @@ impl ForyDefault for Vec { } impl Serializer for VecDeque { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection(self, fory, context, is_field) } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection_type_info(context, is_field, TypeId::LIST as u32); + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection_type_info(context, is_field, TypeId::LIST as u32) } fn fory_read_data( @@ -136,20 +150,24 @@ impl Serializer for VecDeque { read_collection(fory, context) } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_collection_type_info(context, is_field, TypeId::LIST as u32); + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_collection_type_info(context, is_field, TypeId::LIST as u32) } fn fory_reserved_space() -> usize { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::LIST as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::LIST as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::LIST as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::LIST as u32) } fn as_any(&self) -> &dyn std::any::Any { @@ -164,12 +182,21 @@ impl ForyDefault for VecDeque { } impl Serializer for LinkedList { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection(self, fory, context, is_field) } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection_type_info(context, is_field, TypeId::LIST as u32); + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection_type_info(context, is_field, TypeId::LIST as u32) } fn fory_read_data( @@ -180,20 +207,24 @@ impl Serializer for LinkedList { read_collection(fory, context) } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_collection_type_info(context, is_field, TypeId::LIST as u32); + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_collection_type_info(context, is_field, TypeId::LIST as u32) } fn fory_reserved_space() -> usize { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::LIST as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::LIST as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::LIST as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::LIST as u32) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index f7b650a82f..27045f482a 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; @@ -24,7 +25,6 @@ use crate::serializer::{ }; use crate::types::{TypeId, SIZE_OF_REF_AND_TYPE}; use std::collections::{BTreeMap, HashMap}; -use std::mem; const MAX_CHUNK_SIZE: u8 = 255; @@ -41,16 +41,16 @@ fn check_and_write_null( is_field: bool, key: &K, value: &V, -) -> bool { +) -> Result { if key.fory_is_none() && value.fory_is_none() { context.writer.write_u8(KEY_NULL | VALUE_NULL); - return true; + return Ok(true); } if key.fory_is_none() { let mut chunk_header = KEY_NULL; let skip_ref_flag; if is_field { - skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory); + skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory)?; chunk_header |= DECL_VALUE_TYPE; } else { skip_ref_flag = false; @@ -58,8 +58,8 @@ fn check_and_write_null( } context.writer.write_u8(chunk_header); - write_ref_info_data(value, fory, context, is_field, skip_ref_flag, false); - return true; + write_ref_info_data(value, fory, context, is_field, skip_ref_flag, false)?; + return Ok(true); } if value.fory_is_none() { let mut chunk_header = VALUE_NULL; @@ -73,10 +73,10 @@ fn check_and_write_null( chunk_header |= TRACKING_KEY_REF; } context.writer.write_u8(chunk_header); - write_ref_info_data(key, fory, context, is_field, skip_ref_flag, false); - return true; + write_ref_info_data(key, fory, context, is_field, skip_ref_flag, false)?; + return Ok(true); } - false + Ok(false) } fn write_chunk_size(context: &mut WriteContext, header_offset: usize, size: u8) { @@ -89,14 +89,15 @@ fn write_map_data<'a, K, V, I>( fory: &Fory, context: &mut WriteContext, is_field: bool, -) where +) -> Result<(), Error> +where K: Serializer + ForyDefault + 'a + Eq + std::hash::Hash, V: Serializer + ForyDefault + 'a, I: Iterator, { context.writer.write_varuint32(length as u32); if length == 0 { - return; + return Ok(()); } let reserved_space = (K::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) * length + (V::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) * length; @@ -109,7 +110,7 @@ fn write_map_data<'a, K, V, I>( let mut skip_val_ref_flag = false; for (key, value) in iter { if need_write_header { - if check_and_write_null(fory, context, is_field, key, value) { + if check_and_write_null(fory, context, is_field, key, value)? { continue; } header_offset = context.writer.len(); @@ -126,8 +127,8 @@ fn write_map_data<'a, K, V, I>( if !skip_val_ref_flag { chunk_header |= TRACKING_VALUE_REF; } - K::fory_write_type_info(fory, context, is_field); - V::fory_write_type_info(fory, context, is_field); + K::fory_write_type_info(fory, context, is_field)?; + V::fory_write_type_info(fory, context, is_field)?; context.writer.set_bytes(header_offset, &[chunk_header]); need_write_header = false; } @@ -135,18 +136,18 @@ fn write_map_data<'a, K, V, I>( write_chunk_size(context, header_offset, pair_counter); pair_counter = 0; need_write_header = true; - check_and_write_null(fory, context, is_field, key, value); + check_and_write_null(fory, context, is_field, key, value)?; continue; } if K::fory_is_polymorphic() || K::fory_is_shared_ref() { - key.fory_write(fory, context, is_field); + key.fory_write(fory, context, is_field)?; } else { - write_ref_info_data(key, fory, context, is_field, skip_key_ref_flag, true); + write_ref_info_data(key, fory, context, is_field, skip_key_ref_flag, true)?; } if V::fory_is_polymorphic() || V::fory_is_shared_ref() { - value.fory_write(fory, context, is_field); + value.fory_write(fory, context, is_field)?; } else { - write_ref_info_data(value, fory, context, is_field, skip_val_ref_flag, true); + write_ref_info_data(value, fory, context, is_field, skip_val_ref_flag, true)?; } pair_counter += 1; if pair_counter == MAX_CHUNK_SIZE { @@ -158,13 +159,19 @@ fn write_map_data<'a, K, V, I>( if pair_counter > 0 { write_chunk_size(context, header_offset, pair_counter); } + Ok(()) } impl Serializer for HashMap { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_map_data(self.iter(), self.len(), fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), fory, context, is_field) } fn fory_read_data( @@ -172,7 +179,7 @@ impl Result { - let len = context.reader.read_varuint32(); + let len = context.reader.read_varuint32()?; let mut map = HashMap::::with_capacity(len as usize); if len == 0 { return Ok(map); @@ -182,7 +189,7 @@ impl(fory) + crate::serializer::get_skip_ref_flag::(fory)? } else { false }; @@ -204,7 +211,7 @@ impl(fory) + crate::serializer::get_skip_ref_flag::(fory)? } else { false }; @@ -213,9 +220,17 @@ impl usize { - mem::size_of::() + size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::MAP as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::MAP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::MAP as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::MAP as u32) } fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } @@ -271,8 +294,13 @@ impl ForyDefault for HashMap { impl Serializer for BTreeMap { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_map_data(self.iter(), self.len(), fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), fory, context, is_field) } fn fory_read_data( @@ -280,7 +308,7 @@ impl Result { - let len = context.reader.read_varuint32(); + let len = context.reader.read_varuint32()?; let mut map = BTreeMap::::new(); if len == 0 { return Ok(map); @@ -290,7 +318,7 @@ impl(fory) + crate::serializer::get_skip_ref_flag::(fory)? } else { false }; @@ -312,7 +340,7 @@ impl(fory) + crate::serializer::get_skip_ref_flag::(fory)? } else { false }; @@ -321,9 +349,9 @@ impl usize { - mem::size_of::() + size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::MAP as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::MAP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::MAP as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::MAP as u32) } fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 418ae87483..3962f70a87 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta, NAMESPACE_DECODER, TYPE_NAME_DECODER}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::types::{RefFlag, TypeId, PRIMITIVE_TYPES}; -use anyhow::anyhow; use std::any::Any; pub mod any; @@ -54,7 +54,7 @@ pub fn write_ref_info_data( is_field: bool, skip_ref_flag: bool, skip_type_info: bool, -) { +) -> Result<(), Error> { if record.fory_is_none() { context.writer.write_i8(RefFlag::Null as i8); } else { @@ -62,10 +62,11 @@ pub fn write_ref_info_data( context.writer.write_i8(RefFlag::NotNullValue as i8); } if !skip_type_info { - T::fory_write_type_info(fory, context, is_field); + T::fory_write_type_info(fory, context, is_field)?; } - record.fory_write_data(fory, context, is_field); + record.fory_write_data(fory, context, is_field)?; } + Ok(()) } #[inline(always)] @@ -77,18 +78,18 @@ pub fn read_ref_info_data( skip_type_info: bool, ) -> Result { if !skip_ref_flag { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag == RefFlag::Null as i8 { Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { if !skip_type_info { - T::fory_read_type_info(fory, context, is_field); + T::fory_read_type_info(fory, context, is_field)?; } T::fory_read_data(fory, context, is_field) } else if ref_flag == (RefFlag::RefValue as i8) { // First time seeing this referenceable object if !skip_type_info { - T::fory_read_type_info(fory, context, is_field); + T::fory_read_type_info(fory, context, is_field)?; } T::fory_read_data(fory, context, is_field) } else if ref_flag == (RefFlag::Ref as i8) { @@ -100,35 +101,48 @@ pub fn read_ref_info_data( } } else { if !skip_type_info { - T::fory_read_type_info(fory, context, is_field); + T::fory_read_type_info(fory, context, is_field)?; } T::fory_read_data(fory, context, is_field) } } #[inline(always)] -fn write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { +fn write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let type_id = T::fory_get_type_id(fory); + let type_id = T::fory_get_type_id(fory)?; context.writer.write_varuint32(type_id); + Ok(()) } #[inline(always)] -fn read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { +fn read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let local_type_id = T::fory_get_type_id(fory); - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(local_type_id, remote_type_id); + let local_type_id = T::fory_get_type_id(fory)?; + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); + Ok(()) } #[inline(always)] -pub fn get_skip_ref_flag(fory: &Fory) -> bool { - let elem_type_id = T::fory_get_type_id(fory); - !T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id) +pub fn get_skip_ref_flag(fory: &Fory) -> Result { + let elem_type_id = T::fory_get_type_id(fory)?; + Ok(!T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id)) } pub trait ForyDefault: Sized { @@ -146,11 +160,16 @@ pub trait ForyDefault: Sized { pub trait Serializer: 'static { /// Entry point of the serialization. - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> where Self: Sized, { - write_ref_info_data(self, fory, context, is_field, false, false); + write_ref_info_data(self, fory, context, is_field, false, false) } fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result @@ -185,16 +204,17 @@ pub trait Serializer: 'static { false } - fn fory_get_type_id(fory: &Fory) -> u32 + fn fory_get_type_id(fory: &Fory) -> Result where Self: Sized, { - fory.get_type_resolver() - .get_type_info(std::any::TypeId::of::()) - .get_type_id() + Ok(fory + .get_type_resolver() + .get_type_info(std::any::TypeId::of::())? + .get_type_id()) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32; + fn fory_type_id_dyn(&self, fory: &Fory) -> Result; /// The possible max memory size of the type. /// Used to reserve the buffer space to avoid reallocation, which may hurt performance. @@ -205,49 +225,62 @@ pub trait Serializer: 'static { 0 } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, _is_field: bool) + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> where Self: Sized, { // default implementation only for ext/named_ext - let type_id = Self::fory_get_type_id(fory); + let type_id = Self::fory_get_type_id(fory)?; context.writer.write_varuint32(type_id); if type_id & 0xff == TypeId::EXT as u32 { - return; + return Ok(()); } let rs_type_id = std::any::TypeId::of::(); if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id) as u32; + let meta_index = context.push_meta(fory, rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { let type_info = { let type_resolver = fory.get_type_resolver(); - type_resolver.get_type_info(rs_type_id) + type_resolver.get_type_info(rs_type_id)? }; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - context.write_meta_string_bytes(&namespace); - context.write_meta_string_bytes(&type_name); + context.write_meta_string_bytes(&namespace)?; + context.write_meta_string_bytes(&type_name)?; } + Ok(()) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, _is_field: bool) + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, + ) -> Result<(), Error> where Self: Sized, { // default implementation only for ext/named_ext - let local_type_id = Self::fory_get_type_id(fory); - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(local_type_id, remote_type_id); + let local_type_id = Self::fory_get_type_id(fory)?; + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); if local_type_id & 0xff == TypeId::EXT as u32 { - return; + return Ok(()); } if fory.is_share_meta() { - let _meta_index = context.reader.read_varuint32(); + let _meta_index = context.reader.read_varuint32()?; } else { - let _namespace_msb = context.read_meta_string_bytes(); - let _type_name_msb = context.read_meta_string_bytes(); + let _namespace_msb = context.read_meta_string_bytes()?; + let _type_name_msb = context.read_meta_string_bytes()?; } + Ok(()) } // only used by struct/enum/ext @@ -256,45 +289,53 @@ pub trait Serializer: 'static { Self: Sized, { // default logic only for ext/named_ext - let remote_type_id = context.reader.read_varuint32(); - let local_type_id = Self::fory_get_type_id(fory); - assert_eq!(remote_type_id, local_type_id); + let remote_type_id = context.reader.read_varuint32()?; + let local_type_id = Self::fory_get_type_id(fory)?; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); if local_type_id & 0xff == TypeId::EXT as u32 { let type_resolver = fory.get_type_resolver(); type_resolver - .get_ext_harness(local_type_id) + .get_ext_harness(local_type_id)? .get_read_data_fn()(fory, context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) - .map_err(|_| anyhow!("downcast to Self failed").into()) + .map_err(|_| Error::TypeError("downcast to Self failed".into())) }) } else { let (namespace, type_name) = if fory.is_share_meta() { - let meta_index = context.reader.read_varuint32(); - let type_def = context.get_meta(meta_index as usize); - (type_def.get_namespace(), type_def.get_type_name()) + let meta_index = context.reader.read_varuint32()?; + let type_meta = context.get_meta(meta_index as usize); + (type_meta.get_namespace(), type_meta.get_type_name()) } else { - let nsb = context.read_meta_string_bytes(); - let tsb = context.read_meta_string_bytes(); + let nsb = context.read_meta_string_bytes()?; + let tsb = context.read_meta_string_bytes()?; let ns = NAMESPACE_DECODER.decode(&nsb.bytes, nsb.encoding)?; let ts = TYPE_NAME_DECODER.decode(&tsb.bytes, tsb.encoding)?; (ns, ts) }; let type_resolver = fory.get_type_resolver(); type_resolver - .get_ext_name_harness(&namespace, &type_name) + .get_ext_name_harness(&namespace, &type_name)? .get_read_data_fn()(fory, context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) - .map_err(|_| anyhow!("downcast to Self failed").into()) + .map_err(|_| Error::TypeError("downcast to Self failed".into())) }) } } /// Write/Read the data into the buffer. Need to be implemented. - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error>; fn fory_read_data( fory: &Fory, @@ -318,8 +359,8 @@ pub trait StructSerializer: Serializer + 'static { _namespace: MetaString, _type_name: MetaString, _register_by_name: bool, - ) -> (Vec, TypeMeta) { - (Vec::default(), TypeMeta::empty()) + ) -> Result<(Vec, TypeMeta), Error> { + Ok((Vec::default(), TypeMeta::empty())) } fn fory_type_index() -> u32 { diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index a16e22f717..0d0fcf0984 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -52,21 +52,35 @@ use std::sync::Mutex; /// Simply delegates to the serializer for `T`, allowing thread-safe interior mutable /// containers to be included in serialized graphs. impl Serializer for Mutex { - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // Don't add ref tracking for Mutex itself, just delegate to inner type // The inner type will handle its own ref tracking let guard = self.lock().unwrap(); - T::fory_write(&*guard, fory, context, is_field); + T::fory_write(&*guard, fory, context, is_field) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // When called from Rc/Arc, just delegate to inner type's data serialization let guard = self.lock().unwrap(); - T::fory_write_data(&*guard, fory, context, is_field); + T::fory_write_data(&*guard, fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -89,15 +103,19 @@ impl Serializer for Mutex { Ok(Mutex::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { let guard = self.lock().unwrap(); (*guard).fory_type_id_dyn(fory) } diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index e3646cb118..abe4f62390 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -27,8 +27,14 @@ macro_rules! impl_num_serializer { ($ty:ty, $writer:expr, $reader:expr, $field_type:expr) => { impl Serializer for $ty { #[inline] - fn fory_write_data(&self, _fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + _fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { $writer(&mut context.writer, *self); + Ok(()) } #[inline] @@ -37,7 +43,7 @@ macro_rules! impl_num_serializer { context: &mut ReadContext, _is_field: bool, ) -> Result { - Ok($reader(&mut context.reader)) + $reader(&mut context.reader) } #[inline] @@ -46,12 +52,12 @@ macro_rules! impl_num_serializer { } #[inline] - fn fory_get_type_id(_fory: &Fory) -> u32 { - $field_type as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok($field_type as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - $field_type as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok($field_type as u32) } #[inline] @@ -60,13 +66,21 @@ macro_rules! impl_num_serializer { } #[inline] - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } #[inline] - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } impl ForyDefault for $ty { diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 168fdc9046..6ea9c58c21 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -32,12 +32,21 @@ impl Serializer for Option { } #[inline(always)] - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } #[inline(always)] - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if let Some(v) = self { T::fory_write_data(v, fory, context, is_field) } else { @@ -46,8 +55,12 @@ impl Serializer for Option { } #[inline(always)] - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } #[inline(always)] @@ -56,12 +69,12 @@ impl Serializer for Option { } #[inline(always)] - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } #[inline(always)] - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { match self { Some(val) => val.fory_type_id_dyn(fory), None => T::fory_get_type_id(fory), diff --git a/rust/fory-core/src/serializer/primitive_list.rs b/rust/fory-core/src/serializer/primitive_list.rs index 736305e236..41b19161a9 100644 --- a/rust/fory-core/src/serializer/primitive_list.rs +++ b/rust/fory-core/src/serializer/primitive_list.rs @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::types::TypeId; -pub fn fory_write_data(this: &[T], context: &mut WriteContext) { +pub fn fory_write_data(this: &[T], context: &mut WriteContext) -> Result<(), Error> { let len_bytes = std::mem::size_of_val(this); context.writer.write_varuint32(len_bytes as u32); context.writer.reserve(len_bytes); @@ -32,40 +33,60 @@ pub fn fory_write_data(this: &[T], context: &mut WriteContext) { context.writer.write_bytes(slice); } } + Ok(()) } -pub fn fory_write_type_info(context: &mut WriteContext, is_field: bool, type_id: TypeId) { +pub fn fory_write_type_info( + context: &mut WriteContext, + is_field: bool, + type_id: TypeId, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } context.writer.write_varuint32(type_id as u32); + Ok(()) } pub fn fory_read_data(context: &mut ReadContext) -> Result, Error> { - let size_bytes = context.reader.read_varuint32() as usize; + let size_bytes = context.reader.read_varuint32()? as usize; if size_bytes % std::mem::size_of::() != 0 { - panic!("Invalid data length"); + return Err(Error::InvalidData("Invalid data length".into())); } let len = size_bytes / std::mem::size_of::(); let mut vec: Vec = Vec::with_capacity(len); unsafe { let dst_ptr = vec.as_mut_ptr() as *mut u8; - let src = context.reader.read_bytes(size_bytes); + let src = context.reader.read_bytes(size_bytes)?; std::ptr::copy_nonoverlapping(src.as_ptr(), dst_ptr, size_bytes); vec.set_len(len); } Ok(vec) } -pub fn fory_read_type_info(context: &mut ReadContext, is_field: bool, type_id: TypeId) { +pub fn fory_read_type_info( + context: &mut ReadContext, + is_field: bool, + type_id: TypeId, +) -> Result<(), Error> { if is_field { - return; + return Ok(()); } - let remote_type_id = context.reader.read_varuint32(); + let remote_type_id = context.reader.read_varuint32()?; if remote_type_id == TypeId::LIST as u32 { - panic!("Vec belongs to the `number_array` type, and Vec> belongs to the `list` type. You should not read data of type `list` as data of type `number_array`"); + return Err(Error::TypeError( + "Vec belongs to the `number_array` type, \ + and Vec> belongs to the `list` type. \ + You should not read data of type `list` as data of type `number_array`." + .into(), + )); } - assert_eq!(remote_type_id, type_id as u32); + let local_type_id = type_id as u32; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); + Ok(()) } pub fn fory_reserved_space() -> usize { diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index 5d3c3716ff..86953ce683 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -20,7 +20,6 @@ use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; -use anyhow::anyhow; use std::rc::Rc; impl Serializer for Rc { @@ -28,37 +27,50 @@ impl Serializer for Rc { true } - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if !context .ref_writer .try_write_rc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), fory, context, is_field); - } + T::fory_write_data(self.as_ref(), fory, context, is_field)? + }; + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // When Rc is nested inside another shared ref (like Arc>), // the outer ref calls fory_write_data on the inner Rc. // We still need to track the Rc's own references here. - self.fory_write(fory, context, is_field); + self.fory_write(fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); - + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { - RefFlag::Null => Err(anyhow!("Rc cannot be null").into()), + RefFlag::Null => Err(Error::InvalidRef("Rc cannot be null".into())), RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); - context - .ref_reader - .get_rc_ref::(ref_id) - .ok_or_else(|| anyhow!("Rc reference {} not found", ref_id).into()) + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + context.ref_reader.get_rc_ref::(ref_id).ok_or_else(|| { + Error::InvalidRef(format!("Rc reference {ref_id} not found").into()) + }) } RefFlag::NotNullValue => { let inner = T::fory_read_data(fory, context, is_field)?; @@ -84,8 +96,12 @@ impl Serializer for Rc { Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -94,11 +110,11 @@ impl Serializer for Rc { 4 } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { (**self).fory_type_id_dyn(fory) } diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index 9fff58be7b..7dcba16607 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -58,23 +58,41 @@ impl Serializer for RefCell { Ok(RefCell::new(T::fory_read_data(fory, context, is_field)?)) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // Don't add ref tracking for RefCell itself, just delegate to inner type // The inner type will handle its own ref tracking - T::fory_write(&*self.borrow(), fory, context, is_field); + T::fory_write(&*self.borrow(), fory, context, is_field) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { // When called from Rc, just delegate to inner type's data serialization T::fory_write_data(&*self.borrow(), fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -82,11 +100,11 @@ impl Serializer for RefCell { T::fory_reserved_space() } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { (*self.borrow()).fory_type_id_dyn(fory) } diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs index 7755cbaf7e..fc50b958d9 100644 --- a/rust/fory-core/src/serializer/set.rs +++ b/rust/fory-core/src/serializer/set.rs @@ -29,12 +29,21 @@ use std::collections::{BTreeSet, HashSet}; use std::mem; impl Serializer for HashSet { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection(self, fory, context, is_field) } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection_type_info(context, is_field, TypeId::SET as u32); + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection_type_info(context, is_field, TypeId::SET as u32) } fn fory_read_data( @@ -45,7 +54,11 @@ impl Serializer for HashSet< read_collection(fory, context) } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -53,12 +66,12 @@ impl Serializer for HashSet< mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } fn as_any(&self) -> &dyn std::any::Any { @@ -73,12 +86,21 @@ impl ForyDefault for HashSet { } impl Serializer for BTreeSet { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection(self, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection(self, fory, context, is_field) } - fn fory_write_type_info(_fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_collection_type_info(context, is_field, TypeId::SET as u32); + fn fory_write_type_info( + _fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_collection_type_info(context, is_field, TypeId::SET as u32) } fn fory_read_data( @@ -89,7 +111,11 @@ impl Serializer for BTreeSet { read_collection(fory, context) } - fn fory_read_type_info(_fory: &Fory, context: &mut ReadContext, is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -97,12 +123,12 @@ impl Serializer for BTreeSet { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::SET as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::SET as u32) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index 1dd658a682..aa9feda19c 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -21,7 +21,7 @@ use crate::resolver::context::ReadContext; use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE}; use crate::serializer::Serializer; use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPES}; -use crate::Fory; +use crate::{ensure, Fory}; use chrono::{NaiveDate, NaiveDateTime}; pub fn get_read_ref_flag(field_type: &FieldType) -> bool { @@ -33,7 +33,7 @@ macro_rules! basic_type_deserialize { ($fory:expr, $tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { $( if $tid == TypeId::$id { - <$ty as Serializer>::fory_read_type_info($fory, $context, true); + <$ty as Serializer>::fory_read_type_info($fory, $context, true)?; <$ty as Serializer>::fory_read_data($fory, $context, true)?; return Ok(()); } @@ -52,7 +52,7 @@ pub fn skip_field_value( read_ref_flag: bool, ) -> Result<(), Error> { if read_ref_flag { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if field_type.nullable && ref_flag == (RefFlag::Null as i8) { return Ok(()); } @@ -82,11 +82,11 @@ pub fn skip_field_value( ); } else if CONTAINER_TYPES.contains(&type_id) { if type_id == TypeId::LIST || type_id == TypeId::SET { - let length = context.reader.read_varuint32() as usize; + let length = context.reader.read_varuint32()? as usize; if length == 0 { return Ok(()); } - let header = context.reader.read_u8(); + let header = context.reader.read_u8()?; let has_null = (header & HAS_NULL) != 0; let is_same_type = (header & IS_SAME_TYPE) != 0; let skip_ref_flag = is_same_type && !has_null; @@ -97,7 +97,7 @@ pub fn skip_field_value( } context.dec_depth(); } else if type_id == TypeId::MAP { - let length = context.reader.read_varuint32(); + let length = context.reader.read_varuint32()?; if length == 0 { return Ok(()); } @@ -108,7 +108,7 @@ pub fn skip_field_value( if len_counter == length { break; } - let header = context.reader.read_u8(); + let header = context.reader.read_u8()?; if header & crate::serializer::map::KEY_NULL != 0 && header & crate::serializer::map::VALUE_NULL != 0 { @@ -131,7 +131,7 @@ pub fn skip_field_value( len_counter += 1; continue; } - let chunk_size = context.reader.read_u8(); + let chunk_size = context.reader.read_u8()?; context.inc_depth()?; for _ in (0..chunk_size).enumerate() { // let read_ref_flag = get_read_ref_flag(key_type); @@ -145,12 +145,15 @@ pub fn skip_field_value( } Ok(()) } else if type_id == TypeId::NAMED_ENUM { - let _ordinal = context.reader.read_varuint32(); + let _ordinal = context.reader.read_varuint32()?; Ok(()) } else if type_id == TypeId::NAMED_COMPATIBLE_STRUCT { - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(type_id_num, remote_type_id); - let meta_index = context.reader.read_varuint32(); + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + type_id_num == remote_type_id, + Error::TypeMismatch(type_id_num, remote_type_id) + ); + let meta_index = context.reader.read_varuint32()?; let type_meta = context.get_meta(meta_index as usize); let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; @@ -161,13 +164,16 @@ pub fn skip_field_value( context.dec_depth(); Ok(()) } else if type_id == TypeId::NAMED_EXT { - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(type_id_num, remote_type_id); - let meta_index = context.reader.read_varuint32(); + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + type_id_num == remote_type_id, + Error::TypeMismatch(type_id_num, remote_type_id) + ); + let meta_index = context.reader.read_varuint32()?; let type_meta = context.get_meta(meta_index as usize); let type_resolver = fory.get_type_resolver(); type_resolver - .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name()) + .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name())? .get_read_data_fn()(fory, context, true)?; Ok(()) } else { @@ -180,10 +186,13 @@ pub fn skip_field_value( const EXT_ID: u32 = TypeId::EXT as u32; const ENUM_ID: u32 = TypeId::ENUM as u32; if internal_id == COMPATIBLE_STRUCT_ID { - let remote_type_id = context.reader.read_varuint32(); - let meta_index = context.reader.read_varuint32(); + let remote_type_id = context.reader.read_varuint32()?; + let meta_index = context.reader.read_varuint32()?; let type_meta = context.get_meta(meta_index as usize); - assert_eq!(remote_type_id, type_meta.get_type_id()); + ensure!( + type_meta.get_type_id() == remote_type_id, + Error::TypeMismatch(type_meta.get_type_id(), remote_type_id) + ); let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { @@ -192,14 +201,17 @@ pub fn skip_field_value( } context.dec_depth(); } else if internal_id == ENUM_ID { - let _ordinal = context.reader.read_varuint32(); + let _ordinal = context.reader.read_varuint32()?; } else if internal_id == EXT_ID { - let remote_type_id = context.reader.read_varuint32(); - assert_eq!(remote_type_id, type_id_num); + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + type_id_num == remote_type_id, + Error::TypeMismatch(type_id_num, remote_type_id) + ); context.inc_depth()?; let type_resolver = fory.get_type_resolver(); type_resolver - .get_ext_harness(type_id_num) + .get_ext_harness(type_id_num)? .get_read_data_fn()(fory, context, true)?; context.dec_depth(); } else { diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index e6a7a0b699..a0ce9d02a3 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -32,7 +32,12 @@ enum StrEncoding { impl Serializer for String { #[inline] - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { let mut len = get_latin1_length(self); if len >= 0 { let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64; @@ -50,6 +55,7 @@ impl Serializer for String { context.writer.write_varuint36_small(bitor); context.writer.write_utf16_bytes(&utf16); } + Ok(()) } #[inline] @@ -58,7 +64,7 @@ impl Serializer for String { context: &mut ReadContext, _is_field: bool, ) -> Result { - let bitor = context.reader.read_varuint36small(); + let bitor = context.reader.read_varuint36small()?; let len = bitor >> 2; let encoding = bitor & 0b11; let encoding = match encoding { @@ -66,14 +72,16 @@ impl Serializer for String { 1 => StrEncoding::Utf16, 2 => StrEncoding::Utf8, _ => { - panic!("wrong encoding value: {}", encoding); + return Err(Error::EncodingError( + format!("wrong encoding value: {}", encoding).into(), + )) } }; let s = match encoding { StrEncoding::Latin1 => context.reader.read_latin1_string(len as usize), StrEncoding::Utf16 => context.reader.read_utf16_string(len as usize), StrEncoding::Utf8 => context.reader.read_utf8_string(len as usize), - }; + }?; Ok(s) } @@ -83,12 +91,12 @@ impl Serializer for String { } #[inline(always)] - fn fory_get_type_id(_fory: &Fory) -> u32 { - TypeId::STRING as u32 + fn fory_get_type_id(_fory: &Fory) -> Result { + Ok(TypeId::STRING as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> u32 { - TypeId::STRING as u32 + fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + Ok(TypeId::STRING as u32) } #[inline(always)] @@ -97,13 +105,21 @@ impl Serializer for String { } #[inline(always)] - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_type_info::(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_type_info::(fory, context, is_field) } #[inline(always)] - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - read_type_info::(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + read_type_info::(fory, context, is_field) } } diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index b827627f81..4f1003d8db 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; +use crate::error::Error; use crate::fory::Fory; use crate::meta::{FieldInfo, MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; @@ -58,7 +60,7 @@ pub fn type_def( } } if !found { - panic!("Field {} not found in field_infos", name); + unreachable!("Field {} not found in field_infos", name); } } // assign field id in ascending order @@ -77,60 +79,79 @@ pub fn type_def( } #[inline(always)] -pub fn write_type_info(fory: &Fory, context: &mut WriteContext, _is_field: bool) { - let type_id = T::fory_get_type_id(fory); +pub fn write_type_info( + fory: &Fory, + context: &mut WriteContext, + _is_field: bool, +) -> Result<(), Error> { + let type_id = T::fory_get_type_id(fory)?; context.writer.write_varuint32(type_id); let rs_type_id = std::any::TypeId::of::(); if type_id & 0xff == TypeId::NAMED_STRUCT as u32 { if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id) as u32; + let meta_index = context.push_meta(fory, rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = fory.get_type_resolver().get_type_info(rs_type_id); + let type_info = fory.get_type_resolver().get_type_info(rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); - context.write_meta_string_bytes(&namespace); - context.write_meta_string_bytes(&type_name); + context.write_meta_string_bytes(&namespace)?; + context.write_meta_string_bytes(&type_name)?; } } else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { - let meta_index = context.push_meta(fory, rs_type_id) as u32; + let meta_index = context.push_meta(fory, rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } + Ok(()) } #[inline(always)] -pub fn read_type_info(fory: &Fory, context: &mut ReadContext, _is_field: bool) { - let remote_type_id = context.reader.read_varuint32(); - let local_type_id = T::fory_get_type_id(fory); - assert_eq!(remote_type_id, local_type_id); +pub fn read_type_info( + fory: &Fory, + context: &mut ReadContext, + _is_field: bool, +) -> Result<(), Error> { + let remote_type_id = context.reader.read_varuint32()?; + let local_type_id = T::fory_get_type_id(fory)?; + ensure!( + local_type_id == remote_type_id, + Error::TypeMismatch(local_type_id, remote_type_id) + ); if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { if fory.is_share_meta() { - let _meta_index = context.reader.read_varuint32(); + let _meta_index = context.reader.read_varuint32()?; } else { - let _namespace_msb = context.read_meta_string_bytes(); - let _type_name_msb = context.read_meta_string_bytes(); + let _namespace_msb = context.read_meta_string_bytes()?; + let _type_name_msb = context.read_meta_string_bytes()?; } } else if local_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || local_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { let _meta_index = context.reader.read_varuint32(); } + Ok(()) } #[inline(always)] -pub fn write(this: &T, fory: &Fory, context: &mut WriteContext, _is_field: bool) { +pub fn write( + this: &T, + fory: &Fory, + context: &mut WriteContext, + _is_field: bool, +) -> Result<(), Error> { if fory.is_compatible() { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false); - this.fory_write_data(fory, context, true); + T::fory_write_type_info(fory, context, false)?; + this.fory_write_data(fory, context, true)?; } else { // currently same context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false); - this.fory_write_data(fory, context, true); + T::fory_write_type_info(fory, context, false)?; + this.fory_write_data(fory, context, true)?; } + Ok(()) } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 727f88b1e0..6d434f5527 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -28,7 +28,7 @@ pub fn write_trait_object_headers( context: &mut WriteContext, fory_type_id: u32, concrete_type_id: std::any::TypeId, -) { +) -> Result<(), Error> { use crate::types::{RefFlag, TypeId}; context.writer.write_i8(RefFlag::NotNullValue as i8); @@ -38,32 +38,31 @@ pub fn write_trait_object_headers( && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { - let meta_index = context.push_meta(fory, concrete_type_id) as u32; + let meta_index = context.push_meta(fory, concrete_type_id)? as u32; context.writer.write_varuint32(meta_index); - } + }; + Ok(()) } /// Reads common trait object headers and returns the type ID pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Result { use crate::types::{RefFlag, TypeId}; - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag != RefFlag::NotNullValue as i8 { - return Err(Error::Other(crate::error::AnyhowError::msg(format!( - "Expected NotNullValue ref flag, got {}", - ref_flag - )))); + return Err(Error::InvalidRef( + format!("Expected NotNullValue ref flag, got {ref_flag}").into(), + )); } - let fory_type_id = context.reader.read_varuint32(); + let fory_type_id = context.reader.read_varuint32()?; if fory.is_compatible() && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { - let _meta_index = context.reader.read_varuint32(); + let _meta_index = context.reader.read_varuint32()?; } - Ok(fory_type_id) } @@ -74,12 +73,12 @@ macro_rules! downcast_and_serialize { $( if $any_ref.type_id() == std::any::TypeId::of::<$impl_type>() { if let Some(concrete) = $any_ref.downcast_ref::<$impl_type>() { - concrete.fory_write_data($fory, $context, $is_field); - return; + concrete.fory_write_data($fory, $context, $is_field)?; + return Ok(()); } } )* - panic!("Failed to downcast to any registered type for trait {}", stringify!($trait_name)); + return Err($crate::error::Error::TypeError(format!("Failed to downcast to any registered type for trait {}", stringify!($trait_name)).into())); }}; } @@ -95,9 +94,9 @@ macro_rules! resolve_and_deserialize { } } )* - Err($crate::error::Error::Other($crate::error::AnyhowError::msg( - format!("Type ID {} not registered for trait {}", $fory_type_id, stringify!($trait_name)) - ))) + Err($crate::error::Error::TypeError( + format!("Type ID {} not registered for trait {}", $fory_type_id, stringify!($trait_name)).into() + )) }}; } @@ -201,41 +200,43 @@ macro_rules! register_trait_type { // 4. Serializer implementation for Box (existing functionality) impl $crate::serializer::Serializer for Box { - fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); if let Some(fory_type_id) = fory.get_type_resolver().get_fory_type_id(concrete_type_id) { - $crate::serializer::trait_object::write_trait_object_headers(fory, context, fory_type_id, concrete_type_id); + $crate::serializer::trait_object::write_trait_object_headers(fory, context, fory_type_id, concrete_type_id)?; $crate::downcast_and_serialize!(any_ref, fory, context, is_field, $trait_name, $($impl_type),+); } else { - panic!("Type {:?} not registered for Box serialization", concrete_type_id, stringify!($trait_name)); + return Err($crate::error::Error::TypeError(format!("Type {:?} not registered for Box serialization", concrete_type_id, stringify!($trait_name)).into())); } } - fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { // Delegate to fory_write since this handles the polymorphic dispatch - self.fory_write(fory, context, is_field); + self.fory_write(fory, context, is_field) } - fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> Result { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered for trait object") + .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { + fn fory_write_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { // Box is polymorphic - type info is written per element + Ok(()) } - fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { + fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { // Box is polymorphic - type info is read per element + Ok(()) } fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { @@ -256,8 +257,8 @@ macro_rules! register_trait_type { panic!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)); } - fn fory_get_type_id(_fory: &$crate::fory::Fory) -> u32 { - $crate::types::TypeId::STRUCT as u32 + fn fory_get_type_id(_fory: &$crate::fory::Fory) -> Result { + Ok($crate::types::TypeId::STRUCT as u32) } fn fory_reserved_space() -> usize { @@ -287,16 +288,16 @@ macro_rules! register_trait_type { $( if any_box.is::<$impl_type>() { let concrete = any_box.downcast::<$impl_type>() - .map_err(|_| $crate::error::Error::Other( - $crate::error::AnyhowError::msg(format!("Failed to downcast to {}", stringify!($impl_type))) + .map_err(|_| $crate::error::Error::TypeError( + format!("Failed to downcast to {}", stringify!($impl_type)).into() ))?; return Ok(Box::new(*concrete) as Box); } )+ - Err($crate::error::Error::Other($crate::error::AnyhowError::msg( - format!("No matching type found for trait {}", stringify!($trait_name)) - ))) + Err($crate::error::Error::TypeError( + format!("No matching type found for trait {}", stringify!($trait_name)).into() + )) } } } @@ -405,41 +406,42 @@ macro_rules! generate_smart_pointer_wrapper { macro_rules! impl_smart_pointer_serializer { ($wrapper_name:ident, $pointer_type:ty, $constructor_expr:expr, $trait_name:ident, $try_write_ref:ident, $get_ref:ident, $store_ref:ident, $($impl_type:ty),+) => { impl $crate::serializer::Serializer for $wrapper_name { - fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { if !context.ref_writer.$try_write_ref(&mut context.writer, &self.0) { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id); + let harness = context.write_any_typeinfo(fory, concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(any_obj, fory, context, is_field); + serializer_fn(any_obj, fory, context, is_field)?; } + Ok(()) } - fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { let any_obj = ::as_any(&*self.0); - $crate::downcast_and_serialize!(any_obj, fory, context, is_field, $trait_name, $($impl_type),+); + $crate::downcast_and_serialize!(any_obj, fory, context, is_field, $trait_name, $($impl_type),+) } fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { use $crate::types::RefFlag; - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { - RefFlag::Null => Err($crate::error::AnyhowError::msg( - format!("{} cannot be null", stringify!($pointer_type), stringify!($trait_name)) - ).into()), + RefFlag::Null => Err($crate::error::Error::InvalidRef( + format!("{} cannot be null", stringify!($pointer_type), stringify!($trait_name)).into() + )), RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; context.ref_reader.$get_ref::(ref_id) .map(|ptr| Self::from(ptr)) - .ok_or_else(|| $crate::error::AnyhowError::msg( - format!("{} reference {} not found", stringify!($pointer_type), stringify!($trait_name), ref_id) - ).into()) + .ok_or_else(|| $crate::error::Error::InvalidData( + format!("{} reference {} not found", stringify!($pointer_type), stringify!($trait_name), ref_id).into() + )) } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed_any = deserializer_fn(fory, context, is_field)?; context.dec_depth(); @@ -447,19 +449,19 @@ macro_rules! impl_smart_pointer_serializer { $( if boxed_any.is::<$impl_type>() { let concrete = boxed_any.downcast::<$impl_type>() - .map_err(|_| $crate::error::AnyhowError::msg("Downcast failed"))?; + .map_err(|_| $crate::error::Error::TypeError("Downcast failed".into()))?; let ptr = $constructor_expr(*concrete) as $pointer_type; return Ok(Self::from(ptr)); } )* - Err($crate::error::AnyhowError::msg( - format!("Deserialized type does not implement trait {}", stringify!($trait_name)) - ).into()) + Err($crate::error::Error::TypeError( + format!("Deserialized type does not implement trait {}", stringify!($trait_name)).into() + )) } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory); + let harness = context.read_any_typeinfo(fory)?; let deserializer_fn = harness.get_read_data_fn(); let boxed_any = deserializer_fn(fory, context, is_field)?; context.dec_depth(); @@ -467,21 +469,21 @@ macro_rules! impl_smart_pointer_serializer { $( if boxed_any.is::<$impl_type>() { let concrete = boxed_any.downcast::<$impl_type>() - .map_err(|_| $crate::error::AnyhowError::msg("Downcast failed"))?; + .map_err(|_| $crate::error::Error::TypeError("Downcast failed".into()))?; let ptr = $constructor_expr(*concrete) as $pointer_type; context.ref_reader.$store_ref(ptr.clone()); return Ok(Self::from(ptr)); } )* - Err($crate::error::AnyhowError::msg( - format!("Deserialized type does not implement trait {}", stringify!($trait_name)) - ).into()) + Err($crate::error::Error::TypeError( + format!("Deserialized type does not implement trait {}", stringify!($trait_name)).into() + )) } } } fn fory_read_data(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { - let concrete_fory_type_id = context.reader.read_varuint32(); + let concrete_fory_type_id = context.reader.read_varuint32()?; $crate::resolve_and_deserialize!( concrete_fory_type_id, fory, context, is_field, |obj| { @@ -492,26 +494,28 @@ macro_rules! impl_smart_pointer_serializer { ) } - fn fory_get_type_id(_fory: &$crate::fory::Fory) -> u32 { - $crate::types::TypeId::STRUCT as u32 + fn fory_get_type_id(_fory: &$crate::fory::Fory) -> Result { + Ok($crate::types::TypeId::STRUCT as u32) } - fn fory_write_type_info(_fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) { + fn fory_write_type_info(_fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { + Ok(()) } - fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) { + fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { + Ok(()) } fn fory_is_polymorphic() -> bool { true } - fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> Result { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered for trait object") + .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) } fn fory_concrete_type_id(&self) -> std::any::TypeId { @@ -539,19 +543,29 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - let fory_type_id = (**self).fory_type_id_dyn(fory); + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + let fory_type_id = (**self).fory_type_id_dyn(fory)?; let concrete_type_id = (**self).fory_concrete_type_id(); - write_trait_object_headers(fory, context, fory_type_id, concrete_type_id); - (**self).fory_write_data(fory, context, is_field); + write_trait_object_headers(fory, context, fory_type_id, concrete_type_id)?; + (**self).fory_write_data(fory, context, is_field) } - fn fory_write_data(&self, _fory: &Fory, _context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + _fory: &Fory, + _context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { panic!("fory_write_data should not be called directly on Box"); } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { (**self).fory_type_id_dyn(fory) } @@ -563,12 +577,22 @@ impl Serializer for Box { true } - fn fory_write_type_info(_fory: &Fory, _context: &mut WriteContext, _is_field: bool) { + fn fory_write_type_info( + _fory: &Fory, + _context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), Error> { // Box is polymorphic - type info is written per element + Ok(()) } - fn fory_read_type_info(_fory: &Fory, _context: &mut ReadContext, _is_field: bool) { + fn fory_read_type_info( + _fory: &Fory, + _context: &mut ReadContext, + _is_field: bool, + ) -> Result<(), Error> { // Box is polymorphic - type info is read per element + Ok(()) } fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { @@ -588,28 +612,25 @@ impl Serializer for Box { context.dec_depth(); match fory_type_id { id if id == TypeId::LIST as u32 => { - Err(Error::Other(anyhow::anyhow!( - "Cannot deserialize LIST type ID {} as Box without knowing concrete type. \ - Use concrete type instead (e.g., Vec)", - fory_type_id - ))) + Err(Error::TypeError(format!( + "Cannot deserialize LIST type ID {fory_type_id} as Box without knowing concrete type. \ + Use concrete type instead (e.g., Vec)" + ).into())) } id if id == TypeId::MAP as u32 => { - Err(Error::Other(anyhow::anyhow!( - "Cannot deserialize MAP type ID {} as Box without knowing concrete type. \ - Use concrete type instead (e.g., HashMap)", - fory_type_id - ))) + Err(Error::TypeError(format!( + "Cannot deserialize MAP type ID {fory_type_id} as Box without knowing concrete type. \ + Use concrete type instead (e.g., HashMap)" + ).into())) } id if id == TypeId::SET as u32 => { - Err(Error::Other(anyhow::anyhow!( - "Cannot deserialize SET type ID {} as Box without knowing concrete type. \ - Use concrete type instead (e.g., HashSet)", - fory_type_id - ))) + Err(Error::TypeError(format!( + "Cannot deserialize SET type ID {fory_type_id} as Box without knowing concrete type. \ + Use concrete type instead (e.g., HashSet)" + ).into())) } _ => { - Err(Error::Other(anyhow::anyhow!("Type ID {} not registered", fory_type_id))) + Err(Error::TypeError(format!("Type ID {fory_type_id} not registered").into())) } } } diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index f0c8e3fbab..54fc3304f5 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -126,7 +126,6 @@ use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; -use anyhow::anyhow; use std::cell::UnsafeCell; use std::rc::Rc; use std::sync::Arc; @@ -313,30 +312,45 @@ impl Serializer for RcWeak { true } - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if let Some(rc) = self.upgrade() { if context .ref_writer .try_write_rc_ref(&mut context.writer, &rc) { - return; + return Ok(()); } - T::fory_write_data(&*rc, fory, context, is_field); + T::fory_write_data(&*rc, fory, context, is_field)?; } else { context.writer.write_i8(RefFlag::Null as i8); } + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - self.fory_write(fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + self.fory_write(fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { RefFlag::Null => Ok(RcWeak::new()), @@ -350,7 +364,7 @@ impl Serializer for RcWeak { Ok(RcWeak::from(&rc)) } RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; if let Some(rc) = context.ref_reader.get_rc_ref::(ref_id) { Ok(RcWeak::from(&rc)) @@ -367,7 +381,9 @@ impl Serializer for RcWeak { Ok(result_weak) } } - _ => Err(anyhow!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into()), + _ => Err(Error::InvalidRef( + format!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into(), + )), } } @@ -379,8 +395,12 @@ impl Serializer for RcWeak { Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -388,11 +408,11 @@ impl Serializer for RcWeak { 4 } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { if let Some(rc) = self.upgrade() { (*rc).fory_type_id_dyn(fory) } else { @@ -416,7 +436,12 @@ impl Serializer for ArcWeak true } - fn fory_write(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { + fn fory_write( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { if let Some(arc) = self.upgrade() { // IMPORTANT: If the target Arc was serialized already, just write a ref if context @@ -424,25 +449,35 @@ impl Serializer for ArcWeak .try_write_arc_ref(&mut context.writer, &arc) { // Already seen, wrote Ref flag + id, we're done - return; + return Ok(()); } // First time seeing this object, write RefValue and then its data - T::fory_write_data(&*arc, fory, context, is_field); + T::fory_write_data(&*arc, fory, context, is_field)?; } else { context.writer.write_i8(RefFlag::Null as i8); } + Ok(()) } - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - self.fory_write(fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + self.fory_write(fory, context, is_field) } - fn fory_write_type_info(fory: &Fory, context: &mut WriteContext, is_field: bool) { - T::fory_write_type_info(fory, context, is_field); + fn fory_write_type_info( + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_write_type_info(fory, context, is_field) } fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader); + let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { RefFlag::Null => Ok(ArcWeak::new()), @@ -457,7 +492,7 @@ impl Serializer for ArcWeak Ok(weak) } RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader); + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; let weak = ArcWeak::new(); if let Some(arc) = context.ref_reader.get_arc_ref::(ref_id) { @@ -473,10 +508,11 @@ impl Serializer for ArcWeak } })); } - Ok(weak) } - _ => Err(anyhow!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into()), + _ => Err(Error::InvalidRef( + format!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into(), + )), } } fn fory_read_data( @@ -487,8 +523,12 @@ impl Serializer for ArcWeak Self::fory_read(fory, context, is_field) } - fn fory_read_type_info(fory: &Fory, context: &mut ReadContext, is_field: bool) { - T::fory_read_type_info(fory, context, is_field); + fn fory_read_type_info( + fory: &Fory, + context: &mut ReadContext, + is_field: bool, + ) -> Result<(), Error> { + T::fory_read_type_info(fory, context, is_field) } fn fory_reserved_space() -> usize { @@ -496,11 +536,11 @@ impl Serializer for ArcWeak 4 } - fn fory_get_type_id(fory: &Fory) -> u32 { + fn fory_get_type_id(fory: &Fory) -> Result { T::fory_get_type_id(fory) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { if let Some(arc) = self.upgrade() { (*arc).fory_type_id_dyn(fory) } else { diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index 9da9cd705d..14fc585349 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -16,7 +16,6 @@ // under the License. use crate::error::Error; -use anyhow::anyhow; use num_enum::{IntoPrimitive, TryFromPrimitive}; use std::mem; @@ -253,7 +252,9 @@ impl TryFrom for Language { 4 => Ok(Language::Go), 5 => Ok(Language::Javascript), 6 => Ok(Language::Rust), - _ => Err(anyhow!("Unsupported language code, value:{num}"))?, + _ => Err(Error::InvalidData( + format!("Unsupported language code, value:{num}").into(), + )), } } } diff --git a/rust/fory-derive/src/fory_row.rs b/rust/fory-derive/src/fory_row.rs index eea9758abc..d044564a40 100644 --- a/rust/fory-derive/src/fory_row.rs +++ b/rust/fory-derive/src/fory_row.rs @@ -34,7 +34,7 @@ pub fn derive_row(ast: &syn::DeriveInput) -> TokenStream { quote! { let mut callback_info = struct_writer.write_start(#index); - <#ty as fory_core::row::Row<'a>>::write(&v.#ident, struct_writer.get_writer()); + <#ty as fory_core::row::Row<'a>>::write(&v.#ident, struct_writer.get_writer())?; struct_writer.write_end(callback_info); } }); @@ -69,9 +69,10 @@ pub fn derive_row(ast: &syn::DeriveInput) -> TokenStream { type ReadResult = #getter<'a>; - fn write(v: &Self, writer: &mut fory_core::buffer::Writer) { + fn write(v: &Self, writer: &mut fory_core::buffer::Writer) -> Result<(), fory_core::error::Error> { let mut struct_writer = fory_core::row::StructWriter::new(#num_fields, writer); #(#write_exprs);*; + Ok(()) } fn cast(bytes: &'a [u8]) -> Self::ReadResult { diff --git a/rust/fory-derive/src/lib.rs b/rust/fory-derive/src/lib.rs index 7eb3ea78ad..059fca4399 100644 --- a/rust/fory-derive/src/lib.rs +++ b/rust/fory-derive/src/lib.rs @@ -151,7 +151,7 @@ //! text: "Hello, Fory!".to_string(), //! }; //! -//! let serialized = fory.serialize(&data); +//! let serialized = fory.serialize(&data)?; //! let deserialized: MyData = fory.deserialize(&serialized)?; //! //! assert_eq!(data, deserialized); diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 1524fc9453..7221241c99 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -27,7 +27,7 @@ pub fn gen_actual_type_id() -> TokenStream { pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::type_def(fory, type_id, namespace, type_name, register_by_name) + Ok(fory_core::serializer::enum_::type_def(fory, type_id, namespace, type_name, register_by_name)) } } @@ -53,13 +53,13 @@ pub fn gen_write_data(data_enum: &DataEnum) -> TokenStream { let variant_idents: Vec<_> = data_enum.variants.iter().map(|v| &v.ident).collect(); let variant_values: Vec<_> = (0..variant_idents.len()).map(|v| v as u32).collect(); quote! { - match self { + Ok(match self { #( Self::#variant_idents => { context.writer.write_varuint32(#variant_values); } )* - } + }) } } @@ -67,12 +67,12 @@ pub fn gen_read_data(data_enum: &DataEnum) -> TokenStream { let variant_idents: Vec<_> = data_enum.variants.iter().map(|v| &v.ident).collect(); let variant_values: Vec<_> = (0..variant_idents.len()).map(|v| v as u32).collect(); quote! { - let ordinal = context.reader.read_varuint32(); + let ordinal = context.reader.read_varuint32()?; match ordinal { #( #variant_values => Ok(Self::#variant_idents), )* - _ => panic!("unknown value"), + _ => return Err(fory_core::error::Error::UnknownEnum("unknown enum value".into())), } } } diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index 40f52502e6..4bb522cadd 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -127,6 +127,6 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { }); quote! { let field_infos: Vec = vec![#(#field_infos),*]; - fory_core::serializer::struct_::type_def::(fory, type_id, namespace, type_name, register_by_name, field_infos) + Ok(fory_core::serializer::struct_::type_def::(fory, type_id, namespace, type_name, register_by_name, field_infos)) } } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 82a3d8ec77..2a18bda71d 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -26,7 +26,7 @@ use super::util::{ }; fn create_private_field_name(field: &Field) -> Ident { - format_ident!("_{}", field.ident.as_ref().expect("")) + format_ident!("_{}", field.ident.as_ref().unwrap()) } fn need_declared_by_option(field: &Field) -> bool { @@ -108,16 +108,16 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let from_any_fn = format_ident!("from_any_internal_{}", trait_name); let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); quote! { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - panic!("Expected NotNullValue for trait object field"); + return Err(Error::InvalidRef("Expected NotNullValue for trait object field".into())); } - let fory_type_id = context.reader.read_varuint32(); + let fory_type_id = context.reader.read_varuint32()?; let harness = fory.get_type_resolver() .get_harness(fory_type_id) - .expect("Type not registered for trait object field"); + .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; let deserializer_fn = harness.get_read_fn(); let any_box = deserializer_fn(fory, context, true, false)?; @@ -254,14 +254,14 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let from_any_fn = format_ident!("from_any_internal_{}", trait_name); let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); quote! { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - panic!("Expected NotNullValue for trait object field"); + return Err(Error::InvalidRef("Expected NotNullValue for trait object field".into())); } - let fory_type_id = context.reader.read_varuint32(); + let fory_type_id = context.reader.read_varuint32()?; let harness = fory.get_type_resolver() .get_harness(fory_type_id) - .expect("Type not registered for trait object field"); + .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; let deserializer_fn = harness.get_read_fn(); let any_box = deserializer_fn(fory, context, true, false)?; let base_type_id = fory_type_id >> 8; @@ -374,7 +374,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS if !_field.field_type.nullable { #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; } else { - if context.reader.read_i8() == #null_flag { + if context.reader.read_i8()? == #null_flag { #var_name = <#ty as fory_core::serializer::ForyDefault>::fory_default(); } else { #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; @@ -389,21 +389,21 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS pub fn gen_read(struct_ident: &Ident) -> TokenStream { quote! { - let ref_flag = context.reader.read_i8(); + let ref_flag = context.reader.read_i8()?; if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::types::RefFlag::RefValue as i8) { if fory.is_compatible() { <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) } else { - ::fory_read_type_info(fory, context, false); + ::fory_read_type_info(fory, context, false)?; ::fory_read_data(fory, context, false) } } else if ref_flag == (fory_core::types::RefFlag::Null as i8) { Ok(::fory_default()) - // Err(fory_core::error::AnyhowError::msg("Try to read non-option type to null"))? } else if ref_flag == (fory_core::types::RefFlag::Ref as i8) { - Err(fory_core::error::Error::Ref) + // Err(fory_core::error::Error::Ref) + todo!() } else { - Err(fory_core::error::AnyhowError::msg("Unknown ref flag, value:{ref_flag}"))? + Err(fory_core::error::Error::InvalidRef(format!("Unknown ref flag, value:{ref_flag}").into())) } } } @@ -428,8 +428,8 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { .collect(); quote! { - let remote_type_id = context.reader.read_varuint32(); - let meta_index = context.reader.read_varuint32(); + let remote_type_id = context.reader.read_varuint32()?; + let meta_index = context.reader.read_varuint32()?; let meta = context.get_meta(meta_index as usize); let fields = { let meta = context.get_meta(meta_index as usize); @@ -437,7 +437,7 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { }; #(#declare_ts)* - let local_type_hash = fory.get_type_resolver().get_type_info(std::any::TypeId::of::()).get_type_meta().get_hash(); + let local_type_hash = fory.get_type_resolver().get_type_info(std::any::TypeId::of::())?.get_type_meta().get_hash(); if meta.get_hash() == local_type_hash { ::fory_read_data(fory, context, false) } else { @@ -447,7 +447,7 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { _ => { let field_type = &_field.field_type; let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&field_type); - fory_core::serializer::skip::skip_field_value(fory, context, &field_type, read_ref_flag).unwrap(); + fory_core::serializer::skip::skip_field_value(fory, context, &field_type, read_ref_flag)?; } } } diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index daaf1bdfed..23f911452f 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -128,16 +128,16 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #get_sorted_field_names_ts } - fn fory_type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: fory_core::meta::MetaString, type_name: fory_core::meta::MetaString, register_by_name: bool) -> (Vec, fory_core::meta::TypeMeta) { + fn fory_type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: fory_core::meta::MetaString, type_name: fory_core::meta::MetaString, register_by_name: bool) -> Result<(Vec, fory_core::meta::TypeMeta), fory_core::error::Error> { #type_def_ts } } impl fory_core::serializer::Serializer for #name { - fn fory_get_type_id(fory: &fory_core::fory::Fory) -> u32 { + fn fory_get_type_id(fory: &fory_core::fory::Fory) -> Result { fory.get_type_resolver().get_type_id(&std::any::TypeId::of::(), #type_idx) } - fn fory_type_id_dyn(&self, fory: &fory_core::fory::Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &fory_core::fory::Fory) -> Result { Self::fory_get_type_id(fory) } @@ -149,15 +149,15 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #reserved_space_ts } - fn fory_write_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_type_info_ts } - fn fory_read_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) { + fn fory_read_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result<(), fory_core::error::Error> { #read_type_info_ts } - fn fory_write_data(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write_data(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_data_ts } @@ -165,7 +165,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #read_data_ts } - fn fory_write(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) { + fn fory_write(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_ts } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index c0698bf1a6..6f7ca8ef62 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -354,7 +354,7 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { ts } else { quote! { - <#ty as fory_core::serializer::Serializer>::fory_get_type_id(fory) + <#ty as fory_core::serializer::Serializer>::fory_get_type_id(fory)? } }; diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 5736f6aade..18cee31ee5 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -110,18 +110,18 @@ fn gen_write_field(field: &Field) -> TokenStream { let concrete_type_id = any_ref.type_id(); let fory_type_id = fory.get_type_resolver() .get_fory_type_id(concrete_type_id) - .expect("Type not registered for trait object field"); + .ok_or_else(|| fory_core::error::Error::TypeError("Type not registered for trait object field".into()))?; - context.writer.write_i8(fory_core::types::RefFlag::NotNullValue as i8); + context.writer.write_i8(fory_core::types::RefFlag::NotNullValue as i8)?; context.writer.write_varuint32(fory_type_id); let harness = fory .get_type_resolver() .get_harness(fory_type_id) - .expect("Harness not found for trait object field"); + .ok_or_else(|| fory_core::error::Error::TypeError("Harness not found for trait object field".into()))?; let serializer_fn = harness.get_write_fn(); - serializer_fn(any_ref, fory, context, true); + serializer_fn(any_ref, fory, context, true)?; } } } @@ -132,7 +132,7 @@ fn gen_write_field(field: &Field) -> TokenStream { quote! { { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); - fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true)?; } } } @@ -143,7 +143,7 @@ fn gen_write_field(field: &Field) -> TokenStream { quote! { { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); - fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true)?; } } } @@ -156,7 +156,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true)?; } } } @@ -169,7 +169,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true)?; } } } @@ -182,7 +182,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true)?; } } } @@ -195,34 +195,31 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true); + fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true)?; } } } StructField::Forward => { quote! { { - fory_core::serializer::Serializer::fory_write(&self.#ident, fory, context, true); + fory_core::serializer::Serializer::fory_write(&self.#ident, fory, context, true)?; } } } _ => { let skip_ref_flag = skip_ref_flag(ty); quote! { - fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, #skip_ref_flag, false); + fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, #skip_ref_flag, false)?; } } } } pub fn gen_write_data(fields: &[&Field]) -> TokenStream { - if fields.is_empty() { - quote! {} - } else { - let write_fields_ts: Vec<_> = fields.iter().map(|field| gen_write_field(field)).collect(); - quote! { - #(#write_fields_ts)* - } + let write_fields_ts: Vec<_> = fields.iter().map(|field| gen_write_field(field)).collect(); + quote! { + #(#write_fields_ts)* + Ok(()) } } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index 3ba3844f55..10cd983b2c 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -76,7 +76,7 @@ //! email: "alice@example.com".to_string(), //! }; //! -//! let bytes = fory.serialize(&user); +//! let bytes = fory.serialize(&user)?; //! let decoded: User = fory.deserialize(&bytes)?; //! assert_eq!(user, decoded); //! # Ok(()) @@ -149,7 +149,7 @@ //! ]), //! }; //! -//! let bytes = fory.serialize(&person); +//! let bytes = fory.serialize(&person)?; //! let decoded: Person = fory.deserialize(&bytes)?; //! assert_eq!(person, decoded); //! # Ok(()) @@ -190,7 +190,7 @@ //! let shared = Rc::new(String::from("shared_value")); //! let data = vec![shared.clone(), shared.clone(), shared.clone()]; //! -//! let bytes = fory.serialize(&data); +//! let bytes = fory.serialize(&data)?; //! let decoded: Vec> = fory.deserialize(&bytes)?; //! //! assert_eq!(decoded.len(), 3); @@ -212,7 +212,7 @@ //! let shared = Arc::new(String::from("shared_value")); //! let data = vec![shared.clone(), shared.clone()]; //! -//! let bytes = fory.serialize(&data); +//! let bytes = fory.serialize(&data)?; //! let decoded: Vec> = fory.deserialize(&bytes)?; //! //! assert!(Arc::ptr_eq(&decoded[0], &decoded[1])); @@ -260,7 +260,7 @@ //! //! parent.borrow_mut().children.push(child1.clone()); //! -//! let bytes = fory.serialize(&parent); +//! let bytes = fory.serialize(&parent)?; //! let decoded: Rc> = fory.deserialize(&bytes)?; //! //! assert_eq!(decoded.borrow().children.len(), 1); @@ -302,7 +302,7 @@ //! //! parent.lock().unwrap().children.push(child.clone()); //! -//! let bytes = fory.serialize(&parent); +//! let bytes = fory.serialize(&parent)?; //! let decoded: Arc> = fory.deserialize(&bytes)?; //! //! assert_eq!(decoded.lock().unwrap().children.len(), 1); @@ -378,7 +378,7 @@ //! }), //! }; //! -//! let bytes = fory.serialize(&zoo); +//! let bytes = fory.serialize(&zoo)?; //! let decoded: Zoo = fory.deserialize(&bytes)?; //! //! assert_eq!(decoded.star_animal.name(), "Buddy"); @@ -419,7 +419,7 @@ //! name: "Rex".to_string() //! }); //! -//! let bytes = fory.serialize(&dog); +//! let bytes = fory.serialize(&dog)?; //! let decoded: Rc = fory.deserialize(&bytes)?; //! //! let unwrapped = decoded.downcast_ref::().unwrap(); @@ -448,7 +448,7 @@ //! name: "Whiskers".to_string() //! }); //! -//! let bytes = fory.serialize(&cat); +//! let bytes = fory.serialize(&cat)?; //! let decoded: Arc = fory.deserialize(&bytes)?; //! //! let unwrapped = decoded.downcast_ref::().unwrap(); @@ -508,7 +508,7 @@ //! ], //! }; //! -//! let bytes = fory.serialize(&shelter); +//! let bytes = fory.serialize(&shelter)?; //! let decoded: AnimalShelter = fory.deserialize(&bytes)?; //! //! assert_eq!(decoded.animals_rc[0].name(), "Rex"); @@ -554,7 +554,7 @@ //! let dog_rc: Rc = Rc::new(Dog { name: "Rex".to_string() }); //! let wrapper = AnimalRc::from(dog_rc); //! -//! let bytes = fory.serialize(&wrapper); +//! let bytes = fory.serialize(&wrapper)?; //! let decoded: AnimalRc = fory.deserialize(&bytes)?; //! //! // Unwrap back to Rc @@ -565,7 +565,7 @@ //! let dog_arc: Arc = Arc::new(Dog { name: "Buddy".to_string() }); //! let wrapper = AnimalArc::from(dog_arc); //! -//! let bytes = fory.serialize(&wrapper); +//! let bytes = fory.serialize(&wrapper)?; //! let decoded: AnimalArc = fory.deserialize(&bytes)?; //! //! let unwrapped: Arc = decoded.unwrap(); @@ -634,7 +634,7 @@ //! address: "123 Main St".to_string(), //! }; //! -//! let bytes = fory1.serialize(&person_v1); +//! let bytes = fory1.serialize(&person_v1)?; //! let person_v2: PersonV2 = fory2.deserialize(&bytes)?; //! //! assert_eq!(person_v2.name, "Alice"); @@ -682,7 +682,7 @@ //! fory.register::(1); //! //! let status = Status::Active; -//! let bytes = fory.serialize(&status); +//! let bytes = fory.serialize(&status)?; //! let decoded: Status = fory.deserialize(&bytes)?; //! assert_eq!(status, decoded); //! # Ok(()) @@ -717,20 +717,21 @@ //! } //! //! impl Serializer for CustomType { -//! fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { +//! fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { //! context.writer.write_i32(self.value); //! context.writer.write_varuint32(self.name.len() as u32); //! context.writer.write_utf8_string(&self.name); +//! Ok(()) //! } //! //! fn fory_read_data(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { -//! let value = context.reader.read_i32(); -//! let len = context.reader.read_varuint32() as usize; -//! let name = context.reader.read_utf8_string(len); +//! let value = context.reader.read_i32()?; +//! let len = context.reader.read_varuint32()? as usize; +//! let name = context.reader.read_utf8_string(len)?; //! Ok(Self { value, name }) //! } //! -//! fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { +//! fn fory_type_id_dyn(&self, fory: &Fory) -> Result { //! Self::fory_get_type_id(fory) //! } //! @@ -753,7 +754,7 @@ //! value: 42, //! name: "test".to_string(), //! }; -//! let bytes = fory.serialize(&custom); +//! let bytes = fory.serialize(&custom)?; //! let decoded: CustomType = fory.deserialize(&bytes)?; //! assert_eq!(custom, decoded); //! # Ok(()) @@ -828,7 +829,7 @@ //! is_active: true, //! }; //! -//! let row_data = to_row(&profile); +//! let row_data = to_row(&profile).unwrap(); //! let row = from_row::(&row_data); //! //! assert_eq!(row.id(), 12345); @@ -1023,7 +1024,7 @@ //! # fn serialize_data() -> Vec { //! FORY.with(|fory| { //! let data = vec![1, 2, 3]; -//! fory.borrow().serialize(&data) +//! fory.borrow().serialize(&data).unwrap() //! }) //! # } //! ``` diff --git a/rust/tests/tests/compatible/test_basic_type.rs b/rust/tests/tests/compatible/test_basic_type.rs index 1077e913e4..0ae571fcf8 100644 --- a/rust/tests/tests/compatible/test_basic_type.rs +++ b/rust/tests/tests/compatible/test_basic_type.rs @@ -45,64 +45,107 @@ const FLOAT32_ARRAY: [f32; 1] = [52.0]; const FLOAT64_ARRAY: [f64; 1] = [53.0]; fn serialize_non_null(fory: &Fory, context: &mut WriteContext) { - fory.serialize_with_context(&BOOL_VAL, context); - fory.serialize_with_context(&I8_VAL, context); - fory.serialize_with_context(&I16_VAL, context); - fory.serialize_with_context(&I32_VAL, context); - fory.serialize_with_context(&I64_VAL, context); - fory.serialize_with_context(&F32_VAL, context); - fory.serialize_with_context(&F64_VAL, context); - fory.serialize_with_context(&STR_LATIN1_VAL.to_string(), context); - fory.serialize_with_context(&LOCAL_DATE_VAL, context); - fory.serialize_with_context(&TIMESTAMP_VAL, context); + fory.serialize_with_context(&BOOL_VAL, context).unwrap(); + fory.serialize_with_context(&I8_VAL, context).unwrap(); + fory.serialize_with_context(&I16_VAL, context).unwrap(); + fory.serialize_with_context(&I32_VAL, context).unwrap(); + fory.serialize_with_context(&I64_VAL, context).unwrap(); + fory.serialize_with_context(&F32_VAL, context).unwrap(); + fory.serialize_with_context(&F64_VAL, context).unwrap(); + fory.serialize_with_context(&STR_LATIN1_VAL.to_string(), context) + .unwrap(); + fory.serialize_with_context(&LOCAL_DATE_VAL, context) + .unwrap(); + fory.serialize_with_context(&TIMESTAMP_VAL, context) + .unwrap(); - fory.serialize_with_context(&BOOL_ARRAY.to_vec(), context); - fory.serialize_with_context(&INT8_ARRAY.to_vec(), context); - fory.serialize_with_context(&INT16_ARRAY.to_vec(), context); - fory.serialize_with_context(&INT32_ARRAY.to_vec(), context); - fory.serialize_with_context(&INT64_ARRAY.to_vec(), context); - fory.serialize_with_context(&FLOAT32_ARRAY.to_vec(), context); - fory.serialize_with_context(&FLOAT64_ARRAY.to_vec(), context); + fory.serialize_with_context(&BOOL_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&INT8_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&INT16_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&INT32_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&INT64_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&FLOAT32_ARRAY.to_vec(), context) + .unwrap(); + fory.serialize_with_context(&FLOAT64_ARRAY.to_vec(), context) + .unwrap(); } fn serialize_nullable(fory: &Fory, context: &mut WriteContext) { - fory.serialize_with_context(&Some(BOOL_VAL), context); - fory.serialize_with_context(&Some(I8_VAL), context); - fory.serialize_with_context(&Some(I16_VAL), context); - fory.serialize_with_context(&Some(I32_VAL), context); - fory.serialize_with_context(&Some(I64_VAL), context); - fory.serialize_with_context(&Some(F32_VAL), context); - fory.serialize_with_context(&Some(F64_VAL), context); - fory.serialize_with_context(&Some(STR_LATIN1_VAL.to_string()), context); - fory.serialize_with_context(&Some(LOCAL_DATE_VAL), context); - fory.serialize_with_context(&Some(TIMESTAMP_VAL), context); + fory.serialize_with_context(&Some(BOOL_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(I8_VAL), context).unwrap(); + fory.serialize_with_context(&Some(I16_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(I32_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(I64_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(F32_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(F64_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(STR_LATIN1_VAL.to_string()), context) + .unwrap(); + fory.serialize_with_context(&Some(LOCAL_DATE_VAL), context) + .unwrap(); + fory.serialize_with_context(&Some(TIMESTAMP_VAL), context) + .unwrap(); - fory.serialize_with_context(&Some(BOOL_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(INT8_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(INT16_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(INT32_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(INT64_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(FLOAT32_ARRAY.to_vec()), context); - fory.serialize_with_context(&Some(FLOAT64_ARRAY.to_vec()), context); + fory.serialize_with_context(&Some(BOOL_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(INT8_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(INT16_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(INT32_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(INT64_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(FLOAT32_ARRAY.to_vec()), context) + .unwrap(); + fory.serialize_with_context(&Some(FLOAT64_ARRAY.to_vec()), context) + .unwrap(); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); - fory.serialize_with_context(&Option::::None, context); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); + fory.serialize_with_context(&Option::::None, context) + .unwrap(); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); - fory.serialize_with_context(&Option::>::None, context); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, context) + .unwrap(); } fn deserialize_non_null(fory: &Fory, context: &mut ReadContext, auto_conv: bool, to_end: bool) { diff --git a/rust/tests/tests/compatible/test_container.rs b/rust/tests/tests/compatible/test_container.rs index 061a71d1cf..0dec8d01a9 100644 --- a/rust/tests/tests/compatible/test_container.rs +++ b/rust/tests/tests/compatible/test_container.rs @@ -224,9 +224,12 @@ fn container_outer_auto_conv() { // serialize_outer_non-null let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&basic_list(), &mut write_context); - fory.serialize_with_context(&basic_set(), &mut write_context); - fory.serialize_with_context(&basic_map(), &mut write_context); + fory.serialize_with_context(&basic_list(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&basic_set(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&basic_map(), &mut write_context) + .unwrap(); // deserialize_outer_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -250,12 +253,18 @@ fn container_outer_auto_conv() { // serialize_outer_nullable let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&Some(basic_list()), &mut write_context); - fory.serialize_with_context(&Some(basic_set()), &mut write_context); - fory.serialize_with_context(&Some(basic_map()), &mut write_context); - fory.serialize_with_context(&Option::>::None, &mut write_context); - fory.serialize_with_context(&Option::>::None, &mut write_context); - fory.serialize_with_context(&Option::>::None, &mut write_context); + fory.serialize_with_context(&Some(basic_list()), &mut write_context) + .unwrap(); + fory.serialize_with_context(&Some(basic_set()), &mut write_context) + .unwrap(); + fory.serialize_with_context(&Some(basic_map()), &mut write_context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, &mut write_context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, &mut write_context) + .unwrap(); + fory.serialize_with_context(&Option::>::None, &mut write_context) + .unwrap(); // deserialize_outer_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -296,21 +305,29 @@ fn container_outer_auto_conv() { #[test] fn collection_inner() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("item"); + fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { // serialize let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&basic_list(), &mut write_context); - fory.serialize_with_context(&item_list(), &mut write_context); - fory.serialize_with_context(&basic_set(), &mut write_context); - fory.serialize_with_context(&item_set(), &mut write_context); - fory.serialize_with_context(&nullable_basic_list(false), &mut write_context); - fory.serialize_with_context(&nullable_item_list(false), &mut write_context); - fory.serialize_with_context(&nullable_basic_set(false), &mut write_context); - fory.serialize_with_context(&nullable_item_set(false), &mut write_context); + fory.serialize_with_context(&basic_list(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_list(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&basic_set(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_set(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_basic_list(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_list(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_basic_set(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_set(false), &mut write_context) + .unwrap(); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -362,17 +379,21 @@ fn collection_inner() { #[test] fn collection_inner_auto_conv() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("item"); + fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { // serialize_non-null let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&basic_list(), &mut write_context); - fory.serialize_with_context(&item_list(), &mut write_context); - fory.serialize_with_context(&basic_set(), &mut write_context); - fory.serialize_with_context(&item_set(), &mut write_context); + fory.serialize_with_context(&basic_list(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_list(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&basic_set(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_set(), &mut write_context) + .unwrap(); // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -401,10 +422,14 @@ fn collection_inner_auto_conv() { // serialize_nullable let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&nullable_basic_list(false), &mut write_context); - fory.serialize_with_context(&nullable_item_list(false), &mut write_context); - fory.serialize_with_context(&nullable_basic_set(false), &mut write_context); - fory.serialize_with_context(&nullable_item_set(false), &mut write_context); + fory.serialize_with_context(&nullable_basic_list(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_list(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_basic_set(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_set(false), &mut write_context) + .unwrap(); // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -436,17 +461,21 @@ fn collection_inner_auto_conv() { #[test] fn map_inner() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("item"); + fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { // serialize let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&basic_map(), &mut write_context); - fory.serialize_with_context(&item_map(), &mut write_context); - fory.serialize_with_context(&nullable_basic_map(false), &mut write_context); - fory.serialize_with_context(&nullable_item_map(false), &mut write_context); + fory.serialize_with_context(&basic_map(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_map(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_basic_map(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_map(false), &mut write_context) + .unwrap(); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -480,15 +509,17 @@ fn map_inner() { #[test] fn map_inner_auto_conv() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("item"); + fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { // serialize_non-null let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&basic_map(), &mut write_context); - fory.serialize_with_context(&item_map(), &mut write_context); + fory.serialize_with_context(&basic_map(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&item_map(), &mut write_context) + .unwrap(); // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -509,8 +540,10 @@ fn map_inner_auto_conv() { // serialize_nullable let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&nullable_basic_map(false), &mut write_context); - fory.serialize_with_context(&nullable_item_map(false), &mut write_context); + fory.serialize_with_context(&nullable_basic_map(false), &mut write_context) + .unwrap(); + fory.serialize_with_context(&nullable_item_map(false), &mut write_context) + .unwrap(); // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); @@ -532,15 +565,18 @@ fn map_inner_auto_conv() { #[test] fn complex() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("item"); + fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { let writer = Writer::default(); let mut write_context = WriteContext::new(writer); - fory.serialize_with_context(&nested_collection(), &mut write_context); - fory.serialize_with_context(&complex_container1(), &mut write_context); - fory.serialize_with_context(&complex_container2(), &mut write_context); + fory.serialize_with_context(&nested_collection(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&complex_container1(), &mut write_context) + .unwrap(); + fory.serialize_with_context(&complex_container2(), &mut write_context) + .unwrap(); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); let mut read_context = ReadContext::new(reader, 5); diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 95f0fbf8d2..46ccf19720 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -45,8 +45,8 @@ fn simple() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let animal: Animal1 = Animal1 { f1: HashMap::from([(1, vec![2])]), f2: String::from("hello"), @@ -56,7 +56,7 @@ fn simple() { f7: 43, last: 44, }; - let bin = fory1.serialize(&animal); + let bin = fory1.serialize(&animal).unwrap(); let obj: Animal2 = fory2.deserialize(&bin).unwrap(); assert_eq!(animal.f1, obj.f1); assert_eq!(animal.f3, obj.f3); @@ -84,14 +84,14 @@ fn skip_option() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let item1 = Item1 { f1: None, f2: Some(String::from("f2")), last: 42, }; - let bin = fory1.serialize(&item1); + let bin = fory1.serialize(&item1).unwrap(); let item2: Item2 = fory2.deserialize(&bin).unwrap(); assert_eq!(item2.f1, i8::default()); @@ -123,16 +123,16 @@ fn nonexistent_struct() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(899); - fory1.register::(999); - fory2.register::(799); - fory2.register::(999); + fory1.register::(899).unwrap(); + fory1.register::(999).unwrap(); + fory2.register::(799).unwrap(); + fory2.register::(999).unwrap(); let person = Person1 { f2: Item1 { f1: 42 }, f3: 24, last: String::from("foo"), }; - let bin = fory1.serialize(&person); + let bin = fory1.serialize(&person).unwrap(); let obj: Person2 = fory2.deserialize(&bin).unwrap(); assert_eq!(obj.f2, Item2::default()); assert_eq!(obj.f3, i64::default()); @@ -152,7 +152,7 @@ fn option() { last: i64, } let mut fory = Fory::default().compatible(true); - fory.register::(999); + fory.register::(999).unwrap(); let animal: Animal = Animal { f1: Some(String::from("f1")), f2: None, @@ -160,7 +160,7 @@ fn option() { f5: vec![Some(vec![Some(String::from("f1"))])], last: 666, }; - let bin = fory.serialize(&animal); + let bin = fory.serialize(&animal).unwrap(); let obj: Animal = fory.deserialize(&bin).unwrap(); assert_eq!(animal, obj); } @@ -197,8 +197,8 @@ fn nullable() { let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let item1 = Item1 { f2: 43, @@ -209,7 +209,7 @@ fn nullable() { last: 666, }; - let bin = fory1.serialize(&item1); + let bin = fory1.serialize(&item1).unwrap(); let item2: Item2 = fory2.deserialize(&bin).unwrap(); assert_eq!(item2.f2.unwrap(), item1.f2); assert_eq!(item2.f3, item1.f3.unwrap()); @@ -251,8 +251,8 @@ fn nullable_container() { let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let item1 = Item1 { f1: vec![44, 45], @@ -267,7 +267,7 @@ fn nullable_container() { last: 666, }; - let bin = fory1.serialize(&item1); + let bin = fory1.serialize(&item1).unwrap(); let item2: Item2 = fory2.deserialize(&bin).unwrap(); assert_eq!(item2.f1.unwrap(), item1.f1); @@ -301,8 +301,8 @@ fn inner_nullable() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let item1 = Item1 { f1: vec![None, Some("hello".to_string())], @@ -310,7 +310,7 @@ fn inner_nullable() { f3: HashMap::from([(44, None), (45, Some(46))]), last: 666, }; - let bin = fory1.serialize(&item1); + let bin = fory1.serialize(&item1).unwrap(); let item2: Item2 = fory2.deserialize(&bin).unwrap(); assert_eq!(item2.f1, vec![String::default(), "hello".to_string()]); @@ -345,10 +345,10 @@ fn nullable_struct() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(199); - fory1.register::(200); - fory2.register::(199); - fory2.register::(200); + fory1.register::(199).unwrap(); + fory1.register::(200).unwrap(); + fory2.register::(199).unwrap(); + fory2.register::(200).unwrap(); let person1 = Person1 { f1: Item { @@ -364,7 +364,7 @@ fn nullable_struct() { }), last: 46, }; - let bin = fory1.serialize(&person1); + let bin = fory1.serialize(&person1).unwrap(); let person2: Person2 = fory2.deserialize(&bin).unwrap(); assert_eq!(person2.f1.unwrap(), person1.f1); @@ -418,13 +418,13 @@ fn enum_without_payload() { } let mut fory1 = Fory::default().compatible(true).xlang(true); - fory1.register::(101); - fory1.register::(102); - fory1.register::(103); + fory1.register::(101).unwrap(); + fory1.register::(102).unwrap(); + fory1.register::(103).unwrap(); let mut fory2 = Fory::default().compatible(true).xlang(true); - fory2.register::(101); - fory2.register::(102); - fory2.register::(103); + fory2.register::(101).unwrap(); + fory2.register::(102).unwrap(); + fory2.register::(103).unwrap(); let person1 = Person1 { f1: Color1::Blue, @@ -436,7 +436,7 @@ fn enum_without_payload() { f8: Color1::Red, last: 10, }; - let bin = fory1.serialize(&person1); + let bin = fory1.serialize(&person1).unwrap(); let person2: Person2 = fory2.deserialize(&bin).expect(""); assert_eq!(person2.f1, person1.f1); assert_eq!(person2.f2, Color2::default()); @@ -482,11 +482,11 @@ fn named_enum() { last: i8, } let mut fory1 = Fory::default().compatible(true).xlang(true); - fory1.register_by_name::("a"); - fory1.register::(101); + fory1.register_by_name::("a").unwrap(); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true).xlang(true); - fory2.register_by_name::("a"); - fory2.register::(101); + fory2.register_by_name::("a").unwrap(); + fory2.register::(101).unwrap(); let item1 = Item1 { f1: Color::Red, f2: Color::Blue, @@ -508,7 +508,7 @@ fn named_enum() { f6: None, last: 42, }; - let bin = fory1.serialize(&item1); + let bin = fory1.serialize(&item1).unwrap(); let actual_item2: Item2 = fory2.deserialize(&bin).unwrap(); assert_eq!(expected_item2, actual_item2); } @@ -538,9 +538,9 @@ fn boxed() { } let mut fory1 = Fory::default().compatible(true).xlang(true); - fory1.register::(101); + fory1.register::(101).unwrap(); let mut fory2 = Fory::default().compatible(true).xlang(true); - fory2.register::(101); + fory2.register::(101).unwrap(); let f1 = 1; let f2 = 2; @@ -556,7 +556,7 @@ fn boxed() { f5, f6, }; - let bytes = fory1.serialize(&item1); + let bytes = fory1.serialize(&item1).unwrap(); let item2: Item2 = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f1, f1); assert_eq!(item2.f2.unwrap(), f2); @@ -565,27 +565,27 @@ fn boxed() { assert_eq!(item2.f5, i32::default()); assert_eq!(item2.f6, f6); - let bytes = fory1.serialize(&f1); + let bytes = fory1.serialize(&f1).unwrap(); let item2_f1: i32 = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f1, item2_f1); - let bytes = fory1.serialize(&f2); + let bytes = fory1.serialize(&f2).unwrap(); let item2_f2: Option = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f2, item2_f2); - let bytes = fory1.serialize(&f3); + let bytes = fory1.serialize(&f3).unwrap(); let item2_f3: Option = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f3, item2_f3); - let bytes = fory1.serialize(&f4); + let bytes = fory1.serialize(&f4).unwrap(); let item2_f4: i32 = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f4, item2_f4); - let bytes = fory1.serialize(&f5); + let bytes = fory1.serialize(&f5).unwrap(); let item2_f5: i32 = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f5, item2_f5); - let bytes = fory1.serialize(&f6); + let bytes = fory1.serialize(&f6).unwrap(); let item2_f6: Option = fory2.deserialize(&bytes).unwrap(); assert_eq!(item2.f6, item2_f6); } diff --git a/rust/tests/tests/compatible/test_struct_enum.rs b/rust/tests/tests/compatible/test_struct_enum.rs index 288c653d22..39bd8241f6 100644 --- a/rust/tests/tests/compatible/test_struct_enum.rs +++ b/rust/tests/tests/compatible/test_struct_enum.rs @@ -82,19 +82,21 @@ struct Empty {} #[test] fn basic() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); - fory1.register::(102); - fory1.register::(103); + fory1.register::(101).unwrap(); + fory1.register::(102).unwrap(); + fory1.register::(103).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("color"); - fory2.register_by_name::("item"); - fory2.register_by_name::("person"); + fory2.register_by_name::("color").unwrap(); + fory2.register_by_name::("item").unwrap(); + fory2.register_by_name::("person").unwrap(); for fory in [fory1, fory2] { let writer = Writer::default(); let mut write_context = WriteContext::new(writer); let person = Person::default(); - fory.serialize_with_context(&person, &mut write_context); - fory.serialize_with_context(&person, &mut write_context); + fory.serialize_with_context(&person, &mut write_context) + .unwrap(); + fory.serialize_with_context(&person, &mut write_context) + .unwrap(); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); let mut read_context = ReadContext::new(reader, 5); @@ -115,16 +117,16 @@ fn basic() { #[test] fn outer_nullable() { let mut fory1 = Fory::default().compatible(true); - fory1.register::(101); - fory1.register::(102); - fory1.register::(103); + fory1.register::(101).unwrap(); + fory1.register::(102).unwrap(); + fory1.register::(103).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("color"); - fory2.register_by_name::("item"); - fory2.register_by_name::("person"); + fory2.register_by_name::("color").unwrap(); + fory2.register_by_name::("item").unwrap(); + fory2.register_by_name::("person").unwrap(); for fory in [fory1, fory2] { let null_person: Option = None; - let bytes = fory.serialize(&null_person); + let bytes = fory.serialize(&null_person).unwrap(); assert_eq!( fory.deserialize::(&bytes).unwrap(), Person::default() @@ -138,23 +140,23 @@ fn skip_basic() { let person2_default = Empty::default(); let mut id_fory1 = Fory::default().compatible(true); - id_fory1.register::(101); - id_fory1.register::(102); - id_fory1.register::(103); + id_fory1.register::(101).unwrap(); + id_fory1.register::(102).unwrap(); + id_fory1.register::(103).unwrap(); let mut id_fory2 = Fory::default().compatible(true); - id_fory2.register::(103); + id_fory2.register::(103).unwrap(); let mut name_fory1 = Fory::default().compatible(true); - name_fory1.register_by_name::("color"); - name_fory1.register_by_name::("item"); - name_fory1.register_by_name::("person"); + name_fory1.register_by_name::("color").unwrap(); + name_fory1.register_by_name::("item").unwrap(); + name_fory1.register_by_name::("person").unwrap(); let mut name_fory2 = Fory::default().compatible(true); - name_fory2.register_by_name::("person"); + name_fory2.register_by_name::("person").unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { - let bytes = fory1.serialize(&person_default); + let bytes = fory1.serialize(&person_default).unwrap(); assert_eq!(fory2.deserialize::(&bytes).unwrap(), person2_default); - let bytes = fory2.serialize(&person2_default); + let bytes = fory2.serialize(&person2_default).unwrap(); assert_eq!(fory1.deserialize::(&bytes).unwrap(), person_default); } } @@ -181,28 +183,28 @@ fn nested() { f7: Element, } let mut id_fory1 = Fory::default().compatible(true); - id_fory1.register::(101); - id_fory1.register::(102); - id_fory1.register::(103); - id_fory1.register::(104); + id_fory1.register::(101).unwrap(); + id_fory1.register::(102).unwrap(); + id_fory1.register::(103).unwrap(); + id_fory1.register::(104).unwrap(); let mut id_fory2 = Fory::default().compatible(true); - id_fory2.register::(104); + id_fory2.register::(104).unwrap(); let mut name_fory1 = Fory::default().compatible(true); - name_fory1.register_by_name::("item"); - name_fory1.register_by_name::("color"); - name_fory1.register_by_name::("element"); - name_fory1.register_by_name::("nested"); + name_fory1.register_by_name::("item").unwrap(); + name_fory1.register_by_name::("color").unwrap(); + name_fory1.register_by_name::("element").unwrap(); + name_fory1.register_by_name::("nested").unwrap(); let mut name_fory2 = Fory::default().compatible(true); - name_fory2.register_by_name::("nested"); + name_fory2.register_by_name::("nested").unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { - let bytes = fory1.serialize(&Nested::default()); + let bytes = fory1.serialize(&Nested::default()).unwrap(); assert_eq!( fory2.deserialize::(&bytes).unwrap(), Empty::default() ); - let bytes = fory2.serialize(&Empty::default()); + let bytes = fory2.serialize(&Empty::default()).unwrap(); assert_eq!( fory1.deserialize::(&bytes).unwrap(), Nested::default() @@ -288,27 +290,27 @@ fn compatible_nullable() { f29: Some(HashMap::::default()), }; let mut id_fory1 = Fory::default().compatible(true); - id_fory1.register::(101); - id_fory1.register::(102); - id_fory1.register::(103); + id_fory1.register::(101).unwrap(); + id_fory1.register::(102).unwrap(); + id_fory1.register::(103).unwrap(); let mut id_fory2 = Fory::default().compatible(true); - id_fory2.register::(101); - id_fory2.register::(102); - id_fory2.register::(103); + id_fory2.register::(101).unwrap(); + id_fory2.register::(102).unwrap(); + id_fory2.register::(103).unwrap(); let mut name_fory1 = Fory::default().compatible(true); - name_fory1.register_by_name::("color"); - name_fory1.register_by_name::("item"); - name_fory1.register_by_name::("obj"); + name_fory1.register_by_name::("color").unwrap(); + name_fory1.register_by_name::("item").unwrap(); + name_fory1.register_by_name::("obj").unwrap(); let mut name_fory2 = Fory::default().compatible(true); - name_fory2.register_by_name::("color"); - name_fory2.register_by_name::("item"); - name_fory2.register_by_name::("obj"); + name_fory2.register_by_name::("color").unwrap(); + name_fory2.register_by_name::("item").unwrap(); + name_fory2.register_by_name::("obj").unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { - let bytes = fory1.serialize(&Nonnull::default()); + let bytes = fory1.serialize(&Nonnull::default()).unwrap(); assert_eq!(fory2.deserialize::(&bytes).unwrap(), nullable_obj); - let bytes = fory2.serialize(&Nullable::default()); + let bytes = fory2.serialize(&Nullable::default()).unwrap(); assert_eq!( fory1.deserialize::(&bytes).unwrap(), Nonnull::default() @@ -351,30 +353,32 @@ fn name_mismatch() { f30: HashMap, } let mut id_fory1 = Fory::default().compatible(true); - id_fory1.register::(101); - id_fory1.register::(102); - id_fory1.register::(103); + id_fory1.register::(101).unwrap(); + id_fory1.register::(102).unwrap(); + id_fory1.register::(103).unwrap(); let mut id_fory2 = Fory::default().compatible(true); - id_fory2.register::(101); - id_fory2.register::(102); - id_fory2.register::(103); + id_fory2.register::(101).unwrap(); + id_fory2.register::(102).unwrap(); + id_fory2.register::(103).unwrap(); let mut name_fory1 = Fory::default().compatible(true); - name_fory1.register_by_name::("color"); - name_fory1.register_by_name::("item"); - name_fory1.register_by_name::("person"); + name_fory1.register_by_name::("color").unwrap(); + name_fory1.register_by_name::("item").unwrap(); + name_fory1.register_by_name::("person").unwrap(); let mut name_fory2 = Fory::default().compatible(true); - name_fory2.register_by_name::("color"); - name_fory2.register_by_name::("item"); - name_fory2.register_by_name::("person"); + name_fory2.register_by_name::("color").unwrap(); + name_fory2.register_by_name::("item").unwrap(); + name_fory2 + .register_by_name::("person") + .unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { - let bytes = fory1.serialize(&Person::default()); + let bytes = fory1.serialize(&Person::default()).unwrap(); assert_eq!( fory2.deserialize::(&bytes).unwrap(), MismatchPerson::default() ); - let bytes = fory2.serialize(&MismatchPerson::default()); + let bytes = fory2.serialize(&MismatchPerson::default()).unwrap(); assert_eq!( fory1.deserialize::(&bytes).unwrap(), Person::default() @@ -394,8 +398,13 @@ fn ext() { } } impl Serializer for ExtItem { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), fory_core::error::Error> { + write_data(&self.id, fory, context, is_field) } fn fory_read_data( fory: &Fory, @@ -407,7 +416,7 @@ fn ext() { }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { Self::fory_get_type_id(fory) } @@ -421,18 +430,20 @@ fn ext() { } let mut id_fory = Fory::default().compatible(true).xlang(true); - id_fory.register_serializer::(100); - id_fory.register::(101); + id_fory.register_serializer::(100).unwrap(); + id_fory.register::(101).unwrap(); let mut name_fory = Fory::default().compatible(true).xlang(true); - name_fory.register_serializer_by_name::("ext_item"); - name_fory.register::(101); + name_fory + .register_serializer_by_name::("ext_item") + .unwrap(); + name_fory.register::(101).unwrap(); for fory in [id_fory, name_fory] { let wrapper = ExtWrapper { f1: ExtItem { id: 1 }, }; - let bytes = fory.serialize(&wrapper); + let bytes = fory.serialize(&wrapper).unwrap(); assert_eq!(fory.deserialize::(&bytes).unwrap(), wrapper); } } @@ -444,8 +455,13 @@ fn skip_ext() { id: i32, } impl Serializer for ExtItem { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), fory_core::error::Error> { + write_data(&self.id, fory, context, is_field) } fn fory_read_data( fory: &Fory, @@ -457,7 +473,7 @@ fn skip_ext() { }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { Self::fory_get_type_id(fory) } @@ -475,24 +491,28 @@ fn skip_ext() { f1: ExtItem, } let mut id_fory1 = Fory::default().compatible(true).xlang(true); - id_fory1.register_serializer::(100); - id_fory1.register::(101); + id_fory1.register_serializer::(100).unwrap(); + id_fory1.register::(101).unwrap(); let mut id_fory2 = Fory::default().compatible(true).xlang(true); - id_fory2.register_serializer::(100); - id_fory2.register::(101); + id_fory2.register_serializer::(100).unwrap(); + id_fory2.register::(101).unwrap(); let mut name_fory1 = Fory::default().compatible(true).xlang(true); - name_fory1.register_serializer_by_name::("ext_item"); - name_fory1.register::(101); + name_fory1 + .register_serializer_by_name::("ext_item") + .unwrap(); + name_fory1.register::(101).unwrap(); let mut name_fory2 = Fory::default().compatible(true).xlang(true); - name_fory2.register_serializer_by_name::("ext_item"); - name_fory2.register::(101); + name_fory2 + .register_serializer_by_name::("ext_item") + .unwrap(); + name_fory2.register::(101).unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { let wrapper = ExtWrapper { f1: ExtItem { id: 1 }, }; - let bytes = fory1.serialize(&wrapper); + let bytes = fory1.serialize(&wrapper).unwrap(); assert_eq!( fory2.deserialize::(&bytes).unwrap(), Empty::default() @@ -507,8 +527,13 @@ fn compatible_ext() { id: i32, } impl Serializer for ExtItem { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_data(&self.id, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), fory_core::error::Error> { + write_data(&self.id, fory, context, is_field) } fn fory_read_data( fory: &Fory, @@ -519,7 +544,7 @@ fn compatible_ext() { id: read_data(fory, context, is_field)?, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { Self::fory_get_type_id(fory) } @@ -541,24 +566,28 @@ fn compatible_ext() { f1: Option, } let mut id_fory1 = Fory::default().compatible(true).xlang(true); - id_fory1.register_serializer::(100); - id_fory1.register::(101); + id_fory1.register_serializer::(100).unwrap(); + id_fory1.register::(101).unwrap(); let mut id_fory2 = Fory::default().compatible(true).xlang(true); - id_fory2.register_serializer::(100); - id_fory2.register::(101); + id_fory2.register_serializer::(100).unwrap(); + id_fory2.register::(101).unwrap(); let mut name_fory1 = Fory::default().compatible(true).xlang(true); - name_fory1.register_serializer_by_name::("ext_item"); - name_fory1.register::(101); + name_fory1 + .register_serializer_by_name::("ext_item") + .unwrap(); + name_fory1.register::(101).unwrap(); let mut name_fory2 = Fory::default().compatible(true).xlang(true); - name_fory2.register_serializer_by_name::("ext_item"); - name_fory2.register::(101); + name_fory2 + .register_serializer_by_name::("ext_item") + .unwrap(); + name_fory2.register::(101).unwrap(); for (fory1, fory2) in [(id_fory1, id_fory2), (name_fory1, name_fory2)] { let wrapper = ExtWrapper1 { f1: ExtItem { id: 1 }, }; - let bytes = fory1.serialize(&wrapper); + let bytes = fory1.serialize(&wrapper).unwrap(); assert_eq!( fory2 .deserialize::(&bytes) diff --git a/rust/tests/tests/test_any.rs b/rust/tests/tests/test_any.rs index 169876731f..13929f5cc3 100644 --- a/rust/tests/tests/test_any.rs +++ b/rust/tests/tests/test_any.rs @@ -27,7 +27,7 @@ fn test_box_dyn_any() { let fory = Fory::default(); let value: Box = Box::new("hello".to_string()); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Box = fory.deserialize(&bytes).unwrap(); assert_eq!( deserialized.downcast_ref::().unwrap(), @@ -35,17 +35,17 @@ fn test_box_dyn_any() { ); let value2: Box = Box::new(42i32); - let bytes2 = fory.serialize(&value2); + let bytes2 = fory.serialize(&value2).unwrap(); let deserialized2: Box = fory.deserialize(&bytes2).unwrap(); assert_eq!(deserialized2.downcast_ref::().unwrap(), &42i32); let value3: Box = Box::new("".to_string()); - let bytes3 = fory.serialize(&value3); + let bytes3 = fory.serialize(&value3).unwrap(); let deserialized3: Box = fory.deserialize(&bytes3).unwrap(); assert_eq!(deserialized3.downcast_ref::().unwrap(), ""); let value5: Box = Box::new(3.15f64); - let bytes5 = fory.serialize(&value5); + let bytes5 = fory.serialize(&value5).unwrap(); let deserialized5: Box = fory.deserialize(&bytes5).unwrap(); assert_eq!(deserialized5.downcast_ref::().unwrap(), &3.15f64); } @@ -54,7 +54,7 @@ fn test_box_dyn_any() { fn test_rc_dyn_any() { let fory = Fory::default(); let value: Rc = Rc::new("world".to_string()); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Rc = fory.deserialize(&bytes).unwrap(); assert_eq!( deserialized.downcast_ref::().unwrap(), @@ -62,12 +62,12 @@ fn test_rc_dyn_any() { ); let value2: Rc = Rc::new(99i32); - let bytes2 = fory.serialize(&value2); + let bytes2 = fory.serialize(&value2).unwrap(); let deserialized2: Rc = fory.deserialize(&bytes2).unwrap(); assert_eq!(deserialized2.downcast_ref::().unwrap(), &99i32); let value3: Rc = Rc::new(true); - let bytes3 = fory.serialize(&value3); + let bytes3 = fory.serialize(&value3).unwrap(); let deserialized3: Rc = fory.deserialize(&bytes3).unwrap(); assert_eq!(deserialized3.downcast_ref::().unwrap(), &true); } @@ -77,7 +77,7 @@ fn test_arc_dyn_any() { let fory = Fory::default(); let value: Arc = Arc::new("arc test".to_string()); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Arc = fory.deserialize(&bytes).unwrap(); assert_eq!( deserialized.downcast_ref::().unwrap(), @@ -85,12 +85,12 @@ fn test_arc_dyn_any() { ); let value2: Arc = Arc::new(123i32); - let bytes2 = fory.serialize(&value2); + let bytes2 = fory.serialize(&value2).unwrap(); let deserialized2: Arc = fory.deserialize(&bytes2).unwrap(); assert_eq!(deserialized2.downcast_ref::().unwrap(), &123i32); let value3: Arc = Arc::new(vec![1, 2, 3]); - let bytes3 = fory.serialize(&value3); + let bytes3 = fory.serialize(&value3).unwrap(); let deserialized3: Arc = fory.deserialize(&bytes3).unwrap(); assert_eq!( deserialized3.downcast_ref::>().unwrap(), @@ -106,7 +106,7 @@ fn test_rc_dyn_any_shared_reference() { let data = vec![shared_str.clone(), shared_str.clone()]; - let bytes = fory.serialize(&data); + let bytes = fory.serialize(&data).unwrap(); let deserialized: Vec> = fory.deserialize(&bytes).unwrap(); let first_str = deserialized[0].downcast_ref::().unwrap(); @@ -125,7 +125,7 @@ fn test_arc_dyn_any_shared_reference() { let data = vec![shared_vec.clone(), shared_vec.clone()]; - let bytes = fory.serialize(&data); + let bytes = fory.serialize(&data).unwrap(); let deserialized: Vec> = fory.deserialize(&bytes).unwrap(); let first_vec = deserialized[0].downcast_ref::>().unwrap(); @@ -146,7 +146,8 @@ fn test_any_registered_by_name() { } let mut fory = Fory::default(); - fory.register_by_namespace::("test", "Person"); + fory.register_by_namespace::("test", "Person") + .unwrap(); let person = Person { name: "Alice".to_string(), @@ -154,7 +155,7 @@ fn test_any_registered_by_name() { }; let value: Box = Box::new(person); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Box = fory.deserialize(&bytes).unwrap(); let result = deserialized.downcast_ref::().unwrap(); @@ -173,7 +174,7 @@ fn test_mixed_any_types() { } let mut fory = Fory::default(); - fory.register_by_name::("Item"); + fory.register_by_name::("Item").unwrap(); let item = Item { id: 123, @@ -187,7 +188,7 @@ fn test_mixed_any_types() { Box::new(3.15f64), ]; - let bytes = fory.serialize(&mixed); + let bytes = fory.serialize(&mixed).unwrap(); let deserialized: Vec> = fory.deserialize(&bytes).unwrap(); assert_eq!(deserialized[0].downcast_ref::().unwrap(), &42i32); @@ -209,7 +210,7 @@ struct Container { #[test] fn test_arc_by_name() { let mut fory = Fory::default(); - fory.register_by_name::("Container"); + fory.register_by_name::("Container").unwrap(); let container = Container { id: 999, @@ -217,7 +218,7 @@ fn test_arc_by_name() { }; let value: Arc = Arc::new(container); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Arc = fory.deserialize(&bytes).unwrap(); let result = deserialized.downcast_ref::().unwrap(); @@ -225,7 +226,7 @@ fn test_arc_by_name() { assert_eq!(result.items, vec!["a", "b", "c"]); let container_vec: Vec> = vec![value.clone(), value.clone()]; - let bytes_vec = fory.serialize(&container_vec); + let bytes_vec = fory.serialize(&container_vec).unwrap(); let deserialized_vec: Vec> = fory.deserialize(&bytes_vec).unwrap(); assert_eq!(deserialized_vec.len(), 2); let first = deserialized_vec[0].downcast_ref::().unwrap(); @@ -240,7 +241,7 @@ fn test_arc_by_name() { #[test] fn test_rc_by_name() { let mut fory = Fory::default(); - fory.register_by_name::("Container"); + fory.register_by_name::("Container").unwrap(); let container = Container { id: 555, @@ -248,7 +249,7 @@ fn test_rc_by_name() { }; let value: Rc = Rc::new(container); - let bytes = fory.serialize(&value); + let bytes = fory.serialize(&value).unwrap(); let deserialized: Rc = fory.deserialize(&bytes).unwrap(); let result = deserialized.downcast_ref::().unwrap(); @@ -256,7 +257,7 @@ fn test_rc_by_name() { assert_eq!(result.items, vec!["x", "y"]); let container_vec: Vec> = vec![value.clone(), value.clone()]; - let bytes_vec = fory.serialize(&container_vec); + let bytes_vec = fory.serialize(&container_vec).unwrap(); let deserialized_vec: Vec> = fory.deserialize(&bytes_vec).unwrap(); assert_eq!(deserialized_vec.len(), 2); let first = deserialized_vec[0].downcast_ref::().unwrap(); diff --git a/rust/tests/tests/test_box.rs b/rust/tests/tests/test_box.rs index 3f37d5a198..a8a8b9626b 100644 --- a/rust/tests/tests/test_box.rs +++ b/rust/tests/tests/test_box.rs @@ -25,13 +25,13 @@ fn test_box_primitive() { // Test Box let value = Box::new(42i32); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box = fory.deserialize(&bin).expect("Should deserialize Box"); assert_eq!(*value, *deserialized); // Test Box let value = Box::new("Hello, Box!".to_string()); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box = fory .deserialize(&bin) .expect("Should deserialize Box"); @@ -39,7 +39,7 @@ fn test_box_primitive() { // Test Box let value = Box::new(std::f64::consts::PI); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box = fory.deserialize(&bin).expect("Should deserialize Box"); assert_eq!(*value, *deserialized); } @@ -53,14 +53,14 @@ fn test_box_struct() { } let mut fory = Fory::default(); - fory.register::(999); + fory.register::(999).unwrap(); let person = Person { name: "John Doe".to_string(), age: 30, }; let value = Box::new(person); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box = fory .deserialize(&bin) .expect("Should deserialize Box"); @@ -76,7 +76,7 @@ fn test_box_struct_separate() { } let mut fory = Fory::default(); - fory.register::(999); + fory.register::(999).unwrap(); // Test serializing the Box directly, not as a field let person = Person { @@ -84,7 +84,7 @@ fn test_box_struct_separate() { age: 25, }; let boxed_person = Box::new(person); - let bin = fory.serialize(&boxed_person); + let bin = fory.serialize(&boxed_person).unwrap(); let deserialized: Box = fory .deserialize(&bin) .expect("Should deserialize Box"); @@ -97,7 +97,7 @@ fn test_box_collection() { // Test Box> let value = Box::new(vec![1, 2, 3, 4, 5]); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box> = fory .deserialize(&bin) .expect("Should deserialize Box>"); @@ -108,7 +108,7 @@ fn test_box_collection() { map.insert("key1".to_string(), 10); map.insert("key2".to_string(), 20); let value = Box::new(map); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box> = fory .deserialize(&bin) .expect("Should deserialize Box>"); @@ -121,7 +121,7 @@ fn test_box_option() { // Test Box> with Some value let value = Box::new(Some("Hello".to_string())); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box> = fory .deserialize(&bin) .expect("Should deserialize Box>"); @@ -138,7 +138,7 @@ fn test_nested_box() { let fory = Fory::default(); let value = Box::new(Box::new(42i32)); - let bin = fory.serialize(&value); + let bin = fory.serialize(&value).unwrap(); let deserialized: Box> = fory .deserialize(&bin) .expect("Should deserialize Box>"); diff --git a/rust/tests/tests/test_buffer.rs b/rust/tests/tests/test_buffer.rs index c4dee48d07..ef25f78324 100644 --- a/rust/tests/tests/test_buffer.rs +++ b/rust/tests/tests/test_buffer.rs @@ -45,7 +45,7 @@ fn test_varint32() { writer.write_varint32(data); let binding = writer.dump(); let mut reader = Reader::new(binding.as_slice()); - let res = reader.read_varint32(); + let res = reader.read_varint32().unwrap(); assert_eq!(res, data); } for &data in &test_data { @@ -53,7 +53,7 @@ fn test_varint32() { writer.write_varuint32(data as u32); let binding = writer.dump(); let mut reader = Reader::new(binding.as_slice()); - let res = reader.read_varuint32(); + let res = reader.read_varuint32().unwrap(); assert_eq!(res, data as u32); } } @@ -89,7 +89,7 @@ fn test_varuint36_small() { let buf = writer.dump(); let mut reader = Reader::new(buf.as_slice()); - let value = reader.read_varuint36small(); + let value = reader.read_varuint36small().unwrap(); assert_eq!(value, data, "failed for data {}", data); } } diff --git a/rust/tests/tests/test_collection.rs b/rust/tests/tests/test_collection.rs index dc370ba36a..116b397072 100644 --- a/rust/tests/tests/test_collection.rs +++ b/rust/tests/tests/test_collection.rs @@ -29,7 +29,7 @@ fn test_btreeset_roundtrip() { original.insert(3); let trait_obj: Box = Box::new(original.clone()); - let serialized = fory.serialize(&trait_obj); + let serialized = fory.serialize(&trait_obj).unwrap(); let deserialized_concrete: BTreeSet = fory.deserialize(&serialized).unwrap(); @@ -48,7 +48,7 @@ fn test_binaryheap_roundtrip() { original.push(20); original.push(15); - let serialized = fory.serialize(&original); + let serialized = fory.serialize(&original).unwrap(); let deserialized_concrete: BinaryHeap = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized_concrete.len(), 3); @@ -64,7 +64,7 @@ struct SetContainer { #[test] fn test_set_container() { let mut fory: Fory = Fory::default(); - fory.register::(100); + fory.register::(100).unwrap(); let mut btree = BTreeSet::new(); btree.insert("apple".to_string()); @@ -81,7 +81,7 @@ fn test_set_container() { hash_set: hash, }; - let serialized = fory.serialize(&original); + let serialized = fory.serialize(&original).unwrap(); let deserialized: SetContainer = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized, original); @@ -103,7 +103,7 @@ struct HeapContainer { #[test] fn test_heap_container() { let mut fory: Fory = Fory::default(); - fory.register::(100); + fory.register::(100).unwrap(); let mut binary_heap = BinaryHeap::new(); binary_heap.push(3); @@ -112,7 +112,7 @@ fn test_heap_container() { let original = HeapContainer { binary_heap }; - let serialized = fory.serialize(&original); + let serialized = fory.serialize(&original).unwrap(); let deserialized: HeapContainer = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.binary_heap.len(), 3); diff --git a/rust/tests/tests/test_complex_refs.rs b/rust/tests/tests/test_complex_refs.rs index f790f4ce1f..d19da70bc2 100644 --- a/rust/tests/tests/test_complex_refs.rs +++ b/rust/tests/tests/test_complex_refs.rs @@ -35,7 +35,7 @@ fn test_rc_shared_in_nested_vec() { vec![shared1.clone()], ]; - let serialized = fory.serialize(&nested); + let serialized = fory.serialize(&nested).unwrap(); let deserialized: Vec>> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -67,7 +67,7 @@ fn test_arc_shared_in_nested_vec() { vec![shared1.clone()], ]; - let serialized = fory.serialize(&nested); + let serialized = fory.serialize(&nested).unwrap(); let deserialized: Vec>> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -97,8 +97,8 @@ fn test_mixed_rc_arc_sharing() { let rc_vec = vec![shared_rc.clone(), shared_rc.clone()]; let arc_vec = vec![shared_arc.clone(), shared_arc.clone()]; - let serialized_rc = fory.serialize(&rc_vec); - let serialized_arc = fory.serialize(&arc_vec); + let serialized_rc = fory.serialize(&rc_vec).unwrap(); + let serialized_arc = fory.serialize(&arc_vec).unwrap(); let deserialized_rc: Vec> = fory.deserialize(&serialized_rc).unwrap(); let deserialized_arc: Vec> = fory.deserialize(&serialized_arc).unwrap(); @@ -125,7 +125,7 @@ fn test_deep_sharing_stress_test() { vec![vec![shared.clone()]], ]; - let serialized = fory.serialize(&deep_structure); + let serialized = fory.serialize(&deep_structure).unwrap(); let deserialized: Vec>>> = fory.deserialize(&serialized).unwrap(); // Verify structure diff --git a/rust/tests/tests/test_complex_struct.rs b/rust/tests/tests/test_complex_struct.rs index 86d9e14259..accc593a71 100644 --- a/rust/tests/tests/test_complex_struct.rs +++ b/rust/tests/tests/test_complex_struct.rs @@ -58,9 +58,9 @@ fn enum_without_payload() { Blue, } let mut fory = Fory::default(); - fory.register::(999); + fory.register::(999).unwrap(); let color = Color::Red; - let bin = fory.serialize(&color); + let bin = fory.serialize(&color).unwrap(); let color2: Color = fory.deserialize(&bin).expect(""); assert_eq!(color, color2); } @@ -109,15 +109,15 @@ fn complex_struct() { c6: 4.0, }; let mut fory = Fory::default(); - fory.register::(899); - fory.register::(999); - let bin: Vec = fory.serialize(&person); + fory.register::(899).unwrap(); + fory.register::(999).unwrap(); + let bin: Vec = fory.serialize(&person).unwrap(); let obj: Person = fory.deserialize(&bin).expect("should success"); assert_eq!(person, obj); let mut fory = Fory::default(); - fory.register_by_name::("animal"); - fory.register_by_name::("person"); - let bin: Vec = fory.serialize(&person); + fory.register_by_name::("animal").unwrap(); + fory.register_by_name::("person").unwrap(); + let bin: Vec = fory.serialize(&person).unwrap(); let obj: Person = fory.deserialize(&bin).expect("should success"); assert_eq!(person, obj); } @@ -142,19 +142,21 @@ fn encode_to_obin() { f10: HashMap, } let mut fory = Fory::default(); - fory.register::(999); - fory.register::(899); - let bin: Vec = fory.serialize(&Person { - f1: "Hello".to_string(), - f2: HashMap::from([("hello1".to_string(), 1), ("hello2".to_string(), 2)]), - f3: 1, - f4: 2, - f5: 3, - f6: 4, - f7: 5.0, - f8: 6.0, - f10: HashMap::from([(1, 1.0), (2, 2.0)]), - }); + fory.register::(999).unwrap(); + fory.register::(899).unwrap(); + let bin: Vec = fory + .serialize(&Person { + f1: "Hello".to_string(), + f2: HashMap::from([("hello1".to_string(), 1), ("hello2".to_string(), 2)]), + f3: 1, + f4: 2, + f5: 3, + f6: 4, + f7: 5.0, + f8: 6.0, + f10: HashMap::from([(1, 1.0), (2, 2.0)]), + }) + .unwrap(); print!("{bin:?}"); } diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 3b044971c9..274ea803d0 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -68,17 +68,17 @@ fn test_buffer() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let mut reader = Reader::new(bytes.as_slice()); - assert_eq!(reader.read_u8(), 1); - assert_eq!(reader.read_i8(), i8::MAX); - assert_eq!(reader.read_i16(), i16::MAX); - assert_eq!(reader.read_i32(), i32::MAX); - assert_eq!(reader.read_i64(), i64::MAX); - assert_eq!(reader.read_f32(), -1.1f32); - assert_eq!(reader.read_f64(), -1.1f64); - assert_eq!(reader.read_varuint32(), 100); - let bytes_size = reader.read_i32() as usize; + assert_eq!(reader.read_u8().unwrap(), 1); + assert_eq!(reader.read_i8().unwrap(), i8::MAX); + assert_eq!(reader.read_i16().unwrap(), i16::MAX); + assert_eq!(reader.read_i32().unwrap(), i32::MAX); + assert_eq!(reader.read_i64().unwrap(), i64::MAX); + assert_eq!(reader.read_f32().unwrap(), -1.1f32); + assert_eq!(reader.read_f64().unwrap(), -1.1f64); + assert_eq!(reader.read_varuint32().unwrap(), 100); + let bytes_size = reader.read_i32().unwrap() as usize; let binary = b"ab"; - assert_eq!(reader.read_bytes(bytes_size), binary); + assert_eq!(reader.read_bytes(bytes_size).unwrap(), binary); let mut writer = Writer::default(); writer.write_u8(1); @@ -123,7 +123,7 @@ fn test_buffer_var() { i32::MAX, ]; for &expected in &varint32_values { - let value = reader.read_varint32(); + let value = reader.read_varint32().unwrap(); assert_eq!(expected, value, "varint32 value mismatch"); } let varuint32_values = vec![ @@ -141,7 +141,7 @@ fn test_buffer_var() { i32::MAX, ]; for &expected in &varuint32_values { - let value = reader.read_varuint32(); + let value = reader.read_varuint32().unwrap(); assert_eq!(expected, value as i32, "varuint32 value mismatch"); } let varuint64_values = vec![ @@ -166,7 +166,7 @@ fn test_buffer_var() { i64::MAX as u64, ]; for &expected in &varuint64_values { - let value = reader.read_varuint64(); + let value = reader.read_varuint64().unwrap(); assert_eq!(expected, value, "varuint64 value mismatch"); } let varint64_values = vec![ @@ -187,7 +187,7 @@ fn test_buffer_var() { i64::MAX, ]; for &expected in &varint64_values { - let value = reader.read_varint64(); + let value = reader.read_varint64().unwrap(); assert_eq!(expected, value, "varint64 value mismatch"); } @@ -214,8 +214,8 @@ fn test_murmurhash3() { let bytes = fs::read(&data_file_path).unwrap(); let mut reader = Reader::new(bytes.as_slice()); let (h1, h2) = murmurhash3_x64_128(&[1, 2, 8], 47); - assert_eq!(reader.read_i64(), h1 as i64); - assert_eq!(reader.read_i64(), h2 as i64); + assert_eq!(reader.read_i64().unwrap(), h1 as i64); + assert_eq!(reader.read_i64().unwrap(), h2 as i64); } #[test] @@ -262,7 +262,7 @@ fn test_string_serializer() { let fory = Fory::default().compatible(true).xlang(true); let mut context = WriteContext::new(writer); for s in &test_strings { - s.fory_write_data(&fory, &mut context, true); + s.fory_write_data(&fory, &mut context, true).unwrap(); } fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -292,7 +292,7 @@ fn test_cross_language_serializer() { let bytes = fs::read(&data_file_path).unwrap(); let reader = Reader::new(bytes.as_slice()); let mut fory = Fory::default().compatible(true).xlang(true); - fory.register::(101); + fory.register::(101).unwrap(); let mut context = ReadContext::new(reader, 5); assert_de!(fory, context, bool, true); assert_de!(fory, context, bool, false); @@ -323,32 +323,46 @@ fn test_cross_language_serializer() { let writer = Writer::default(); let mut context = WriteContext::new(writer); - fory.serialize_with_context(&true, &mut context); - fory.serialize_with_context(&false, &mut context); - fory.serialize_with_context(&-1, &mut context); - fory.serialize_with_context(&i8::MAX, &mut context); - fory.serialize_with_context(&i8::MIN, &mut context); - fory.serialize_with_context(&i16::MAX, &mut context); - fory.serialize_with_context(&i16::MIN, &mut context); - fory.serialize_with_context(&i32::MAX, &mut context); - fory.serialize_with_context(&i32::MIN, &mut context); - fory.serialize_with_context(&i64::MAX, &mut context); - fory.serialize_with_context(&i64::MIN, &mut context); - fory.serialize_with_context(&-1f32, &mut context); - fory.serialize_with_context(&-1f64, &mut context); - fory.serialize_with_context(&"str".to_string(), &mut context); - fory.serialize_with_context(&day, &mut context); - fory.serialize_with_context(&instant, &mut context); - fory.serialize_with_context(&vec![true, false], &mut context); - fory.serialize_with_context(&vec![1, i16::MAX], &mut context); - fory.serialize_with_context(&vec![1, i32::MAX], &mut context); - fory.serialize_with_context(&vec![1, i64::MAX], &mut context); - fory.serialize_with_context(&vec![1f32, 2f32], &mut context); - fory.serialize_with_context(&vec![1f64, 2f64], &mut context); - fory.serialize_with_context(&str_list, &mut context); - fory.serialize_with_context(&str_set, &mut context); - fory.serialize_with_context(&str_map, &mut context); - fory.serialize_with_context(&color, &mut context); + fory.serialize_with_context(&true, &mut context).unwrap(); + fory.serialize_with_context(&false, &mut context).unwrap(); + fory.serialize_with_context(&-1, &mut context).unwrap(); + fory.serialize_with_context(&i8::MAX, &mut context).unwrap(); + fory.serialize_with_context(&i8::MIN, &mut context).unwrap(); + fory.serialize_with_context(&i16::MAX, &mut context) + .unwrap(); + fory.serialize_with_context(&i16::MIN, &mut context) + .unwrap(); + fory.serialize_with_context(&i32::MAX, &mut context) + .unwrap(); + fory.serialize_with_context(&i32::MIN, &mut context) + .unwrap(); + fory.serialize_with_context(&i64::MAX, &mut context) + .unwrap(); + fory.serialize_with_context(&i64::MIN, &mut context) + .unwrap(); + fory.serialize_with_context(&-1f32, &mut context).unwrap(); + fory.serialize_with_context(&-1f64, &mut context).unwrap(); + fory.serialize_with_context(&"str".to_string(), &mut context) + .unwrap(); + fory.serialize_with_context(&day, &mut context).unwrap(); + fory.serialize_with_context(&instant, &mut context).unwrap(); + fory.serialize_with_context(&vec![true, false], &mut context) + .unwrap(); + fory.serialize_with_context(&vec![1, i16::MAX], &mut context) + .unwrap(); + fory.serialize_with_context(&vec![1, i32::MAX], &mut context) + .unwrap(); + fory.serialize_with_context(&vec![1, i64::MAX], &mut context) + .unwrap(); + fory.serialize_with_context(&vec![1f32, 2f32], &mut context) + .unwrap(); + fory.serialize_with_context(&vec![1f64, 2f64], &mut context) + .unwrap(); + fory.serialize_with_context(&str_list, &mut context) + .unwrap(); + fory.serialize_with_context(&str_set, &mut context).unwrap(); + fory.serialize_with_context(&str_map, &mut context).unwrap(); + fory.serialize_with_context(&color, &mut context).unwrap(); fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -358,9 +372,9 @@ fn test_simple_struct() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true).xlang(true); - fory.register::(101); - fory.register::(102); - fory.register::(103); + fory.register::(101).unwrap(); + fory.register::(102).unwrap(); + fory.register::(103).unwrap(); let local_obj = SimpleStruct { f1: HashMap::from([(1, 1.0f64), (2, 2.0f64)]), @@ -377,7 +391,7 @@ fn test_simple_struct() { }; let remote_obj: SimpleStruct = fory.deserialize(&bytes).unwrap(); assert_eq!(remote_obj, local_obj); - let new_bytes = fory.serialize(&remote_obj); + let new_bytes = fory.serialize(&remote_obj).unwrap(); let new_local_obj: SimpleStruct = fory.deserialize(&new_bytes).unwrap(); assert_eq!(new_local_obj, local_obj); fs::write(&data_file_path, new_bytes).unwrap(); @@ -389,9 +403,11 @@ fn test_simple_named_struct() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true).xlang(true); - fory.register_by_namespace::("demo", "color"); - fory.register_by_namespace::("demo", "item"); - fory.register_by_namespace::("demo", "simple_struct"); + fory.register_by_namespace::("demo", "color") + .unwrap(); + fory.register_by_namespace::("demo", "item").unwrap(); + fory.register_by_namespace::("demo", "simple_struct") + .unwrap(); let local_obj = SimpleStruct { f1: HashMap::from([(1, 1.0f64), (2, 2.0f64)]), @@ -408,7 +424,7 @@ fn test_simple_named_struct() { }; let remote_obj: SimpleStruct = fory.deserialize(&bytes).unwrap(); assert_eq!(remote_obj, local_obj); - let new_bytes = fory.serialize(&remote_obj); + let new_bytes = fory.serialize(&remote_obj).unwrap(); let new_local_obj: SimpleStruct = fory.deserialize(&new_bytes).unwrap(); assert_eq!(new_local_obj, local_obj); fs::write(&data_file_path, new_bytes).unwrap(); @@ -421,7 +437,7 @@ fn test_list() { let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true); - fory.register::(102); + fory.register::(102).unwrap(); let reader = Reader::new(bytes.as_slice()); let mut context = ReadContext::new(reader, 5); @@ -451,10 +467,14 @@ fn test_list() { let writer = Writer::default(); let mut context = WriteContext::new(writer); - fory.serialize_with_context(&remote_str_list, &mut context); - fory.serialize_with_context(&remote_str_list2, &mut context); - fory.serialize_with_context(&remote_item_list, &mut context); - fory.serialize_with_context(&remote_item_list2, &mut context); + fory.serialize_with_context(&remote_str_list, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_str_list2, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_item_list, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_item_list2, &mut context) + .unwrap(); fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -466,7 +486,7 @@ fn test_map() { let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true); - fory.register::(102); + fory.register::(102).unwrap(); let reader = Reader::new(bytes.as_slice()); let mut context = ReadContext::new(reader, 5); @@ -501,7 +521,7 @@ fn test_map() { let remote_str_map: HashMap, Option> = fory.deserialize_with_context(&mut context).unwrap(); assert_eq!(remote_str_map, str_map); - let data_bytes1 = fory.serialize(&remote_str_map); + let data_bytes1 = fory.serialize(&remote_str_map).unwrap(); let new_local_str_map: HashMap, Option> = fory.deserialize(&data_bytes1).unwrap(); assert_eq!(new_local_str_map, str_map); @@ -509,7 +529,7 @@ fn test_map() { let remote_item_map: HashMap, Option> = fory.deserialize_with_context(&mut context).unwrap(); assert_eq!(remote_item_map, item_map); - let data_bytes2 = fory.serialize(&remote_item_map); + let data_bytes2 = fory.serialize(&remote_item_map).unwrap(); let new_local_item_map: HashMap, Option> = fory.deserialize(&data_bytes2).unwrap(); assert_eq!(new_local_item_map, item_map); @@ -535,7 +555,7 @@ fn test_integer() { let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true); - fory.register::(101); + fory.register::(101).unwrap(); let reader = Reader::new(bytes.as_slice()); let mut context = ReadContext::new(reader, 5); @@ -571,13 +591,20 @@ fn test_integer() { let writer = Writer::default(); let mut context = WriteContext::new(writer); - fory.serialize_with_context(&remote_item2, &mut context); - fory.serialize_with_context(&remote_f1, &mut context); - fory.serialize_with_context(&remote_f2, &mut context); - fory.serialize_with_context(&remote_f3, &mut context); - fory.serialize_with_context(&remote_f4, &mut context); - fory.serialize_with_context(&remote_f5, &mut context); - fory.serialize_with_context(&remote_f6, &mut context); + fory.serialize_with_context(&remote_item2, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f1, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f2, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f3, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f4, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f5, &mut context) + .unwrap(); + fory.serialize_with_context(&remote_f6, &mut context) + .unwrap(); fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -590,9 +617,14 @@ struct MyExt { id: i32, } impl Serializer for MyExt { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, _is_field: bool) { + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + _is_field: bool, + ) -> Result<(), fory_core::error::Error> { // set is_field=false to write type_id like in java - write_data(&self.id, fory, context, false); + write_data(&self.id, fory, context, false) } fn fory_read_data( @@ -606,7 +638,7 @@ impl Serializer for MyExt { }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { Self::fory_get_type_id(fory) } @@ -638,26 +670,26 @@ fn _test_skip_custom(fory1: &Fory, fory2: &Fory) { my_struct: MyStruct { id: 42 }, my_ext: MyExt { id: 43 }, }; - let bytes = fory2.serialize(&wrapper); + let bytes = fory2.serialize(&wrapper).unwrap(); fs::write(&data_file_path, bytes).unwrap(); } #[test] fn test_kankankan() { let mut fory1 = Fory::default().compatible(true); - fory1.register_serializer::(103); - fory1.register::(104); + fory1.register_serializer::(103).unwrap(); + fory1.register::(104).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register::(101); - fory2.register::(102); - fory2.register_serializer::(103); - fory2.register::(104); + fory2.register::(101).unwrap(); + fory2.register::(102).unwrap(); + fory2.register_serializer::(103).unwrap(); + fory2.register::(104).unwrap(); let wrapper = MyWrapper { color: Color::White, my_struct: MyStruct { id: 42 }, my_ext: MyExt { id: 43 }, }; - let bytes = fory2.serialize(&wrapper); + let bytes = fory2.serialize(&wrapper).unwrap(); fory1.deserialize::(&bytes).unwrap(); } @@ -665,13 +697,13 @@ fn test_kankankan() { #[ignore] fn test_skip_id_custom() { let mut fory1 = Fory::default().compatible(true); - fory1.register_serializer::(103); - fory1.register::(104); + fory1.register_serializer::(103).unwrap(); + fory1.register::(104).unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register::(101); - fory2.register::(102); - fory2.register_serializer::(103); - fory2.register::(104); + fory2.register::(101).unwrap(); + fory2.register::(102).unwrap(); + fory2.register_serializer::(103).unwrap(); + fory2.register::(104).unwrap(); _test_skip_custom(&fory1, &fory2); } @@ -679,23 +711,27 @@ fn test_skip_id_custom() { #[ignore] fn test_skip_name_custom() { let mut fory1 = Fory::default().compatible(true); - fory1.register_serializer_by_name::("my_ext"); - fory1.register_by_name::("my_wrapper"); + fory1 + .register_serializer_by_name::("my_ext") + .unwrap(); + fory1.register_by_name::("my_wrapper").unwrap(); let mut fory2 = Fory::default().compatible(true); - fory2.register_by_name::("color"); - fory2.register_by_name::("my_struct"); - fory2.register_serializer_by_name::("my_ext"); - fory2.register_by_name::("my_wrapper"); + fory2.register_by_name::("color").unwrap(); + fory2.register_by_name::("my_struct").unwrap(); + fory2 + .register_serializer_by_name::("my_ext") + .unwrap(); + fory2.register_by_name::("my_wrapper").unwrap(); _test_skip_custom(&fory1, &fory2); } #[test] #[ignore] fn test_consistent_named() { - let mut fory = Fory::default().compatible(false); - fory.register_by_name::("color"); - fory.register_by_name::("my_struct"); - fory.register_serializer_by_name::("my_ext"); + let mut fory = Fory::default().compatible(true); + fory.register_by_name::("color").unwrap(); + fory.register_by_name::("my_struct").unwrap(); + fory.register_serializer_by_name::("my_ext").unwrap(); let color = Color::White; let _my_struct = MyStruct { id: 42 }; @@ -740,13 +776,13 @@ fn test_consistent_named() { let writer = Writer::default(); let mut context = WriteContext::new(writer); - fory.serialize_with_context(&color, &mut context); - fory.serialize_with_context(&color, &mut context); - fory.serialize_with_context(&color, &mut context); + fory.serialize_with_context(&color, &mut context).unwrap(); + fory.serialize_with_context(&color, &mut context).unwrap(); + fory.serialize_with_context(&color, &mut context).unwrap(); // todo: checkVersion // fory.serialize_with_context(&my_struct, &mut context); - fory.serialize_with_context(&my_ext, &mut context); - fory.serialize_with_context(&my_ext, &mut context); - fory.serialize_with_context(&my_ext, &mut context); + fory.serialize_with_context(&my_ext, &mut context).unwrap(); + fory.serialize_with_context(&my_ext, &mut context).unwrap(); + fory.serialize_with_context(&my_ext, &mut context).unwrap(); fs::write(&data_file_path, context.writer.dump()).unwrap(); } diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index 5faf86c486..7fa2024396 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -59,8 +59,13 @@ fn test_use() { } impl Serializer for Item { - fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) { - write_data(&self.f1, fory, context, is_field); + fn fory_write_data( + &self, + fory: &Fory, + context: &mut WriteContext, + is_field: bool, + ) -> Result<(), Error> { + write_data(&self.f1, fory, context, is_field) } fn fory_read_data( @@ -74,7 +79,7 @@ fn test_use() { }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { + fn fory_type_id_dyn(&self, fory: &Fory) -> Result { Self::fory_get_type_id(fory) } @@ -84,8 +89,8 @@ fn test_use() { } let mut fory = Fory::default().compatible(true).xlang(true); let item = Item { f1: 1, f2: 2 }; - fory.register_serializer::(100); - let bytes = fory.serialize(&item); + fory.register_serializer::(100).unwrap(); + let bytes = fory.serialize(&item).unwrap(); let new_item: Item = fory.deserialize(&bytes).unwrap(); assert_eq!(new_item.f1, item.f1); assert_eq!(new_item.f2, 0); diff --git a/rust/tests/tests/test_list.rs b/rust/tests/tests/test_list.rs index 218b3d61ec..87311cf354 100644 --- a/rust/tests/tests/test_list.rs +++ b/rust/tests/tests/test_list.rs @@ -26,7 +26,7 @@ fn test_vecdeque_i32() { deque.push_back(1); deque.push_back(2); deque.push_back(3); - let bin = fory.serialize(&deque); + let bin = fory.serialize(&deque).unwrap(); let obj: VecDeque = fory.deserialize(&bin).expect("deserialize"); assert_eq!(deque, obj); } @@ -35,7 +35,7 @@ fn test_vecdeque_i32() { fn test_vecdeque_empty() { let fory = Fory::default(); let deque: VecDeque = VecDeque::new(); - let bin = fory.serialize(&deque); + let bin = fory.serialize(&deque).unwrap(); let obj: VecDeque = fory.deserialize(&bin).expect("deserialize"); assert_eq!(deque, obj); } @@ -46,7 +46,7 @@ fn test_vecdeque_string() { let mut deque = VecDeque::new(); deque.push_back("hello".to_string()); deque.push_back("world".to_string()); - let bin = fory.serialize(&deque); + let bin = fory.serialize(&deque).unwrap(); let obj: VecDeque = fory.deserialize(&bin).expect("deserialize"); assert_eq!(deque, obj); } @@ -58,7 +58,7 @@ fn test_vecdeque_f64() { deque.push_back(1.5); deque.push_back(2.5); deque.push_back(3.5); - let bin = fory.serialize(&deque); + let bin = fory.serialize(&deque).unwrap(); let obj: VecDeque = fory.deserialize(&bin).expect("deserialize"); assert_eq!(deque, obj); } @@ -70,7 +70,7 @@ fn test_linkedlist_i32() { list.push_back(1); list.push_back(2); list.push_back(3); - let bin = fory.serialize(&list); + let bin = fory.serialize(&list).unwrap(); let obj: LinkedList = fory.deserialize(&bin).expect("deserialize"); assert_eq!(list, obj); } @@ -79,7 +79,7 @@ fn test_linkedlist_i32() { fn test_linkedlist_empty() { let fory = Fory::default(); let list: LinkedList = LinkedList::new(); - let bin = fory.serialize(&list); + let bin = fory.serialize(&list).unwrap(); let obj: LinkedList = fory.deserialize(&bin).expect("deserialize"); assert_eq!(list, obj); } @@ -90,7 +90,7 @@ fn test_linkedlist_string() { let mut list = LinkedList::new(); list.push_back("foo".to_string()); list.push_back("bar".to_string()); - let bin = fory.serialize(&list); + let bin = fory.serialize(&list).unwrap(); let obj: LinkedList = fory.deserialize(&bin).expect("deserialize"); assert_eq!(list, obj); } @@ -102,7 +102,7 @@ fn test_linkedlist_bool() { list.push_back(true); list.push_back(false); list.push_back(true); - let bin = fory.serialize(&list); + let bin = fory.serialize(&list).unwrap(); let obj: LinkedList = fory.deserialize(&bin).expect("deserialize"); assert_eq!(list, obj); } @@ -117,7 +117,8 @@ struct CollectionStruct { #[test] fn test_struct_with_collections() { let mut fory = Fory::default(); - fory.register_by_name::("CollectionStruct"); + fory.register_by_name::("CollectionStruct") + .unwrap(); let mut deque = VecDeque::new(); deque.push_back("hello".to_string()); @@ -133,7 +134,7 @@ fn test_struct_with_collections() { list_field: list, }; - let bin = fory.serialize(&data); + let bin = fory.serialize(&data).unwrap(); let obj: CollectionStruct = fory.deserialize(&bin).expect("deserialize"); assert_eq!(data, obj); } diff --git a/rust/tests/tests/test_map.rs b/rust/tests/tests/test_map.rs index 38b2c18f74..f619800a77 100644 --- a/rust/tests/tests/test_map.rs +++ b/rust/tests/tests/test_map.rs @@ -25,7 +25,7 @@ fn test_hashmap_string() { let mut map = HashMap::new(); map.insert("key1".to_string(), "value1".to_string()); map.insert("key2".to_string(), "value2".to_string()); - let bin = fory.serialize(&map); + let bin = fory.serialize(&map).unwrap(); let obj: HashMap = fory.deserialize(&bin).expect("deserialize"); assert_eq!(map, obj); } @@ -36,7 +36,7 @@ fn test_btreemap_string() { let mut map = BTreeMap::new(); map.insert("key1".to_string(), "value1".to_string()); map.insert("key2".to_string(), "value2".to_string()); - let bin = fory.serialize(&map); + let bin = fory.serialize(&map).unwrap(); let obj: BTreeMap = fory.deserialize(&bin).expect("deserialize"); assert_eq!(map, obj); } @@ -50,7 +50,8 @@ struct MapContainer { #[test] fn test_struct_with_maps() { let mut fory = Fory::default(); - fory.register_by_name::("MapContainer"); + fory.register_by_name::("MapContainer") + .unwrap(); let mut hash_map = HashMap::new(); hash_map.insert("foo".to_string(), "bar".to_string()); let mut btree_map = BTreeMap::new(); @@ -62,7 +63,7 @@ fn test_struct_with_maps() { btree_map, }; - let bin = fory.serialize(&container); + let bin = fory.serialize(&container).unwrap(); let obj: MapContainer = fory.deserialize(&bin).expect("deserialize"); assert_eq!(container, obj); } diff --git a/rust/tests/tests/test_max_dyn_depth.rs b/rust/tests/tests/test_max_dyn_depth.rs index e4a1bff9fc..2205c95afe 100644 --- a/rust/tests/tests/test_max_dyn_depth.rs +++ b/rust/tests/tests/test_max_dyn_depth.rs @@ -29,7 +29,7 @@ struct Container { fn test_max_dyn_depth_exceeded_box_dyn_any() { for compatible in [false, true] { let mut fory = Fory::default().max_dyn_depth(2).compatible(compatible); - fory.register::(100); + fory.register::(100).unwrap(); let level3 = Container { value: 3, @@ -45,7 +45,7 @@ fn test_max_dyn_depth_exceeded_box_dyn_any() { }; let outer: Box = Box::new(level1); - let bytes = fory.serialize(&outer); + let bytes = fory.serialize(&outer).unwrap(); let result: Result, _> = fory.deserialize(&bytes); assert!( result.is_err(), @@ -60,7 +60,7 @@ fn test_max_dyn_depth_exceeded_box_dyn_any() { #[test] fn test_max_dyn_depth_within_limit_box_dyn_any() { let mut fory = Fory::default().max_dyn_depth(3); - fory.register::(100); + fory.register::(100).unwrap(); let level3 = Container { value: 3, @@ -76,7 +76,7 @@ fn test_max_dyn_depth_within_limit_box_dyn_any() { }; let outer: Box = Box::new(level1); - let bytes = fory.serialize(&outer); + let bytes = fory.serialize(&outer).unwrap(); let result: Result, _> = fory.deserialize(&bytes); assert!(result.is_ok()); } @@ -84,7 +84,7 @@ fn test_max_dyn_depth_within_limit_box_dyn_any() { #[test] fn test_max_dyn_depth_default_exceeded() { let mut fory = Fory::default(); - fory.register::(100); + fory.register::(100).unwrap(); let mut current = Container { value: 6, @@ -99,7 +99,7 @@ fn test_max_dyn_depth_default_exceeded() { } let outer: Box = Box::new(current); - let bytes = fory.serialize(&outer); + let bytes = fory.serialize(&outer).unwrap(); let result: Result, _> = fory.deserialize(&bytes); assert!(result.is_err()); @@ -112,7 +112,7 @@ fn test_max_dyn_depth_default_exceeded() { #[test] fn test_max_dyn_depth_default_within_limit() { let mut fory = Fory::default(); - fory.register::(100); + fory.register::(100).unwrap(); let mut current = Container { value: 5, @@ -127,7 +127,7 @@ fn test_max_dyn_depth_default_within_limit() { } let outer: Box = Box::new(current); - let bytes = fory.serialize(&outer); + let bytes = fory.serialize(&outer).unwrap(); let result: Result, _> = fory.deserialize(&bytes); assert!(result.is_ok()); diff --git a/rust/tests/tests/test_multi_thread.rs b/rust/tests/tests/test_multi_thread.rs index d1055caf6b..a8a7678c50 100644 --- a/rust/tests/tests/test_multi_thread.rs +++ b/rust/tests/tests/test_multi_thread.rs @@ -30,7 +30,7 @@ fn test_simple_multi_thread() { for item in &src { let fory_clone = Arc::clone(&fory); let item = *item; - let handle = thread::spawn(move || fory_clone.serialize(&item)); + let handle = thread::spawn(move || fory_clone.serialize(&item).unwrap()); handles.push(handle); } let mut serialized_data = vec![]; @@ -61,7 +61,7 @@ fn test_struct_multi_thread() { f1: i32, } let mut fory = Fory::default(); - fory.register::(101); + fory.register::(101).unwrap(); let fory = Arc::new(fory); let src: HashSet<_> = [ Item1 { f1: 42 }, @@ -77,7 +77,7 @@ fn test_struct_multi_thread() { for item in &src { let fory_clone = Arc::clone(&fory); let item = *item; - let handle = thread::spawn(move || fory_clone.serialize(&item)); + let handle = thread::spawn(move || fory_clone.serialize(&item).unwrap()); handles.push(handle); } let mut serialized_data = vec![]; diff --git a/rust/tests/tests/test_mutex.rs b/rust/tests/tests/test_mutex.rs index af2b759179..e22c68ba0c 100644 --- a/rust/tests/tests/test_mutex.rs +++ b/rust/tests/tests/test_mutex.rs @@ -22,7 +22,7 @@ use std::sync::{Arc, Mutex}; fn test_mutex_basic_serialization() { let fory = Fory::default(); let m = Mutex::new(42i32); - let serialized = fory.serialize(&m); + let serialized = fory.serialize(&m).unwrap(); let deserialized: Mutex = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.lock().unwrap().clone(), 42); } @@ -31,7 +31,7 @@ fn test_mutex_basic_serialization() { fn test_arc_mutex_serialization() { let fory = Fory::default(); let arc_mutex = Arc::new(Mutex::new(String::from("hello"))); - let serialized = fory.serialize(&arc_mutex); + let serialized = fory.serialize(&arc_mutex).unwrap(); let deserialized: Arc> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.lock().unwrap().as_str(), "hello"); } @@ -43,7 +43,7 @@ fn test_arc_mutex_sharing_preserved() { let data = Arc::new(Mutex::new(123i32)); let list = vec![data.clone(), data.clone()]; - let serialized = fory.serialize(&list); + let serialized = fory.serialize(&list).unwrap(); let deserialized: Vec>> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 2); diff --git a/rust/tests/tests/test_rc_arc.rs b/rust/tests/tests/test_rc_arc.rs index 7b6604051b..135d14f2bd 100644 --- a/rust/tests/tests/test_rc_arc.rs +++ b/rust/tests/tests/test_rc_arc.rs @@ -29,7 +29,7 @@ fn test_rc_string_serialization() { let data = String::from("Hello, Rc!"); let rc_data = Rc::new(data); - let serialized = fory.serialize(&rc_data); + let serialized = fory.serialize(&rc_data).unwrap(); let deserialized: Rc = fory.deserialize(&serialized).unwrap(); assert_eq!(*rc_data, *deserialized); @@ -43,7 +43,7 @@ fn test_arc_string_serialization() { let data = String::from("Hello, Arc!"); let arc_data = Arc::new(data); - let serialized = fory.serialize(&arc_data); + let serialized = fory.serialize(&arc_data).unwrap(); let deserialized: Arc = fory.deserialize(&serialized).unwrap(); assert_eq!(*arc_data, *deserialized); @@ -56,7 +56,7 @@ fn test_rc_number_serialization() { let rc_number = Rc::new(42i32); - let serialized = fory.serialize(&rc_number); + let serialized = fory.serialize(&rc_number).unwrap(); let deserialized: Rc = fory.deserialize(&serialized).unwrap(); assert_eq!(*rc_number, *deserialized); @@ -69,7 +69,7 @@ fn test_arc_number_serialization() { let arc_number = Arc::new(100i64); - let serialized = fory.serialize(&arc_number); + let serialized = fory.serialize(&arc_number).unwrap(); let deserialized: Arc = fory.deserialize(&serialized).unwrap(); assert_eq!(*arc_number, *deserialized); @@ -85,7 +85,7 @@ fn test_rc_in_collections() { let strings = vec![string1.clone(), string2.clone(), string1.clone()]; - let serialized = fory.serialize(&strings); + let serialized = fory.serialize(&strings).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(strings.len(), deserialized.len()); @@ -106,7 +106,7 @@ fn test_arc_in_collections() { let numbers = vec![number1.clone(), number2.clone(), number1.clone()]; - let serialized = fory.serialize(&numbers); + let serialized = fory.serialize(&numbers).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(numbers.len(), deserialized.len()); @@ -125,7 +125,7 @@ fn test_rc_vec_serialization() { let data = vec![1, 2, 3, 4, 5]; let rc_data = Rc::new(data); - let serialized = fory.serialize(&rc_data); + let serialized = fory.serialize(&rc_data).unwrap(); let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); assert_eq!(*rc_data, *deserialized); @@ -139,7 +139,7 @@ fn test_arc_vec_serialization() { let data = vec![String::from("a"), String::from("b"), String::from("c")]; let arc_data = Arc::new(data); - let serialized = fory.serialize(&arc_data); + let serialized = fory.serialize(&arc_data).unwrap(); let deserialized: Arc> = fory.deserialize(&serialized).unwrap(); assert_eq!(*arc_data, *deserialized); @@ -154,8 +154,8 @@ fn test_mixed_rc_arc_serialization() { let rc_number = Rc::new(42i32); let arc_number = Arc::new(100i64); - let rc_serialized = fory.serialize(&rc_number); - let arc_serialized = fory.serialize(&arc_number); + let rc_serialized = fory.serialize(&rc_number).unwrap(); + let arc_serialized = fory.serialize(&arc_number).unwrap(); let rc_deserialized: Rc = fory.deserialize(&rc_serialized).unwrap(); let arc_deserialized: Arc = fory.deserialize(&arc_serialized).unwrap(); @@ -172,7 +172,7 @@ fn test_nested_rc_arc() { let inner_data = Arc::new(String::from("nested")); let outer_data = Rc::new(inner_data.clone()); - let serialized = fory.serialize(&outer_data); + let serialized = fory.serialize(&outer_data).unwrap(); let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); assert_eq!(**outer_data, **deserialized); @@ -188,7 +188,7 @@ fn test_rc_arc_with_hashmaps() { map.insert("key1".to_string(), string_data.clone()); map.insert("key2".to_string(), string_data.clone()); - let serialized = fory.serialize(&map); + let serialized = fory.serialize(&map).unwrap(); let deserialized: HashMap> = fory.deserialize(&serialized).unwrap(); assert_eq!(map.len(), deserialized.len()); @@ -205,7 +205,7 @@ fn test_arc_serialization_basic() { let fory = Fory::default(); let arc = Arc::new(42i32); - let serialized = fory.serialize(&arc); + let serialized = fory.serialize(&arc).unwrap(); let deserialized: Arc = fory.deserialize(&serialized).unwrap(); assert_eq!(*deserialized, 42); @@ -216,7 +216,7 @@ fn test_arc_shared_reference() { let fory = Fory::default(); let arc1 = Arc::new(String::from("shared")); - let serialized = fory.serialize(&arc1); + let serialized = fory.serialize(&arc1).unwrap(); let deserialized: Arc = fory.deserialize(&serialized).unwrap(); assert_eq!(*deserialized, "shared"); @@ -229,7 +229,7 @@ fn test_arc_shared_reference_in_vec() { let shared = Arc::new(String::from("shared_value")); let vec = vec![shared.clone(), shared.clone(), shared.clone()]; - let serialized = fory.serialize(&vec); + let serialized = fory.serialize(&vec).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -257,7 +257,7 @@ fn test_arc_multiple_shared_references() { shared1.clone(), ]; - let serialized = fory.serialize(&vec); + let serialized = fory.serialize(&vec).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 5); @@ -280,7 +280,7 @@ fn test_arc_thread_safety() { let fory = Fory::default(); let arc = Arc::new(vec![1, 2, 3, 4, 5]); - let serialized = fory.serialize(&arc); + let serialized = fory.serialize(&arc).unwrap(); // Test that Arc can be sent across threads let handle = thread::spawn(move || { @@ -297,7 +297,7 @@ fn test_rc_serialization_basic() { let fory = Fory::default(); let rc = Rc::new(42i32); - let serialized = fory.serialize(&rc); + let serialized = fory.serialize(&rc).unwrap(); let deserialized: Rc = fory.deserialize(&serialized).unwrap(); assert_eq!(*deserialized, 42); @@ -308,7 +308,7 @@ fn test_rc_shared_reference() { let fory = Fory::default(); let rc1 = Rc::new(String::from("shared")); - let serialized = fory.serialize(&rc1); + let serialized = fory.serialize(&rc1).unwrap(); let deserialized: Rc = fory.deserialize(&serialized).unwrap(); assert_eq!(*deserialized, "shared"); @@ -321,7 +321,7 @@ fn test_rc_shared_reference_in_vec() { let shared = Rc::new(String::from("shared_value")); let vec = vec![shared.clone(), shared.clone(), shared.clone()]; - let serialized = fory.serialize(&vec); + let serialized = fory.serialize(&vec).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -349,7 +349,7 @@ fn test_rc_multiple_shared_references() { shared1.clone(), ]; - let serialized = fory.serialize(&vec); + let serialized = fory.serialize(&vec).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 5); diff --git a/rust/tests/tests/test_rc_arc_trait_object.rs b/rust/tests/tests/test_rc_arc_trait_object.rs index 0dd5505cf6..68ccaa872b 100644 --- a/rust/tests/tests/test_rc_arc_trait_object.rs +++ b/rust/tests/tests/test_rc_arc_trait_object.rs @@ -228,8 +228,8 @@ fn test_conversion_helper_macros() { #[test] fn test_nested_wrapper_collections() { let mut fory = fory_compatible(); - fory.register::(100); - fory.register::(101); + fory.register::(100).unwrap(); + fory.register::(101).unwrap(); // Wrapper types are not registered since they're transparent @@ -244,7 +244,7 @@ fn test_nested_wrapper_collections() { color: "Color1".to_string(), }) as Rc), ]; - let serialized_simple = fory.serialize(&simple_wrappers); + let serialized_simple = fory.serialize(&simple_wrappers).unwrap(); let deserialized_simple: Vec = fory.deserialize(&serialized_simple).unwrap(); assert_eq!(deserialized_simple.len(), 2); @@ -269,7 +269,7 @@ fn test_nested_wrapper_collections() { }) as Rc)], ]; - let serialized = fory.serialize(&nested_wrappers); + let serialized = fory.serialize(&nested_wrappers).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 2); @@ -286,22 +286,22 @@ fn test_nested_wrapper_collections() { #[test] fn test_empty_wrapper_collections() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); // Test empty collections let empty_rc_vec: Vec = vec![]; - let serialized = fory.serialize(&empty_rc_vec); + let serialized = fory.serialize(&empty_rc_vec).unwrap(); let deserialized: Vec = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 0); let empty_arc_vec: Vec = vec![]; - let serialized = fory.serialize(&empty_arc_vec); + let serialized = fory.serialize(&empty_arc_vec).unwrap(); let deserialized: Vec = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 0); let empty_map: HashMap = HashMap::new(); - let serialized = fory.serialize(&empty_map); + let serialized = fory.serialize(&empty_map).unwrap(); let deserialized: HashMap = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 0); } @@ -316,9 +316,9 @@ struct AnimalShelter { #[test] fn test_collections_of_wrappers() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); - fory.register::(8011); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); + fory.register::(8011).unwrap(); let shelter = AnimalShelter { animals_rc: vec![ @@ -355,7 +355,7 @@ fn test_collections_of_wrappers() { }, }; - let serialized = fory.serialize(&shelter); + let serialized = fory.serialize(&shelter).unwrap(); let deserialized: AnimalShelter = fory.deserialize(&serialized).unwrap(); // Test Vec @@ -413,8 +413,8 @@ fn test_collections_of_wrappers() { #[test] fn test_rc_shared_ref_tracking() { let mut fory = fory_compatible(); - fory.register::(200); - fory.register::(201); + fory.register::(200).unwrap(); + fory.register::(201).unwrap(); let dog = Rc::new(Dog { name: "Rex".to_string(), @@ -427,7 +427,7 @@ fn test_rc_shared_ref_tracking() { AnimalRc::from(dog.clone()), ]; - let serialized = fory.serialize(&shared_animals); + let serialized = fory.serialize(&shared_animals).unwrap(); let deserialized: Vec = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -445,8 +445,8 @@ fn test_rc_shared_ref_tracking() { #[test] fn test_arc_shared_ref_tracking() { let mut fory = fory_compatible(); - fory.register::(200); - fory.register::(201); + fory.register::(200).unwrap(); + fory.register::(201).unwrap(); let cat = Arc::new(Cat { name: "Whiskers".to_string(), @@ -459,7 +459,7 @@ fn test_arc_shared_ref_tracking() { AnimalArc::from(cat.clone()), ]; - let serialized = fory.serialize(&shared_animals); + let serialized = fory.serialize(&shared_animals).unwrap(); let deserialized: Vec = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); diff --git a/rust/tests/tests/test_refcell.rs b/rust/tests/tests/test_refcell.rs index da7d91f1bc..8531f2505e 100644 --- a/rust/tests/tests/test_refcell.rs +++ b/rust/tests/tests/test_refcell.rs @@ -28,10 +28,10 @@ struct Simple { #[test] fn test_rc_refcell_simple() { let mut fory = Fory::default(); - fory.register::(3000); + fory.register::(3000).unwrap(); let node = Rc::new(RefCell::new(Simple { value: 42 })); - let serialized = fory.serialize(&node); + let serialized = fory.serialize(&node).unwrap(); let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.borrow().value, 42); } @@ -45,15 +45,15 @@ struct Parent { #[test] fn test_rc_refcell_in_struct() { let mut fory = Fory::default(); - fory.register::(3001); - fory.register::(3002); + fory.register::(3001).unwrap(); + fory.register::(3002).unwrap(); let child = Rc::new(RefCell::new(Simple { value: 99 })); let parent = Parent { value: 1, child: Some(child), }; - let serialized = fory.serialize(&parent); + let serialized = fory.serialize(&parent).unwrap(); let deserialized: Parent = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.value, 1); assert_eq!(deserialized.child.unwrap().borrow().value, 99); diff --git a/rust/tests/tests/test_row.rs b/rust/tests/tests/test_row.rs index c6de3ba24e..41ced830b8 100644 --- a/rust/tests/tests/test_row.rs +++ b/rust/tests/tests/test_row.rs @@ -48,7 +48,8 @@ fn row() { f4: vec![-1, 2, -3], f5, }, - }); + }) + .unwrap(); let obj = from_row::(&row); let f1: &str = obj.f3().f1(); diff --git a/rust/tests/tests/test_simple_struct.rs b/rust/tests/tests/test_simple_struct.rs index 9a9bbf53d0..1024a6facc 100644 --- a/rust/tests/tests/test_simple_struct.rs +++ b/rust/tests/tests/test_simple_struct.rs @@ -47,8 +47,8 @@ fn test_simple() { } let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999); - fory2.register::(999); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); let animal: Animal1 = Animal1 { f1: HashMap::from([(1, vec![2])]), f2: String::from("hello"), @@ -59,7 +59,7 @@ fn test_simple() { last: 44, }; - let bin = fory1.serialize(&animal); + let bin = fory1.serialize(&animal).unwrap(); let obj: Animal2 = fory2.deserialize(&bin).unwrap(); assert_eq!(animal.f1, obj.f1); assert_eq!(animal.f3, obj.f3); diff --git a/rust/tests/tests/test_trait_object.rs b/rust/tests/tests/test_trait_object.rs index 15bc57b853..0919625ac9 100644 --- a/rust/tests/tests/test_trait_object.rs +++ b/rust/tests/tests/test_trait_object.rs @@ -37,9 +37,9 @@ fn test_multiple_types_in_sequence() { let val2: Box = Box::new(original2.clone()); let val3: Box = Box::new(original3.clone()); - let ser1 = fory.serialize(&val1); - let ser2 = fory.serialize(&val2); - let ser3 = fory.serialize(&val3); + let ser1 = fory.serialize(&val1).unwrap(); + let ser2 = fory.serialize(&val2).unwrap(); + let ser3 = fory.serialize(&val3).unwrap(); let de1_trait: Box = fory.deserialize(&ser1).unwrap(); let de2_trait: Box = fory.deserialize(&ser2).unwrap(); @@ -53,9 +53,9 @@ fn test_multiple_types_in_sequence() { assert_eq!(de2_concrete, original2); assert_eq!(de3_concrete, original3); - assert_eq!(ser1, fory.serialize(&de1_trait)); - assert_eq!(ser2, fory.serialize(&de2_trait)); - assert_eq!(ser3, fory.serialize(&de3_trait)); + assert_eq!(ser1, fory.serialize(&de1_trait).unwrap()); + assert_eq!(ser2, fory.serialize(&de2_trait).unwrap()); + assert_eq!(ser3, fory.serialize(&de3_trait).unwrap()); } #[test] @@ -64,19 +64,20 @@ fn test_option_some_roundtrip() { let original = Some(42); let trait_obj: Box = Box::new(original); - let serialized = fory.serialize(&trait_obj); + let serialized = fory.serialize(&trait_obj).unwrap(); let deserialized_trait: Box = fory.deserialize(&serialized).unwrap(); let deserialized_concrete: Option = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized_concrete, original); - assert_eq!(fory.serialize(&deserialized_trait), serialized); + assert_eq!(fory.serialize(&deserialized_trait).unwrap(), serialized); } #[test] fn test_hashmap_roundtrip() { let mut fory = fory_compatible(); - fory.register_serializer::>(1001); + fory.register_serializer::>(1001) + .unwrap(); let mut original = HashMap::new(); original.insert(String::from("one"), 1); @@ -84,7 +85,7 @@ fn test_hashmap_roundtrip() { original.insert(String::from("three"), 3); let trait_obj: Box = Box::new(original.clone()); - let serialized = fory.serialize(&trait_obj); + let serialized = fory.serialize(&trait_obj).unwrap(); let deserialized_concrete: HashMap = fory.deserialize(&serialized).unwrap(); @@ -97,7 +98,7 @@ fn test_hashmap_roundtrip() { #[test] fn test_hashset_roundtrip() { let mut fory = fory_compatible(); - fory.register_serializer::>(1002); + fory.register_serializer::>(1002).unwrap(); let mut original = HashSet::new(); original.insert(1); @@ -105,7 +106,7 @@ fn test_hashset_roundtrip() { original.insert(3); let trait_obj: Box = Box::new(original.clone()); - let serialized = fory.serialize(&trait_obj); + let serialized = fory.serialize(&trait_obj).unwrap(); let deserialized_concrete: HashSet = fory.deserialize(&serialized).unwrap(); @@ -118,7 +119,8 @@ fn test_hashset_roundtrip() { #[test] fn test_vec_of_trait_objects() { let mut fory = fory_compatible(); - fory.register_serializer::>>(3000); + fory.register_serializer::>>(3000) + .unwrap(); let vec_of_trait_objects: Vec> = vec![ Box::new(42i32), @@ -126,7 +128,7 @@ fn test_vec_of_trait_objects() { Box::new(2.71f64), ]; - let serialized = fory.serialize(&vec_of_trait_objects); + let serialized = fory.serialize(&vec_of_trait_objects).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -135,14 +137,15 @@ fn test_vec_of_trait_objects() { #[test] fn test_hashmap_string_to_trait_objects() { let mut fory = fory_compatible(); - fory.register_serializer::>>(3002); + fory.register_serializer::>>(3002) + .unwrap(); let mut map: HashMap> = HashMap::new(); map.insert(String::from("int"), Box::new(42i32)); map.insert(String::from("string"), Box::new(String::from("hello"))); map.insert(String::from("float"), Box::new(2.71f64)); - let serialized = fory.serialize(&map); + let serialized = fory.serialize(&map).unwrap(); let deserialized: HashMap> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -163,14 +166,14 @@ struct Company { #[test] fn test_fory_derived_struct_as_trait_object() { let mut fory = fory_compatible(); - fory.register::(5000); + fory.register::(5000).unwrap(); let person = Person { name: String::from("Alice"), age: 30, }; let trait_obj: Box = Box::new(person.clone()); - let serialized = fory.serialize(&trait_obj); + let serialized = fory.serialize(&trait_obj).unwrap(); let deserialized_trait: Box = fory.deserialize(&serialized).unwrap(); let deserialized_concrete: Person = fory.deserialize(&serialized).unwrap(); @@ -178,16 +181,17 @@ fn test_fory_derived_struct_as_trait_object() { assert_eq!(deserialized_concrete.name, person.name); assert_eq!(deserialized_concrete.age, person.age); - let reserialized = fory.serialize(&deserialized_trait); + let reserialized = fory.serialize(&deserialized_trait).unwrap(); assert_eq!(serialized, reserialized); } #[test] fn test_vec_of_fory_derived_trait_objects() { let mut fory = fory_compatible(); - fory.register::(5000); - fory.register::(5001); - fory.register_serializer::>>(3000); + fory.register::(5000).unwrap(); + fory.register::(5001).unwrap(); + fory.register_serializer::>>(3000) + .unwrap(); let vec_of_trait_objects: Vec> = vec![ Box::new(Person { @@ -204,7 +208,7 @@ fn test_vec_of_fory_derived_trait_objects() { Box::new(42i32), ]; - let serialized = fory.serialize(&vec_of_trait_objects); + let serialized = fory.serialize(&vec_of_trait_objects).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -213,9 +217,10 @@ fn test_vec_of_fory_derived_trait_objects() { #[test] fn test_hashmap_with_fory_derived_values() { let mut fory = fory_compatible(); - fory.register::(5000); - fory.register::(5001); - fory.register_serializer::>>(3002); + fory.register::(5000).unwrap(); + fory.register::(5001).unwrap(); + fory.register_serializer::>>(3002) + .unwrap(); let mut map: HashMap> = HashMap::new(); map.insert( @@ -234,7 +239,7 @@ fn test_hashmap_with_fory_derived_values() { ); map.insert(String::from("number"), Box::new(42i32)); - let serialized = fory.serialize(&map); + let serialized = fory.serialize(&map).unwrap(); let deserialized: HashMap> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -289,9 +294,9 @@ struct Zoo { #[test] fn test_custom_trait_object_basic() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); - fory.register::(8003); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); + fory.register::(8003).unwrap(); let zoo_dog = Zoo { star_animal: Box::new(Dog { @@ -307,8 +312,8 @@ fn test_custom_trait_object_basic() { }), }; - let serialized_dog = fory.serialize(&zoo_dog); - let serialized_cat = fory.serialize(&zoo_cat); + let serialized_dog = fory.serialize(&zoo_dog).unwrap(); + let serialized_cat = fory.serialize(&zoo_cat).unwrap(); let deserialized_dog: Zoo = fory.deserialize(&serialized_dog).unwrap(); let deserialized_cat: Zoo = fory.deserialize(&serialized_cat).unwrap(); @@ -347,9 +352,9 @@ struct PetOwner { #[test] fn test_multiple_traits() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); - fory.register::(9001); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); + fory.register::(9001).unwrap(); let owner = PetOwner { pets: vec![ @@ -374,7 +379,7 @@ fn test_multiple_traits() { ], }; - let serialized = fory.serialize(&owner); + let serialized = fory.serialize(&owner).unwrap(); let deserialized: PetOwner = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.pets.len(), 2); @@ -393,15 +398,15 @@ fn test_multiple_traits() { #[test] fn test_single_custom_trait_object_direct() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); let animal: Box = Box::new(Dog { name: "Rex".to_string(), breed: "Golden Retriever".to_string(), }); - let serialized = fory.serialize(&animal); + let serialized = fory.serialize(&animal).unwrap(); let deserialized: Box = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.name(), "Rex"); @@ -411,8 +416,8 @@ fn test_single_custom_trait_object_direct() { #[test] fn test_vec_custom_trait_objects_direct() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); let animals: Vec> = vec![ Box::new(Dog { @@ -429,7 +434,7 @@ fn test_vec_custom_trait_objects_direct() { }), ]; - let serialized = fory.serialize(&animals); + let serialized = fory.serialize(&animals).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -444,8 +449,8 @@ fn test_vec_custom_trait_objects_direct() { #[test] fn test_hashmap_custom_trait_objects_direct() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); let mut animal_map: std::collections::HashMap> = std::collections::HashMap::new(); @@ -471,7 +476,7 @@ fn test_hashmap_custom_trait_objects_direct() { }), ); - let serialized = fory.serialize(&animal_map); + let serialized = fory.serialize(&animal_map).unwrap(); let deserialized: std::collections::HashMap> = fory.deserialize(&serialized).unwrap(); @@ -487,8 +492,8 @@ fn test_hashmap_custom_trait_objects_direct() { #[test] fn test_nested_custom_trait_object_collections() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); // Test Vec>> let nested_animals: Vec>> = vec![ @@ -508,7 +513,7 @@ fn test_nested_custom_trait_object_collections() { })], ]; - let serialized = fory.serialize(&nested_animals); + let serialized = fory.serialize(&nested_animals).unwrap(); let deserialized: Vec>> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 2); @@ -525,8 +530,8 @@ fn test_nested_custom_trait_object_collections() { #[test] fn test_mixed_trait_object_collections() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); // Test HashMap>> let mut groups: std::collections::HashMap>> = @@ -554,7 +559,7 @@ fn test_mixed_trait_object_collections() { })], ); - let serialized = fory.serialize(&groups); + let serialized = fory.serialize(&groups).unwrap(); let deserialized: std::collections::HashMap>> = fory.deserialize(&serialized).unwrap(); @@ -576,19 +581,19 @@ fn test_mixed_trait_object_collections() { #[test] fn test_empty_trait_object_collections() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); // Test empty Vec> let empty_animals: Vec> = vec![]; - let serialized = fory.serialize(&empty_animals); + let serialized = fory.serialize(&empty_animals).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 0); // Test empty HashMap> let empty_map: std::collections::HashMap> = std::collections::HashMap::new(); - let serialized = fory.serialize(&empty_map); + let serialized = fory.serialize(&empty_map).unwrap(); let deserialized: std::collections::HashMap> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 0); @@ -597,15 +602,15 @@ fn test_empty_trait_object_collections() { #[test] fn test_single_item_trait_object_collections() { let mut fory = fory_compatible(); - fory.register::(8001); - fory.register::(8002); + fory.register::(8001).unwrap(); + fory.register::(8002).unwrap(); // Test single item Vec> let single_animal: Vec> = vec![Box::new(Dog { name: "Solo".to_string(), breed: "Bulldog".to_string(), })]; - let serialized = fory.serialize(&single_animal); + let serialized = fory.serialize(&single_animal).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 1); assert_eq!(deserialized[0].name(), "Solo"); @@ -621,7 +626,7 @@ fn test_single_item_trait_object_collections() { color: "Gray".to_string(), }), ); - let serialized = fory.serialize(&single_map); + let serialized = fory.serialize(&single_map).unwrap(); let deserialized: std::collections::HashMap> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 1); diff --git a/rust/tests/tests/test_weak.rs b/rust/tests/tests/test_weak.rs index d994030a17..1a5490480a 100644 --- a/rust/tests/tests/test_weak.rs +++ b/rust/tests/tests/test_weak.rs @@ -29,7 +29,7 @@ fn test_rc_weak_null_serialization() { let weak: RcWeak = RcWeak::new(); - let serialized = fory.serialize(&weak); + let serialized = fory.serialize(&weak).unwrap(); let deserialized: RcWeak = fory.deserialize(&serialized).unwrap(); assert!(deserialized.upgrade().is_none()); @@ -41,7 +41,7 @@ fn test_arc_weak_null_serialization() { let weak: ArcWeak = ArcWeak::new(); - let serialized = fory.serialize(&weak); + let serialized = fory.serialize(&weak).unwrap(); let deserialized: ArcWeak = fory.deserialize(&serialized).unwrap(); assert!(deserialized.upgrade().is_none()); @@ -61,7 +61,7 @@ fn test_rc_weak_dead_pointer_serializes_as_null() { assert!(weak.upgrade().is_none()); // Should serialize as Null - let serialized = fory.serialize(&weak); + let serialized = fory.serialize(&weak).unwrap(); let deserialized: RcWeak = fory.deserialize(&serialized).unwrap(); assert!(deserialized.upgrade().is_none()); @@ -81,7 +81,7 @@ fn test_arc_weak_dead_pointer_serializes_as_null() { assert!(weak.upgrade().is_none()); // Should serialize as Null - let serialized = fory.serialize(&weak); + let serialized = fory.serialize(&weak).unwrap(); let deserialized: ArcWeak = fory.deserialize(&serialized).unwrap(); assert!(deserialized.upgrade().is_none()); @@ -99,7 +99,7 @@ fn test_rc_weak_in_vec_circular_reference() { let weak3 = weak1.clone(); let weaks = vec![weak1, weak2, weak3]; - let serialized = fory.serialize(&weaks); + let serialized = fory.serialize(&weaks).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -117,7 +117,7 @@ fn test_arc_weak_in_vec_circular_reference() { let weak3 = weak1.clone(); let weaks = vec![weak1, weak2, weak3]; - let serialized = fory.serialize(&weaks); + let serialized = fory.serialize(&weaks).unwrap(); let deserialized: Vec> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.len(), 3); @@ -134,7 +134,7 @@ fn test_rc_weak_field_in_struct() { } let mut fory = Fory::default(); - fory.register::(1000); + fory.register::(1000).unwrap(); let data = Rc::new(42i32); let node = SimpleNode { @@ -142,7 +142,7 @@ fn test_rc_weak_field_in_struct() { weak_ref: RcWeak::from(&data), }; - let serialized = fory.serialize(&node); + let serialized = fory.serialize(&node).unwrap(); let deserialized: SimpleNode = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.value, 1); @@ -161,7 +161,7 @@ struct Node { fn test_node_circular_reference_with_parent_children() { // Register the Node type with Fory let mut fory = Fory::default(); - fory.register::(2000); + fory.register::(2000).unwrap(); // Create parent let parent = Rc::new(RefCell::new(Node { @@ -192,7 +192,7 @@ fn test_node_circular_reference_with_parent_children() { child2.borrow_mut().parent = RcWeak::from(&parent); // --- Serialize the parent node (will include children recursively) --- - let serialized = fory.serialize(&parent); + let serialized = fory.serialize(&parent).unwrap(); // --- Deserialize --- let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); @@ -219,7 +219,7 @@ fn test_arc_mutex_circular_reference() { } let mut fory = Fory::default(); - fory.register::(6000); + fory.register::(6000).unwrap(); let parent = Arc::new(Mutex::new(Node { val: 10, @@ -242,7 +242,7 @@ fn test_arc_mutex_circular_reference() { parent.lock().unwrap().children.push(child1.clone()); parent.lock().unwrap().children.push(child2.clone()); - let serialized = fory.serialize(&parent); + let serialized = fory.serialize(&parent).unwrap(); let deserialized: Arc> = fory.deserialize(&serialized).unwrap(); assert_eq!(deserialized.lock().unwrap().children.len(), 2); From 846531f18ca3866d9702acf5396be210e64b1b55 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Wed, 15 Oct 2025 21:20:36 +0530 Subject: [PATCH 25/37] feat(rust): refactor rust serialization system (#2774) ## Why? Simplify the serialization interface for better ergonomics API ## What does this PR do? - Remove `Fory` from `Serializer` trait - Make `WriteContext` as only enviroment for serialization - Make `ReadContext` as only enviroment for deserialization - Use `OnceLock` for thread-safe, lazily initialization for `Pool` ## Related issues #2737 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 4 +- rust/fory-core/src/error.rs | 4 +- rust/fory-core/src/fory.rs | 144 +++++++---- rust/fory-core/src/resolver/context.rs | 212 +++++++++++++--- rust/fory-core/src/resolver/meta_resolver.rs | 14 +- rust/fory-core/src/resolver/type_resolver.rs | 238 +++++++++--------- rust/fory-core/src/serializer/any.rs | 173 ++++--------- rust/fory-core/src/serializer/arc.rs | 58 ++--- rust/fory-core/src/serializer/bool.rs | 35 +-- rust/fory-core/src/serializer/box_.rs | 43 +--- rust/fory-core/src/serializer/collection.rs | 27 +- rust/fory-core/src/serializer/datetime.rs | 68 ++--- rust/fory-core/src/serializer/enum_.rs | 36 ++- rust/fory-core/src/serializer/heap.rs | 35 +-- rust/fory-core/src/serializer/list.rs | 109 +++----- rust/fory-core/src/serializer/map.rs | 139 ++++------ rust/fory-core/src/serializer/mod.rs | 127 ++++------ rust/fory-core/src/serializer/mutex.rs | 56 ++--- rust/fory-core/src/serializer/number.rs | 25 +- rust/fory-core/src/serializer/option.rs | 45 ++-- rust/fory-core/src/serializer/rc.rs | 58 ++--- rust/fory-core/src/serializer/refcell.rs | 56 ++--- rust/fory-core/src/serializer/set.rs | 76 ++---- rust/fory-core/src/serializer/skip.rs | 33 ++- rust/fory-core/src/serializer/string.rs | 37 +-- rust/fory-core/src/serializer/struct_.rs | 71 +----- rust/fory-core/src/serializer/trait_object.rs | 135 +++++----- rust/fory-core/src/serializer/weak.rs | 114 +++------ rust/fory-derive/src/object/derive_enum.rs | 14 +- rust/fory-derive/src/object/misc.rs | 4 +- rust/fory-derive/src/object/read.rs | 70 +++--- rust/fory-derive/src/object/serializer.rs | 36 +-- rust/fory-derive/src/object/util.rs | 2 +- rust/fory-derive/src/object/write.rs | 26 +- rust/fory/src/lib.rs | 12 +- .../tests/tests/compatible/test_basic_type.rs | 16 +- rust/tests/tests/compatible/test_container.rs | 40 +-- .../tests/compatible/test_struct_enum.rs | 59 ++--- rust/tests/tests/test_cross_language.rs | 53 ++-- rust/tests/tests/test_ext.rs | 25 +- 40 files changed, 1068 insertions(+), 1461 deletions(-) diff --git a/rust/README.md b/rust/README.md index 5891cd95ea..607c556bea 100644 --- a/rust/README.md +++ b/rust/README.md @@ -616,8 +616,8 @@ impl Serializer for CustomType { Ok(Self { value, name }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> u32 { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> u32 { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn Any { diff --git a/rust/fory-core/src/error.rs b/rust/fory-core/src/error.rs index 4cbee205c0..291608033e 100644 --- a/rust/fory-core/src/error.rs +++ b/rust/fory-core/src/error.rs @@ -78,7 +78,7 @@ impl Error { } } -/// Ensures a condition is true; otherwise returns an [`Error`]. +/// Ensures a condition is true; otherwise returns an [`enum@Error`]. /// /// # Examples /// ``` @@ -110,7 +110,7 @@ macro_rules! ensure { }; } -/// Returns early with an [`Error`]. +/// Returns early with an [`enum@Error`]. /// /// # Examples /// ``` diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 2a6322ff42..c80b1a2dc2 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -29,6 +29,7 @@ use crate::types::{ Language, MAGIC_NUMBER, SIZE_OF_REF_AND_TYPE, }; use crate::util::get_ext_actual_type_id; +use std::sync::OnceLock; static EMPTY_STRING: String = String::new(); @@ -82,21 +83,13 @@ pub struct Fory { type_resolver: TypeResolver, compress_string: bool, max_dyn_depth: u32, - write_context_pool: Pool, - read_context_pool: Pool, + // Lazy-initialized pools (thread-safe, one-time initialization) + write_context_pool: OnceLock>, + read_context_pool: OnceLock>, } impl Default for Fory { fn default() -> Self { - let write_context_constructor = || { - let writer = Writer::default(); - WriteContext::new(writer) - }; - let read_context_constructor = || { - let reader = Reader::new(&[]); - // when context is popped out, max_dyn_depth will be assigned a valid value - ReadContext::new(reader, 0) - }; Fory { compatible: false, xlang: true, @@ -104,8 +97,8 @@ impl Default for Fory { type_resolver: TypeResolver::default(), compress_string: false, max_dyn_depth: 5, - write_context_pool: Pool::new(write_context_constructor), - read_context_pool: Pool::new(read_context_constructor), + write_context_pool: OnceLock::new(), + read_context_pool: OnceLock::new(), } } } @@ -248,6 +241,11 @@ impl Fory { self } + /// Returns whether cross-language serialization is enabled. + pub fn is_xlang(&self) -> bool { + self.xlang + } + /// Returns the current serialization mode. /// /// # Returns @@ -275,12 +273,13 @@ impl Fory { self.share_meta } - /// Returns a reference to the type resolver. - /// - /// # Returns - /// - /// A reference to the internal `TypeResolver` used for type registration and lookup. - pub fn get_type_resolver(&self) -> &TypeResolver { + /// Returns the maximum depth for nested dynamic object serialization. + pub fn get_max_dyn_depth(&self) -> u32 { + self.max_dyn_depth + } + + /// Returns a type resolver for type lookups. + pub(crate) fn get_type_resolver(&self) -> &TypeResolver { &self.type_resolver } @@ -384,14 +383,34 @@ impl Fory { /// let deserialized: Point = fory.deserialize(&bytes).unwrap(); /// ``` pub fn deserialize(&self, bf: &[u8]) -> Result { - let mut context = self.read_context_pool.get(); + let pool = self.read_context_pool.get_or_init(|| { + let type_resolver = self.type_resolver.clone(); + let compatible = self.compatible; + let share_meta = self.share_meta; + let xlang = self.xlang; + let max_dyn_depth = self.max_dyn_depth; + + let factory = move || { + let reader = Reader::new(&[]); + ReadContext::new( + reader, + type_resolver.clone(), + compatible, + share_meta, + xlang, + max_dyn_depth, + ) + }; + Pool::new(factory) + }); + let mut context = pool.get(); context.init(bf, self.max_dyn_depth); let result = self.deserialize_with_context(&mut context); if result.is_ok() { assert_eq!(context.reader.slice_after_cursor().len(), 0); } context.reset(); - self.read_context_pool.put(context); + pool.put(context); result } @@ -404,14 +423,13 @@ impl Fory { return Ok(T::fory_default()); } let mut bytes_to_skip = 0; - if self.compatible { + if context.is_compatible() { let meta_offset = context.reader.read_i32()?; if meta_offset != -1 { - bytes_to_skip = - context.load_meta(self.get_type_resolver(), meta_offset as usize)?; + bytes_to_skip = context.load_meta(meta_offset as usize)?; } } - let result = ::fory_read(self, context, false); + let result = ::fory_read(context, false); if bytes_to_skip > 0 { context.reader.skip(bytes_to_skip)?; } @@ -447,10 +465,30 @@ impl Fory { /// let bytes = fory.serialize(&point); /// ``` pub fn serialize(&self, record: &T) -> Result, Error> { - let mut context = self.write_context_pool.get(); + let pool = self.write_context_pool.get_or_init(|| { + let type_resolver = self.type_resolver.clone(); + let compatible = self.compatible; + let share_meta = self.share_meta; + let compress_string = self.compress_string; + let xlang = self.xlang; + + let factory = move || { + let writer = Writer::default(); + WriteContext::new( + writer, + type_resolver.clone(), + compatible, + share_meta, + compress_string, + xlang, + ) + }; + Pool::new(factory) + }); + let mut context = pool.get(); let result = self.serialize_with_context(record, &mut context)?; context.reset(); - self.write_context_pool.put(context); + pool.put(context); Ok(result) } @@ -463,11 +501,11 @@ impl Fory { self.write_head::(is_none, &mut context.writer); let meta_start_offset = context.writer.len(); if !is_none { - if self.compatible { + if context.is_compatible() { context.writer.write_i32(-1); }; - ::fory_write(record, self, context, false)?; - if self.compatible && !context.empty() { + ::fory_write(record, context, false)?; + if context.is_compatible() && !context.empty() { context.write_meta(meta_start_offset); } } @@ -506,9 +544,15 @@ impl Fory { id: u32, ) -> Result<(), Error> { let actual_type_id = T::fory_actual_type_id(id, false, self.compatible); - let type_info = - TypeInfo::new::(self, actual_type_id, &EMPTY_STRING, &EMPTY_STRING, false)?; - self.type_resolver.register::(&type_info) + let type_info = TypeInfo::new::( + &self.type_resolver, + actual_type_id, + &EMPTY_STRING, + &EMPTY_STRING, + false, + )?; + self.type_resolver.register::(&type_info)?; + Ok(()) } /// Registers a struct type with a namespace and type name for cross-language serialization. @@ -547,8 +591,15 @@ impl Fory { type_name: &str, ) -> Result<(), Error> { let actual_type_id = T::fory_actual_type_id(0, true, self.compatible); - let type_info = TypeInfo::new::(self, actual_type_id, namespace, type_name, true)?; - self.type_resolver.register::(&type_info) + let type_info = TypeInfo::new::( + &self.type_resolver, + actual_type_id, + namespace, + type_name, + true, + )?; + self.type_resolver.register::(&type_info)?; + Ok(()) } /// Registers a struct type with a type name (using the default namespace). @@ -617,13 +668,14 @@ impl Fory { ) -> Result<(), Error> { let actual_type_id = get_ext_actual_type_id(id, false); let type_info = TypeInfo::new_with_empty_fields::( - self, + &self.type_resolver, actual_type_id, &EMPTY_STRING, &EMPTY_STRING, false, )?; - self.type_resolver.register_serializer::(&type_info) + self.type_resolver.register_serializer::(&type_info)?; + Ok(()) } /// Registers a custom serializer type with a namespace and type name. @@ -648,9 +700,15 @@ impl Fory { type_name: &str, ) -> Result<(), Error> { let actual_type_id = get_ext_actual_type_id(0, true); - let type_info = - TypeInfo::new_with_empty_fields::(self, actual_type_id, namespace, type_name, true)?; - self.type_resolver.register_serializer::(&type_info) + let type_info = TypeInfo::new_with_empty_fields::( + &self.type_resolver, + actual_type_id, + namespace, + type_name, + true, + )?; + self.type_resolver.register_serializer::(&type_info)?; + Ok(()) } /// Registers a custom serializer type with a type name (using the default namespace). @@ -676,17 +734,15 @@ impl Fory { pub fn write_data( this: &T, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { - T::fory_write_data(this, fory, context, is_field) + T::fory_write_data(this, context, is_field) } pub fn read_data( - fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result { - T::fory_read_data(fory, context, is_field) + T::fory_read_data(context, is_field) } diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index e95ad95f32..94aaa45b3d 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -16,9 +16,9 @@ // under the License. use crate::buffer::{Reader, Writer}; -use crate::fory::Fory; use crate::error::Error; +use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta}; use crate::resolver::meta_resolver::{MetaReaderResolver, MetaWriterResolver}; use crate::resolver::metastring_resolver::{ @@ -29,6 +29,14 @@ use crate::resolver::type_resolver::{Harness, TypeResolver}; use std::sync::{Arc, Mutex}; pub struct WriteContext { + // Replicated environment fields (direct access, no Arc indirection for flags) + type_resolver: TypeResolver, + compatible: bool, + share_meta: bool, + compress_string: bool, + xlang: bool, + + // Context-specific fields pub writer: Writer, meta_resolver: MetaWriterResolver, meta_string_resolver: MetaStringWriterResolver, @@ -36,8 +44,34 @@ pub struct WriteContext { } impl WriteContext { - pub fn new(writer: Writer) -> WriteContext { + pub fn new( + writer: Writer, + type_resolver: TypeResolver, + compatible: bool, + share_meta: bool, + compress_string: bool, + xlang: bool, + ) -> WriteContext { + WriteContext { + type_resolver, + compatible, + share_meta, + compress_string, + xlang, + writer, + meta_resolver: MetaWriterResolver::default(), + meta_string_resolver: MetaStringWriterResolver::default(), + ref_writer: RefWriter::new(), + } + } + + pub fn new_from_fory(writer: Writer, fory: &Fory) -> WriteContext { WriteContext { + type_resolver: fory.get_type_resolver().clone(), + compatible: fory.is_compatible(), + share_meta: fory.is_share_meta(), + compress_string: fory.is_compress_string(), + xlang: fory.is_xlang(), writer, meta_resolver: MetaWriterResolver::default(), meta_string_resolver: MetaStringWriterResolver::default(), @@ -45,14 +79,44 @@ impl WriteContext { } } + /// Get type resolver + #[inline(always)] + pub fn get_type_resolver(&self) -> &TypeResolver { + &self.type_resolver + } + + /// Check if compatible mode is enabled + #[inline(always)] + pub fn is_compatible(&self) -> bool { + self.compatible + } + + /// Check if meta sharing is enabled + #[inline(always)] + pub fn is_share_meta(&self) -> bool { + self.share_meta + } + + /// Check if string compression is enabled + #[inline(always)] + pub fn is_compress_string(&self) -> bool { + self.compress_string + } + + /// Check if cross-language mode is enabled + #[inline(always)] + pub fn is_xlang(&self) -> bool { + self.xlang + } + #[inline(always)] pub fn empty(&mut self) -> bool { self.meta_resolver.empty() } #[inline(always)] - pub fn push_meta(&mut self, fory: &Fory, type_id: std::any::TypeId) -> Result { - self.meta_resolver.push(type_id, fory) + pub fn push_meta(&mut self, type_id: std::any::TypeId) -> Result { + self.meta_resolver.push(type_id, &self.type_resolver) } #[inline(always)] @@ -66,46 +130,49 @@ impl WriteContext { pub fn write_any_typeinfo( &mut self, - fory: &Fory, concrete_type_id: std::any::TypeId, ) -> Result, Error> { use crate::types::TypeId as ForyTypeId; - let type_resolver = fory.get_type_resolver(); - let type_info = type_resolver.get_type_info(concrete_type_id)?; + let type_info = self.type_resolver.get_type_info(concrete_type_id)?; let fory_type_id = type_info.get_type_id(); - if type_info.is_registered_by_name() { + // Clone out the data we need from `type_info` to break the borrow + let namespace = type_info.get_namespace().clone(); + let type_name = type_info.get_type_name().clone(); + let registered_by_name = type_info.is_registered_by_name(); + + if registered_by_name { if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - if fory.is_share_meta() { - let meta_index = self.push_meta(fory, concrete_type_id)? as u32; + if self.is_share_meta() { + let meta_index = self.push_meta(concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { - type_info.get_namespace().write_to(&mut self.writer); - type_info.get_type_name().write_to(&mut self.writer); + namespace.write_to(&mut self.writer); + type_name.write_to(&mut self.writer); } } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(fory, concrete_type_id)? as u32; + let meta_index = self.push_meta(concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(u32::MAX); - type_info.get_namespace().write_to(&mut self.writer); - type_info.get_type_name().write_to(&mut self.writer); + namespace.write_to(&mut self.writer); + type_name.write_to(&mut self.writer); } - type_resolver - .get_name_harness(type_info.get_namespace(), type_info.get_type_name()) + self.type_resolver + .get_name_harness(&namespace, &type_name) .ok_or_else(|| Error::TypeError("Name harness not found".into())) } else { if fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(fory, concrete_type_id)? as u32; + let meta_index = self.push_meta(concrete_type_id)? as u32; self.writer.write_varuint32(meta_index); } else { self.writer.write_varuint32(fory_type_id); } - type_resolver + self.type_resolver .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("ID harness not found".into())) } @@ -126,26 +193,89 @@ impl WriteContext { } pub struct ReadContext { + // Replicated environment fields (direct access, no Arc indirection for flags) + type_resolver: TypeResolver, + compatible: bool, + share_meta: bool, + xlang: bool, + max_dyn_depth: u32, + + // Context-specific fields pub reader: Reader, pub meta_resolver: MetaReaderResolver, meta_string_resolver: MetaStringReaderResolver, pub ref_reader: RefReader, - max_dyn_depth: u32, current_depth: u32, } impl ReadContext { - pub fn new(reader: Reader, max_dyn_depth: u32) -> ReadContext { + pub fn new( + reader: Reader, + type_resolver: TypeResolver, + compatible: bool, + share_meta: bool, + xlang: bool, + max_dyn_depth: u32, + ) -> ReadContext { ReadContext { + type_resolver, + compatible, + share_meta, + xlang, + max_dyn_depth, reader, meta_resolver: MetaReaderResolver::default(), meta_string_resolver: MetaStringReaderResolver::default(), ref_reader: RefReader::new(), - max_dyn_depth, current_depth: 0, } } + pub fn new_from_fory(reader: Reader, fory: &Fory) -> ReadContext { + ReadContext { + type_resolver: fory.get_type_resolver().clone(), + compatible: fory.is_compatible(), + share_meta: fory.is_share_meta(), + xlang: fory.is_xlang(), + max_dyn_depth: fory.get_max_dyn_depth(), + reader, + meta_resolver: MetaReaderResolver::default(), + meta_string_resolver: MetaStringReaderResolver::default(), + ref_reader: RefReader::new(), + current_depth: 0, + } + } + + /// Get type resolver + #[inline(always)] + pub fn get_type_resolver(&self) -> &TypeResolver { + &self.type_resolver + } + + /// Check if compatible mode is enabled + #[inline(always)] + pub fn is_compatible(&self) -> bool { + self.compatible + } + + /// Check if meta sharing is enabled + #[inline(always)] + pub fn is_share_meta(&self) -> bool { + self.share_meta + } + + /// Check if cross-language mode is enabled + #[inline(always)] + pub fn is_xlang(&self) -> bool { + self.xlang + } + + /// Get maximum dynamic depth + #[inline(always)] + pub fn max_dyn_depth(&self) -> u32 { + self.max_dyn_depth + } + #[inline(always)] pub fn init(&mut self, bytes: &[u8], max_dyn_depth: u32) { self.reader.init(bytes); @@ -159,51 +289,47 @@ impl ReadContext { } #[inline(always)] - pub fn load_meta( - &mut self, - type_resolver: &TypeResolver, - offset: usize, - ) -> Result { + pub fn load_meta(&mut self, offset: usize) -> Result { self.meta_resolver.load( - type_resolver, + &self.type_resolver, &mut Reader::new(&self.reader.slice_after_cursor()[offset..]), ) } - pub fn read_any_typeinfo(&mut self, fory: &Fory) -> Result, Error> { + pub fn read_any_typeinfo(&mut self) -> Result, Error> { use crate::types::TypeId as ForyTypeId; let fory_type_id = self.reader.read_varuint32()?; - let type_resolver = fory.get_type_resolver(); if fory_type_id == u32::MAX { let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; - type_resolver + self.type_resolver .get_name_harness(&namespace, &type_name) .ok_or_else(|| Error::TypeError("Name harness not found".into())) } else if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { - if fory.is_share_meta() { + if self.is_share_meta() { let _meta_index = self.reader.read_varuint32(); } else { let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; - return type_resolver + return self + .type_resolver .get_name_harness(&namespace, &type_name) .ok_or_else(|| Error::TypeError("Name harness not found".into())); } - type_resolver + self.type_resolver .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("ID harness not found".into())) } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { let _meta_index = self.reader.read_varuint32(); - type_resolver + self.type_resolver .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("ID harness not found".into())) } else { - type_resolver + self.type_resolver .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("ID harness not found".into())) } @@ -217,13 +343,14 @@ impl ReadContext { #[inline(always)] pub fn inc_depth(&mut self) -> Result<(), Error> { self.current_depth += 1; - if self.current_depth > self.max_dyn_depth { + if self.current_depth > self.max_dyn_depth() { return Err(Error::DepthExceed( format!( "Maximum dynamic object nesting depth ({}) exceeded. Current depth: {}. \ This may indicate a circular reference or overly deep object graph. \ Consider increasing max_dyn_depth if this is expected.", - self.max_dyn_depth, self.current_depth + self.max_dyn_depth(), + self.current_depth ) .into(), )); @@ -246,14 +373,17 @@ impl ReadContext { pub struct Pool { items: Mutex>, - factory: fn() -> T, + factory: Box T + Send + Sync>, } impl Pool { - pub fn new(factory: fn() -> T) -> Self { + pub fn new(factory: F) -> Self + where + F: Fn() -> T + Send + Sync + 'static, + { Pool { items: Mutex::new(vec![]), - factory, + factory: Box::new(factory), } } diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 2fb0b5b084..2e1e09bab5 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -17,7 +17,6 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; -use crate::fory::Fory; use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER}; use crate::TypeResolver; use std::collections::HashMap; @@ -33,15 +32,16 @@ const MAX_PARSED_NUM_TYPE_DEFS: usize = 8192; #[allow(dead_code)] impl MetaWriterResolver { - pub fn push(&mut self, type_id: std::any::TypeId, fory: &Fory) -> Result { + pub fn push( + &mut self, + type_id: std::any::TypeId, + type_resolver: &TypeResolver, + ) -> Result { match self.type_id_index_map.get(&type_id) { None => { let index = self.type_defs.len(); - self.type_defs.push( - fory.get_type_resolver() - .get_type_info(type_id)? - .get_type_def(), - ); + self.type_defs + .push(type_resolver.get_type_info(type_id)?.get_type_def()); self.type_id_index_map.insert(type_id, index); Ok(index) } diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index 7cd89cd66e..8321efc60c 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -17,9 +17,8 @@ use super::context::{ReadContext, WriteContext}; use crate::error::Error; -use crate::fory::Fory; use crate::meta::{ - MetaString, TypeMeta, NAMESPACE_ENCODER, NAMESPACE_ENCODINGS, TYPE_NAME_ENCODER, + FieldInfo, MetaString, TypeMeta, NAMESPACE_ENCODER, NAMESPACE_ENCODINGS, TYPE_NAME_ENCODER, TYPE_NAME_ENCODINGS, }; use crate::serializer::{ForyDefault, Serializer, StructSerializer}; @@ -27,17 +26,12 @@ use crate::Reader; use std::sync::Arc; use std::{any::Any, collections::HashMap}; -type WriteFn = fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool) -> Result<(), Error>; -type ReadFn = fn( - fory: &Fory, - &mut ReadContext, - is_field: bool, - skip_ref_flag: bool, -) -> Result, Error>; - -type WriteDataFn = - fn(&dyn Any, fory: &Fory, &mut WriteContext, is_field: bool) -> Result<(), Error>; -type ReadDataFn = fn(fory: &Fory, &mut ReadContext, is_field: bool) -> Result, Error>; +type WriteFn = fn(&dyn Any, &mut WriteContext, is_field: bool) -> Result<(), Error>; +type ReadFn = + fn(&mut ReadContext, is_field: bool, skip_ref_flag: bool) -> Result, Error>; + +type WriteDataFn = fn(&dyn Any, &mut WriteContext, is_field: bool) -> Result<(), Error>; +type ReadDataFn = fn(&mut ReadContext, is_field: bool) -> Result, Error>; type ToSerializerFn = fn(Box) -> Result, Error>; #[derive(Clone)] @@ -99,7 +93,7 @@ pub struct TypeInfo { impl TypeInfo { pub fn new( - fory: &Fory, + type_resolver: &TypeResolver, type_id: u32, namespace: &str, type_name: &str, @@ -109,16 +103,38 @@ impl TypeInfo { NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; let type_name_metastring = TYPE_NAME_ENCODER.encode_with_encodings(type_name, TYPE_NAME_ENCODINGS)?; - let (type_def_bytes, type_meta) = T::fory_type_def( - fory, + let mut fields_info = T::fory_fields_info(type_resolver)?; + let sorted_field_names = T::fory_get_sorted_field_names(); + let mut sorted_field_infos: Vec = Vec::with_capacity(fields_info.len()); + for name in sorted_field_names.iter() { + let mut found = false; + for i in 0..fields_info.len() { + if &fields_info[i].field_name == name { + // swap_remove is faster + sorted_field_infos.push(fields_info.swap_remove(i)); + found = true; + break; + } + } + if !found { + unreachable!("Field {} not found in fields_info", name); + } + } + // assign field id in ascending order + for (i, field_info) in sorted_field_infos.iter_mut().enumerate() { + field_info.field_id = i as i16; + } + let type_meta = Arc::new(TypeMeta::from_fields( type_id, namespace_metastring.clone(), type_name_metastring.clone(), register_by_name, - )?; + sorted_field_infos, + )); + let type_def_bytes = type_meta.to_bytes()?; Ok(TypeInfo { type_def: Arc::from(type_def_bytes), - type_meta: Arc::new(type_meta), + type_meta, type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -127,7 +143,7 @@ impl TypeInfo { } pub fn new_with_empty_fields( - _fory: &Fory, + type_resolver: &TypeResolver, type_id: u32, namespace: &str, type_name: &str, @@ -145,7 +161,6 @@ impl TypeInfo { vec![], ); let type_def = meta.to_bytes()?; - let type_resolver = _fory.get_type_resolver(); let meta = TypeMeta::from_bytes(&mut Reader::new(&type_def), type_resolver)?; Ok(TypeInfo { type_def: Arc::from(type_def), @@ -182,6 +197,8 @@ impl TypeInfo { } } +/// TypeResolver is a resolver for fast type/serializer dispatch. +#[derive(Clone)] pub struct TypeResolver { serializer_map: HashMap>, name_serializer_map: HashMap<(MetaString, MetaString), Arc>, @@ -198,7 +215,7 @@ const NO_TYPE_ID: u32 = 1000000000; impl Default for TypeResolver { fn default() -> Self { - let mut resolver = TypeResolver { + let mut registry = TypeResolver { serializer_map: HashMap::new(), name_serializer_map: HashMap::new(), type_id_map: HashMap::new(), @@ -208,12 +225,86 @@ impl Default for TypeResolver { type_info_map_by_name: HashMap::new(), type_id_index: Vec::new(), }; - resolver.register_builtin_types().unwrap(); - resolver + registry.register_builtin_types().unwrap(); + registry } } impl TypeResolver { + pub fn get_type_info(&self, type_id: std::any::TypeId) -> Result<&TypeInfo, Error> { + self.type_info_cache.get(&type_id) + .ok_or_else(|| Error::TypeError(format!( + "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", + type_id + ).into())) + } + + pub fn get_type_info_by_id(&self, id: u32) -> Option<&TypeInfo> { + self.type_info_map_by_id.get(&id) + } + + pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option<&TypeInfo> { + self.type_info_map_by_name + .get(&(namespace.to_owned(), type_name.to_owned())) + } + + /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) + pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> Result { + let id_usize = id as usize; + if id_usize < self.type_id_index.len() { + let type_id = self.type_id_index[id_usize]; + if type_id != NO_TYPE_ID { + return Ok(type_id); + } + } + Err(Error::TypeError( + format!( + "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", + type_id + ) + .into(), + )) + } + + pub fn get_harness(&self, id: u32) -> Option> { + self.serializer_map.get(&id).cloned() + } + + pub fn get_name_harness( + &self, + namespace: &MetaString, + type_name: &MetaString, + ) -> Option> { + let key = (namespace.clone(), type_name.clone()); + self.name_serializer_map.get(&key).cloned() + } + + pub fn get_ext_harness(&self, id: u32) -> Result, Error> { + self.serializer_map + .get(&id) + .cloned() + .ok_or_else(|| Error::TypeError("ext type must be registered in both peers".into())) + } + + pub fn get_ext_name_harness( + &self, + namespace: &MetaString, + type_name: &MetaString, + ) -> Result, Error> { + let key = (namespace.clone(), type_name.clone()); + self.name_serializer_map.get(&key).cloned().ok_or_else(|| { + Error::TypeError("named_ext type must be registered in both peers".into()) + }) + } + + pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { + if let Some(type_info) = self.type_info_cache.get(&rust_type_id) { + Some(type_info.get_type_id()) + } else { + self.type_id_map.get(&rust_type_id).copied() + } + } + fn register_builtin_types(&mut self) -> Result<(), Error> { use crate::types::TypeId; let namespace = NAMESPACE_ENCODER.encode_with_encodings("", NAMESPACE_ENCODINGS)?; @@ -253,58 +344,22 @@ impl TypeResolver { Ok(()) } - pub fn get_type_info(&self, type_id: std::any::TypeId) -> Result<&TypeInfo, Error> { - self.type_info_cache.get(&type_id) - .ok_or_else(|| Error::TypeError(format!( - "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", - type_id - ).into())) - } - - pub fn get_type_info_by_id(&self, id: u32) -> Option<&TypeInfo> { - self.type_info_map_by_id.get(&id) - } - - pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option<&TypeInfo> { - self.type_info_map_by_name - .get(&(namespace.to_owned(), type_name.to_owned())) - } - - /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) - pub fn get_type_id(&self, type_id: &std::any::TypeId, id: u32) -> Result { - let id_usize = id as usize; - if id_usize < self.type_id_index.len() { - let type_id = self.type_id_index[id_usize]; - if type_id != NO_TYPE_ID { - return Ok(type_id); - } - } - Err(Error::TypeError( - format!( - "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", - type_id - ) - .into(), - )) - } - pub fn register( &mut self, type_info: &TypeInfo, ) -> Result<(), Error> { fn write( this: &dyn Any, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { Some(v) => { - let skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory)?; + let skip_ref_flag = + crate::serializer::get_skip_ref_flag::(context.get_type_resolver())?; crate::serializer::write_ref_info_data( v, - fory, context, is_field, skip_ref_flag, @@ -317,13 +372,11 @@ impl TypeResolver { } fn read( - fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, ) -> Result, Error> { match crate::serializer::read_ref_info_data::( - fory, context, is_field, skip_ref_flag, @@ -336,23 +389,21 @@ impl TypeResolver { fn write_data( this: &dyn Any, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => T2::fory_write_data(v, fory, context, is_field), + Some(v) => T2::fory_write_data(v, context, is_field), None => todo!(), } } fn read_data( - fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(fory, context, is_field) { + match T2::fory_read_data(context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -443,30 +494,28 @@ impl TypeResolver { ) -> Result<(), Error> { fn write( this: &dyn Any, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => Ok(v.fory_write(fory, context, is_field)?), + Some(v) => Ok(v.fory_write(context, is_field)?), None => todo!(), } } fn read( - fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, ) -> Result, Error> { if skip_ref_flag { - match T2::fory_read_data(fory, context, is_field) { + match T2::fory_read_data(context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } } else { - match T2::fory_read(fory, context, is_field) { + match T2::fory_read(context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -475,23 +524,21 @@ impl TypeResolver { fn write_data( this: &dyn Any, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => T2::fory_write_data(v, fory, context, is_field), + Some(v) => T2::fory_write_data(v, context, is_field), None => todo!(), } } fn read_data( - fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(fory, context, is_field) { + match T2::fory_read_data(context, is_field) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -560,43 +607,4 @@ impl TypeResolver { } Ok(()) } - - pub fn get_harness(&self, id: u32) -> Option> { - self.serializer_map.get(&id).cloned() - } - - pub fn get_name_harness( - &self, - namespace: &MetaString, - type_name: &MetaString, - ) -> Option> { - let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map.get(&key).cloned() - } - - pub fn get_ext_harness(&self, id: u32) -> Result, Error> { - self.serializer_map - .get(&id) - .cloned() - .ok_or_else(|| Error::TypeError("ext type must be registered in both peers".into())) - } - - pub fn get_ext_name_harness( - &self, - namespace: &MetaString, - type_name: &MetaString, - ) -> Result, Error> { - let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map.get(&key).cloned().ok_or_else(|| { - Error::TypeError("named_ext type must be registered in both peers".into()) - }) - } - - pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { - if let Some(type_info) = self.type_info_cache.get(&rust_type_id) { - Some(type_info.get_type_id()) - } else { - self.type_id_map.get(&rust_type_id).copied() - } - } } diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index 52e63675f9..1bbfd5d3a8 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -16,8 +16,8 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use std::any::Any; @@ -27,20 +27,19 @@ use std::sync::Arc; /// Helper function to serialize a `Box` pub fn serialize_any_box( any_box: &Box, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { context.writer.write_i8(RefFlag::NotNullValue as i8); let concrete_type_id = (**any_box).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id)?; + let harness = context.write_any_typeinfo(concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**any_box, fory, context, is_field) + serializer_fn(&**any_box, context, is_field) } /// Helper function to deserialize to `Box` -pub fn deserialize_any_box(fory: &Fory, context: &mut ReadContext) -> Result, Error> { +pub fn deserialize_any_box(context: &mut ReadContext) -> Result, Error> { context.inc_depth()?; let ref_flag = context.reader.read_i8()?; if ref_flag != RefFlag::NotNullValue as i8 { @@ -48,9 +47,9 @@ pub fn deserialize_any_box(fory: &Fory, context: &mut ReadContext) -> Result".into(), )); } - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let result = deserializer_fn(fory, context, true); + let result = deserializer_fn(context, true); context.dec_depth(); result } @@ -62,43 +61,29 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - serialize_any_box(self, fory, context, is_field) + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + serialize_any_box(self, context, is_field) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - serialize_any_box(self, fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + serialize_any_box(self, context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { - deserialize_any_box(fory, context) + fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { + deserialize_any_box(context) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - deserialize_any_box(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + deserialize_any_box(context) } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { unreachable!("Box has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); - fory.get_type_resolver() + type_resolver .get_fory_type_id(concrete_type_id) .ok_or_else(|| Error::TypeError("Type not registered".into())) } @@ -107,20 +92,12 @@ impl Serializer for Box { true } - fn fory_write_type_info( - _fory: &Fory, - _context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { // Rc is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info( - _fory: &Fory, - _context: &mut ReadContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { // Rc is polymorphic - type info is read per element Ok(()) } @@ -137,34 +114,24 @@ impl ForyDefault for Rc { } impl Serializer for Rc { - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if !context .ref_writer .try_write_rc_ref(&mut context.writer, self) { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id)?; + let harness = context.write_any_typeinfo(concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, fory, context, is_field)? + serializer_fn(&**self, context, is_field)? }; Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - self.fory_write(fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + self.fory_write(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { @@ -182,17 +149,17 @@ impl Serializer for Rc { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(fory, context, true)?; + let boxed = deserializer_fn(context, true)?; context.dec_depth(); Ok(Rc::::from(boxed)) } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(fory, context, true)?; + let boxed = deserializer_fn(context, true)?; context.dec_depth(); let rc: Rc = Rc::from(boxed); context.ref_reader.store_rc_ref(rc.clone()); @@ -201,21 +168,17 @@ impl Serializer for Rc { } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Self::fory_read(fory, context, is_field) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Self::fory_read(context, is_field) } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { unreachable!("Rc has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); - fory.get_type_resolver() + type_resolver .get_fory_type_id(concrete_type_id) .ok_or_else(|| Error::TypeError("Type not registered".into())) } @@ -224,20 +187,12 @@ impl Serializer for Rc { true } - fn fory_write_type_info( - _fory: &Fory, - _context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { // Rc is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info( - _fory: &Fory, - _context: &mut ReadContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { // Rc is polymorphic - type info is read per element Ok(()) } @@ -254,34 +209,24 @@ impl ForyDefault for Arc { } impl Serializer for Arc { - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if !context .ref_writer .try_write_arc_ref(&mut context.writer, self) { let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id)?; + let harness = context.write_any_typeinfo(concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, fory, context, is_field)?; + serializer_fn(&**self, context, is_field)?; } Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - self.fory_write(fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + self.fory_write(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { @@ -299,17 +244,17 @@ impl Serializer for Arc { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(fory, context, true)?; + let boxed = deserializer_fn(context, true)?; context.dec_depth(); Ok(Arc::::from(boxed)) } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(fory, context, true)?; + let boxed = deserializer_fn(context, true)?; context.dec_depth(); let arc: Arc = Arc::from(boxed); context.ref_reader.store_arc_ref(arc.clone()); @@ -318,21 +263,17 @@ impl Serializer for Arc { } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Self::fory_read(fory, context, is_field) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Self::fory_read(context, is_field) } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_type_resolver: &TypeResolver) -> Result { unreachable!("Arc has no static type ID - use fory_type_id_dyn") } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); - fory.get_type_resolver() + type_resolver .get_fory_type_id(concrete_type_id) .ok_or_else(|| Error::TypeError("Type not registered".into())) } @@ -341,20 +282,12 @@ impl Serializer for Arc { true } - fn fory_write_type_info( - _fory: &Fory, - _context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { // Arc is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info( - _fory: &Fory, - _context: &mut ReadContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { // Arc is polymorphic - type info is read per element Ok(()) } diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index 41377a98ac..e4cbda85e3 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -16,8 +16,8 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use std::sync::Arc; @@ -27,42 +27,28 @@ impl Serializer for Arc true } - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if !context .ref_writer .try_write_arc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), fory, context, is_field)? + T::fory_write_data(self.as_ref(), context, is_field)? }; Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // When Arc is nested inside another shared ref (like Rc>), // the outer ref calls fory_write_data on the inner Arc. // We still need to track the Arc's own references here. - self.fory_write(fory, context, is_field) + self.fory_write(context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; Ok(match ref_flag { @@ -77,12 +63,12 @@ impl Serializer for Arc ))? } RefFlag::NotNullValue => { - let inner = T::fory_read_data(fory, context, is_field)?; + let inner = T::fory_read_data(context, is_field)?; Arc::new(inner) } RefFlag::RefValue => { let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(fory, context, is_field)?; + let inner = T::fory_read_data(context, is_field)?; let arc = Arc::new(inner); context.ref_reader.store_arc_ref_at(ref_id, arc.clone()); arc @@ -90,22 +76,14 @@ impl Serializer for Arc }) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { // When Arc is nested inside another shared ref, fory_read_data is called. // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(fory, context, is_field) + Self::fory_read(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -114,12 +92,12 @@ impl Serializer for Arc 4 } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - (**self).fory_type_id_dyn(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + (**self).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index 7095038427..9c1642bcd4 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -16,31 +16,22 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; use crate::types::TypeId; use std::mem; impl Serializer for bool { #[inline(always)] - fn fory_write_data( - &self, - _fory: &Fory, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { context.writer.write_u8(if *self { 1 } else { 0 }); Ok(()) } #[inline(always)] - fn fory_read_data( - _fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { Ok(context.reader.read_u8()? == 1) } @@ -50,11 +41,11 @@ impl Serializer for bool { } #[inline(always)] - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::BOOL as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::BOOL as u32) } @@ -64,21 +55,13 @@ impl Serializer for bool { } #[inline(always)] - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } #[inline(always)] - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } diff --git a/rust/fory-core/src/serializer/box_.rs b/rust/fory-core/src/serializer/box_.rs index 63063649d0..6cf8730079 100644 --- a/rust/fory-core/src/serializer/box_.rs +++ b/rust/fory-core/src/serializer/box_.rs @@ -16,55 +16,38 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; impl Serializer for Box { - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Ok(Box::new(T::fory_read_data(fory, context, is_field)?)) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Ok(Box::new(T::fory_read_data(context, is_field)?)) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_data(self.as_ref(), fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_data(self.as_ref(), context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } fn fory_reserved_space() -> usize { T::fory_reserved_space() } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - (**self).fory_type_id_dyn(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + (**self).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 45938f4beb..25469c5898 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; use crate::types::PRIMITIVE_ARRAY_TYPES; -use crate::{ensure, Fory}; // const TRACKING_REF: u8 = 0b1; @@ -46,7 +46,7 @@ pub fn write_collection_type_info( pub fn write_collection<'a, T, I>( iter: I, - fory: &Fory, + context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> @@ -83,26 +83,19 @@ where header |= IS_SAME_TYPE; } context.writer.write_u8(header); - T::fory_write_type_info(fory, context, is_field)?; + T::fory_write_type_info(context, is_field)?; // context.writer.reserve((T::reserved_space() + SIZE_OF_REF_AND_TYPE) * len); if T::fory_is_polymorphic() || T::fory_is_shared_ref() { // TOTO: make it xlang compatible for item in iter { - item.fory_write(fory, context, is_field)?; + item.fory_write(context, is_field)?; } Ok(()) } else { // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); let skip_ref_flag = is_same_type && !has_null; for item in iter { - crate::serializer::write_ref_info_data( - item, - fory, - context, - is_field, - skip_ref_flag, - true, - )?; + crate::serializer::write_ref_info_data(item, context, is_field, skip_ref_flag, true)?; } Ok(()) } @@ -132,7 +125,7 @@ pub fn read_collection_type_info( Ok(()) } -pub fn read_collection(fory: &Fory, context: &mut ReadContext) -> Result +pub fn read_collection(context: &mut ReadContext) -> Result where T: Serializer + ForyDefault, C: FromIterator, @@ -143,20 +136,18 @@ where } let header = context.reader.read_u8()?; let declared = (header & DECL_ELEMENT_TYPE) != 0; - T::fory_read_type_info(fory, context, declared)?; + T::fory_read_type_info(context, declared)?; let has_null = (header & HAS_NULL) != 0; let is_same_type = (header & IS_SAME_TYPE) != 0; if T::fory_is_polymorphic() || T::fory_is_shared_ref() { (0..len) - .map(|_| T::fory_read(fory, context, declared)) + .map(|_| T::fory_read(context, declared)) .collect::>() } else { let skip_ref_flag = is_same_type && !has_null; // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); (0..len) - .map(|_| { - crate::serializer::read_ref_info_data(fory, context, declared, skip_ref_flag, true) - }) + .map(|_| crate::serializer::read_ref_info_data(context, declared, skip_ref_flag, true)) .collect::>() } } diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs index 4c49e26cc0..7ef84a3055 100644 --- a/rust/fory-core/src/serializer/datetime.rs +++ b/rust/fory-core/src/serializer/datetime.rs @@ -16,9 +16,9 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::Serializer; use crate::serializer::{read_type_info, write_type_info, ForyDefault}; use crate::types::TypeId; @@ -27,23 +27,14 @@ use chrono::{DateTime, Days, NaiveDate, NaiveDateTime}; use std::mem; impl Serializer for NaiveDateTime { - fn fory_write_data( - &self, - _fory: &Fory, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { let dt = self.and_utc(); let micros = dt.timestamp() * 1_000_000 + dt.timestamp_subsec_micros() as i64; context.writer.write_i64(micros); Ok(()) } - fn fory_read_data( - _fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { let micros = context.reader.read_i64()?; let seconds = micros / 1_000_000; let subsec_micros = (micros % 1_000_000) as u32; @@ -59,11 +50,11 @@ impl Serializer for NaiveDateTime { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::TIMESTAMP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::TIMESTAMP as u32) } @@ -71,40 +62,23 @@ impl Serializer for NaiveDateTime { self } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } impl Serializer for NaiveDate { - fn fory_write_data( - &self, - _fory: &Fory, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { let days_since_epoch = self.signed_duration_since(EPOCH).num_days(); context.writer.write_i32(days_since_epoch as i32); Ok(()) } - fn fory_read_data( - _fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { let days = context.reader.read_i32()?; EPOCH .checked_add_days(Days::new(days as u64)) @@ -117,11 +91,11 @@ impl Serializer for NaiveDate { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::LOCAL_DATE as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::LOCAL_DATE as u32) } @@ -129,20 +103,12 @@ impl Serializer for NaiveDate { self } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index 60d6930fb9..3cb3232112 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -17,7 +17,6 @@ use crate::ensure; use crate::error::Error; -use crate::fory::Fory; use crate::meta::{MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; @@ -34,7 +33,6 @@ pub fn actual_type_id(type_id: u32, register_by_name: bool, _compatible: bool) - #[inline(always)] pub fn type_def( - _fory: &Fory, type_id: u32, namespace: MetaString, type_name: MetaString, @@ -47,25 +45,24 @@ pub fn type_def( #[inline(always)] pub fn write_type_info( - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { if is_field { return Ok(()); } - let type_id = T::fory_get_type_id(fory)?; + let type_id = T::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); let is_named_enum = type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { return Ok(()); } let rs_type_id = std::any::TypeId::of::(); - if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id)? as u32; + if context.is_share_meta() { + let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = fory.get_type_resolver().get_type_info(rs_type_id)?; + let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); context.write_meta_string_bytes(&namespace)?; @@ -76,14 +73,13 @@ pub fn write_type_info( #[inline(always)] pub fn read_type_info( - fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result<(), Error> { if is_field { return Ok(()); } - let local_type_id = T::fory_get_type_id(fory)?; + let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; let remote_type_id = context.reader.read_varuint32()?; ensure!( local_type_id == remote_type_id, @@ -93,7 +89,7 @@ pub fn read_type_info( if !is_named_enum { return Ok(()); } - if fory.is_share_meta() { + if context.is_share_meta() { let _meta_index = context.reader.read_varuint32()?; } else { let _namespace_msb = context.read_meta_string_bytes()?; @@ -103,29 +99,25 @@ pub fn read_type_info( } #[inline(always)] -pub fn read_compatible( - fory: &Fory, - context: &mut ReadContext, -) -> Result { - T::fory_read_type_info(fory, context, true)?; - T::fory_read_data(fory, context, true) +pub fn read_compatible(context: &mut ReadContext) -> Result { + T::fory_read_type_info(context, true)?; + T::fory_read_data(context, true) } #[inline(always)] pub fn write( this: &T, - fory: &Fory, + context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, is_field)?; - this.fory_write_data(fory, context, is_field) + T::fory_write_type_info(context, is_field)?; + this.fory_write_data(context, is_field) } #[inline(always)] pub fn read( - fory: &Fory, context: &mut ReadContext, is_field: bool, ) -> Result { @@ -133,8 +125,8 @@ pub fn read( if ref_flag == RefFlag::Null as i8 { Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { - T::fory_read_type_info(fory, context, false)?; - T::fory_read_data(fory, context, is_field) + T::fory_read_type_info(context, false)?; + T::fory_read_data(context, is_field) } else { unimplemented!() } diff --git a/rust/fory-core/src/serializer/heap.rs b/rust/fory-core/src/serializer/heap.rs index 94532ec07b..7c9af1871a 100644 --- a/rust/fory-core/src/serializer/heap.rs +++ b/rust/fory-core/src/serializer/heap.rs @@ -16,9 +16,9 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::collection::{ read_collection, read_collection_type_info, write_collection, write_collection_type_info, }; @@ -29,36 +29,19 @@ use std::collections::BinaryHeap; use std::mem; impl Serializer for BinaryHeap { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_collection(self, fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_collection(self, context, is_field) } - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { write_collection_type_info(context, is_field, TypeId::SET as u32) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - read_collection(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + read_collection(context) } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -66,11 +49,11 @@ impl Serializer for BinaryHeap { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs index 399ad8c511..ba3a954aff 100644 --- a/rust/fory-core/src/serializer/list.rs +++ b/rust/fory-core/src/serializer/list.rs @@ -16,9 +16,9 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::primitive_list; use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; @@ -44,45 +44,28 @@ fn check_primitive() -> Option { } impl Serializer for Vec { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { match check_primitive::() { Some(_) => primitive_list::fory_write_data(self, context), - None => write_collection(self, fory, context, is_field), + None => write_collection(self, context, is_field), } } - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { match check_primitive::() { Some(type_id) => primitive_list::fory_write_type_info(context, is_field, type_id), None => write_collection_type_info(context, is_field, TypeId::LIST as u32), } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { match check_primitive::() { Some(_) => primitive_list::fory_read_data(context), - None => read_collection(fory, context), + None => read_collection(context), } } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { match check_primitive::() { Some(type_id) => primitive_list::fory_read_type_info(context, is_field, type_id), None => read_collection_type_info(context, is_field, TypeId::LIST as u32), @@ -99,14 +82,14 @@ impl Serializer for Vec { } } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(match check_primitive::() { Some(type_id) => type_id as u32, None => TypeId::LIST as u32, }) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(match check_primitive::() { Some(type_id) => type_id as u32, None => TypeId::LIST as u32, @@ -125,36 +108,19 @@ impl ForyDefault for Vec { } impl Serializer for VecDeque { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_collection(self, fory, context, is_field) - } - - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_collection(self, context, is_field) + } + + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { write_collection_type_info(context, is_field, TypeId::LIST as u32) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - read_collection(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + read_collection(context) } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::LIST as u32) } @@ -162,11 +128,11 @@ impl Serializer for VecDeque { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::LIST as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::LIST as u32) } @@ -182,36 +148,19 @@ impl ForyDefault for VecDeque { } impl Serializer for LinkedList { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_collection(self, fory, context, is_field) - } - - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_collection(self, context, is_field) + } + + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { write_collection_type_info(context, is_field, TypeId::LIST as u32) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - read_collection(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + read_collection(context) } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::LIST as u32) } @@ -219,11 +168,11 @@ impl Serializer for LinkedList { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::LIST as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::LIST as u32) } diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index 27045f482a..526ef8927e 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -17,8 +17,8 @@ use crate::ensure; use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ read_ref_info_data, read_type_info, write_ref_info_data, write_type_info, ForyDefault, Serializer, @@ -36,7 +36,6 @@ pub const VALUE_NULL: u8 = 0b10000; const DECL_VALUE_TYPE: u8 = 0b100000; fn check_and_write_null( - fory: &Fory, context: &mut WriteContext, is_field: bool, key: &K, @@ -50,7 +49,7 @@ fn check_and_write_null( let mut chunk_header = KEY_NULL; let skip_ref_flag; if is_field { - skip_ref_flag = crate::serializer::get_skip_ref_flag::(fory)?; + skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_type_resolver())?; chunk_header |= DECL_VALUE_TYPE; } else { skip_ref_flag = false; @@ -58,7 +57,7 @@ fn check_and_write_null( } context.writer.write_u8(chunk_header); - write_ref_info_data(value, fory, context, is_field, skip_ref_flag, false)?; + write_ref_info_data(value, context, is_field, skip_ref_flag, false)?; return Ok(true); } if value.fory_is_none() { @@ -73,7 +72,7 @@ fn check_and_write_null( chunk_header |= TRACKING_KEY_REF; } context.writer.write_u8(chunk_header); - write_ref_info_data(key, fory, context, is_field, skip_ref_flag, false)?; + write_ref_info_data(key, context, is_field, skip_ref_flag, false)?; return Ok(true); } Ok(false) @@ -86,7 +85,7 @@ fn write_chunk_size(context: &mut WriteContext, header_offset: usize, size: u8) fn write_map_data<'a, K, V, I>( iter: I, length: usize, - fory: &Fory, + context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> @@ -110,7 +109,7 @@ where let mut skip_val_ref_flag = false; for (key, value) in iter { if need_write_header { - if check_and_write_null(fory, context, is_field, key, value)? { + if check_and_write_null(context, is_field, key, value)? { continue; } header_offset = context.writer.len(); @@ -127,8 +126,8 @@ where if !skip_val_ref_flag { chunk_header |= TRACKING_VALUE_REF; } - K::fory_write_type_info(fory, context, is_field)?; - V::fory_write_type_info(fory, context, is_field)?; + K::fory_write_type_info(context, is_field)?; + V::fory_write_type_info(context, is_field)?; context.writer.set_bytes(header_offset, &[chunk_header]); need_write_header = false; } @@ -136,18 +135,18 @@ where write_chunk_size(context, header_offset, pair_counter); pair_counter = 0; need_write_header = true; - check_and_write_null(fory, context, is_field, key, value)?; + check_and_write_null(context, is_field, key, value)?; continue; } if K::fory_is_polymorphic() || K::fory_is_shared_ref() { - key.fory_write(fory, context, is_field)?; + key.fory_write(context, is_field)?; } else { - write_ref_info_data(key, fory, context, is_field, skip_key_ref_flag, true)?; + write_ref_info_data(key, context, is_field, skip_key_ref_flag, true)?; } if V::fory_is_polymorphic() || V::fory_is_shared_ref() { - value.fory_write(fory, context, is_field)?; + value.fory_write(context, is_field)?; } else { - write_ref_info_data(value, fory, context, is_field, skip_val_ref_flag, true)?; + write_ref_info_data(value, context, is_field, skip_val_ref_flag, true)?; } pair_counter += 1; if pair_counter == MAX_CHUNK_SIZE { @@ -165,20 +164,11 @@ where impl Serializer for HashMap { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_map_data(self.iter(), self.len(), fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { let len = context.reader.read_varuint32()?; let mut map = HashMap::::with_capacity(len as usize); if len == 0 { @@ -199,30 +189,29 @@ impl(fory)? + crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? } else { false }; - let value = - read_ref_info_data(fory, context, value_declared, skip_ref_flag, false)?; + let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(fory)? + crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? } else { false }; - let key = read_ref_info_data(fory, context, key_declared, skip_ref_flag, false)?; + let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8()?; - K::fory_read_type_info(fory, context, key_declared)?; - V::fory_read_type_info(fory, context, value_declared)?; + K::fory_read_type_info(context, key_declared)?; + V::fory_read_type_info(context, value_declared)?; let cur_len = len_counter + chunk_size as u32; ensure!( @@ -234,16 +223,16 @@ impl(context.get_fory()); - read_ref_info_data(fory, context, key_declared, true, true)? + read_ref_info_data(context, key_declared, true, true)? }; let value = if V::fory_is_polymorphic() { - V::fory_read(fory, context, value_declared)? + V::fory_read(context, value_declared)? } else { // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); - read_ref_info_data(fory, context, value_declared, true, true)? + read_ref_info_data(context, value_declared, true, true)? }; map.insert(key, value); } @@ -256,11 +245,11 @@ impl() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::MAP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::MAP as u32) } @@ -268,20 +257,12 @@ impl Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } @@ -294,20 +275,11 @@ impl ForyDefault for HashMap { impl Serializer for BTreeMap { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_map_data(self.iter(), self.len(), fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { let len = context.reader.read_varuint32()?; let mut map = BTreeMap::::new(); if len == 0 { @@ -328,41 +300,40 @@ impl(fory)? + crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? } else { false }; - let value = - read_ref_info_data(fory, context, value_declared, skip_ref_flag, false)?; + let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(fory)? + crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? } else { false }; - let key = read_ref_info_data(fory, context, key_declared, skip_ref_flag, false)?; + let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8()?; - K::fory_read_type_info(fory, context, key_declared)?; - V::fory_read_type_info(fory, context, value_declared)?; + K::fory_read_type_info(context, key_declared)?; + V::fory_read_type_info(context, value_declared)?; assert!(len_counter + chunk_size as u32 <= len); for _ in 0..chunk_size { let key = if K::fory_is_polymorphic() { - K::fory_read(fory, context, key_declared)? + K::fory_read(context, key_declared)? } else { - read_ref_info_data(fory, context, key_declared, true, true)? + read_ref_info_data(context, key_declared, true, true)? }; let value = if V::fory_is_polymorphic() { - V::fory_read(fory, context, value_declared)? + V::fory_read(context, value_declared)? } else { - read_ref_info_data(fory, context, value_declared, true, true)? + read_ref_info_data(context, value_declared, true, true)? }; map.insert(key, value); } @@ -375,11 +346,11 @@ impl() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::MAP as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::MAP as u32) } @@ -387,20 +358,12 @@ impl Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 3962f70a87..4e1eca4044 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use crate::ensure; use crate::error::Error; -use crate::fory::Fory; -use crate::meta::{MetaString, TypeMeta, NAMESPACE_DECODER, TYPE_NAME_DECODER}; +use crate::meta::{FieldInfo, NAMESPACE_DECODER, TYPE_NAME_DECODER}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::types::{RefFlag, TypeId, PRIMITIVE_TYPES}; +use crate::{ensure, TypeResolver}; use std::any::Any; pub mod any; @@ -49,7 +48,6 @@ pub mod weak; #[inline(always)] pub fn write_ref_info_data( record: &T, - fory: &Fory, context: &mut WriteContext, is_field: bool, skip_ref_flag: bool, @@ -62,16 +60,15 @@ pub fn write_ref_info_data( context.writer.write_i8(RefFlag::NotNullValue as i8); } if !skip_type_info { - T::fory_write_type_info(fory, context, is_field)?; + T::fory_write_type_info(context, is_field)?; } - record.fory_write_data(fory, context, is_field)?; + record.fory_write_data(context, is_field)?; } Ok(()) } #[inline(always)] pub fn read_ref_info_data( - fory: &Fory, context: &mut ReadContext, is_field: bool, skip_ref_flag: bool, @@ -83,15 +80,15 @@ pub fn read_ref_info_data( Ok(T::fory_default()) } else if ref_flag == (RefFlag::NotNullValue as i8) { if !skip_type_info { - T::fory_read_type_info(fory, context, is_field)?; + T::fory_read_type_info(context, is_field)?; } - T::fory_read_data(fory, context, is_field) + T::fory_read_data(context, is_field) } else if ref_flag == (RefFlag::RefValue as i8) { // First time seeing this referenceable object if !skip_type_info { - T::fory_read_type_info(fory, context, is_field)?; + T::fory_read_type_info(context, is_field)?; } - T::fory_read_data(fory, context, is_field) + T::fory_read_data(context, is_field) } else if ref_flag == (RefFlag::Ref as i8) { // This is a reference to a previously deserialized object // For now, just return default - this should be handled by specific types @@ -101,36 +98,34 @@ pub fn read_ref_info_data( } } else { if !skip_type_info { - T::fory_read_type_info(fory, context, is_field)?; + T::fory_read_type_info(context, is_field)?; } - T::fory_read_data(fory, context, is_field) + T::fory_read_data(context, is_field) } } #[inline(always)] -fn write_type_info( - fory: &Fory, +pub fn write_type_info( context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { if is_field { return Ok(()); } - let type_id = T::fory_get_type_id(fory)?; + let type_id = T::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); Ok(()) } #[inline(always)] -fn read_type_info( - fory: &Fory, +pub fn read_type_info( context: &mut ReadContext, is_field: bool, ) -> Result<(), Error> { if is_field { return Ok(()); } - let local_type_id = T::fory_get_type_id(fory)?; + let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; let remote_type_id = context.reader.read_varuint32()?; ensure!( local_type_id == remote_type_id, @@ -140,8 +135,8 @@ fn read_type_info( } #[inline(always)] -pub fn get_skip_ref_flag(fory: &Fory) -> Result { - let elem_type_id = T::fory_get_type_id(fory)?; +pub fn get_skip_ref_flag(type_resolver: &TypeResolver) -> Result { + let elem_type_id = T::fory_get_type_id(type_resolver)?; Ok(!T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id)) } @@ -160,23 +155,18 @@ pub trait ForyDefault: Sized { pub trait Serializer: 'static { /// Entry point of the serialization. - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> where Self: Sized, { - write_ref_info_data(self, fory, context, is_field, false, false) + write_ref_info_data(self, context, is_field, false, false) } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - read_ref_info_data(fory, context, is_field, false, false) + read_ref_info_data(context, is_field, false, false) } fn fory_is_option() -> bool @@ -204,17 +194,16 @@ pub trait Serializer: 'static { false } - fn fory_get_type_id(fory: &Fory) -> Result + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result where Self: Sized, { - Ok(fory - .get_type_resolver() + Ok(type_resolver .get_type_info(std::any::TypeId::of::())? .get_type_id()) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result; + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result; /// The possible max memory size of the type. /// Used to reserve the buffer space to avoid reallocation, which may hurt performance. @@ -225,29 +214,22 @@ pub trait Serializer: 'static { 0 } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> + fn fory_write_type_info(context: &mut WriteContext, _is_field: bool) -> Result<(), Error> where Self: Sized, { // default implementation only for ext/named_ext - let type_id = Self::fory_get_type_id(fory)?; + let type_id = Self::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); if type_id & 0xff == TypeId::EXT as u32 { return Ok(()); } let rs_type_id = std::any::TypeId::of::(); - if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id)? as u32; + if context.is_share_meta() { + let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = { - let type_resolver = fory.get_type_resolver(); - type_resolver.get_type_info(rs_type_id)? - }; + let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); context.write_meta_string_bytes(&namespace)?; @@ -256,16 +238,12 @@ pub trait Serializer: 'static { Ok(()) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result<(), Error> + fn fory_read_type_info(context: &mut ReadContext, _is_field: bool) -> Result<(), Error> where Self: Sized, { // default implementation only for ext/named_ext - let local_type_id = Self::fory_get_type_id(fory)?; + let local_type_id = Self::fory_get_type_id(context.get_type_resolver())?; let remote_type_id = context.reader.read_varuint32()?; ensure!( local_type_id == remote_type_id, @@ -274,7 +252,7 @@ pub trait Serializer: 'static { if local_type_id & 0xff == TypeId::EXT as u32 { return Ok(()); } - if fory.is_share_meta() { + if context.is_share_meta() { let _meta_index = context.reader.read_varuint32()?; } else { let _namespace_msb = context.read_meta_string_bytes()?; @@ -284,29 +262,29 @@ pub trait Serializer: 'static { } // only used by struct/enum/ext - fn fory_read_compatible(fory: &Fory, context: &mut ReadContext) -> Result + fn fory_read_compatible(context: &mut ReadContext) -> Result where Self: Sized, { // default logic only for ext/named_ext let remote_type_id = context.reader.read_varuint32()?; - let local_type_id = Self::fory_get_type_id(fory)?; + let local_type_id = Self::fory_get_type_id(context.get_type_resolver())?; ensure!( local_type_id == remote_type_id, Error::TypeMismatch(local_type_id, remote_type_id) ); if local_type_id & 0xff == TypeId::EXT as u32 { - let type_resolver = fory.get_type_resolver(); - type_resolver + context + .get_type_resolver() .get_ext_harness(local_type_id)? - .get_read_data_fn()(fory, context, true) + .get_read_data_fn()(context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) .map_err(|_| Error::TypeError("downcast to Self failed".into())) }) } else { - let (namespace, type_name) = if fory.is_share_meta() { + let (namespace, type_name) = if context.is_share_meta() { let meta_index = context.reader.read_varuint32()?; let type_meta = context.get_meta(meta_index as usize); (type_meta.get_namespace(), type_meta.get_type_name()) @@ -317,10 +295,10 @@ pub trait Serializer: 'static { let ts = TYPE_NAME_DECODER.decode(&tsb.bytes, tsb.encoding)?; (ns, ts) }; - let type_resolver = fory.get_type_resolver(); - type_resolver + context + .get_type_resolver() .get_ext_name_harness(&namespace, &type_name)? - .get_read_data_fn()(fory, context, true) + .get_read_data_fn()(context, true) .and_then(|b: Box| { b.downcast::() .map(|boxed_self| *boxed_self) @@ -330,18 +308,9 @@ pub trait Serializer: 'static { } /// Write/Read the data into the buffer. Need to be implemented. - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error>; + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error>; - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault; @@ -353,14 +322,8 @@ pub trait Serializer: 'static { } pub trait StructSerializer: Serializer + 'static { - fn fory_type_def( - _fory: &Fory, - _type_id: u32, - _namespace: MetaString, - _type_name: MetaString, - _register_by_name: bool, - ) -> Result<(Vec, TypeMeta), Error> { - Ok((Vec::default(), TypeMeta::empty())) + fn fory_fields_info(_: &TypeResolver) -> Result, Error> { + Ok(Vec::default()) } fn fory_type_index() -> u32 { @@ -370,7 +333,7 @@ pub trait StructSerializer: Serializer + 'static { struct_::actual_type_id(type_id, register_by_name, compatible) } - fn fory_get_sorted_field_names(_fory: &Fory) -> &'static [&'static str] { + fn fory_get_sorted_field_names() -> &'static [&'static str] { &[] } } diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index 0d0fcf0984..b3bb0cb45a 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -42,8 +42,8 @@ //! - A poisoned mutex (from a panicked holder) will cause `.lock().unwrap()` to panic //! during serialization — it is assumed this is a programmer error. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use std::sync::Mutex; @@ -52,35 +52,21 @@ use std::sync::Mutex; /// Simply delegates to the serializer for `T`, allowing thread-safe interior mutable /// containers to be included in serialized graphs. impl Serializer for Mutex { - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // Don't add ref tracking for Mutex itself, just delegate to inner type // The inner type will handle its own ref tracking let guard = self.lock().unwrap(); - T::fory_write(&*guard, fory, context, is_field) + T::fory_write(&*guard, context, is_field) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // When called from Rc/Arc, just delegate to inner type's data serialization let guard = self.lock().unwrap(); - T::fory_write_data(&*guard, fory, context, is_field) + T::fory_write_data(&*guard, context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -88,36 +74,28 @@ impl Serializer for Mutex { T::fory_reserved_space() } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - Ok(Mutex::new(T::fory_read(fory, context, is_field)?)) + Ok(Mutex::new(T::fory_read(context, is_field)?)) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Ok(Mutex::new(T::fory_read_data(fory, context, is_field)?)) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Ok(Mutex::new(T::fory_read_data(context, is_field)?)) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let guard = self.lock().unwrap(); - (*guard).fory_type_id_dyn(fory) + (*guard).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index abe4f62390..c923bb15f4 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -17,9 +17,9 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; use crate::types::TypeId; @@ -29,7 +29,7 @@ macro_rules! impl_num_serializer { #[inline] fn fory_write_data( &self, - _fory: &Fory, + context: &mut WriteContext, _is_field: bool, ) -> Result<(), Error> { @@ -38,11 +38,7 @@ macro_rules! impl_num_serializer { } #[inline] - fn fory_read_data( - _fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { $reader(&mut context.reader) } @@ -52,11 +48,11 @@ macro_rules! impl_num_serializer { } #[inline] - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok($field_type as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok($field_type as u32) } @@ -67,20 +63,15 @@ macro_rules! impl_num_serializer { #[inline] fn fory_write_type_info( - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), Error> { - write_type_info::(fory, context, is_field) + write_type_info::(context, is_field) } #[inline] - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } impl ForyDefault for $ty { diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 6ea9c58c21..1a6dc0339f 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -16,51 +16,34 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; impl Serializer for Option { #[inline(always)] - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Ok(Some(T::fory_read_data(fory, context, is_field)?)) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Ok(Some(T::fory_read_data(context, is_field)?)) } #[inline(always)] - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } #[inline(always)] - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if let Some(v) = self { - T::fory_write_data(v, fory, context, is_field) + T::fory_write_data(v, context, is_field) } else { unreachable!("write should be call by serialize") } } #[inline(always)] - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } #[inline(always)] @@ -69,15 +52,15 @@ impl Serializer for Option { } #[inline(always)] - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } #[inline(always)] - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { match self { - Some(val) => val.fory_type_id_dyn(fory), - None => T::fory_get_type_id(fory), + Some(val) => val.fory_type_id_dyn(type_resolver), + None => T::fory_get_type_id(type_resolver), } } diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index 86953ce683..c8750b4a97 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -16,8 +16,8 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use std::rc::Rc; @@ -27,42 +27,28 @@ impl Serializer for Rc { true } - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if !context .ref_writer .try_write_rc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), fory, context, is_field)? + T::fory_write_data(self.as_ref(), context, is_field)? }; Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // When Rc is nested inside another shared ref (like Arc>), // the outer ref calls fory_write_data on the inner Rc. // We still need to track the Rc's own references here. - self.fory_write(fory, context, is_field) + self.fory_write(context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { RefFlag::Null => Err(Error::InvalidRef("Rc cannot be null".into())), @@ -73,12 +59,12 @@ impl Serializer for Rc { }) } RefFlag::NotNullValue => { - let inner = T::fory_read_data(fory, context, is_field)?; + let inner = T::fory_read_data(context, is_field)?; Ok(Rc::new(inner)) } RefFlag::RefValue => { let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(fory, context, is_field)?; + let inner = T::fory_read_data(context, is_field)?; let rc = Rc::new(inner); context.ref_reader.store_rc_ref_at(ref_id, rc.clone()); Ok(rc) @@ -86,22 +72,14 @@ impl Serializer for Rc { } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { // When Rc is nested inside another shared ref, fory_read_data is called. // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(fory, context, is_field) + Self::fory_read(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -110,12 +88,12 @@ impl Serializer for Rc { 4 } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - (**self).fory_type_id_dyn(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + (**self).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index 7dcba16607..e1c194d8d8 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -33,8 +33,8 @@ //! // Can be serialized by the Fory framework //! ``` use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use std::cell::RefCell; @@ -43,56 +43,34 @@ use std::cell::RefCell; /// Simply delegates to the serializer for `T`, allowing interior mutable /// containers to be included in serialized graphs. impl Serializer for RefCell { - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result where Self: Sized + ForyDefault, { - Ok(RefCell::new(T::fory_read(fory, context, is_field)?)) + Ok(RefCell::new(T::fory_read(context, is_field)?)) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Ok(RefCell::new(T::fory_read_data(fory, context, is_field)?)) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Ok(RefCell::new(T::fory_read_data(context, is_field)?)) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // Don't add ref tracking for RefCell itself, just delegate to inner type // The inner type will handle its own ref tracking - T::fory_write(&*self.borrow(), fory, context, is_field) + T::fory_write(&*self.borrow(), context, is_field) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { // When called from Rc, just delegate to inner type's data serialization - T::fory_write_data(&*self.borrow(), fory, context, is_field) + T::fory_write_data(&*self.borrow(), context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -100,12 +78,12 @@ impl Serializer for RefCell { T::fory_reserved_space() } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - (*self.borrow()).fory_type_id_dyn(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + (*self.borrow()).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs index fc50b958d9..dd0bfaa0d6 100644 --- a/rust/fory-core/src/serializer/set.rs +++ b/rust/fory-core/src/serializer/set.rs @@ -16,9 +16,9 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::collection::{ read_collection, read_collection_type_info, write_collection, write_collection_type_info, }; @@ -29,36 +29,19 @@ use std::collections::{BTreeSet, HashSet}; use std::mem; impl Serializer for HashSet { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_collection(self, fory, context, is_field) - } - - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_collection(self, context, is_field) + } + + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { write_collection_type_info(context, is_field, TypeId::SET as u32) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - read_collection(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + read_collection(context) } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -66,11 +49,11 @@ impl Serializer for HashSet< mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } @@ -86,36 +69,19 @@ impl ForyDefault for HashSet { } impl Serializer for BTreeSet { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_collection(self, fory, context, is_field) - } - - fn fory_write_type_info( - _fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_collection(self, context, is_field) + } + + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { write_collection_type_info(context, is_field, TypeId::SET as u32) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { - read_collection(fory, context) + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + read_collection(context) } - fn fory_read_type_info( - _fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { read_collection_type_info(context, is_field, TypeId::SET as u32) } @@ -123,11 +89,11 @@ impl Serializer for BTreeSet { mem::size_of::() } - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::SET as u32) } diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index aa9feda19c..8baf7eaaa6 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::meta::FieldType; use crate::resolver::context::ReadContext; use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE}; use crate::serializer::Serializer; use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPES}; -use crate::{ensure, Fory}; use chrono::{NaiveDate, NaiveDateTime}; pub fn get_read_ref_flag(field_type: &FieldType) -> bool { @@ -30,11 +30,11 @@ pub fn get_read_ref_flag(field_type: &FieldType) -> bool { } macro_rules! basic_type_deserialize { - ($fory:expr, $tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { + ($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { $( if $tid == TypeId::$id { - <$ty as Serializer>::fory_read_type_info($fory, $context, true)?; - <$ty as Serializer>::fory_read_data($fory, $context, true)?; + <$ty as Serializer>::fory_read_type_info($context, true)?; + <$ty as Serializer>::fory_read_data($context, true)?; return Ok(()); } )+else { @@ -46,7 +46,6 @@ macro_rules! basic_type_deserialize { // call when is_field && is_compatible_mode #[allow(unreachable_code)] pub fn skip_field_value( - fory: &Fory, context: &mut ReadContext, field_type: &FieldType, read_ref_flag: bool, @@ -61,7 +60,7 @@ pub fn skip_field_value( match TypeId::try_from(type_id_num as i16) { Ok(type_id) => { if BASIC_TYPES.contains(&type_id) { - basic_type_deserialize!(fory, type_id, context; + basic_type_deserialize!(type_id, context; (bool, BOOL), (i8, INT8), (i16, INT16), @@ -93,7 +92,7 @@ pub fn skip_field_value( let elem_type = field_type.generics.first().unwrap(); context.inc_depth()?; for _ in 0..length { - skip_field_value(fory, context, elem_type, !skip_ref_flag)?; + skip_field_value(context, elem_type, !skip_ref_flag)?; } context.dec_depth(); } else if type_id == TypeId::MAP { @@ -118,7 +117,7 @@ pub fn skip_field_value( if header & crate::serializer::map::KEY_NULL != 0 { // let read_ref_flag = get_read_ref_flag(value_type); context.inc_depth()?; - skip_field_value(fory, context, value_type, false)?; + skip_field_value(context, value_type, false)?; context.dec_depth(); len_counter += 1; continue; @@ -126,7 +125,7 @@ pub fn skip_field_value( if header & crate::serializer::map::VALUE_NULL != 0 { // let read_ref_flag = get_read_ref_flag(key_type); context.inc_depth()?; - skip_field_value(fory, context, key_type, false)?; + skip_field_value(context, key_type, false)?; context.dec_depth(); len_counter += 1; continue; @@ -135,9 +134,9 @@ pub fn skip_field_value( context.inc_depth()?; for _ in (0..chunk_size).enumerate() { // let read_ref_flag = get_read_ref_flag(key_type); - skip_field_value(fory, context, key_type, false)?; + skip_field_value(context, key_type, false)?; // let read_ref_flag = get_read_ref_flag(value_type); - skip_field_value(fory, context, value_type, false)?; + skip_field_value(context, value_type, false)?; } context.dec_depth(); len_counter += chunk_size as u32; @@ -159,7 +158,7 @@ pub fn skip_field_value( context.inc_depth()?; for field_info in field_infos.iter() { let read_ref_flag = get_read_ref_flag(&field_info.field_type); - skip_field_value(fory, context, &field_info.field_type, read_ref_flag)?; + skip_field_value(context, &field_info.field_type, read_ref_flag)?; } context.dec_depth(); Ok(()) @@ -171,10 +170,10 @@ pub fn skip_field_value( ); let meta_index = context.reader.read_varuint32()?; let type_meta = context.get_meta(meta_index as usize); - let type_resolver = fory.get_type_resolver(); + let type_resolver = context.get_type_resolver(); type_resolver .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name())? - .get_read_data_fn()(fory, context, true)?; + .get_read_data_fn()(context, true)?; Ok(()) } else { unreachable!("unimplemented type: {:?}", type_id); @@ -197,7 +196,7 @@ pub fn skip_field_value( context.inc_depth()?; for field_info in field_infos.iter() { let read_ref_flag = get_read_ref_flag(&field_info.field_type); - skip_field_value(fory, context, &field_info.field_type, read_ref_flag)?; + skip_field_value(context, &field_info.field_type, read_ref_flag)?; } context.dec_depth(); } else if internal_id == ENUM_ID { @@ -209,10 +208,10 @@ pub fn skip_field_value( Error::TypeMismatch(type_id_num, remote_type_id) ); context.inc_depth()?; - let type_resolver = fory.get_type_resolver(); + let type_resolver = context.get_type_resolver(); type_resolver .get_ext_harness(type_id_num)? - .get_read_data_fn()(fory, context, true)?; + .get_read_data_fn()(context, true)?; context.dec_depth(); } else { unreachable!("unimplemented skipped type: {:?}", type_id_num); diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index a0ce9d02a3..ad9b2c25ce 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -16,10 +16,10 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::meta::get_latin1_length; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; use crate::types::TypeId; use std::mem; @@ -32,18 +32,13 @@ enum StrEncoding { impl Serializer for String { #[inline] - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { let mut len = get_latin1_length(self); if len >= 0 { let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64; context.writer.write_varuint36_small(bitor); context.writer.write_latin1_string(self); - } else if fory.is_compress_string() { + } else if context.is_compress_string() { // todo: support `writeNumUtf16BytesForUtf8Encoding` like in java len = self.len() as i32; let bitor = (len as u64) << 2 | StrEncoding::Utf8 as u64; @@ -59,11 +54,7 @@ impl Serializer for String { } #[inline] - fn fory_read_data( - _fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { let bitor = context.reader.read_varuint36small()?; let len = bitor >> 2; let encoding = bitor & 0b11; @@ -91,11 +82,11 @@ impl Serializer for String { } #[inline(always)] - fn fory_get_type_id(_fory: &Fory) -> Result { + fn fory_get_type_id(_: &TypeResolver) -> Result { Ok(TypeId::STRING as u32) } - fn fory_type_id_dyn(&self, _fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::STRING as u32) } @@ -105,21 +96,13 @@ impl Serializer for String { } #[inline(always)] - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_type_info::(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_type_info::(context, is_field) } #[inline(always)] - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - read_type_info::(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + read_type_info::(context, is_field) } } diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 4f1003d8db..5ec0137d6d 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -17,10 +17,8 @@ use crate::ensure; use crate::error::Error; -use crate::fory::Fory; -use crate::meta::{FieldInfo, MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::serializer::{Serializer, StructSerializer}; +use crate::serializer::Serializer; use crate::types::{RefFlag, TypeId}; #[inline(always)] @@ -38,62 +36,21 @@ pub fn actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> } } -#[inline(always)] -pub fn type_def( - fory: &Fory, - type_id: u32, - namespace: MetaString, - type_name: MetaString, - register_by_name: bool, - mut field_infos: Vec, -) -> (Vec, TypeMeta) { - let sorted_field_names = T::fory_get_sorted_field_names(fory); - let mut sorted_field_infos: Vec = Vec::with_capacity(field_infos.len()); - for name in sorted_field_names.iter() { - let mut found = false; - for i in 0..field_infos.len() { - if &field_infos[i].field_name == name { - // swap_remove is faster - sorted_field_infos.push(field_infos.swap_remove(i)); - found = true; - break; - } - } - if !found { - unreachable!("Field {} not found in field_infos", name); - } - } - // assign field id in ascending order - for (i, field_info) in sorted_field_infos.iter_mut().enumerate() { - field_info.field_id = i as i16; - } - let meta = TypeMeta::from_fields( - type_id, - namespace, - type_name, - register_by_name, - sorted_field_infos, - ); - let bytes = meta.to_bytes().unwrap(); - (bytes, meta) -} - #[inline(always)] pub fn write_type_info( - fory: &Fory, context: &mut WriteContext, _is_field: bool, ) -> Result<(), Error> { - let type_id = T::fory_get_type_id(fory)?; + let type_id = T::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); let rs_type_id = std::any::TypeId::of::(); if type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if fory.is_share_meta() { - let meta_index = context.push_meta(fory, rs_type_id)? as u32; + if context.is_share_meta() { + let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = fory.get_type_resolver().get_type_info(rs_type_id)?; + let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); context.write_meta_string_bytes(&namespace)?; @@ -102,7 +59,7 @@ pub fn write_type_info( } else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { - let meta_index = context.push_meta(fory, rs_type_id)? as u32; + let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } Ok(()) @@ -110,19 +67,18 @@ pub fn write_type_info( #[inline(always)] pub fn read_type_info( - fory: &Fory, context: &mut ReadContext, _is_field: bool, ) -> Result<(), Error> { let remote_type_id = context.reader.read_varuint32()?; - let local_type_id = T::fory_get_type_id(fory)?; + let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; ensure!( local_type_id == remote_type_id, Error::TypeMismatch(local_type_id, remote_type_id) ); if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { - if fory.is_share_meta() { + if context.is_share_meta() { let _meta_index = context.reader.read_varuint32()?; } else { let _namespace_msb = context.read_meta_string_bytes()?; @@ -139,19 +95,18 @@ pub fn read_type_info( #[inline(always)] pub fn write( this: &T, - fory: &Fory, context: &mut WriteContext, _is_field: bool, ) -> Result<(), Error> { - if fory.is_compatible() { + if context.is_compatible() { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false)?; - this.fory_write_data(fory, context, true)?; + T::fory_write_type_info(context, false)?; + this.fory_write_data(context, true)?; } else { // currently same context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(fory, context, false)?; - this.fory_write_data(fory, context, true)?; + T::fory_write_type_info(context, false)?; + this.fory_write_data(context, true)?; } Ok(()) } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 6d434f5527..7ee6daf929 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -16,15 +16,14 @@ // under the License. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; /// Helper functions for trait object serialization to reduce code duplication /// /// Writes common trait object headers (ref flag, type ID, compatibility metadata) pub fn write_trait_object_headers( - fory: &Fory, context: &mut WriteContext, fory_type_id: u32, concrete_type_id: std::any::TypeId, @@ -34,18 +33,18 @@ pub fn write_trait_object_headers( context.writer.write_i8(RefFlag::NotNullValue as i8); context.writer.write_varuint32(fory_type_id); - if fory.is_compatible() + if context.is_compatible() && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { - let meta_index = context.push_meta(fory, concrete_type_id)? as u32; + let meta_index = context.push_meta(concrete_type_id)? as u32; context.writer.write_varuint32(meta_index); }; Ok(()) } /// Reads common trait object headers and returns the type ID -pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Result { +pub fn read_trait_object_headers(context: &mut ReadContext) -> Result { use crate::types::{RefFlag, TypeId}; let ref_flag = context.reader.read_i8()?; @@ -57,7 +56,7 @@ pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Resu let fory_type_id = context.reader.read_varuint32()?; - if fory.is_compatible() + if context.is_compatible() && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) { @@ -69,11 +68,11 @@ pub fn read_trait_object_headers(fory: &Fory, context: &mut ReadContext) -> Resu /// Helper macro for common type resolution and downcasting pattern #[macro_export] macro_rules! downcast_and_serialize { - ($any_ref:expr, $fory:expr, $context:expr, $is_field:expr, $trait_name:ident, $($impl_type:ty),+) => {{ + ($any_ref:expr, $context:expr, $is_field:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( if $any_ref.type_id() == std::any::TypeId::of::<$impl_type>() { if let Some(concrete) = $any_ref.downcast_ref::<$impl_type>() { - concrete.fory_write_data($fory, $context, $is_field)?; + concrete.fory_write_data($context, $is_field)?; return Ok(()); } } @@ -85,11 +84,11 @@ macro_rules! downcast_and_serialize { /// Helper macro for common type resolution and deserialization pattern #[macro_export] macro_rules! resolve_and_deserialize { - ($fory_type_id:expr, $fory:expr, $context:expr, $is_field:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ + ($fory_type_id:expr, $context:expr, $is_field:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( - if let Some(registered_type_id) = $fory.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { + if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { if $fory_type_id == registered_type_id { - let concrete_obj = <$impl_type as $crate::serializer::Serializer>::fory_read_data($fory, $context, $is_field)?; + let concrete_obj = <$impl_type as $crate::serializer::Serializer>::fory_read_data($context, $is_field)?; return Ok($constructor(concrete_obj)); } } @@ -200,27 +199,27 @@ macro_rules! register_trait_type { // 4. Serializer implementation for Box (existing functionality) impl $crate::serializer::Serializer for Box { - fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); - if let Some(fory_type_id) = fory.get_type_resolver().get_fory_type_id(concrete_type_id) { - $crate::serializer::trait_object::write_trait_object_headers(fory, context, fory_type_id, concrete_type_id)?; - $crate::downcast_and_serialize!(any_ref, fory, context, is_field, $trait_name, $($impl_type),+); + if let Some(fory_type_id) = context.get_type_resolver().get_fory_type_id(concrete_type_id) { + $crate::serializer::trait_object::write_trait_object_headers(context, fory_type_id, concrete_type_id)?; + $crate::downcast_and_serialize!(any_ref, context, is_field, $trait_name, $($impl_type),+); } else { return Err($crate::error::Error::TypeError(format!("Type {:?} not registered for Box serialization", concrete_type_id, stringify!($trait_name)).into())); } } - fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { // Delegate to fory_write since this handles the polymorphic dispatch - self.fory_write(fory, context, is_field) + self.fory_write(context, is_field) } - fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); - fory.get_type_resolver() + type_resolver .get_fory_type_id(concrete_type_id) .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) } @@ -229,21 +228,21 @@ macro_rules! register_trait_type { true } - fn fory_write_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { // Box is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { // Box is polymorphic - type info is read per element Ok(()) } - fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { context.inc_depth()?; - let fory_type_id = $crate::serializer::trait_object::read_trait_object_headers(fory, context)?; + let fory_type_id = $crate::serializer::trait_object::read_trait_object_headers(context)?; let result = $crate::resolve_and_deserialize!( - fory_type_id, fory, context, is_field, + fory_type_id, context, is_field, |obj| Box::new(obj) as Box, $trait_name, $($impl_type),+ ); @@ -251,13 +250,13 @@ macro_rules! register_trait_type { result } - fn fory_read_data(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result { + fn fory_read_data(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result { // This should not be called for polymorphic types like Box // The fory_read method handles the polymorphic dispatch panic!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)); } - fn fory_get_type_id(_fory: &$crate::fory::Fory) -> Result { + fn fory_get_type_id(_type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { Ok($crate::types::TypeId::STRUCT as u32) } @@ -406,23 +405,23 @@ macro_rules! generate_smart_pointer_wrapper { macro_rules! impl_smart_pointer_serializer { ($wrapper_name:ident, $pointer_type:ty, $constructor_expr:expr, $trait_name:ident, $try_write_ref:ident, $get_ref:ident, $store_ref:ident, $($impl_type:ty),+) => { impl $crate::serializer::Serializer for $wrapper_name { - fn fory_write(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { if !context.ref_writer.$try_write_ref(&mut context.writer, &self.0) { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); - let harness = context.write_any_typeinfo(fory, concrete_type_id)?; + let harness = context.write_any_typeinfo(concrete_type_id)?; let serializer_fn = harness.get_write_data_fn(); - serializer_fn(any_obj, fory, context, is_field)?; + serializer_fn(any_obj, context, is_field)?; } Ok(()) } - fn fory_write_data(&self, fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { let any_obj = ::as_any(&*self.0); - $crate::downcast_and_serialize!(any_obj, fory, context, is_field, $trait_name, $($impl_type),+) + $crate::downcast_and_serialize!(any_obj, context, is_field, $trait_name, $($impl_type),+) } - fn fory_read(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { use $crate::types::RefFlag; let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; @@ -441,9 +440,9 @@ macro_rules! impl_smart_pointer_serializer { } RefFlag::NotNullValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(fory, context, is_field)?; + let boxed_any = deserializer_fn(context, is_field)?; context.dec_depth(); $( @@ -461,9 +460,9 @@ macro_rules! impl_smart_pointer_serializer { } RefFlag::RefValue => { context.inc_depth()?; - let harness = context.read_any_typeinfo(fory)?; + let harness = context.read_any_typeinfo()?; let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(fory, context, is_field)?; + let boxed_any = deserializer_fn(context, is_field)?; context.dec_depth(); $( @@ -482,10 +481,10 @@ macro_rules! impl_smart_pointer_serializer { } } } - fn fory_read_data(fory: &$crate::fory::Fory, context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read_data(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { let concrete_fory_type_id = context.reader.read_varuint32()?; $crate::resolve_and_deserialize!( - concrete_fory_type_id, fory, context, is_field, + concrete_fory_type_id, context, is_field, |obj| { let pointer = $constructor_expr(obj) as $pointer_type; Self::from(pointer) @@ -494,15 +493,15 @@ macro_rules! impl_smart_pointer_serializer { ) } - fn fory_get_type_id(_fory: &$crate::fory::Fory) -> Result { + fn fory_get_type_id(_type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { Ok($crate::types::TypeId::STRUCT as u32) } - fn fory_write_type_info(_fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { Ok(()) } - fn fory_read_type_info(fory: &$crate::fory::Fory, _context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { Ok(()) } @@ -510,10 +509,10 @@ macro_rules! impl_smart_pointer_serializer { true } - fn fory_type_id_dyn(&self, fory: &$crate::fory::Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); - fory.get_type_resolver() + type_resolver .get_fory_type_id(concrete_type_id) .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) } @@ -543,30 +542,20 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - let fory_type_id = (**self).fory_type_id_dyn(fory)?; + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + let fory_type_id = (**self).fory_type_id_dyn(context.get_type_resolver())?; let concrete_type_id = (**self).fory_concrete_type_id(); - write_trait_object_headers(fory, context, fory_type_id, concrete_type_id)?; - (**self).fory_write_data(fory, context, is_field) + write_trait_object_headers(context, fory_type_id, concrete_type_id)?; + (**self).fory_write_data(context, is_field) } - fn fory_write_data( - &self, - _fory: &Fory, - _context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, _context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { panic!("fory_write_data should not be called directly on Box"); } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - (**self).fory_type_id_dyn(fory) + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { + (**self).fory_type_id_dyn(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { @@ -577,33 +566,25 @@ impl Serializer for Box { true } - fn fory_write_type_info( - _fory: &Fory, - _context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { // Box is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info( - _fory: &Fory, - _context: &mut ReadContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { // Box is polymorphic - type info is read per element Ok(()) } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { - let fory_type_id = read_trait_object_headers(fory, context)?; + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { + let fory_type_id = read_trait_object_headers(context)?; context.inc_depth()?; - let type_resolver = fory.get_type_resolver(); + let type_resolver = context.get_type_resolver(); if let Some(harness) = type_resolver.get_harness(fory_type_id) { let deserializer_fn = harness.get_read_fn(); let to_serializer_fn = harness.get_to_serializer(); - let boxed_any = deserializer_fn(fory, context, is_field, true)?; + let boxed_any = deserializer_fn(context, is_field, true)?; let trait_object = to_serializer_fn(boxed_any)?; context.dec_depth(); Ok(trait_object) @@ -635,11 +616,7 @@ impl Serializer for Box { } } } - fn fory_read_data( - _fory: &Fory, - _context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(_context: &mut ReadContext, _is_field: bool) -> Result { panic!("fory_read_data should not be called directly on Box"); } } diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 54fc3304f5..c04735901b 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -122,8 +122,8 @@ //! once the strong pointer becomes available. use crate::error::Error; -use crate::fory::Fory; use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use std::cell::UnsafeCell; @@ -312,12 +312,7 @@ impl Serializer for RcWeak { true } - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if let Some(rc) = self.upgrade() { if context .ref_writer @@ -325,38 +320,29 @@ impl Serializer for RcWeak { { return Ok(()); } - T::fory_write_data(&*rc, fory, context, is_field)?; + T::fory_write_data(&*rc, context, is_field)?; } else { context.writer.write_i8(RefFlag::Null as i8); } Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - self.fory_write(fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + self.fory_write(context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { RefFlag::Null => Ok(RcWeak::new()), RefFlag::RefValue => { context.inc_depth()?; - let data = T::fory_read_data(fory, context, is_field)?; + let data = T::fory_read_data(context, is_field)?; context.dec_depth(); let rc = Rc::new(data); let ref_id = context.ref_reader.store_rc_ref(rc); @@ -387,20 +373,12 @@ impl Serializer for RcWeak { } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Self::fory_read(fory, context, is_field) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Self::fory_read(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -408,15 +386,15 @@ impl Serializer for RcWeak { 4 } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { if let Some(rc) = self.upgrade() { - (*rc).fory_type_id_dyn(fory) + (*rc).fory_type_id_dyn(type_resolver) } else { - T::fory_get_type_id(fory) + T::fory_get_type_id(type_resolver) } } @@ -436,12 +414,7 @@ impl Serializer for ArcWeak true } - fn fory_write( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { + fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { if let Some(arc) = self.upgrade() { // IMPORTANT: If the target Arc was serialized already, just write a ref if context @@ -452,38 +425,29 @@ impl Serializer for ArcWeak return Ok(()); } // First time seeing this object, write RefValue and then its data - T::fory_write_data(&*arc, fory, context, is_field)?; + T::fory_write_data(&*arc, context, is_field)?; } else { context.writer.write_i8(RefFlag::Null as i8); } Ok(()) } - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - self.fory_write(fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + self.fory_write(context, is_field) } - fn fory_write_type_info( - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_write_type_info(fory, context, is_field) + fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + T::fory_write_type_info(context, is_field) } - fn fory_read(fory: &Fory, context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; match ref_flag { RefFlag::Null => Ok(ArcWeak::new()), RefFlag::RefValue => { context.inc_depth()?; - let data = T::fory_read_data(fory, context, _is_field)?; + let data = T::fory_read_data(context, _is_field)?; context.dec_depth(); let arc = Arc::new(data); let ref_id = context.ref_reader.store_arc_ref(arc); @@ -515,20 +479,12 @@ impl Serializer for ArcWeak )), } } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { - Self::fory_read(fory, context, is_field) + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + Self::fory_read(context, is_field) } - fn fory_read_type_info( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result<(), Error> { - T::fory_read_type_info(fory, context, is_field) + fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + T::fory_read_type_info(context, is_field) } fn fory_reserved_space() -> usize { @@ -536,15 +492,15 @@ impl Serializer for ArcWeak 4 } - fn fory_get_type_id(fory: &Fory) -> Result { - T::fory_get_type_id(fory) + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { + T::fory_get_type_id(type_resolver) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { if let Some(arc) = self.upgrade() { - (*arc).fory_type_id_dyn(fory) + (*arc).fory_type_id_dyn(type_resolver) } else { - T::fory_get_type_id(fory) + T::fory_get_type_id(type_resolver) } } diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 7221241c99..61f7c02340 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -25,9 +25,9 @@ pub fn gen_actual_type_id() -> TokenStream { } } -pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream { +pub fn gen_field_fields_info(_data_enum: &DataEnum) -> TokenStream { quote! { - Ok(fory_core::serializer::enum_::type_def(fory, type_id, namespace, type_name, register_by_name)) + Ok(Vec::new()) } } @@ -39,13 +39,13 @@ pub fn gen_reserved_space() -> TokenStream { pub fn gen_write_type_info() -> TokenStream { quote! { - fory_core::serializer::enum_::write_type_info::(fory, context, is_field) + fory_core::serializer::enum_::write_type_info::(context, is_field) } } pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::enum_::read_type_info::(fory, context, is_field) + fory_core::serializer::enum_::read_type_info::(context, is_field) } } @@ -79,18 +79,18 @@ pub fn gen_read_data(data_enum: &DataEnum) -> TokenStream { pub fn gen_read_compatible() -> TokenStream { quote! { - fory_core::serializer::enum_::read_compatible::(fory, context) + fory_core::serializer::enum_::read_compatible::(context) } } pub fn gen_write(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::write::(self, fory, context, is_field) + fory_core::serializer::enum_::write::(self, context, is_field) } } pub fn gen_read(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::read::(fory, context, is_field) + fory_core::serializer::enum_::read::(context, is_field) } } diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index 4bb522cadd..f122198c89 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -71,7 +71,7 @@ pub fn gen_get_sorted_field_names(fields: &[&Field]) -> TokenStream { } } -pub fn gen_type_def(fields: &[&Field]) -> TokenStream { +pub fn gen_field_fields_info(fields: &[&Field]) -> TokenStream { let field_infos = fields.iter().map(|field| { let ty = &field.ty; let name = format!("{}", field.ident.as_ref().expect("should be field name")); @@ -127,6 +127,6 @@ pub fn gen_type_def(fields: &[&Field]) -> TokenStream { }); quote! { let field_infos: Vec = vec![#(#field_infos),*]; - Ok(fory_core::serializer::struct_::type_def::(fory, type_id, namespace, type_name, register_by_name, field_infos)) + Ok(field_infos) } } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 2a18bda71d..e72ff3eed6 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -115,12 +115,12 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let fory_type_id = context.reader.read_varuint32()?; - let harness = fory.get_type_resolver() + let harness = context.get_type_resolver() .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(fory, context, true, false)?; + let any_box = deserializer_fn(context, true, false)?; let base_type_id = fory_type_id >> 8; let #private_ident = #helper_mod::#from_any_fn(any_box, base_type_id)?; @@ -132,7 +132,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = std::rc::Rc::::from(wrapper); } } @@ -141,7 +141,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = std::sync::Arc::::from(wrapper); } } @@ -150,7 +150,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect(); @@ -161,7 +161,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect(); @@ -172,7 +172,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect(); @@ -183,7 +183,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; let #private_ident = wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect(); @@ -191,13 +191,13 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { } StructField::Forward => { quote! { - let #private_ident = fory_core::serializer::Serializer::fory_read(fory, context, true)?; + let #private_ident = fory_core::serializer::Serializer::fory_read(context, true)?; } } _ => { let skip_ref_flag = skip_ref_flag(ty); quote! { - let #private_ident = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, #skip_ref_flag, false)?; + let #private_ident = fory_core::serializer::read_ref_info_data::<#ty>(context, true, #skip_ref_flag, false)?; } } } @@ -205,7 +205,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::read_type_info::(fory, context, is_field) + fory_core::serializer::struct_::read_type_info::(context, is_field) } } @@ -259,11 +259,11 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS return Err(Error::InvalidRef("Expected NotNullValue for trait object field".into())); } let fory_type_id = context.reader.read_varuint32()?; - let harness = fory.get_type_resolver() + let harness = context.get_type_resolver() .get_harness(fory_type_id) .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(fory, context, true, false)?; + let any_box = deserializer_fn(context, true, false)?; let base_type_id = fory_type_id >> 8; #var_name = #helper_mod::#from_any_fn(any_box, base_type_id)?; } @@ -273,7 +273,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(std::rc::Rc::::from(wrapper)); } } @@ -282,7 +282,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(std::sync::Arc::::from(wrapper)); } } @@ -291,7 +291,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect()); @@ -302,7 +302,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect()); @@ -313,7 +313,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect()); @@ -324,7 +324,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(fory, context, true)?; + let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect()); @@ -332,13 +332,13 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS } StructField::ContainsTraitObject => { quote! { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(fory); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, skip_ref_flag, false)?); + let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_type_resolver()); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, skip_ref_flag, false)?); } } StructField::Forward => { quote! { - #var_name = Some(fory_core::serializer::Serializer::fory_read(fory, context, true)?); + #var_name = Some(fory_core::serializer::Serializer::fory_read(context, true)?); } } StructField::None => { @@ -351,10 +351,10 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS if local_nullable { quote! { if _field.field_type.nullable { - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, false, false)?); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, false, false)?); } else { #var_name = Some( - fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)? + fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)? ); } } @@ -363,21 +363,21 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS if dec_by_option { quote! { if !_field.field_type.nullable { - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?); + #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?); } else { - #var_name = fory_core::serializer::read_ref_info_data::>(fory, context, true, false, false)? + #var_name = fory_core::serializer::read_ref_info_data::>(context, true, false, false)? } } } else { let null_flag = RefFlag::Null as i8; quote! { if !_field.field_type.nullable { - #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; + #var_name = fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?; } else { if context.reader.read_i8()? == #null_flag { #var_name = <#ty as fory_core::serializer::ForyDefault>::fory_default(); } else { - #var_name = fory_core::serializer::read_ref_info_data::<#ty>(fory, context, true, true, false)?; + #var_name = fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?; } } } @@ -391,11 +391,11 @@ pub fn gen_read(struct_ident: &Ident) -> TokenStream { quote! { let ref_flag = context.reader.read_i8()?; if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::types::RefFlag::RefValue as i8) { - if fory.is_compatible() { - <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(fory, context) + if context.is_compatible() { + <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(context) } else { - ::fory_read_type_info(fory, context, false)?; - ::fory_read_data(fory, context, false) + ::fory_read_type_info(context, false)?; + ::fory_read_data(context, false) } } else if ref_flag == (fory_core::types::RefFlag::Null as i8) { Ok(::fory_default()) @@ -437,9 +437,9 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { }; #(#declare_ts)* - let local_type_hash = fory.get_type_resolver().get_type_info(std::any::TypeId::of::())?.get_type_meta().get_hash(); + let local_type_hash = context.get_type_resolver().get_type_info(std::any::TypeId::of::())?.get_type_meta().get_hash(); if meta.get_hash() == local_type_hash { - ::fory_read_data(fory, context, false) + ::fory_read_data(context, false) } else { for _field in fields.iter() { match _field.field_id { @@ -447,7 +447,7 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { _ => { let field_type = &_field.field_type; let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&field_type); - fory_core::serializer::skip::skip_field_value(fory, context, &field_type, read_ref_flag)?; + fory_core::serializer::skip::skip_field_value(context, &field_type, read_ref_flag)?; } } } diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 23f911452f..2ba9b9513c 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -50,21 +50,21 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { }; // StructSerializer - let (actual_type_id_ts, get_sorted_field_names_ts, type_def_ts, read_compatible_ts) = + let (actual_type_id_ts, get_sorted_field_names_ts, fields_info_ts, read_compatible_ts) = match &ast.data { syn::Data::Struct(s) => { let fields = sorted_fields(&s.fields); ( misc::gen_actual_type_id(), misc::gen_get_sorted_field_names(&fields), - misc::gen_type_def(&fields), + misc::gen_field_fields_info(&fields), read::gen_read_compatible(&fields), ) } syn::Data::Enum(s) => ( derive_enum::gen_actual_type_id(), - quote! { unreachable!() }, - derive_enum::gen_type_def(s), + quote! { &[] }, + derive_enum::gen_field_fields_info(s), derive_enum::gen_read_compatible(), ), syn::Data::Union(_) => { @@ -124,21 +124,21 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #actual_type_id_ts } - fn fory_get_sorted_field_names(_fory: &fory_core::fory::Fory) -> &'static [&'static str] { + fn fory_get_sorted_field_names() -> &'static [&'static str] { #get_sorted_field_names_ts } - fn fory_type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: fory_core::meta::MetaString, type_name: fory_core::meta::MetaString, register_by_name: bool) -> Result<(Vec, fory_core::meta::TypeMeta), fory_core::error::Error> { - #type_def_ts + fn fory_fields_info(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result, fory_core::error::Error> { + #fields_info_ts } } impl fory_core::serializer::Serializer for #name { - fn fory_get_type_id(fory: &fory_core::fory::Fory) -> Result { - fory.get_type_resolver().get_type_id(&std::any::TypeId::of::(), #type_idx) + fn fory_get_type_id(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { + type_resolver.get_type_id(&std::any::TypeId::of::(), #type_idx) } - fn fory_type_id_dyn(&self, fory: &fory_core::fory::Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn(&self, type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { @@ -149,31 +149,31 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { #reserved_space_ts } - fn fory_write_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { + fn fory_write_type_info(context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_type_info_ts } - fn fory_read_type_info(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result<(), fory_core::error::Error> { + fn fory_read_type_info(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result<(), fory_core::error::Error> { #read_type_info_ts } - fn fory_write_data(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { + fn fory_write_data(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_data_ts } - fn fory_read_data(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read_data( context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { #read_data_ts } - fn fory_write(&self, fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { + fn fory_write(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { #write_ts } - fn fory_read(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { + fn fory_read(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { #read_ts } - fn fory_read_compatible(fory: &fory_core::fory::Fory, context: &mut fory_core::resolver::context::ReadContext) -> Result { + fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext) -> Result { #read_compatible_ts } } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 6f7ca8ef62..ee5e0a5856 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -354,7 +354,7 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { ts } else { quote! { - <#ty as fory_core::serializer::Serializer>::fory_get_type_id(fory)? + <#ty as fory_core::serializer::Serializer>::fory_get_type_id(type_resolver)? } }; diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 18cee31ee5..035562ecd9 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -95,7 +95,7 @@ pub fn gen_reserved_space(fields: &[&Field]) -> TokenStream { pub fn gen_write_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::write_type_info::(fory, context, is_field) + fory_core::serializer::struct_::write_type_info::(context, is_field) } } @@ -108,20 +108,20 @@ fn gen_write_field(field: &Field) -> TokenStream { { let any_ref = self.#ident.as_any(); let concrete_type_id = any_ref.type_id(); - let fory_type_id = fory.get_type_resolver() + let fory_type_id = context.get_type_resolver() .get_fory_type_id(concrete_type_id) .ok_or_else(|| fory_core::error::Error::TypeError("Type not registered for trait object field".into()))?; context.writer.write_i8(fory_core::types::RefFlag::NotNullValue as i8)?; context.writer.write_varuint32(fory_type_id); - let harness = fory + let harness = context .get_type_resolver() .get_harness(fory_type_id) .ok_or_else(|| fory_core::error::Error::TypeError("Harness not found for trait object field".into()))?; let serializer_fn = harness.get_write_fn(); - serializer_fn(any_ref, fory, context, true)?; + serializer_fn(any_ref, context, true)?; } } } @@ -132,7 +132,7 @@ fn gen_write_field(field: &Field) -> TokenStream { quote! { { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); - fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper, context, true)?; } } } @@ -143,7 +143,7 @@ fn gen_write_field(field: &Field) -> TokenStream { quote! { { let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); - fory_core::serializer::Serializer::fory_write(&wrapper, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper, context, true)?; } } } @@ -156,7 +156,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true)?; } } } @@ -169,7 +169,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true)?; } } } @@ -182,7 +182,7 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true)?; } } } @@ -195,21 +195,21 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true)?; } } } StructField::Forward => { quote! { { - fory_core::serializer::Serializer::fory_write(&self.#ident, fory, context, true)?; + fory_core::serializer::Serializer::fory_write(&self.#ident, context, true)?; } } } _ => { let skip_ref_flag = skip_ref_flag(ty); quote! { - fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, fory, context, true, #skip_ref_flag, false)?; + fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, context, true, #skip_ref_flag, false)?; } } } @@ -225,6 +225,6 @@ pub fn gen_write_data(fields: &[&Field]) -> TokenStream { pub fn gen_write() -> TokenStream { quote! { - fory_core::serializer::struct_::write::(self, fory, context, is_field) + fory_core::serializer::struct_::write::(self, context, is_field) } } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index 10cd983b2c..b8e6c538fe 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -707,7 +707,7 @@ //! the binary buffer. //! //! ```rust -//! use fory::{Fory, ReadContext, WriteContext, Serializer, ForyDefault, Error}; +//! use fory::{Fory, TypeResolver, ReadContext, WriteContext, Serializer, ForyDefault, Error}; //! use std::any::Any; //! //! #[derive(Debug, PartialEq, Default)] @@ -717,22 +717,22 @@ //! } //! //! impl Serializer for CustomType { -//! fn fory_write_data(&self, fory: &Fory, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { +//! fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { //! context.writer.write_i32(self.value); //! context.writer.write_varuint32(self.name.len() as u32); //! context.writer.write_utf8_string(&self.name); //! Ok(()) //! } //! -//! fn fory_read_data(fory: &Fory, context: &mut ReadContext, is_field: bool) -> Result { +//! fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { //! let value = context.reader.read_i32()?; //! let len = context.reader.read_varuint32()? as usize; //! let name = context.reader.read_utf8_string(len)?; //! Ok(Self { value, name }) //! } //! -//! fn fory_type_id_dyn(&self, fory: &Fory) -> Result { -//! Self::fory_get_type_id(fory) +//! fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { +//! Self::fory_get_type_id(type_resolver) //! } //! //! fn as_any(&self) -> &dyn Any { @@ -1048,6 +1048,6 @@ pub use fory_core::{ error::Error, fory::Fory, register_trait_type, row::from_row, row::to_row, types::TypeId, - ArcWeak, ForyDefault, RcWeak, ReadContext, Serializer, WriteContext, + ArcWeak, ForyDefault, RcWeak, ReadContext, Serializer, TypeResolver, WriteContext, }; pub use fory_derive::{ForyObject, ForyRow}; diff --git a/rust/tests/tests/compatible/test_basic_type.rs b/rust/tests/tests/compatible/test_basic_type.rs index 0ae571fcf8..9c1d008c66 100644 --- a/rust/tests/tests/compatible/test_basic_type.rs +++ b/rust/tests/tests/compatible/test_basic_type.rs @@ -482,12 +482,12 @@ fn basic() { let fory = Fory::default().compatible(true); // serialize let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); serialize_non_null(&fory, &mut write_context); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); deserialize_non_null(&fory, &mut read_context, false, true); } @@ -497,12 +497,12 @@ fn basic_nullable() { let fory = Fory::default().compatible(true); // serialize let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); serialize_nullable(&fory, &mut write_context); // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); deserialize_nullable(&fory, &mut read_context, false, true); } @@ -512,20 +512,20 @@ fn auto_conv() { let fory = Fory::default().compatible(true); // serialize_non-null let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); serialize_non_null(&fory, &mut write_context); // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context: ReadContext = ReadContext::new(reader, 5); + let mut read_context: ReadContext = ReadContext::new_from_fory(reader, &fory); deserialize_nullable(&fory, &mut read_context, true, true); // serialize_nullable let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); serialize_nullable(&fory, &mut write_context); // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); deserialize_non_null(&fory, &mut read_context, true, true); } diff --git a/rust/tests/tests/compatible/test_container.rs b/rust/tests/tests/compatible/test_container.rs index 0dec8d01a9..9fa1969a50 100644 --- a/rust/tests/tests/compatible/test_container.rs +++ b/rust/tests/tests/compatible/test_container.rs @@ -223,7 +223,7 @@ fn container_outer_auto_conv() { let fory = Fory::default().compatible(true); // serialize_outer_non-null let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&basic_list(), &mut write_context) .unwrap(); fory.serialize_with_context(&basic_set(), &mut write_context) @@ -233,7 +233,7 @@ fn container_outer_auto_conv() { // deserialize_outer_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( Some(basic_list()), fory.deserialize_with_context::>>(&mut read_context) @@ -252,7 +252,7 @@ fn container_outer_auto_conv() { assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_outer_nullable let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&Some(basic_list()), &mut write_context) .unwrap(); fory.serialize_with_context(&Some(basic_set()), &mut write_context) @@ -268,7 +268,7 @@ fn container_outer_auto_conv() { // deserialize_outer_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -311,7 +311,7 @@ fn collection_inner() { for fory in [fory1, fory2] { // serialize let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&basic_list(), &mut write_context) .unwrap(); fory.serialize_with_context(&item_list(), &mut write_context) @@ -331,7 +331,7 @@ fn collection_inner() { // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -383,9 +383,9 @@ fn collection_inner_auto_conv() { let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { - // serialize_non-null + // serialize_non_null let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&basic_list(), &mut write_context) .unwrap(); fory.serialize_with_context(&item_list(), &mut write_context) @@ -397,7 +397,7 @@ fn collection_inner_auto_conv() { // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( nullable_basic_list(true), fory.deserialize_with_context::>>(&mut read_context) @@ -421,7 +421,7 @@ fn collection_inner_auto_conv() { assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_nullable let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&nullable_basic_list(false), &mut write_context) .unwrap(); fory.serialize_with_context(&nullable_item_list(false), &mut write_context) @@ -433,7 +433,7 @@ fn collection_inner_auto_conv() { // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( basic_list(), fory.deserialize_with_context::>(&mut read_context) @@ -467,7 +467,7 @@ fn map_inner() { for fory in [fory1, fory2] { // serialize let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&basic_map(), &mut write_context) .unwrap(); fory.serialize_with_context(&item_map(), &mut write_context) @@ -479,7 +479,7 @@ fn map_inner() { // deserialize let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( basic_map(), fory.deserialize_with_context::>(&mut read_context) @@ -513,9 +513,9 @@ fn map_inner_auto_conv() { let mut fory2 = Fory::default().compatible(true); fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { - // serialize_non-null + // serialize_non_null let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&basic_map(), &mut write_context) .unwrap(); fory.serialize_with_context(&item_map(), &mut write_context) @@ -523,7 +523,7 @@ fn map_inner_auto_conv() { // deserialize_nullable let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( nullable_basic_map(true), fory.deserialize_with_context::, Option>>( @@ -539,7 +539,7 @@ fn map_inner_auto_conv() { assert_eq!(read_context.reader.slice_after_cursor().len(), 0); // serialize_nullable let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&nullable_basic_map(false), &mut write_context) .unwrap(); fory.serialize_with_context(&nullable_item_map(false), &mut write_context) @@ -547,7 +547,7 @@ fn map_inner_auto_conv() { // deserialize_non-null let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( basic_map(), fory.deserialize_with_context::>(&mut read_context) @@ -570,7 +570,7 @@ fn complex() { fory2.register_by_name::("item").unwrap(); for fory in [fory1, fory2] { let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&nested_collection(), &mut write_context) .unwrap(); fory.serialize_with_context(&complex_container1(), &mut write_context) @@ -579,7 +579,7 @@ fn complex() { .unwrap(); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( nested_collection(), fory.deserialize_with_context::>>(&mut read_context) diff --git a/rust/tests/tests/compatible/test_struct_enum.rs b/rust/tests/tests/compatible/test_struct_enum.rs index 39bd8241f6..ca25a0aba8 100644 --- a/rust/tests/tests/compatible/test_struct_enum.rs +++ b/rust/tests/tests/compatible/test_struct_enum.rs @@ -21,6 +21,7 @@ use fory_core::error::Error; use fory_core::fory::{read_data, write_data, Fory}; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; +use fory_core::TypeResolver; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; @@ -91,7 +92,7 @@ fn basic() { fory2.register_by_name::("person").unwrap(); for fory in [fory1, fory2] { let writer = Writer::default(); - let mut write_context = WriteContext::new(writer); + let mut write_context = WriteContext::new_from_fory(writer, &fory); let person = Person::default(); fory.serialize_with_context(&person, &mut write_context) .unwrap(); @@ -99,7 +100,7 @@ fn basic() { .unwrap(); let bytes = write_context.writer.dump(); let reader = Reader::new(bytes.as_slice()); - let mut read_context = ReadContext::new(reader, 5); + let mut read_context = ReadContext::new_from_fory(reader, &fory); assert_eq!( person, fory.deserialize_with_context::(&mut read_context) @@ -400,24 +401,22 @@ fn ext() { impl Serializer for ExtItem { fn fory_write_data( &self, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, fory, context, is_field) + write_data(&self.id, context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { Ok(Self { - id: read_data(fory, context, is_field)?, + id: read_data(context, is_field)?, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { @@ -457,24 +456,22 @@ fn skip_ext() { impl Serializer for ExtItem { fn fory_write_data( &self, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, fory, context, is_field) + write_data(&self.id, context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { Ok(Self { - id: read_data(fory, context, is_field)?, + id: read_data(context, is_field)?, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { @@ -529,23 +526,21 @@ fn compatible_ext() { impl Serializer for ExtItem { fn fory_write_data( &self, - fory: &Fory, context: &mut WriteContext, is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, fory, context, is_field) + write_data(&self.id, context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { Ok(Self { - id: read_data(fory, context, is_field)?, + id: read_data(context, is_field)?, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 274ea803d0..f6bfdc4084 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -22,6 +22,7 @@ use fory_core::fory::{read_data, write_data, Fory}; use fory_core::meta::murmurhash3_x64_128; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; +use fory_core::TypeResolver; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; use std::{fs, vec}; @@ -228,13 +229,13 @@ fn test_string_serializer() { .compatible(true) .xlang(true) .compress_string(false); - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); let reader_compress = Reader::new(bytes.as_slice()); let fory_compress = Fory::default() .compatible(true) .xlang(true) .compress_string(true); - let mut context_compress = ReadContext::new(reader_compress, 5); + let mut context_compress = ReadContext::new_from_fory(reader_compress, &fory_compress); let test_strings: Vec = vec![ // Latin1 "ab".to_string(), @@ -249,20 +250,17 @@ fn test_string_serializer() { ]; for s in &test_strings { // make is_field=true to skip read/write type_id + assert_eq!(*s, String::fory_read_data(&mut context, true).unwrap()); assert_eq!( *s, - String::fory_read_data(&fory, &mut context, true).unwrap() - ); - assert_eq!( - *s, - String::fory_read_data(&fory_compress, &mut context_compress, true).unwrap() + String::fory_read_data(&mut context_compress, true).unwrap() ); } let writer = Writer::default(); let fory = Fory::default().compatible(true).xlang(true); - let mut context = WriteContext::new(writer); + let mut context = WriteContext::new_from_fory(writer, &fory); for s in &test_strings { - s.fory_write_data(&fory, &mut context, true).unwrap(); + s.fory_write_data(&mut context, true).unwrap(); } fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -293,7 +291,7 @@ fn test_cross_language_serializer() { let reader = Reader::new(bytes.as_slice()); let mut fory = Fory::default().compatible(true).xlang(true); fory.register::(101).unwrap(); - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); assert_de!(fory, context, bool, true); assert_de!(fory, context, bool, false); assert_de!(fory, context, i32, -1); @@ -322,7 +320,7 @@ fn test_cross_language_serializer() { assert_de!(fory, context, Color, color); let writer = Writer::default(); - let mut context = WriteContext::new(writer); + let mut context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&true, &mut context).unwrap(); fory.serialize_with_context(&false, &mut context).unwrap(); fory.serialize_with_context(&-1, &mut context).unwrap(); @@ -439,7 +437,7 @@ fn test_list() { let mut fory = Fory::default().compatible(true); fory.register::(102).unwrap(); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); let str_list = vec![Some("a".to_string()), Some("b".to_string())]; let str_list2 = vec![None, Some("b".to_string())]; @@ -466,7 +464,7 @@ fn test_list() { assert_eq!(remote_item_list2, item_list2); let writer = Writer::default(); - let mut context = WriteContext::new(writer); + let mut context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&remote_str_list, &mut context) .unwrap(); fory.serialize_with_context(&remote_str_list2, &mut context) @@ -488,7 +486,7 @@ fn test_map() { let mut fory = Fory::default().compatible(true); fory.register::(102).unwrap(); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); let str_map = HashMap::from([ (Some("k1".to_string()), Some("v1".to_string())), @@ -557,8 +555,7 @@ fn test_integer() { let mut fory = Fory::default().compatible(true); fory.register::(101).unwrap(); let reader = Reader::new(bytes.as_slice()); - - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); let f1 = 1; let f2 = Some(2); let f3 = Some(3); @@ -590,7 +587,7 @@ fn test_integer() { assert_eq!(remote_f6, f6); let writer = Writer::default(); - let mut context = WriteContext::new(writer); + let mut context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&remote_item2, &mut context) .unwrap(); fory.serialize_with_context(&remote_f1, &mut context) @@ -619,27 +616,25 @@ struct MyExt { impl Serializer for MyExt { fn fory_write_data( &self, - fory: &Fory, context: &mut WriteContext, _is_field: bool, ) -> Result<(), fory_core::error::Error> { // set is_field=false to write type_id like in java - write_data(&self.id, fory, context, false) + write_data(&self.id, context, false) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - _is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { Ok(Self { // set is_field=false to write type_id like in java - id: read_data(fory, context, false)?, + id: read_data(context, false)?, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { @@ -740,7 +735,7 @@ fn test_consistent_named() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let reader = Reader::new(bytes.as_slice()); - let mut context = ReadContext::new(reader, 5); + let mut context = ReadContext::new_from_fory(reader, &fory); assert_eq!( fory.deserialize_with_context::(&mut context) @@ -775,7 +770,7 @@ fn test_consistent_named() { ); let writer = Writer::default(); - let mut context = WriteContext::new(writer); + let mut context = WriteContext::new_from_fory(writer, &fory); fory.serialize_with_context(&color, &mut context).unwrap(); fory.serialize_with_context(&color, &mut context).unwrap(); fory.serialize_with_context(&color, &mut context).unwrap(); diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index 7fa2024396..3e023622a3 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -19,6 +19,7 @@ use fory_core::error::Error; use fory_core::fory::Fory; use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_core::serializer::{ForyDefault, Serializer}; +use fory_core::TypeResolver; use fory_derive::ForyObject; #[test] @@ -59,28 +60,22 @@ fn test_use() { } impl Serializer for Item { - fn fory_write_data( - &self, - fory: &Fory, - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_data(&self.f1, fory, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + write_data(&self.f1, context, is_field) } - fn fory_read_data( - fory: &Fory, - context: &mut ReadContext, - is_field: bool, - ) -> Result { + fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { Ok(Self { - f1: read_data(fory, context, is_field)?, + f1: read_data(context, is_field)?, f2: 0, }) } - fn fory_type_id_dyn(&self, fory: &Fory) -> Result { - Self::fory_get_type_id(fory) + fn fory_type_id_dyn( + &self, + type_resolver: &TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) } fn as_any(&self) -> &dyn std::any::Any { From 4604a5f503e9c85586615a726d2dabb834986bcb Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Wed, 15 Oct 2025 23:19:34 +0530 Subject: [PATCH 26/37] feat(python): support optional typehint for dataclass fields (#2766) ## Why? ## What does this PR do? - support optional typehint for dataclass fields - fastpath for numeric and string fields serialization ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- ci/run_ci.py | 2 +- .../org/apache/fory/CrossLanguageTest.java | 18 +- python/pyfory/_fory.py | 28 +-- python/pyfory/_serialization.pyx | 46 ++-- python/pyfory/_serializer.py | 14 +- python/pyfory/_struct.py | 10 +- python/pyfory/_util.pxd | 8 + python/pyfory/_util.pyx | 20 ++ python/pyfory/meta/typedef.py | 48 ++-- python/pyfory/meta/typedef_decoder.py | 12 +- python/pyfory/meta/typedef_encoder.py | 22 +- python/pyfory/serializer.py | 222 +++++++++++++----- python/pyfory/tests/test_cross_language.py | 13 +- python/pyfory/tests/test_reduce_serializer.py | 2 +- python/pyfory/tests/test_struct.py | 162 +++++++++++-- python/pyfory/type.py | 24 +- 16 files changed, 479 insertions(+), 172 deletions(-) diff --git a/ci/run_ci.py b/ci/run_ci.py index 7818a9e606..cdd8f7fbc1 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -293,7 +293,7 @@ def parse_args(): if USE_PYTHON_GO: func() else: - run_shell_script("go") + # run_shell_script("go") pass elif command == "format": if USE_PYTHON_FORMAT: diff --git a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java b/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java index d2397a50fa..ce357e273d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java @@ -102,7 +102,7 @@ private boolean executeCommand(List command, int waitTimeoutSeconds) { @Data public static class A { - public Integer f1; + public int f1; public Map f2; public static A create() { @@ -184,7 +184,7 @@ public void testMurmurHash3() throws IOException { /** Keep this in sync with `foo_schema` in test_cross_language.py */ @Data public static class Foo { - public Integer f1; + public int f1; public String f2; public List f3; public Map f4; @@ -214,7 +214,7 @@ public static Foo create() { /** Keep this in sync with `bar_schema` in test_cross_language.py */ @Data public static class Bar { - public Integer f1; + public int f1; public String f2; public static Bar create() { @@ -446,12 +446,12 @@ public static class ComplexObject1 { String f2; List f3; Map f4; - Byte f5; - Short f6; - Integer f7; - Long f8; - Float f9; - Double f10; + byte f5; + short f6; + int f7; + long f8; + float f9; + double f10; short[] f11; List f12; } diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index dec22fe61d..e89f8ffa7f 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -322,7 +322,7 @@ def _serialize( if self.language == Language.PYTHON: self.serialize_ref(buffer, obj) else: - self.xserialize_ref(buffer, obj) + self.xwrite_ref(buffer, obj) # Write type definitions at the end, similar to Java implementation if self.serialization_context.scoped_meta_share_enabled: @@ -379,18 +379,18 @@ def serialize_nonref(self, buffer, obj): self.type_resolver.write_typeinfo(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) - def xserialize_ref(self, buffer, obj, serializer=None): + def xwrite_ref(self, buffer, obj, serializer=None): if serializer is None or serializer.need_to_write_ref: if not self.ref_resolver.write_ref_or_null(buffer, obj): - self.xserialize_nonref(buffer, obj, serializer=serializer) + self.xwrite_no_ref(buffer, obj, serializer=serializer) else: if obj is None: buffer.write_int8(NULL_FLAG) else: buffer.write_int8(NOT_NULL_VALUE_FLAG) - self.xserialize_nonref(buffer, obj, serializer=serializer) + self.xwrite_no_ref(buffer, obj, serializer=serializer) - def xserialize_nonref(self, buffer, obj, serializer=None): + def xwrite_no_ref(self, buffer, obj, serializer=None): if serializer is not None: serializer.xwrite(buffer, obj) return @@ -458,12 +458,12 @@ def _deserialize( buffer.reader_index = current_reader_index if is_target_x_lang: - obj = self.xdeserialize_ref(buffer) + obj = self.xread_ref(buffer) else: - obj = self.deserialize_ref(buffer) + obj = self.read_ref(buffer) return obj - def deserialize_ref(self, buffer): + def read_ref(self, buffer): ref_resolver = self.ref_resolver ref_id = ref_resolver.try_preserve_ref_id(buffer) # indicates that the object is first read. @@ -477,7 +477,7 @@ def deserialize_ref(self, buffer): else: return ref_resolver.get_read_object() - def deserialize_nonref(self, buffer): + def read_no_ref(self, buffer): """Deserialize not-null and non-reference object from buffer.""" typeinfo = self.type_resolver.read_typeinfo(buffer) self.inc_depth() @@ -485,13 +485,13 @@ def deserialize_nonref(self, buffer): self.dec_depth() return o - def xdeserialize_ref(self, buffer, serializer=None): + def xread_ref(self, buffer, serializer=None): if serializer is None or serializer.need_to_write_ref: ref_resolver = self.ref_resolver ref_id = ref_resolver.try_preserve_ref_id(buffer) # indicates that the object is first read. if ref_id >= NOT_NULL_VALUE_FLAG: - o = self.xdeserialize_nonref(buffer, serializer=serializer) + o = self.xread_no_ref(buffer, serializer=serializer) ref_resolver.set_read_object(ref_id, o) return o else: @@ -499,9 +499,9 @@ def xdeserialize_ref(self, buffer, serializer=None): head_flag = buffer.read_int8() if head_flag == NULL_FLAG: return None - return self.xdeserialize_nonref(buffer, serializer=serializer) + return self.xread_no_ref(buffer, serializer=serializer) - def xdeserialize_nonref(self, buffer, serializer=None): + def xread_no_ref(self, buffer, serializer=None): if serializer is None: serializer = self.type_resolver.read_typeinfo(buffer).serializer self.inc_depth() @@ -551,7 +551,7 @@ def write_ref_pyobject(self, buffer, value, typeinfo=None): typeinfo.serializer.write(buffer, value) def read_ref_pyobject(self, buffer): - return self.deserialize_ref(buffer) + return self.read_ref(buffer) def reset_write(self): self.ref_resolver.reset_write() diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx index 8339ae2ba8..8be215235e 100644 --- a/python/pyfory/_serialization.pyx +++ b/python/pyfory/_serialization.pyx @@ -923,7 +923,7 @@ cdef class Fory: Serialize an object to bytes, alias for `serialize` method. """ return self.serialize(obj, buffer, buffer_callback, unsupported_callback) - + def loads( self, buffer: Union[Buffer, bytes], @@ -995,7 +995,7 @@ cdef class Fory: if self.language == Language.PYTHON: self.serialize_ref(buffer, obj) else: - self.xserialize_ref(buffer, obj) + self.xwrite_ref(buffer, obj) # Write type definitions at the end, similar to Java implementation if self.serialization_context.scoped_meta_share_enabled: @@ -1059,11 +1059,11 @@ cdef class Fory: self.type_resolver.write_typeinfo(buffer, typeinfo) typeinfo.serializer.write(buffer, obj) - cpdef inline xserialize_ref( + cpdef inline xwrite_ref( self, Buffer buffer, obj, Serializer serializer=None): if serializer is None or serializer.need_to_write_ref: if not self.ref_resolver.write_ref_or_null(buffer, obj): - self.xserialize_nonref( + self.xwrite_no_ref( buffer, obj, serializer=serializer ) else: @@ -1071,11 +1071,11 @@ cdef class Fory: buffer.write_int8(NULL_FLAG) else: buffer.write_int8(NOT_NULL_VALUE_FLAG) - self.xserialize_nonref( + self.xwrite_no_ref( buffer, obj, serializer=serializer ) - cpdef inline xserialize_nonref( + cpdef inline xwrite_no_ref( self, Buffer buffer, obj, Serializer serializer=None): if serializer is None: typeinfo = self.type_resolver.get_typeinfo(type(obj)) @@ -1149,10 +1149,10 @@ cdef class Fory: buffer.reader_index = current_reader_index if not is_target_x_lang: - return self.deserialize_ref(buffer) - return self.xdeserialize_ref(buffer) + return self.read_ref(buffer) + return self.xread_ref(buffer) - cpdef inline deserialize_ref(self, Buffer buffer): + cpdef inline read_ref(self, Buffer buffer): cdef MapRefResolver ref_resolver = self.ref_resolver cdef int32_t ref_id = ref_resolver.try_preserve_ref_id(buffer) if ref_id < NOT_NULL_VALUE_FLAG: @@ -1174,7 +1174,7 @@ cdef class Fory: ref_resolver.set_read_object(ref_id, o) return o - cpdef inline deserialize_nonref(self, Buffer buffer): + cpdef inline read_no_ref(self, Buffer buffer): """Deserialize not-null and non-reference object from buffer.""" cdef TypeInfo typeinfo = self.type_resolver.read_typeinfo(buffer) cls = typeinfo.cls @@ -1191,7 +1191,7 @@ cdef class Fory: self.depth -= 1 return o - cpdef inline xdeserialize_ref(self, Buffer buffer, Serializer serializer=None): + cpdef inline xread_ref(self, Buffer buffer, Serializer serializer=None): cdef MapRefResolver ref_resolver cdef int32_t ref_id if serializer is None or serializer.need_to_write_ref: @@ -1199,7 +1199,7 @@ cdef class Fory: ref_id = ref_resolver.try_preserve_ref_id(buffer) # indicates that the object is first read. if ref_id >= NOT_NULL_VALUE_FLAG: - o = self.xdeserialize_nonref( + o = self.xread_no_ref( buffer, serializer=serializer ) ref_resolver.set_read_object(ref_id, o) @@ -1209,11 +1209,11 @@ cdef class Fory: cdef int8_t head_flag = buffer.read_int8() if head_flag == NULL_FLAG: return None - return self.xdeserialize_nonref( + return self.xread_no_ref( buffer, serializer=serializer ) - cpdef inline xdeserialize_nonref( + cpdef inline xread_no_ref( self, Buffer buffer, Serializer serializer=None): if serializer is None: serializer = self.type_resolver.read_typeinfo(buffer).serializer @@ -2087,7 +2087,7 @@ cdef class MapSerializer(Serializer): if is_py: fory.serialize_ref(buffer, key) else: - fory.xserialize_ref(buffer, key) + fory.xwrite_ref(buffer, key) else: if value is not None: if value_serializer is not None: @@ -2114,7 +2114,7 @@ cdef class MapSerializer(Serializer): if is_py: fory.serialize_ref(buffer, value) else: - fory.xserialize_ref(buffer, value) + fory.xwrite_ref(buffer, value) else: buffer.write_int8(KV_NULL) has_next = PyDict_Next(obj, &pos, &key_addr, &value_addr) @@ -2250,9 +2250,9 @@ cdef class MapSerializer(Serializer): key = key_serializer.xread(buffer) else: if is_py: - key = fory.deserialize_ref(buffer) + key = fory.read_ref(buffer) else: - key = fory.xdeserialize_ref(buffer) + key = fory.xread_ref(buffer) map_[key] = None else: if not value_has_null: @@ -2270,9 +2270,9 @@ cdef class MapSerializer(Serializer): ref_resolver.set_read_object(ref_id, value) else: if is_py: - value = fory.deserialize_ref(buffer) + value = fory.read_ref(buffer) else: - value = fory.xdeserialize_ref(buffer) + value = fory.xread_ref(buffer) map_[None] = value else: map_[None] = None @@ -2514,15 +2514,15 @@ cdef class SliceSerializer(Serializer): if buffer.read_int8() == NULL_FLAG: start = None else: - start = self.fory.deserialize_nonref(buffer) + start = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: stop = None else: - stop = self.fory.deserialize_nonref(buffer) + stop = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: step = None else: - step = self.fory.deserialize_nonref(buffer) + step = self.fory.read_no_ref(buffer) return slice(start, stop, step) cpdef xwrite(self, Buffer buffer, value): diff --git a/python/pyfory/_serializer.py b/python/pyfory/_serializer.py index 9b702e8407..5950e1ecec 100644 --- a/python/pyfory/_serializer.py +++ b/python/pyfory/_serializer.py @@ -464,7 +464,7 @@ def write(self, buffer, o): items_iter = iter(obj.items()) key, value = next(items_iter) has_next = True - serialize_ref = fory.serialize_ref if self.fory.is_py else fory.xserialize_ref + serialize_ref = fory.serialize_ref if self.fory.is_py else fory.xwrite_ref while has_next: while True: if key is not None: @@ -567,7 +567,7 @@ def read(self, buffer): if size != 0: chunk_header = buffer.read_uint8() key_serializer, value_serializer = self.key_serializer, self.value_serializer - deserialize_ref = fory.deserialize_ref if self.fory.is_py else fory.xdeserialize_ref + read_ref = fory.read_ref if self.fory.is_py else fory.xread_ref fory.inc_depth() while size > 0: while True: @@ -589,7 +589,7 @@ def read(self, buffer): else: key = self._read_obj(key_serializer, buffer) else: - key = deserialize_ref(buffer) + key = read_ref(buffer) map_[key] = None else: if not value_has_null: @@ -603,7 +603,7 @@ def read(self, buffer): value = self._read_obj(value_serializer, buffer) ref_resolver.set_read_object(ref_id, value) else: - value = deserialize_ref(buffer) + value = read_ref(buffer) map_[None] = value else: map_[None] = None @@ -733,15 +733,15 @@ def read(self, buffer): if buffer.read_int8() == NULL_FLAG: start = None else: - start = self.fory.deserialize_nonref(buffer) + start = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: stop = None else: - stop = self.fory.deserialize_nonref(buffer) + stop = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: step = None else: - step = self.fory.deserialize_nonref(buffer) + step = self.fory.read_no_ref(buffer) return slice(start, stop, step) def xwrite(self, buffer, value): diff --git a/python/pyfory/_struct.py b/python/pyfory/_struct.py index 2b9ae87145..481e695fe4 100644 --- a/python/pyfory/_struct.py +++ b/python/pyfory/_struct.py @@ -121,8 +121,10 @@ def _get_hash(fory, field_names: list, type_hints: dict): _time_types = {datetime.date, datetime.datetime, datetime.timedelta} -def _sort_fields(type_resolver, field_names, serializers): +def _sort_fields(type_resolver, field_names, serializers, nullable_map=None): + nullable_map = nullable_map or {} boxed_types = [] + nullable_boxed_types = [] collection_types = [] set_types = [] map_types = [] @@ -141,8 +143,9 @@ def _sort_fields(type_resolver, field_names, serializers): ) ) for type_id, serializer, field_name in type_ids: + is_nullable = nullable_map.get(field_name, False) if is_primitive_type(type_id): - container = boxed_types + container = nullable_boxed_types if is_nullable else boxed_types elif type_id == TypeId.SET: container = set_types elif is_list_type(serializer.type_): @@ -174,11 +177,12 @@ def numeric_sorter(item): return int(compress), -get_primitive_type_size(id_), item[2] boxed_types = sorted(boxed_types, key=numeric_sorter) + nullable_boxed_types = sorted(nullable_boxed_types, key=numeric_sorter) collection_types = sorted(collection_types, key=sorter) internal_types = sorted(internal_types, key=sorter) map_types = sorted(map_types, key=sorter) other_types = sorted(other_types, key=lambda item: item[2]) - all_types = boxed_types + internal_types + collection_types + set_types + map_types + other_types + all_types = boxed_types + nullable_boxed_types + internal_types + collection_types + set_types + map_types + other_types return [t[2] for t in all_types], [t[1] for t in all_types] diff --git a/python/pyfory/_util.pxd b/python/pyfory/_util.pxd index f705800a8c..5108a6f657 100644 --- a/python/pyfory/_util.pxd +++ b/python/pyfory/_util.pxd @@ -108,8 +108,12 @@ cdef class Buffer: cpdef inline write_float(self, float value) + cpdef inline write_float32(self, float value) + cpdef inline write_double(self, double value) + cpdef inline write_float64(self, double value) + cpdef inline skip(self, int32_t length) cpdef inline c_bool read_bool(self) @@ -128,8 +132,12 @@ cdef class Buffer: cpdef inline float read_float(self) + cpdef inline float read_float32(self) + cpdef inline double read_double(self) + cpdef inline double read_float64(self) + cpdef inline write_varint64(self, int64_t v) cpdef inline write_varuint64(self, int64_t v) diff --git a/python/pyfory/_util.pyx b/python/pyfory/_util.pyx index 5a7e684784..fa4eef19be 100644 --- a/python/pyfory/_util.pyx +++ b/python/pyfory/_util.pyx @@ -206,11 +206,21 @@ cdef class Buffer: self.c_buffer.get().UnsafePut(self.writer_index, value) self.writer_index += 4 + cpdef inline write_float32(self, float value): + self.grow(4) + self.c_buffer.get().UnsafePut(self.writer_index, value) + self.writer_index += 4 + cpdef inline write_double(self, double value): self.grow(8) self.c_buffer.get().UnsafePut(self.writer_index, value) self.writer_index += 8 + cpdef inline write_float64(self, double value): + self.grow(8) + self.c_buffer.get().UnsafePut(self.writer_index, value) + self.writer_index += 8 + cpdef put_buffer(self, uint32_t offset, v, int32_t src_index, int32_t length): if length == 0: # access an emtpy buffer may raise out-of-bound exception. return @@ -351,11 +361,21 @@ cdef class Buffer: self.reader_index += 4 return value + cpdef inline float read_float32(self): + value = self.get_float(self.reader_index) + self.reader_index += 4 + return value + cpdef inline double read_double(self): value = self.get_double(self.reader_index) self.reader_index += 8 return value + cpdef inline double read_float64(self): + value = self.get_double(self.reader_index) + self.reader_index += 8 + return value + cpdef inline bytes read(self, int32_t length): return self.read_bytes(length) diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 9a797abee1..252a5535da 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -18,7 +18,7 @@ import enum import typing from typing import List -from pyfory.type import TypeId +from pyfory.type import TypeId, is_primitive_type from pyfory._util import Buffer from pyfory.type import infer_field, is_polymorphic_type from pyfory.meta.metastring import Encoding @@ -28,7 +28,8 @@ # Constants from the specification SMALL_NUM_FIELDS_THRESHOLD = 0b11111 REGISTER_BY_NAME_FLAG = 0b100000 -FIELD_NAME_SIZE_THRESHOLD = 0b1111 +FIELD_NAME_SIZE_THRESHOLD = 0b1111 # 4-bit threshold for field names +BIG_NAME_THRESHOLD = 0b111111 # 6-bit threshold for namespace/typename COMPRESS_META_FLAG = 0b1 << 13 HAS_FIELDS_META_FLAG = 0b1 << 12 META_SIZE_MASKS = 0xFFF @@ -69,8 +70,14 @@ def create_serializer(self, resolver): from pyfory.serializer import DataClassSerializer fory = resolver.fory + nullable_fields = {f.name: f.field_type.is_nullable for f in self.fields} return DataClassSerializer( - fory, self.cls, xlang=not fory.is_py, field_names=self.get_field_names(), serializers=self.create_fields_serializer(resolver) + fory, + self.cls, + xlang=not fory.is_py, + field_names=self.get_field_names(), + serializers=self.create_fields_serializer(resolver), + nullable_fields=nullable_fields, ) def __repr__(self): @@ -180,7 +187,7 @@ def __init__( def create_serializer(self, resolver, type_): from pyfory.serializer import ListSerializer, SetSerializer - elem_type = type_[1] if len(type_) >= 2 else None + elem_type = type_[1] if type_ and len(type_) >= 2 else None elem_serializer = self.element_type.create_serializer(resolver, elem_type) if self.type_id == TypeId.LIST: return ListSerializer(resolver.fory, list, elem_serializer) @@ -206,9 +213,9 @@ def __init__( def create_serializer(self, resolver, type_): key_type, value_type = None, None - if len(type_) >= 2: + if type_ and len(type_) >= 2: key_type = type_[1] - if len(type_) >= 3: + if type_ and len(type_) >= 3: value_type = type_[2] key_serializer = self.key_type.create_serializer(resolver, key_type) value_serializer = self.value_type.create_serializer(resolver, value_type) @@ -237,22 +244,27 @@ def __repr__(self): def build_field_infos(type_resolver, cls): """Build field information for the class.""" from pyfory._struct import _sort_fields, StructTypeIdVisitor, get_field_names + from pyfory.type import unwrap_optional field_names = get_field_names(cls) type_hints = typing.get_type_hints(cls) field_infos = [] + nullable_map = {} visitor = StructTypeIdVisitor(type_resolver.fory, cls) for field_name in field_names: field_type_hint = type_hints.get(field_name, typing.Any) - field_type = build_field_type(type_resolver, field_name, field_type_hint, visitor) + unwrapped_type, is_nullable = unwrap_optional(field_type_hint) + is_nullable = is_nullable or not is_primitive_type(unwrapped_type) + nullable_map[field_name] = is_nullable + field_type = build_field_type(type_resolver, field_name, unwrapped_type, visitor, is_nullable) field_info = FieldInfo(field_name, field_type, cls.__name__) field_infos.append(field_info) field_types = infer_field_types(cls) serializers = [field_info.field_type.create_serializer(type_resolver, field_types.get(field_info.name, None)) for field_info in field_infos] - field_names, serializers = _sort_fields(type_resolver, field_names, serializers) + field_names, serializers = _sort_fields(type_resolver, field_names, serializers, nullable_map) field_infos_map = {field_info.name: field_info for field_info in field_infos} new_field_infos = [] for field_name in field_names: @@ -261,16 +273,16 @@ def build_field_infos(type_resolver, cls): return new_field_infos -def build_field_type(type_resolver, field_name: str, type_hint, visitor): +def build_field_type(type_resolver, field_name: str, type_hint, visitor, is_nullable=False): """Build field type from type hint.""" type_ids = infer_field(field_name, type_hint, visitor) try: - return build_field_type_from_type_ids(type_resolver, field_name, type_ids, visitor) + return build_field_type_from_type_ids(type_resolver, field_name, type_ids, visitor, is_nullable) except Exception as e: raise TypeError(f"Error building field type for field: {field_name} with type hint: {type_hint} in class: {visitor.cls}") from e -def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, visitor): +def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, visitor, is_nullable=False): tracking_ref = type_resolver.fory.ref_tracking type_id = type_ids[0] if type_id is None: @@ -279,15 +291,15 @@ def build_field_type_from_type_ids(type_resolver, field_name: str, type_ids, vis type_id = type_id & 0xFF morphic = not is_polymorphic_type(type_id) if type_id in [TypeId.SET, TypeId.LIST]: - elem_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[1], visitor) - return CollectionFieldType(type_id, morphic, True, tracking_ref, elem_type) + elem_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[1], visitor, is_nullable=False) + return CollectionFieldType(type_id, morphic, is_nullable, tracking_ref, elem_type) elif type_id == TypeId.MAP: - key_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[1], visitor) - value_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[2], visitor) - return MapFieldType(type_id, morphic, True, tracking_ref, key_type, value_type) + key_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[1], visitor, is_nullable=False) + value_type = build_field_type_from_type_ids(type_resolver, field_name, type_ids[2], visitor, is_nullable=False) + return MapFieldType(type_id, morphic, is_nullable, tracking_ref, key_type, value_type) elif type_id in [TypeId.UNKNOWN, TypeId.EXT, TypeId.STRUCT, TypeId.NAMED_STRUCT, TypeId.COMPATIBLE_STRUCT, TypeId.NAMED_COMPATIBLE_STRUCT]: - return DynamicFieldType(type_id, False, True, tracking_ref) + return DynamicFieldType(type_id, False, is_nullable, tracking_ref) else: if type_id <= 0 or type_id >= TypeId.BOUND: raise TypeError(f"Unknown type: {type_id} for field: {field_name}") - return FieldType(type_id, morphic, True, tracking_ref) + return FieldType(type_id, morphic, is_nullable, tracking_ref) diff --git a/python/pyfory/meta/typedef_decoder.py b/python/pyfory/meta/typedef_decoder.py index 9446e3c457..8b2f4b794c 100644 --- a/python/pyfory/meta/typedef_decoder.py +++ b/python/pyfory/meta/typedef_decoder.py @@ -28,6 +28,7 @@ SMALL_NUM_FIELDS_THRESHOLD, REGISTER_BY_NAME_FLAG, FIELD_NAME_SIZE_THRESHOLD, + BIG_NAME_THRESHOLD, COMPRESS_META_FLAG, HAS_FIELDS_META_FLAG, META_SIZE_MASKS, @@ -149,7 +150,7 @@ def read_typename(buffer: Buffer) -> str: def read_meta_string(buffer: Buffer, decoder: MetaStringDecoder, encodings: List[Encoding]) -> str: - """Read a meta string from the buffer.""" + """Read a big meta string (namespace/typename) from the buffer using 6-bit size field.""" # Read encoding and length combined in first byte header = buffer.read_uint8() @@ -161,8 +162,8 @@ def read_meta_string(buffer: Buffer, decoder: MetaStringDecoder, encodings: List # Read length - same logic as encoder length = 0 - if size_value >= FIELD_NAME_SIZE_THRESHOLD: - length = size_value - FIELD_NAME_SIZE_THRESHOLD + buffer.read_varuint32() + if size_value >= BIG_NAME_THRESHOLD: + length = size_value - BIG_NAME_THRESHOLD + buffer.read_varuint32() else: length = size_value @@ -202,6 +203,7 @@ def read_field_info(buffer: Buffer, resolver, defined_class: str) -> FieldInfo: xtype_id = buffer.read_varuint32() field_type = FieldType.xread_with_type(buffer, resolver, xtype_id, is_nullable, is_tracking_ref) - # Read field name - field_name = FIELD_NAME_DECODER.decode(buffer.read_bytes(field_name_size), encoding) + # Read field name - it comes AFTER the type info in the encoding + field_name_bytes = buffer.read_bytes(field_name_size) + field_name = FIELD_NAME_DECODER.decode(field_name_bytes, encoding) return FieldInfo(field_name, field_type, defined_class) diff --git a/python/pyfory/meta/typedef_encoder.py b/python/pyfory/meta/typedef_encoder.py index 29b5cc8431..954c389c86 100644 --- a/python/pyfory/meta/typedef_encoder.py +++ b/python/pyfory/meta/typedef_encoder.py @@ -24,6 +24,7 @@ SMALL_NUM_FIELDS_THRESHOLD, REGISTER_BY_NAME_FLAG, FIELD_NAME_SIZE_THRESHOLD, + BIG_NAME_THRESHOLD, COMPRESS_META_FLAG, HAS_FIELDS_META_FLAG, META_SIZE_MASKS, @@ -66,18 +67,21 @@ def encode_typedef(type_resolver, cls): buffer = Buffer.allocate(64) - # Write placeholder for header - buffer.write_uint8(0) - # Write meta header header = len(field_infos) if len(field_infos) >= SMALL_NUM_FIELDS_THRESHOLD: header = SMALL_NUM_FIELDS_THRESHOLD + if type_resolver.is_registered_by_name(cls): + header |= REGISTER_BY_NAME_FLAG + buffer.write_uint8(header) buffer.write_varuint32(len(field_infos) - SMALL_NUM_FIELDS_THRESHOLD) + else: + if type_resolver.is_registered_by_name(cls): + header |= REGISTER_BY_NAME_FLAG + buffer.write_uint8(header) # Write type info if type_resolver.is_registered_by_name(cls): - header |= REGISTER_BY_NAME_FLAG namespace, typename = type_resolver.get_registered_name(cls) write_namespace(buffer, namespace) write_typename(buffer, typename) @@ -87,8 +91,6 @@ def encode_typedef(type_resolver, cls): assert type_resolver.is_registered_by_id(cls), "Class must be registered by name or id" type_id = type_resolver.get_registered_id(cls) buffer.write_varuint32(type_id) - # Update header byte - buffer.put_uint8(0, header) # Write fields info write_fields_info(type_resolver, buffer, field_infos) @@ -162,15 +164,15 @@ def write_typename(buffer: Buffer, typename: str): def write_meta_string(buffer: Buffer, meta_string, encoding_value: int): - """Write a meta string to the buffer.""" + """Write a big meta string (namespace/typename) to the buffer using 6-bit size field.""" # Write encoding and length combined in first byte length = len(meta_string.encoded_data) - if length >= FIELD_NAME_SIZE_THRESHOLD: + if length >= BIG_NAME_THRESHOLD: # Use threshold value and write additional length - header = (FIELD_NAME_SIZE_THRESHOLD << 2) | encoding_value + header = (BIG_NAME_THRESHOLD << 2) | encoding_value buffer.write_uint8(header) - buffer.write_varuint32(length - FIELD_NAME_SIZE_THRESHOLD) + buffer.write_varuint32(length - BIG_NAME_THRESHOLD) else: # Combine length and encoding in single byte header = (length << 2) | encoding_value diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index a1732d2b4a..7b66fa5bbe 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -27,7 +27,7 @@ import pickle import types import typing -from typing import List +from typing import List, Dict import warnings from pyfory.buffer import Buffer @@ -40,6 +40,8 @@ from pyfory.resolver import NULL_FLAG, NOT_NULL_VALUE_FLAG from pyfory import Language +from pyfory.type import is_primitive_type + try: import numpy as np except ImportError: @@ -229,7 +231,7 @@ def _deserialize_local_class(self, buffer): # Read base classes num_bases = buffer.read_varuint32() - bases = tuple([fory.deserialize_ref(buffer) for _ in range(num_bases)]) + bases = tuple([fory.read_ref(buffer) for _ in range(num_bases)]) # Create the class using type() constructor cls = type(name, bases, {}) # `class_dict` may reference to `cls`, which is a circular reference @@ -238,12 +240,12 @@ def _deserialize_local_class(self, buffer): # classmethods for i in range(buffer.read_varuint32()): attr_name = buffer.read_string() - func = fory.deserialize_ref(buffer) + func = fory.read_ref(buffer) method = types.MethodType(func, cls) setattr(cls, attr_name, method) # Read class dictionary # Fory's normal deserialization will handle methods via FunctionSerializer - class_dict = fory.deserialize_ref(buffer) + class_dict = fory.read_ref(buffer) for k, v in class_dict.items(): setattr(cls, k, v) @@ -275,7 +277,7 @@ def write(self, buffer, value): self.fory.serialize_ref(buffer, dict(value)) def read(self, buffer): - return types.MappingProxyType(self.fory.deserialize_ref(buffer)) + return types.MappingProxyType(self.fory.read_ref(buffer)) class PandasRangeIndexSerializer(Serializer): @@ -325,17 +327,17 @@ def read(self, buffer): if buffer.read_int8() == NULL_FLAG: start = None else: - start = self.fory.deserialize_nonref(buffer) + start = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: stop = None else: - stop = self.fory.deserialize_nonref(buffer) + stop = self.fory.read_no_ref(buffer) if buffer.read_int8() == NULL_FLAG: step = None else: - step = self.fory.deserialize_nonref(buffer) - dtype = self.fory.deserialize_ref(buffer) - name = self.fory.deserialize_ref(buffer) + step = self.fory.read_no_ref(buffer) + dtype = self.fory.read_ref(buffer) + name = self.fory.read_ref(buffer) return self.type_(start, stop, step, dtype=dtype, name=name) def xwrite(self, buffer, value): @@ -365,22 +367,32 @@ def __init__( xlang: bool = False, field_names: List[str] = None, serializers: List[Serializer] = None, + nullable_fields: Dict[str, bool] = None, ): super().__init__(fory, clz) self._xlang = xlang - # This will get superclass type hints too. + from pyfory.type import unwrap_optional + self._type_hints = typing.get_type_hints(clz) self._field_names = field_names or self._get_field_names(clz) self._has_slots = hasattr(clz, "__slots__") + self._nullable_fields = nullable_fields or {} + if self._field_names and not self._nullable_fields: + for field_name in self._field_names: + if field_name in self._type_hints: + unwrapped_type, is_nullable = unwrap_optional(self._type_hints[field_name]) + is_nullable = is_nullable or not is_primitive_type(unwrapped_type) + self._nullable_fields[field_name] = is_nullable if self._xlang: self._serializers = serializers or [None] * len(self._field_names) if serializers is None: visitor = StructFieldSerializerVisitor(fory) for index, key in enumerate(self._field_names): - serializer = infer_field(key, self._type_hints[key], visitor, types_path=[]) + unwrapped_type, _ = unwrap_optional(self._type_hints[key]) + serializer = infer_field(key, unwrapped_type, visitor, types_path=[]) self._serializers[index] = serializer - self._field_names, self._serializers = _sort_fields(fory.type_resolver, self._field_names, self._serializers) + self._field_names, self._serializers = _sort_fields(fory.type_resolver, self._field_names, self._serializers, self._nullable_fields) self._hash = 0 # Will be computed on first xwrite/xread self._generated_xwrite_method = self._gen_xwrite_method() self._generated_xread_method = self._gen_xread_method() @@ -510,27 +522,64 @@ def _gen_xwrite_method(self): context[fory] = self.fory context[get_hash_func] = _get_hash context["_field_names"] = self._field_names - context["_type_hints"] = self._type_hints + from pyfory.type import unwrap_optional + + unwrapped_hints = {} + for field_name, hint in self._type_hints.items(): + unwrapped, _ = unwrap_optional(hint) + unwrapped_hints[field_name] = unwrapped + context["_type_hints"] = unwrapped_hints context["_serializers"] = self._serializers stmts = [ f'"""xwrite method for {self.type_}"""', ] if not self.fory.compatible: - # Compute hash at generation time since we're in xlang mode if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, self._type_hints) + self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) stmts.append(f"{buffer}.write_int32({self._hash})") if not self._has_slots: stmts.append(f"{value_dict} = {value}.__dict__") for index, field_name in enumerate(self._field_names): field_value = f"field_value{next(counter)}" serializer_var = f"serializer{index}" - context[serializer_var] = self._serializers[index] + serializer = self._serializers[index] + context[serializer_var] = serializer if not self._has_slots: stmts.append(f"{field_value} = {value_dict}['{field_name}']") else: stmts.append(f"{field_value} = {value}.{field_name}") - stmts.append(f"{fory}.xserialize_ref({buffer}, {field_value}, serializer={serializer_var})") + is_nullable = self._nullable_fields.get(field_name, False) + if is_nullable: + if isinstance(serializer, StringSerializer): + stmts.extend( + [ + f"if {field_value} is None:", + f" {buffer}.write_int8({NULL_FLAG})", + "else:", + f" {buffer}.write_int8({NOT_NULL_VALUE_FLAG})", + f" {buffer}.write_string({field_value})", + ] + ) + else: + stmts.append(f"{fory}.xwrite_ref({buffer}, {field_value}, serializer={serializer_var})") + else: + if isinstance(serializer, BooleanSerializer): + stmt = f"{buffer}.write_bool({field_value})" + elif isinstance(serializer, ByteSerializer): + stmt = f"{buffer}.write_int8({field_value})" + elif isinstance(serializer, Int16Serializer): + stmt = f"{buffer}.write_int16({field_value})" + elif isinstance(serializer, Int32Serializer): + stmt = f"{buffer}.write_varint32({field_value})" + elif isinstance(serializer, Int64Serializer): + stmt = f"{buffer}.write_varint64({field_value})" + elif isinstance(serializer, Float32Serializer): + stmt = f"{buffer}.write_float32({field_value})" + elif isinstance(serializer, Float64Serializer): + stmt = f"{buffer}.write_float64({field_value})" + else: + stmt = f"{fory}.xwrite_no_ref({buffer}, {field_value}, serializer={serializer_var})" + stmts.append(stmt) self._xwrite_method_code, func = compile_function( f"xwrite_{self.type_.__module__}_{self.type_.__qualname__}".replace(".", "_"), [buffer, value], @@ -555,16 +604,21 @@ def _gen_xread_method(self): context[ref_resolver] = self.fory.ref_resolver context[get_hash_func] = _get_hash context["_field_names"] = self._field_names - context["_type_hints"] = self._type_hints + from pyfory.type import unwrap_optional + + unwrapped_hints = {} + for field_name, hint in self._type_hints.items(): + unwrapped, _ = unwrap_optional(hint) + unwrapped_hints[field_name] = unwrapped + context["_type_hints"] = unwrapped_hints context["_serializers"] = self._serializers current_class_field_names = set(self._get_field_names(self.type_)) stmts = [ f'"""xread method for {self.type_}"""', ] if not self.fory.compatible: - # Compute hash at generation time since we're in xlang mode if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, self._type_hints) + self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) stmts.extend( [ f"read_hash = {buffer}.read_int32()", @@ -585,9 +639,40 @@ def _gen_xread_method(self): for index, field_name in enumerate(self._field_names): serializer_var = f"serializer{index}" - context[serializer_var] = self._serializers[index] + serializer = self._serializers[index] + context[serializer_var] = serializer field_value = f"field_value{index}" - stmts.append(f"{field_value} = {fory}.xdeserialize_ref({buffer}, serializer={serializer_var})") + is_nullable = self._nullable_fields.get(field_name, False) + if is_nullable: + if isinstance(serializer, StringSerializer): + stmts.extend( + [ + f"if {buffer}.read_int8() >= {NOT_NULL_VALUE_FLAG}:", + f" {field_value} = {buffer}.read_string()", + "else:", + f" {field_value} = None", + ] + ) + else: + stmts.append(f"{field_value} = {fory}.xread_ref({buffer}, serializer={serializer_var})") + else: + if isinstance(serializer, BooleanSerializer): + stmt = f"{field_value} = {buffer}.read_bool()" + elif isinstance(serializer, ByteSerializer): + stmt = f"{field_value} = {buffer}.read_int8()" + elif isinstance(serializer, Int16Serializer): + stmt = f"{field_value} = {buffer}.read_int16()" + elif isinstance(serializer, Int32Serializer): + stmt = f"{field_value} = {buffer}.read_varint32()" + elif isinstance(serializer, Int64Serializer): + stmt = f"{field_value} = {buffer}.read_varint64()" + elif isinstance(serializer, Float32Serializer): + stmt = f"{field_value} = {buffer}.read_float32()" + elif isinstance(serializer, Float64Serializer): + stmt = f"{field_value} = {buffer}.read_float64()" + else: + stmt = f"{field_value} = {fory}.xread_no_ref({buffer}, serializer={serializer_var})" + stmts.append(stmt) if field_name not in current_class_field_names: stmts.append(f"# {field_name} is not in {self.type_}") continue @@ -619,7 +704,7 @@ def read(self, buffer): obj = self.type_.__new__(self.type_) self.fory.ref_resolver.reference(obj) for field_name in self._field_names: - field_value = self.fory.deserialize_ref(buffer) + field_value = self.fory.read_ref(buffer) setattr( obj, field_name, @@ -631,33 +716,62 @@ def xwrite(self, buffer: Buffer, value): if not self._xlang: raise TypeError("xwrite can only be called when DataClassSerializer is in xlang mode") if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, self._type_hints) - buffer.write_int32(self._hash) + from pyfory.type import unwrap_optional + + unwrapped_hints = {} + for field_name, hint in self._type_hints.items(): + unwrapped, _ = unwrap_optional(hint) + unwrapped_hints[field_name] = unwrapped + self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + if not self.fory.compatible: + buffer.write_int32(self._hash) for index, field_name in enumerate(self._field_names): field_value = getattr(value, field_name) serializer = self._serializers[index] - self.fory.xserialize_ref(buffer, field_value, serializer=serializer) + is_nullable = self._nullable_fields.get(field_name, False) + if is_nullable and field_value is None: + buffer.write_int8(-3) + else: + self.fory.xwrite_ref(buffer, field_value, serializer=serializer) def xread(self, buffer): if not self._xlang: raise TypeError("xread can only be called when DataClassSerializer is in xlang mode") if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, self._type_hints) - hash_ = buffer.read_int32() - if hash_ != self._hash: - raise TypeNotCompatibleError( - f"Hash {hash_} is not consistent with {self._hash} for type {self.type_}", - ) + from pyfory.type import unwrap_optional + + unwrapped_hints = {} + for field_name, hint in self._type_hints.items(): + unwrapped, _ = unwrap_optional(hint) + unwrapped_hints[field_name] = unwrapped + self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + if not self.fory.compatible: + hash_ = buffer.read_int32() + if hash_ != self._hash: + raise TypeNotCompatibleError( + f"Hash {hash_} is not consistent with {self._hash} for type {self.type_}", + ) obj = self.type_.__new__(self.type_) self.fory.ref_resolver.reference(obj) + current_class_field_names = set(self._get_field_names(self.type_)) for index, field_name in enumerate(self._field_names): serializer = self._serializers[index] - field_value = self.fory.xdeserialize_ref(buffer, serializer=serializer) - setattr( - obj, - field_name, - field_value, - ) + is_nullable = self._nullable_fields.get(field_name, False) + if is_nullable: + ref_id = buffer.read_int8() + if ref_id == -3: + field_value = None + else: + buffer.reader_index -= 1 + field_value = self.fory.xread_ref(buffer, serializer=serializer) + else: + field_value = self.fory.xread_ref(buffer, serializer=serializer) + if field_name in current_class_field_names: + setattr( + obj, + field_name, + field_value, + ) return obj @@ -902,12 +1016,12 @@ def write(self, buffer, value): def read(self, buffer): fory = self.fory - dtype = fory.deserialize_ref(buffer) + dtype = fory.read_ref(buffer) ndim = buffer.read_varuint32() shape = tuple(buffer.read_varuint32() for _ in range(ndim)) if dtype.kind == "O": length = buffer.read_varint32() - items = [fory.deserialize_ref(buffer) for _ in range(length)] + items = [fory.read_ref(buffer) for _ in range(length)] return np.array(items, dtype=object) fory_buf = fory.read_buffer_object(buffer) if isinstance(fory_buf, memoryview): @@ -1037,9 +1151,9 @@ def write(self, buffer, value): self.fory.serialize_ref(buffer, state) def read(self, buffer): - args = self.fory.deserialize_ref(buffer) - kwargs = self.fory.deserialize_ref(buffer) - state = self.fory.deserialize_ref(buffer) + args = self.fory.read_ref(buffer) + kwargs = self.fory.read_ref(buffer) + state = self.fory.read_ref(buffer) if args or kwargs: # Case 1: __getnewargs__ was used. Re-create by calling __init__. @@ -1126,7 +1240,7 @@ def read(self, buffer): reduce_data = [None] * 6 fory = self.fory for i in range(reduce_data_num_items): - reduce_data[i] = fory.deserialize_ref(buffer) + reduce_data[i] = fory.read_ref(buffer) if reduce_data[0] == "global": # Case 1: Global name @@ -1193,7 +1307,7 @@ class FunctionSerializer(CrossLanguageCompatibleSerializer): The code object is serialized with marshal, and all other components (defaults, globals, closure cells, attrs) go through Fory’s own - serialize_ref/deserialize_ref pipeline to ensure proper type registration + serialize_ref/read_ref pipeline to ensure proper type registration and reference tracking. """ @@ -1317,7 +1431,7 @@ def _deserialize_function(self, buffer): func_type_id = buffer.read_int8() if func_type_id == 0: # Handle bound methods - self_obj = self.fory.deserialize_ref(buffer) + self_obj = self.fory.read_ref(buffer) method_name = buffer.read_string() return getattr(self_obj, method_name) @@ -1347,7 +1461,7 @@ def _deserialize_function(self, buffer): # Deserialize each default value default_values = [] for _ in range(num_defaults): - default_values.append(self.fory.deserialize_ref(buffer)) + default_values.append(self.fory.read_ref(buffer)) defaults = tuple(default_values) # Handle closure @@ -1359,7 +1473,7 @@ def _deserialize_function(self, buffer): closure_values = [] if has_closure: for _ in range(num_freevars): - closure_values.append(self.fory.deserialize_ref(buffer)) + closure_values.append(self.fory.read_ref(buffer)) # Create closure cells closure = tuple(types.CellType(value) for value in closure_values) @@ -1371,7 +1485,7 @@ def _deserialize_function(self, buffer): freevars.append(buffer.read_string()) # Handle globals - globals_dict = self.fory.deserialize_ref(buffer) + globals_dict = self.fory.read_ref(buffer) # Create a globals dictionary with module's globals as the base func_globals = {} @@ -1397,7 +1511,7 @@ def _deserialize_function(self, buffer): func.__qualname__ = qualname # Deserialize and set additional attributes - attrs = self.fory.deserialize_ref(buffer) + attrs = self.fory.read_ref(buffer) for attr_name, attr_value in attrs.items(): setattr(func, attr_name, attr_value) @@ -1438,7 +1552,7 @@ def read(self, buffer): mod = importlib.import_module(module) return getattr(mod, name) else: - obj = self.fory.deserialize_ref(buffer) + obj = self.fory.read_ref(buffer) return getattr(obj, name) @@ -1458,7 +1572,7 @@ def write(self, buffer, value): buffer.write_string(method_name) def read(self, buffer): - instance = self.fory.deserialize_ref(buffer) + instance = self.fory.read_ref(buffer) method_name = buffer.read_string() return getattr(instance, method_name) @@ -1505,7 +1619,7 @@ def read(self, buffer): num_fields = buffer.read_varuint32() for _ in range(num_fields): field_name = buffer.read_string() - field_value = self.fory.deserialize_ref(buffer) + field_value = self.fory.read_ref(buffer) setattr(obj, field_name, field_value) return obj diff --git a/python/pyfory/tests/test_cross_language.py b/python/pyfory/tests/test_cross_language.py index 33cd2f18ad..3cb9c9daab 100644 --- a/python/pyfory/tests/test_cross_language.py +++ b/python/pyfory/tests/test_cross_language.py @@ -597,16 +597,16 @@ def read(self, buffer): return self.xread(buffer) def xwrite(self, buffer, value): - self.fory.xserialize_ref(buffer, value.f1) - self.fory.xserialize_ref(buffer, value.f2) - self.fory.xserialize_ref(buffer, value.f3) + self.fory.xwrite_ref(buffer, value.f1) + self.fory.xwrite_ref(buffer, value.f2) + self.fory.xwrite_ref(buffer, value.f3) def xread(self, buffer): obj = ComplexObject1(*([None] * len(typing.get_type_hints(ComplexObject1).keys()))) self.fory.ref_resolver.reference(obj) - obj.f1 = self.fory.xdeserialize_ref(buffer) - obj.f2 = self.fory.xdeserialize_ref(buffer) - obj.f3 = self.fory.xdeserialize_ref(buffer) + obj.f1 = self.fory.xread_ref(buffer) + obj.f2 = self.fory.xread_ref(buffer) + obj.f3 = self.fory.xread_ref(buffer) return obj @@ -817,6 +817,7 @@ class CompatTestV1: # Serialize back new_serialized = fory.serialize(obj) debug_print(f"Re-serialized data length: {len(new_serialized)}") + assert fory.deserialize(new_serialized) == obj # Write back for Java to verify with open(data_file_path, "wb") as f: diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 9f6d7c3edb..c7703c61b8 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -303,5 +303,5 @@ def test_cross_language_compatibility(): assert deserialized == obj # The serialized data should use Fory's native format, not pickle - # This is verified by the fact that we're using serialize_ref/deserialize_ref + # This is verified by the fact that we're using serialize_ref/read_ref # in the ReduceSerializer implementation diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index 2e522964ad..cf25365276 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -17,7 +17,7 @@ from dataclasses import dataclass import datetime -from typing import Dict, Any, List, Set +from typing import Dict, Any, List, Set, Optional import os import pytest @@ -159,23 +159,7 @@ class TestClass: fory = Fory(xlang=True, ref=True) serializer = DataClassSerializer(fory, TestClass, xlang=True) - assert serializer._field_names == [ - "f13", - "f5", - "f11", - "f7", - "f12", - "f1", - "f4", - "f15", - "f6", - "f10", - "f2", - "f14", - "f3", - "f9", - "f8", - ] + assert serializer._field_names == ["f13", "f5", "f11", "f12", "f1", "f7", "f4", "f15", "f6", "f10", "f2", "f14", "f3", "f9", "f8"] def test_data_class_serializer_xlang(): @@ -376,12 +360,12 @@ def test_data_class_serializer_xlang_codegen_generated_code(): # Check that xwrite code contains expected elements assert "def xwrite_" in xwrite_code assert "buffer.write_int32" in xwrite_code # Hash writing - assert "fory.xserialize_ref" in xwrite_code # Field serialization + assert "fory.xwrite_ref" in xwrite_code # Field serialization # Check that xread code contains expected elements assert "def xread_" in xread_code assert "buffer.read_int32" in xread_code # Hash reading - assert "fory.xdeserialize_ref" in xread_code # Field deserialization + assert "fory.xread_ref" in xread_code # Field deserialization assert "TypeNotCompatibleError" in xread_code # Hash validation # Check that field names are referenced in the code @@ -419,3 +403,141 @@ def test_data_class_serializer_xlang_vs_non_xlang(): # They should have different method implementations assert serializer_xlang._generated_xwrite_method != serializer_python._generated_write_method assert serializer_xlang._generated_xread_method != serializer_python._generated_read_method + + +@dataclass +class OptionalFieldsObject: + f1: Optional[int] = None + f2: Optional[str] = None + f3: Optional[List[int]] = None + f4: int = 0 + f5: str = "" + + +@pytest.mark.parametrize("compatible", [False, True]) +def test_optional_fields(compatible): + fory = Fory(xlang=True, ref=True, compatible=compatible) + fory.register_type(OptionalFieldsObject, typename="example.OptionalFieldsObject") + + obj_with_none = OptionalFieldsObject(f1=None, f2=None, f3=None, f4=42, f5="test") + result = ser_de(fory, obj_with_none) + assert result.f1 is None + assert result.f2 is None + assert result.f3 is None + assert result.f4 == 42 + assert result.f5 == "test" + + obj_with_values = OptionalFieldsObject(f1=100, f2="hello", f3=[1, 2, 3], f4=42, f5="test") + result = ser_de(fory, obj_with_values) + assert result.f1 == 100 + assert result.f2 == "hello" + assert result.f3 == [1, 2, 3] + assert result.f4 == 42 + assert result.f5 == "test" + + obj_mixed = OptionalFieldsObject(f1=100, f2=None, f3=[1, 2, 3], f4=42, f5="test") + result = ser_de(fory, obj_mixed) + assert result.f1 == 100 + assert result.f2 is None + assert result.f3 == [1, 2, 3] + assert result.f4 == 42 + assert result.f5 == "test" + + +@dataclass +class NestedOptionalObject: + f1: Optional[ComplexObject] = None + f2: Optional[Dict[str, int]] = None + f3: str = "" + + +@pytest.mark.parametrize("compatible", [False, True]) +def test_nested_optional_fields(compatible): + fory = Fory(xlang=True, ref=True, compatible=compatible) + fory.register_type(ComplexObject, typename="example.ComplexObject") + fory.register_type(NestedOptionalObject, typename="example.NestedOptionalObject") + + obj_with_none = NestedOptionalObject(f1=None, f2=None, f3="test") + result = ser_de(fory, obj_with_none) + assert result.f1 is None + assert result.f2 is None + assert result.f3 == "test" + + complex_obj = ComplexObject(f1="nested", f5=100, f8=3.14) + obj_with_values = NestedOptionalObject(f1=complex_obj, f2={"a": 1, "b": 2}, f3="test") + result = ser_de(fory, obj_with_values) + assert result.f1.f1 == "nested" + assert result.f1.f5 == 100 + assert result.f2 == {"a": 1, "b": 2} + assert result.f3 == "test" + + +@dataclass +class OptionalV1: + f1: Optional[int] = None + f2: str = "" + f3: Optional[List[int]] = None + + +@dataclass +class OptionalV2: + f1: Optional[int] = None + f2: str = "" + f3: Optional[List[int]] = None + f4: Optional[str] = None + + +@dataclass +class OptionalV3: + f1: Optional[int] = None + f2: str = "" + + +def test_optional_compatible_mode_evolution(): + fory_v1 = Fory(xlang=True, ref=True, compatible=True) + fory_v2 = Fory(xlang=True, ref=True, compatible=True) + fory_v3 = Fory(xlang=True, ref=True, compatible=True) + + fory_v1.register_type(OptionalV1, typename="example.OptionalVersioned") + fory_v2.register_type(OptionalV2, typename="example.OptionalVersioned") + fory_v3.register_type(OptionalV3, typename="example.OptionalVersioned") + + v1_obj = OptionalV1(f1=100, f2="test", f3=[1, 2, 3]) + v1_binary = fory_v1.serialize(v1_obj) + + v2_result = fory_v2.deserialize(v1_binary) + assert v2_result.f1 == 100 + assert v2_result.f2 == "test" + assert v2_result.f3 == [1, 2, 3] + assert v2_result.f4 is None + + v1_obj_with_none = OptionalV1(f1=None, f2="test", f3=None) + v1_binary_with_none = fory_v1.serialize(v1_obj_with_none) + + v2_result_with_none = fory_v2.deserialize(v1_binary_with_none) + assert v2_result_with_none.f1 is None + assert v2_result_with_none.f2 == "test" + assert v2_result_with_none.f3 is None + assert v2_result_with_none.f4 is None + + v2_obj = OptionalV2(f1=200, f2="test2", f3=[4, 5], f4="extra") + v2_binary = fory_v2.serialize(v2_obj) + + v3_result = fory_v3.deserialize(v2_binary) + assert v3_result.f1 == 200 + assert v3_result.f2 == "test2" + + v2_obj_partial_none = OptionalV2(f1=None, f2="test2", f3=None, f4=None) + v2_binary_partial_none = fory_v2.serialize(v2_obj_partial_none) + + v3_result_partial_none = fory_v3.deserialize(v2_binary_partial_none) + assert v3_result_partial_none.f1 is None + assert v3_result_partial_none.f2 == "test2" + + v3_obj = OptionalV3(f1=300, f2="test3") + v3_binary = fory_v3.serialize(v3_obj) + + v1_result = fory_v1.deserialize(v3_binary) + assert v1_result.f1 == 300 + assert v1_result.f2 == "test3" + assert v1_result.f3 is None diff --git a/python/pyfory/type.py b/python/pyfory/type.py index dc25dd6b65..ecd505d136 100644 --- a/python/pyfory/type.py +++ b/python/pyfory/type.py @@ -420,7 +420,29 @@ def infer_field_types(type_): from pyfory._struct import StructTypeVisitor visitor = StructTypeVisitor(type_) - return {name: infer_field(name, type_, visitor) for name, type_ in sorted(type_hints.items())} + result = {} + for name, hint in sorted(type_hints.items()): + unwrapped, _ = unwrap_optional(hint) + result[name] = infer_field(name, unwrapped, visitor) + return result + + +def is_optional_type(type_): + origin = typing.get_origin(type_) if hasattr(typing, "get_origin") else getattr(type_, "__origin__", None) + if origin is typing.Union: + args = typing.get_args(type_) if hasattr(typing, "get_args") else getattr(type_, "__args__", ()) + return type(None) in args + return False + + +def unwrap_optional(type_): + if not is_optional_type(type_): + return type_, False + args = typing.get_args(type_) if hasattr(typing, "get_args") else getattr(type_, "__args__", ()) + non_none_types = [arg for arg in args if arg is not type(None)] + if len(non_none_types) == 1: + return non_none_types[0], True + return typing.Union[tuple(non_none_types)], True def infer_field(field_name, type_, visitor: TypeVisitor, types_path=None): From 09793c7f24d336fcb6e9bcb39d96b931b549549e Mon Sep 17 00:00:00 2001 From: Zhong Junjie Date: Sun, 19 Oct 2025 01:49:48 +0800 Subject: [PATCH 27/37] docs(go): Update README.md (#2675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? ## What does this PR do? ## Related issues #2192 ## Does this PR introduce any user-facing change? - [x] Does this PR introduce any public API change? no - [x] Does this PR introduce any binary protocol compatibility change? no ## Benchmark --- go/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/README.md b/go/README.md index 8a58e3d4ca..ddf4bd1534 100644 --- a/go/README.md +++ b/go/README.md @@ -170,12 +170,12 @@ fory --force -file Fory Go supports several configuration options through the functional options pattern: -### Compatible Mode (Metashare) +### Compatible Mode (Meta share mode) Compatible mode enables meta information sharing, which allows for schema evolution: ```go -// Enable compatible mode with metashare +// Enable compatible mode with meta share fory := NewForyWithOptions(WithCompatible(true)) ``` From 16d4db9c25a878f2781c6c01a17ae109b7a69805 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 02:56:05 +0800 Subject: [PATCH 28/37] feat(rust): dynamic rust serializer system (#2778) ## What does this PR do? This PR rewrite whole rust serializer to provide a much more clean, dynamic and flexible serializer system. Main changes inlucde: - New Serializer trait inferface: privide new `fory_write`/`fory_write_generic`/`fory_read`/`fory_read_with_type_info` API,, to allow control nested serialization behaviour - Provide `fory_static_type_id` to allow fast compile-time serializer dispatch - Implement fory collection/map dynamic xlang serialization protocol, make it work with trait object and shared/circular reference - Removed `is_field` parameter, it inotroduce lots of confusion to users - Rewrite whole error processing system, use static method instead, and support dump backtrace in https://github.com/apache/fory/issues/2780 - Add struct field read/write hook, support dump field write/read info and support register customized debug hook - Refactored the derive macro with better generated code - Move `fory_read_compatible` to `StrucSerializer` ## Related issues - Closes #2777 - Closes #2775 - Closes #2780 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- AGENTS.md | 5 + CONTRIBUTING.md | 4 +- docs/guide/DEVELOPMENT.md | 2 + rust/README.md | 9 + rust/fory-core/src/buffer.rs | 11 +- rust/fory-core/src/error.rs | 380 +++++++++- rust/fory-core/src/fory.rs | 96 +-- rust/fory-core/src/lib.rs | 5 +- rust/fory-core/src/meta/meta_string.rs | 55 +- rust/fory-core/src/meta/type_meta.rs | 16 +- rust/fory-core/src/resolver/context.rs | 160 ++-- rust/fory-core/src/resolver/meta_resolver.rs | 6 +- .../src/resolver/metastring_resolver.rs | 8 +- rust/fory-core/src/resolver/ref_resolver.rs | 17 +- rust/fory-core/src/resolver/type_resolver.rs | 570 +++++++++------ rust/fory-core/src/row/row.rs | 12 +- rust/fory-core/src/serializer/any.rs | 514 +++++++++---- rust/fory-core/src/serializer/arc.rs | 162 ++-- rust/fory-core/src/serializer/bool.rs | 20 +- rust/fory-core/src/serializer/box_.rs | 24 +- rust/fory-core/src/serializer/collection.rs | 251 ++++++- rust/fory-core/src/serializer/core.rs | 276 +++++++ rust/fory-core/src/serializer/datetime.rs | 62 +- rust/fory-core/src/serializer/enum_.rs | 101 ++- rust/fory-core/src/serializer/heap.rs | 23 +- rust/fory-core/src/serializer/list.rs | 100 ++- rust/fory-core/src/serializer/map.rs | 690 ++++++++++++++---- rust/fory-core/src/serializer/mod.rs | 303 +------- rust/fory-core/src/serializer/mutex.rs | 75 +- rust/fory-core/src/serializer/number.rs | 28 +- rust/fory-core/src/serializer/option.rs | 97 ++- .../src/serializer/primitive_list.rs | 27 +- rust/fory-core/src/serializer/rc.rs | 160 ++-- rust/fory-core/src/serializer/refcell.rs | 79 +- rust/fory-core/src/serializer/set.rs | 49 +- rust/fory-core/src/serializer/skip.rs | 82 ++- rust/fory-core/src/serializer/string.rs | 32 +- rust/fory-core/src/serializer/struct_.rs | 170 ++++- rust/fory-core/src/serializer/trait_object.rs | 624 ++++++++-------- rust/fory-core/src/serializer/util.rs | 78 ++ rust/fory-core/src/serializer/weak.rs | 323 +++++--- rust/fory-core/src/types.rs | 86 ++- rust/fory-derive/src/lib.rs | 9 +- rust/fory-derive/src/object/derive_enum.rs | 45 +- rust/fory-derive/src/object/read.rs | 340 ++++++--- rust/fory-derive/src/object/serializer.rs | 105 +-- rust/fory-derive/src/object/util.rs | 262 +++++-- rust/fory-derive/src/object/write.rs | 167 +++-- rust/fory/src/lib.rs | 4 +- rust/tests/tests/compatible/test_container.rs | 1 + rust/tests/tests/compatible/test_struct.rs | 10 + .../tests/compatible/test_struct_enum.rs | 29 +- rust/tests/tests/mod.rs | 2 + rust/tests/tests/test_collection.rs | 3 +- rust/tests/tests/test_cross_language.rs | 21 +- rust/tests/tests/test_debug.rs | 195 +++++ rust/tests/tests/test_ext.rs | 8 +- rust/tests/tests/test_max_dyn_depth.rs | 13 + rust/tests/tests/test_meta_string.rs | 4 + rust/tests/tests/test_one_struct.rs | 72 ++ rust/tests/tests/test_simple_struct.rs | 188 ++++- rust/tests/tests/test_trait_object.rs | 22 +- 62 files changed, 5034 insertions(+), 2258 deletions(-) create mode 100644 rust/fory-core/src/serializer/core.rs create mode 100644 rust/fory-core/src/serializer/util.rs create mode 100644 rust/tests/tests/test_debug.rs create mode 100644 rust/tests/tests/test_one_struct.rs diff --git a/AGENTS.md b/AGENTS.md index 7e2cd126e8..18786ac256 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,6 +123,8 @@ go generate ./... - All cargo commands must be executed within the `rust` directory. - All changes to `rust` must pass the clippy check and tests. +- You must set `RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1` when debuging rust tests to get backtrace. +- You must not set `FORY_PANIC_ON_ERROR=1` when runing all rust tests to check whether all tests pass, some tests will check Error content, which will fail if error just panic. ```bash # Check code @@ -143,6 +145,9 @@ cargo test -p fory-tests --test $test_file $test_method # run specific test under subdirectory cargo test --test mod $dir$::$test_file::$test_method +# debug specific test under subdirectory and get backtrace +RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1 ENABLE_FORY_DEBUG_OUTPUT=1 cargo test --test mod $dir$::$test_file::$test_method + # inspect generated code by fory derive macro cargo expand --test mod $mod$::$file$ > expanded.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 180cd26469..b89a77ca7d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,9 @@ cargo test # run test with specific test file and method cargo test -p fory-tests --test $test_file $test_method # run specific test under subdirectory - cargo test --test mod $dir$::$test_file::$test_method +cargo test --test mod $dir$::$test_file::$test_method +# debug specific test under subdirectory and get backtrace +RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1 cargo test --test mod $dir$::$test_file::$test_method ``` ### JavaScript diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index dc08f8a00c..921143aaad 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -100,6 +100,8 @@ cargo test cargo test -p fory-tests --test $test_file $test_method # run specific test under subdirectory cargo test --test mod $dir$::$test_file::$test_method +# debug specific test under subdirectory and get backtrace +RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1 cargo test --test mod $dir$::$test_file::$test_method # inspect generated code by fory derive macro cargo expand --test mod $mod$::$file$ > expanded.rs ``` diff --git a/rust/README.md b/rust/README.md index 607c556bea..849afd9355 100644 --- a/rust/README.md +++ b/rust/README.md @@ -920,6 +920,15 @@ let fory = Fory::default().max_dyn_depth(10); // Allow up to 10 levels Note: Static data types (non-dynamic types) are secure by nature and not subject to depth limits, as their structure is known at compile time. +## 🧪 Troubleshooting + +- **Type registry errors**: An error like `TypeId ... not found in type_info registry` means the type was never registered with the current `Fory` instance. Confirm that every serializable struct or trait implementation calls `fory.register::(type_id)` before serialization and that the same IDs are reused on the deserialize side. +- **Quick error lookup**: Prefer the static constructors on `fory_core::error::Error` (`Error::type_mismatch`, `Error::invalid_data`, `Error::unknown`, etc.) rather than instantiating variants manually. This keeps diagnostics consistent and makes opt-in panics work. +- **Panic on error for backtraces**: Toggle `FORY_PANIC_ON_ERROR=1` (or `true`) alongside `RUST_BACKTRACE=1` when running tests or binaries to panic at the exact site an error is constructed. Reset the variable afterwards to avoid aborting user-facing code paths. +- **Struct field tracing**: Add the `#[fory_debug]` attribute alongside `#[derive(ForyObject)]` to tell the macro to emit hook invocations for that type. Once compiled with debug hooks, call `set_before_write_field_func`, `set_after_write_field_func`, `set_before_read_field_func`, or `set_after_read_field_func` (from `fory-core/src/serializer/struct_.rs`) to plug in custom callbacks, and use `reset_struct_debug_hooks()` when you want the defaults back. +- **Lightweight logging**: Without custom hooks, enable `ENABLE_FORY_DEBUG_OUTPUT=1` to print field-level read/write events emitted by the default hook functions. This is especially useful when investigating alignment or cursor mismatches. +- **Test-time hygiene**: Some integration tests expect `FORY_PANIC_ON_ERROR` to remain unset. Export it only for focused debugging sessions, and prefer `cargo test --features tests -p fory-tests --test ` when isolating failing scenarios. + ## 🛠️ Development ### Building diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index 5769377388..d38bf4fcb2 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -374,6 +374,11 @@ impl Reader { self.cursor += additional; } + #[inline(always)] + pub(crate) fn move_back(&mut self, additional: usize) { + self.cursor -= additional; + } + #[inline(always)] pub(crate) fn ptr_at(&self, offset: usize) -> *const u8 { unsafe { self.bf.add(offset) } @@ -395,10 +400,10 @@ impl Reader { fn check_bound(&self, n: usize) -> Result<(), Error> { // The upper layer guarantees it is non-null // if self.bf.is_null() { - // return Err(Error::InvalidData("buffer pointer is null".into())); + // return Err(Error::invalid_data("buffer pointer is null")); // } if self.cursor + n > self.len { - Err(Error::BufferOutOfBound(self.cursor, n, self.len)) + Err(Error::buffer_out_of_bound(self.cursor, n, self.len)) } else { Ok(()) } @@ -673,7 +678,7 @@ impl Reader { } shift += 7; if shift >= 36 { - return Err(Error::EncodeError("varuint36small overflow".into())); + return Err(Error::encode_error("varuint36small overflow")); } } Ok(result) diff --git a/rust/fory-core/src/error.rs b/rust/fory-core/src/error.rs index 291608033e..84606e6038 100644 --- a/rust/fory-core/src/error.rs +++ b/rust/fory-core/src/error.rs @@ -17,54 +17,402 @@ use std::borrow::Cow; use std::io; +use std::sync::OnceLock; use thiserror::Error; +/// Global flag to check if FORY_PANIC_ON_ERROR environment variable is set. +static PANIC_ON_ERROR: OnceLock = OnceLock::new(); + +/// Check if FORY_PANIC_ON_ERROR environment variable is set. +#[inline] +pub fn should_panic_on_error() -> bool { + *PANIC_ON_ERROR.get_or_init(|| { + std::env::var("FORY_PANIC_ON_ERROR") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) + }) +} + +/// Error type for Fory serialization and deserialization operations. +/// +/// # IMPORTANT: Always Use Static Constructor Functions +/// +/// **DO NOT** construct error variants directly using the enum syntax. +/// **ALWAYS** use the provided static constructor functions instead. +/// +/// ## Why Use Static Functions? +/// +/// The static constructor functions provide: +/// - Automatic type conversion via `Into>` +/// - Consistent error creation across the codebase +/// - Better ergonomics (no need for manual `.into()` calls) +/// - Future-proof API if error construction logic needs to change +/// +/// ## Examples +/// +/// ```rust +/// use fory_core::error::Error; +/// +/// // ✅ CORRECT: Use static functions +/// let err = Error::type_error("Expected string type"); +/// let err = Error::invalid_data(format!("Invalid value: {}", 42)); +/// let err = Error::type_mismatch(1, 2); +/// +/// // ❌ WRONG: Do not construct directly +/// // let err = Error::TypeError("Expected string type".into()); +/// // let err = Error::InvalidData(format!("Invalid value: {}", 42).into()); +/// ``` +/// +/// ## Available Constructor Functions +/// +/// - [`Error::type_mismatch`] - For type ID mismatches +/// - [`Error::buffer_out_of_bound`] - For buffer boundary violations +/// - [`Error::encode_error`] - For encoding failures +/// - [`Error::invalid_data`] - For invalid or corrupted data +/// - [`Error::invalid_ref`] - For invalid reference IDs +/// - [`Error::unknown_enum`] - For unknown enum variants +/// - [`Error::type_error`] - For general type errors +/// - [`Error::encoding_error`] - For encoding format errors +/// - [`Error::depth_exceed`] - For exceeding maximum nesting depth +/// - [`Error::unsupported`] - For unsupported operations +/// - [`Error::not_allowed`] - For disallowed operations +/// - [`Error::unknown`] - For generic errors +/// +/// ## Debug Mode: FORY_PANIC_ON_ERROR +/// +/// For easier debugging, you can set the `FORY_PANIC_ON_ERROR` environment variable to make +/// the program panic at the exact location where an error is created. This helps identify +/// the error source with a full stack trace. +/// +/// ```bash +/// RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1 cargo run +/// # or +/// RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=true cargo test +/// ``` +/// +/// When enabled, any error created via the static constructor functions will panic immediately +/// with the error message, allowing you to see the exact call stack in your debugger or +/// panic output. Use `RUST_BACKTRACE=1` together with `FORY_PANIC_ON_ERROR` to get a full +/// stack trace showing exactly where the error was created. #[derive(Error, Debug)] #[non_exhaustive] pub enum Error { + /// Type mismatch between local and remote type IDs. + /// + /// Do not construct this variant directly; use [`Error::type_mismatch`] instead. #[error("Type mismatch: type_a = {0}, type_b = {1}")] TypeMismatch(u32, u32), + /// Buffer boundary violation during read/write operations. + /// + /// Do not construct this variant directly; use [`Error::buffer_out_of_bound`] instead. #[error("Buffer out of bound: {0} + {1} > {2}")] BufferOutOfBound(usize, usize, usize), + /// IO error from underlying operations. #[error("IO error: {0}")] Io(#[from] io::Error), + /// Error during data encoding. + /// + /// Do not construct this variant directly; use [`Error::encode_error`] instead. #[error("{0}")] EncodeError(Cow<'static, str>), + /// Invalid or corrupted data encountered. + /// + /// Do not construct this variant directly; use [`Error::invalid_data`] instead. #[error("{0}")] InvalidData(Cow<'static, str>), + /// Invalid reference ID encountered. + /// + /// Do not construct this variant directly; use [`Error::invalid_ref`] instead. #[error("{0}")] InvalidRef(Cow<'static, str>), + /// Unknown enum variant encountered. + /// + /// Do not construct this variant directly; use [`Error::unknown_enum`] instead. #[error("{0}")] UnknownEnum(Cow<'static, str>), + /// General type-related error. + /// + /// Do not construct this variant directly; use [`Error::type_error`] instead. #[error("{0}")] TypeError(Cow<'static, str>), + /// Error in encoding format or conversion. + /// + /// Do not construct this variant directly; use [`Error::encoding_error`] instead. #[error("{0}")] EncodingError(Cow<'static, str>), + /// Maximum nesting depth exceeded. + /// + /// Do not construct this variant directly; use [`Error::depth_exceed`] instead. #[error("{0}")] DepthExceed(Cow<'static, str>), + /// Unsupported operation or feature. + /// + /// Do not construct this variant directly; use [`Error::unsupported`] instead. + #[error("{0}")] + Uunsupported(Cow<'static, str>), + + /// Operation not allowed in current context. + /// + /// Do not construct this variant directly; use [`Error::not_allowed`] instead. + #[error("{0}")] + NotAllowed(Cow<'static, str>), + + /// Generic unknown error. + /// /// Do not construct this variant directly; use [`Error::unknown`] instead. #[error("{0}")] Unknown(Cow<'static, str>), } impl Error { + /// Creates a new [`Error::TypeMismatch`] with the given type IDs. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::type_mismatch(1, 2); + /// ``` + #[inline(always)] + #[track_caller] + pub fn type_mismatch(type_a: u32, type_b: u32) -> Self { + let err = Error::TypeMismatch(type_a, type_b); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::BufferOutOfBound`] with the given bounds. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::buffer_out_of_bound(10, 20, 25); + /// ``` + #[inline(always)] + #[track_caller] + pub fn buffer_out_of_bound(offset: usize, length: usize, capacity: usize) -> Self { + let err = Error::BufferOutOfBound(offset, length, capacity); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::EncodeError`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::encode_error("Failed to encode"); + /// let err = Error::encode_error(format!("Failed to encode field {}", "name")); + /// ``` + #[inline(always)] + #[track_caller] + pub fn encode_error>>(s: S) -> Self { + let err = Error::EncodeError(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::InvalidData`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::invalid_data("Invalid data format"); + /// let err = Error::invalid_data(format!("Invalid data at position {}", 42)); + /// ``` + #[inline(always)] + #[track_caller] + pub fn invalid_data>>(s: S) -> Self { + let err = Error::InvalidData(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::InvalidRef`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::invalid_ref("Invalid reference"); + /// let err = Error::invalid_ref(format!("Invalid ref id {}", 123)); + /// ``` + #[inline(always)] + #[track_caller] + pub fn invalid_ref>>(s: S) -> Self { + let err = Error::InvalidRef(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::UnknownEnum`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::unknown_enum("Unknown enum variant"); + /// let err = Error::unknown_enum(format!("Unknown variant {}", 5)); + /// ``` + #[inline(always)] + #[track_caller] + pub fn unknown_enum>>(s: S) -> Self { + let err = Error::UnknownEnum(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::TypeError`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::type_error("Type error"); + /// let err = Error::type_error(format!("Expected type {}", "String")); + /// ``` + #[inline(always)] + #[track_caller] + pub fn type_error>>(s: S) -> Self { + let err = Error::TypeError(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::EncodingError`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::encoding_error("Encoding failed"); + /// let err = Error::encoding_error(format!("Failed to encode as {}", "UTF-8")); + /// ``` + #[inline(always)] + #[track_caller] + pub fn encoding_error>>(s: S) -> Self { + let err = Error::EncodingError(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::DepthExceed`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::depth_exceed("Max depth exceeded"); + /// let err = Error::depth_exceed(format!("Depth {} exceeds max {}", 100, 64)); + /// ``` + #[inline(always)] + #[track_caller] + pub fn depth_exceed>>(s: S) -> Self { + let err = Error::DepthExceed(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::Uunsupported`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::unsupported("Unsupported operation"); + /// let err = Error::unsupported(format!("Type {} not supported", "MyType")); + /// ``` + #[inline(always)] + #[track_caller] + pub fn unsupported>>(s: S) -> Self { + let err = Error::Uunsupported(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + + /// Creates a new [`Error::NotAllowed`] from a string or static message. + /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// + /// # Example + /// ``` + /// use fory_core::error::Error; + /// + /// let err = Error::not_allowed("Operation not allowed"); + /// let err = Error::not_allowed(format!("Cannot perform {}", "delete")); + /// ``` + #[inline(always)] + #[track_caller] + pub fn not_allowed>>(s: S) -> Self { + let err = Error::NotAllowed(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err + } + /// Creates a new [`Error::Unknown`] from a string or static message. /// /// This function is a convenient way to produce an error message /// from a literal, `String`, or any type that can be converted into /// a [`Cow<'static, str>`]. /// + /// If `FORY_PANIC_ON_ERROR` environment variable is set, this will panic with the error message. + /// /// # Example /// ``` /// use fory_core::error::Error; @@ -73,8 +421,13 @@ impl Error { /// let err = Error::unknown(format!("ID:{} not found", 1)); /// ``` #[inline(always)] + #[track_caller] pub fn unknown>>(s: S) -> Self { - Error::Unknown(s.into()) + let err = Error::Unknown(s.into()); + if should_panic_on_error() { + panic!("FORY_PANIC_ON_ERROR: {}", err); + } + err } } @@ -130,3 +483,28 @@ macro_rules! bail { return Err($crate::error::Error::unknown(format!($fmt, $($arg)*))) }; } + +/// Returns early with a [`Error::NotAllowed`]. +/// +/// # Examples +/// ``` +/// use fory_core::not_allowed; +/// use fory_core::error::Error; +/// +/// fn check_operation() -> Result<(), Error> { +/// not_allowed!("operation not allowed"); +/// } +/// +/// fn check_operation_with_context(op: &str) -> Result<(), Error> { +/// not_allowed!("operation {} not allowed", op); +/// } +/// ``` +#[macro_export] +macro_rules! not_allowed { + ($err:expr) => { + return Err($crate::error::Error::not_allowed($err)) + }; + ($fmt:expr, $($arg:tt)*) => { + return Err($crate::error::Error::not_allowed(format!($fmt, $($arg)*))) + }; +} diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index c80b1a2dc2..bb1983ee3a 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -20,7 +20,7 @@ use crate::ensure; use crate::error::Error; use crate::resolver::context::WriteContext; use crate::resolver::context::{Pool, ReadContext}; -use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; +use crate::resolver::type_resolver::TypeResolver; use crate::serializer::ForyDefault; use crate::serializer::{Serializer, StructSerializer}; use crate::types::config_flags::IS_NULL_FLAG; @@ -28,11 +28,8 @@ use crate::types::{ config_flags::{IS_CROSS_LANGUAGE_FLAG, IS_LITTLE_ENDIAN_FLAG}, Language, MAGIC_NUMBER, SIZE_OF_REF_AND_TYPE, }; -use crate::util::get_ext_actual_type_id; use std::sync::OnceLock; -static EMPTY_STRING: String = String::new(); - /// The main Fory serialization framework instance. /// /// `Fory` provides high-performance cross-language serialization and deserialization @@ -135,6 +132,7 @@ impl Fory { // Setting share_meta individually is not supported currently self.share_meta = compatible; self.compatible = compatible; + self.type_resolver.set_compatible(compatible); self } @@ -313,29 +311,25 @@ impl Fory { let magic_numer = reader.read_u16()?; ensure!( magic_numer == MAGIC_NUMBER, - Error::InvalidData( - format!( - "The fory xlang serialization must start with magic number {:X}. \ + Error::invalid_data(format!( + "The fory xlang serialization must start with magic number {:X}. \ Please check whether the serialization is based on the xlang protocol \ and the data didn't corrupt.", - MAGIC_NUMBER - ) - .into() - ) + MAGIC_NUMBER + )) ) } let bitmap = reader.read_u8()?; let peer_is_xlang = (bitmap & IS_CROSS_LANGUAGE_FLAG) != 0; ensure!( self.xlang == peer_is_xlang, - Error::InvalidData("header bitmap mismatch at xlang bit".into()) + Error::invalid_data("header bitmap mismatch at xlang bit") ); let is_little_endian = (bitmap & IS_LITTLE_ENDIAN_FLAG) != 0; ensure!( is_little_endian, - Error::InvalidData( + Error::invalid_data( "Big endian is not supported for now, please ensure peer machine is little endian." - .into() ) ); let is_none = (bitmap & IS_NULL_FLAG) != 0; @@ -429,7 +423,7 @@ impl Fory { bytes_to_skip = context.load_meta(meta_offset as usize)?; } } - let result = ::fory_read(context, false); + let result = ::fory_read(context, true, true); if bytes_to_skip > 0 { context.reader.skip(bytes_to_skip)?; } @@ -504,7 +498,7 @@ impl Fory { if context.is_compatible() { context.writer.write_i32(-1); }; - ::fory_write(record, context, false)?; + ::fory_write(record, context, true, true, false)?; if context.is_compatible() && !context.empty() { context.write_meta(meta_start_offset); } @@ -543,16 +537,7 @@ impl Fory { &mut self, id: u32, ) -> Result<(), Error> { - let actual_type_id = T::fory_actual_type_id(id, false, self.compatible); - let type_info = TypeInfo::new::( - &self.type_resolver, - actual_type_id, - &EMPTY_STRING, - &EMPTY_STRING, - false, - )?; - self.type_resolver.register::(&type_info)?; - Ok(()) + self.type_resolver.register_by_id::(id) } /// Registers a struct type with a namespace and type name for cross-language serialization. @@ -590,16 +575,8 @@ impl Fory { namespace: &str, type_name: &str, ) -> Result<(), Error> { - let actual_type_id = T::fory_actual_type_id(0, true, self.compatible); - let type_info = TypeInfo::new::( - &self.type_resolver, - actual_type_id, - namespace, - type_name, - true, - )?; - self.type_resolver.register::(&type_info)?; - Ok(()) + self.type_resolver + .register_by_namespace::(namespace, type_name) } /// Registers a struct type with a type name (using the default namespace). @@ -666,16 +643,7 @@ impl Fory { &mut self, id: u32, ) -> Result<(), Error> { - let actual_type_id = get_ext_actual_type_id(id, false); - let type_info = TypeInfo::new_with_empty_fields::( - &self.type_resolver, - actual_type_id, - &EMPTY_STRING, - &EMPTY_STRING, - false, - )?; - self.type_resolver.register_serializer::(&type_info)?; - Ok(()) + self.type_resolver.register_serializer_by_id::(id) } /// Registers a custom serializer type with a namespace and type name. @@ -699,16 +667,8 @@ impl Fory { namespace: &str, type_name: &str, ) -> Result<(), Error> { - let actual_type_id = get_ext_actual_type_id(0, true); - let type_info = TypeInfo::new_with_empty_fields::( - &self.type_resolver, - actual_type_id, - namespace, - type_name, - true, - )?; - self.type_resolver.register_serializer::(&type_info)?; - Ok(()) + self.type_resolver + .register_serializer_by_namespace::(namespace, type_name) } /// Registers a custom serializer type with a type name (using the default namespace). @@ -730,19 +690,21 @@ impl Fory { ) -> Result<(), Error> { self.register_serializer_by_namespace::("", type_name) } + + /// Registers a generic trait object type for serialization. + /// This method should be used to register collection types such as `Vec`, `HashMap`, etc. + /// Don't register concrete struct types with this method. Use `register()` instead. + pub fn register_generic_trait( + &mut self, + ) -> Result<(), Error> { + self.type_resolver.register_generic_trait::() + } } -pub fn write_data( - this: &T, - context: &mut WriteContext, - is_field: bool, -) -> Result<(), Error> { - T::fory_write_data(this, context, is_field) +pub fn write_data(this: &T, context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_data(this, context) } -pub fn read_data( - context: &mut ReadContext, - is_field: bool, -) -> Result { - T::fory_read_data(context, is_field) +pub fn read_data(context: &mut ReadContext) -> Result { + T::fory_read_data(context) } diff --git a/rust/fory-core/src/lib.rs b/rust/fory-core/src/lib.rs index f4f9f5d6f0..cbb2778980 100644 --- a/rust/fory-core/src/lib.rs +++ b/rust/fory-core/src/lib.rs @@ -190,9 +190,10 @@ pub mod util; pub use paste; pub use crate::buffer::{Reader, Writer}; +pub use crate::error::Error; pub use crate::fory::Fory; pub use crate::resolver::context::{ReadContext, WriteContext}; -pub use crate::resolver::type_resolver::TypeResolver; +pub use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; pub use crate::serializer::weak::{ArcWeak, RcWeak}; -pub use crate::serializer::{ForyDefault, Serializer}; +pub use crate::serializer::{ForyDefault, Serializer, StructSerializer}; pub use crate::types::{RefFlag, TypeId}; diff --git a/rust/fory-core/src/meta/meta_string.rs b/rust/fory-core/src/meta/meta_string.rs index 357849361a..9391b94e4e 100644 --- a/rust/fory-core/src/meta/meta_string.rs +++ b/rust/fory-core/src/meta/meta_string.rs @@ -77,7 +77,7 @@ impl MetaString { let mut strip_last_char = false; if encoding != Encoding::Utf8 { if bytes.is_empty() { - return Err(Error::EncodeError("Encoded data cannot be empty".into())); + return Err(Error::encode_error("Encoded data cannot be empty")); } strip_last_char = (bytes[0] & 0x80) != 0; } @@ -142,13 +142,10 @@ impl MetaStringEncoder { ensure!( input.len() < SHORT_MAX_VALUE, - Error::EncodeError( - format!( - "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", - input.len() - ) - .into() - ) + Error::encode_error(format!( + "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", + input.len() + )) ); if !self.is_latin(input) { @@ -263,17 +260,14 @@ impl MetaStringEncoder { } ensure!( input.len() < SHORT_MAX_VALUE, - Error::EncodeError( - format!( - "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", - input.len() - ) - .into() - ) + Error::encode_error(format!( + "Meta string is too long, max:{SHORT_MAX_VALUE}, current:{}", + input.len() + )) ); ensure!( encoding == Encoding::Utf8 || self.is_latin(input), - Error::EncodeError("Non-ASCII characters in meta string are not allowed".into()) + Error::encode_error("Non-ASCII characters in meta string are not allowed") ); if input.is_empty() { @@ -421,10 +415,9 @@ impl MetaStringEncoder { '_' => Ok(27), '$' => Ok(28), '|' => Ok(29), - _ => Err(Error::EncodeError( - format!("Unsupported character for LOWER_UPPER_DIGIT_SPECIAL encoding: {c}",) - .into(), - ))?, + _ => Err(Error::encode_error(format!( + "Unsupported character for LOWER_UPPER_DIGIT_SPECIAL encoding: {c}", + )))?, }, 6 => match c { 'a'..='z' => Ok(c as u8 - b'a'), @@ -436,10 +429,9 @@ impl MetaStringEncoder { } else if c == self.special_char2 { Ok(63) } else { - Err(Error::EncodeError( - format!("Invalid character value for LOWER_SPECIAL decoding: {c:?}",) - .into(), - ))? + Err(Error::encode_error(format!( + "Invalid character value for LOWER_SPECIAL decoding: {c:?}", + )))? } } }, @@ -547,9 +539,9 @@ impl MetaStringDecoder { 27 => Ok('_'), 28 => Ok('$'), 29 => Ok('|'), - _ => Err(Error::EncodeError( - format!("Invalid character value for LOWER_SPECIAL decoding: {char_value}",).into(), - ))?, + _ => Err(Error::encode_error(format!( + "Invalid character value for LOWER_SPECIAL decoding: {char_value}", + )))?, } } @@ -560,12 +552,9 @@ impl MetaStringDecoder { 52..=61 => Ok((b'0' + char_value - 52) as char), 62 => Ok(self.special_char1), 63 => Ok(self.special_char2), - _ => Err(Error::EncodeError( - format!( - "Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {char_value}", - ) - .into(), - ))?, + _ => Err(Error::encode_error(format!( + "Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {char_value}", + )))?, } } diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs index 8e0d3e9e1d..a52f94d4a8 100644 --- a/rust/fory-core/src/meta/type_meta.rs +++ b/rust/fory-core/src/meta/type_meta.rs @@ -86,13 +86,13 @@ impl FieldType { match self.type_id { x if x == TypeId::LIST as u32 || x == TypeId::SET as u32 => { let generic = self.generics.first().unwrap(); - generic.to_bytes(writer, true, false)?; + generic.to_bytes(writer, true, generic.nullable)?; } x if x == TypeId::MAP as u32 => { let key_generic = self.generics.first().unwrap(); let val_generic = self.generics.get(1).unwrap(); - key_generic.to_bytes(writer, true, false)?; - val_generic.to_bytes(writer, true, false)?; + key_generic.to_bytes(writer, true, key_generic.nullable)?; + val_generic.to_bytes(writer, true, val_generic.nullable)?; } _ => {} } @@ -163,9 +163,9 @@ impl FieldInfo { 0x00 => Ok(Encoding::Utf8), 0x01 => Ok(Encoding::AllToLowerSpecial), 0x02 => Ok(Encoding::LowerUpperDigitSpecial), - _ => Err(Error::EncodingError( - format!("Unsupported encoding of field name in type meta, value:{value}").into(), - ))?, + _ => Err(Error::encoding_error(format!( + "Unsupported encoding of field name in type meta, value:{value}" + )))?, } } @@ -494,10 +494,10 @@ impl TypeMetaLayer { if let Some(type_info_current) = type_resolver.get_type_info_by_name(&namespace.original, &type_name.original) { - Self::assign_field_ids(type_info_current, &mut sorted_field_infos); + Self::assign_field_ids(&type_info_current, &mut sorted_field_infos); } } else if let Some(type_info_current) = type_resolver.get_type_info_by_id(type_id) { - Self::assign_field_ids(type_info_current, &mut sorted_field_infos); + Self::assign_field_ids(&type_info_current, &mut sorted_field_infos); } // if no type found, keep all fields id as -1 to be skipped. Ok(TypeMetaLayer::new( diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 94aaa45b3d..714f7dda89 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -25,7 +25,8 @@ use crate::resolver::metastring_resolver::{ MetaStringBytes, MetaStringReaderResolver, MetaStringWriterResolver, }; use crate::resolver::ref_resolver::{RefReader, RefWriter}; -use crate::resolver::type_resolver::{Harness, TypeResolver}; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; +use crate::types; use std::sync::{Arc, Mutex}; pub struct WriteContext { @@ -85,6 +86,11 @@ impl WriteContext { &self.type_resolver } + #[inline(always)] + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + self.type_resolver.get_type_info(type_id) + } + /// Check if compatible mode is enabled #[inline(always)] pub fn is_compatible(&self) -> bool { @@ -130,52 +136,46 @@ impl WriteContext { pub fn write_any_typeinfo( &mut self, + fory_type_id: u32, concrete_type_id: std::any::TypeId, - ) -> Result, Error> { - use crate::types::TypeId as ForyTypeId; - - let type_info = self.type_resolver.get_type_info(concrete_type_id)?; + ) -> Result, Error> { + if types::is_internal_type(fory_type_id) { + self.writer.write_varuint32(fory_type_id); + return self + .type_resolver + .get_type_info_by_id(fory_type_id) + .ok_or_else(|| Error::type_error("Type info for internal type not found")); + } + let type_info = self.type_resolver.get_type_info(&concrete_type_id)?; let fory_type_id = type_info.get_type_id(); - - // Clone out the data we need from `type_info` to break the borrow - let namespace = type_info.get_namespace().clone(); - let type_name = type_info.get_type_name().clone(); - let registered_by_name = type_info.is_registered_by_name(); - - if registered_by_name { - if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { - self.writer.write_varuint32(fory_type_id); + let namespace = type_info.get_namespace(); + let type_name = type_info.get_type_name(); + self.writer.write_varuint32(fory_type_id); + // should be compiled to jump table generation + match fory_type_id & 0xff { + types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { + let meta_index = + self.meta_resolver + .push(concrete_type_id, &self.type_resolver)? as u32; + self.writer.write_varuint32(meta_index); + } + types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { if self.is_share_meta() { - let meta_index = self.push_meta(concrete_type_id)? as u32; + let meta_index = self + .meta_resolver + .push(concrete_type_id, &self.type_resolver)? + as u32; self.writer.write_varuint32(meta_index); } else { namespace.write_to(&mut self.writer); type_name.write_to(&mut self.writer); } - } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 { - self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(concrete_type_id)? as u32; - self.writer.write_varuint32(meta_index); - } else { - self.writer.write_varuint32(u32::MAX); - namespace.write_to(&mut self.writer); - type_name.write_to(&mut self.writer); } - self.type_resolver - .get_name_harness(&namespace, &type_name) - .ok_or_else(|| Error::TypeError("Name harness not found".into())) - } else { - if fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 { - self.writer.write_varuint32(fory_type_id); - let meta_index = self.push_meta(concrete_type_id)? as u32; - self.writer.write_varuint32(meta_index); - } else { - self.writer.write_varuint32(fory_type_id); + _ => { + // default case: do nothing } - self.type_resolver - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("ID harness not found".into())) } + Ok(type_info) } #[inline(always)] @@ -284,8 +284,10 @@ impl ReadContext { } #[inline(always)] - pub fn get_meta(&self, type_index: usize) -> &Arc { - self.meta_resolver.get(type_index) + pub fn get_meta(&self, type_index: usize) -> Result<&Arc, Error> { + self.meta_resolver.get(type_index).ok_or_else(|| { + Error::type_error(format!("Meta not found for type index: {}", type_index)) + }) } #[inline(always)] @@ -296,45 +298,48 @@ impl ReadContext { ) } - pub fn read_any_typeinfo(&mut self) -> Result, Error> { - use crate::types::TypeId as ForyTypeId; - + pub fn read_any_typeinfo(&mut self) -> Result, Error> { let fory_type_id = self.reader.read_varuint32()?; - - if fory_type_id == u32::MAX { - let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; - let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; - self.type_resolver - .get_name_harness(&namespace, &type_name) - .ok_or_else(|| Error::TypeError("Name harness not found".into())) - } else if fory_type_id & 0xff == ForyTypeId::NAMED_STRUCT as u32 { - if self.is_share_meta() { - let _meta_index = self.reader.read_varuint32(); - } else { - let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; - let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; - return self + // should be compiled to jump table generation + match fory_type_id & 0xff { + types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { + let meta_index = self.reader.read_varuint32()? as usize; + let remote_meta = self.get_meta(meta_index)?.clone(); + let local_type_info = self .type_resolver - .get_name_harness(&namespace, &type_name) - .ok_or_else(|| Error::TypeError("Name harness not found".into())); + .get_type_info_by_id(fory_type_id) + .ok_or_else(|| Error::type_error("ID harness not found"))?; + // Create a new TypeInfo with remote metadata but local harness + // TODO: make self.get_meta return type info + Ok(Arc::new(local_type_info.with_remote_meta(remote_meta))) + } + types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { + if self.is_share_meta() { + let meta_index = self.reader.read_varuint32()? as usize; + self.get_meta(meta_index)?; + self.type_resolver + .get_type_info_by_id(fory_type_id) + .ok_or_else(|| Error::type_error("ID harness not found")) + } else { + let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; + let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; + self.type_resolver + .get_type_info_by_msname(&namespace, &type_name) + .ok_or_else(|| Error::type_error("Name harness not found")) + } } - self.type_resolver - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("ID harness not found".into())) - } else if fory_type_id & 0xff == ForyTypeId::NAMED_COMPATIBLE_STRUCT as u32 - || fory_type_id & 0xff == ForyTypeId::COMPATIBLE_STRUCT as u32 - { - let _meta_index = self.reader.read_varuint32(); - self.type_resolver - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("ID harness not found".into())) - } else { - self.type_resolver - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("ID harness not found".into())) + _ => self + .type_resolver + .get_type_info_by_id(fory_type_id) + .ok_or_else(|| Error::type_error("ID harness not found")), } } + #[inline(always)] + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + self.type_resolver.get_type_info(type_id) + } + pub fn read_meta_string_bytes(&mut self) -> Result { self.meta_string_resolver .read_meta_string_bytes(&mut self.reader) @@ -344,16 +349,13 @@ impl ReadContext { pub fn inc_depth(&mut self) -> Result<(), Error> { self.current_depth += 1; if self.current_depth > self.max_dyn_depth() { - return Err(Error::DepthExceed( - format!( - "Maximum dynamic object nesting depth ({}) exceeded. Current depth: {}. \ + return Err(Error::depth_exceed(format!( + "Maximum dynamic object nesting depth ({}) exceeded. Current depth: {}. \ This may indicate a circular reference or overly deep object graph. \ Consider increasing max_dyn_depth if this is expected.", - self.max_dyn_depth(), - self.current_depth - ) - .into(), - )); + self.max_dyn_depth(), + self.current_depth + ))); } Ok(()) } diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 2e1e09bab5..4cb8b4d90a 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -41,7 +41,7 @@ impl MetaWriterResolver { None => { let index = self.type_defs.len(); self.type_defs - .push(type_resolver.get_type_info(type_id)?.get_type_def()); + .push(type_resolver.get_type_info(&type_id)?.get_type_def()); self.type_id_index_map.insert(type_id, index); Ok(index) } @@ -73,8 +73,8 @@ pub struct MetaReaderResolver { } impl MetaReaderResolver { - pub fn get(&self, index: usize) -> &Arc { - unsafe { self.reading_type_defs.get_unchecked(index) } + pub fn get(&self, index: usize) -> Option<&Arc> { + self.reading_type_defs.get(index) } pub fn load( diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs b/rust/fory-core/src/resolver/metastring_resolver.rs index ddc206927f..beb6f4445d 100644 --- a/rust/fory-core/src/resolver/metastring_resolver.rs +++ b/rust/fory-core/src/resolver/metastring_resolver.rs @@ -98,12 +98,12 @@ impl MetaStringBytes { } let first8: [u8; 8] = bytes[0..8].try_into().map_err(|_| { - Error::InvalidData(format!("expected at least 8 bytes, got {}", bytes.len()).into()) + Error::invalid_data(format!("expected at least 8 bytes, got {}", bytes.len())) })?; let first8 = u64::from_le_bytes(first8); let second8: [u8; 8] = bytes[8..16].try_into().map_err(|_| { - Error::InvalidData(format!("expected at least 16 bytes, got {}", bytes.len()).into()) + Error::invalid_data(format!("expected at least 16 bytes, got {}", bytes.len())) })?; let second8 = u64::from_le_bytes(second8); @@ -267,7 +267,7 @@ impl MetaStringReaderResolver { self.dynamic_read .get(idx) .and_then(|opt| opt.clone()) - .ok_or_else(|| Error::InvalidData("dynamic id not found".into())) + .ok_or_else(|| Error::invalid_data("dynamic id not found")) } } @@ -293,7 +293,7 @@ impl MetaStringReaderResolver { self.dynamic_read .get(idx) .and_then(|opt| opt.clone()) - .ok_or_else(|| Error::InvalidData("dynamic id not found".into())) + .ok_or_else(|| Error::invalid_data("dynamic id not found")) } } diff --git a/rust/fory-core/src/resolver/ref_resolver.rs b/rust/fory-core/src/resolver/ref_resolver.rs index 3ea8ad6c05..84eaa4b26c 100644 --- a/rust/fory-core/src/resolver/ref_resolver.rs +++ b/rust/fory-core/src/resolver/ref_resolver.rs @@ -84,7 +84,7 @@ impl RefWriter { if let Some(&ref_id) = self.refs.get(&ptr_addr) { writer.write_i8(RefFlag::Ref as i8); - writer.write_u32(ref_id); + writer.write_varuint32(ref_id); true } else { let ref_id = self.next_ref_id; @@ -116,7 +116,7 @@ impl RefWriter { if let Some(&ref_id) = self.refs.get(&ptr_addr) { // This object has been seen before, write a reference writer.write_i8(RefFlag::Ref as i8); - writer.write_u32(ref_id); + writer.write_varuint32(ref_id); true } else { // First time seeing this object, register it and return false @@ -286,9 +286,9 @@ impl RefReader { /// /// The RefFlag indicating what type of reference this is /// - /// # Panics + /// # Errors /// - /// Panics if an invalid reference flag value is encountered + /// Errors if an invalid reference flag value is encountered pub fn read_ref_flag(&self, reader: &mut Reader) -> Result { let flag_value = reader.read_i8()?; Ok(match flag_value { @@ -296,9 +296,10 @@ impl RefReader { -2 => RefFlag::Ref, -1 => RefFlag::NotNullValue, 0 => RefFlag::RefValue, - _ => Err(Error::InvalidRef( - format!("Invalid reference flag: {}", flag_value).into(), - ))?, + _ => Err(Error::invalid_ref(format!( + "Invalid reference flag: {}", + flag_value + )))?, }) } @@ -312,7 +313,7 @@ impl RefReader { /// /// The reference ID as a u32 pub fn read_ref_id(&self, reader: &mut Reader) -> Result { - reader.read_u32() + reader.read_varuint32() } /// Execute all pending callbacks to resolve weak pointer references. diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index 8321efc60c..0fdd8fda54 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -22,19 +22,28 @@ use crate::meta::{ TYPE_NAME_ENCODINGS, }; use crate::serializer::{ForyDefault, Serializer, StructSerializer}; -use crate::Reader; +use crate::util::get_ext_actual_type_id; +use crate::{Reader, TypeId}; +use std::collections::{HashSet, LinkedList}; use std::sync::Arc; use std::{any::Any, collections::HashMap}; -type WriteFn = fn(&dyn Any, &mut WriteContext, is_field: bool) -> Result<(), Error>; +type WriteFn = fn( + &dyn Any, + &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_enerics: bool, +) -> Result<(), Error>; type ReadFn = - fn(&mut ReadContext, is_field: bool, skip_ref_flag: bool) -> Result, Error>; + fn(&mut ReadContext, read_ref_info: bool, read_type_info: bool) -> Result, Error>; -type WriteDataFn = fn(&dyn Any, &mut WriteContext, is_field: bool) -> Result<(), Error>; -type ReadDataFn = fn(&mut ReadContext, is_field: bool) -> Result, Error>; +type WriteDataFn = fn(&dyn Any, &mut WriteContext, has_generics: bool) -> Result<(), Error>; +type ReadDataFn = fn(&mut ReadContext) -> Result, Error>; type ToSerializerFn = fn(Box) -> Result, Error>; +static EMPTY_STRING: String = String::new(); -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Harness { write_fn: WriteFn, read_fn: ReadFn, @@ -89,6 +98,7 @@ pub struct TypeInfo { namespace: MetaString, type_name: MetaString, register_by_name: bool, + harness: Harness, } impl TypeInfo { @@ -98,6 +108,7 @@ impl TypeInfo { namespace: &str, type_name: &str, register_by_name: bool, + harness: Harness, ) -> Result { let namespace_metastring = NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; @@ -139,6 +150,7 @@ impl TypeInfo { namespace: namespace_metastring, type_name: type_name_metastring, register_by_name, + harness, }) } @@ -148,6 +160,7 @@ impl TypeInfo { namespace: &str, type_name: &str, register_by_name: bool, + harness: Harness, ) -> Result { let namespace_metastring = NAMESPACE_ENCODER.encode_with_encodings(namespace, NAMESPACE_ENCODINGS)?; @@ -169,6 +182,7 @@ impl TypeInfo { namespace: namespace_metastring, type_name: type_name_metastring, register_by_name, + harness, }) } @@ -195,20 +209,37 @@ impl TypeInfo { pub fn is_registered_by_name(&self) -> bool { self.register_by_name } + + pub fn get_harness(&self) -> &Harness { + &self.harness + } + + /// Create a new TypeInfo with the same properties but different type_meta. + /// This is used during deserialization to create a TypeInfo with remote metadata + /// while keeping the local harness for deserialization functions. + pub fn with_remote_meta(&self, remote_meta: Arc) -> TypeInfo { + TypeInfo { + type_def: self.type_def.clone(), + type_meta: remote_meta, + type_id: self.type_id, + namespace: self.namespace.clone(), + type_name: self.type_name.clone(), + register_by_name: self.register_by_name, + harness: self.harness.clone(), + } + } } /// TypeResolver is a resolver for fast type/serializer dispatch. #[derive(Clone)] pub struct TypeResolver { - serializer_map: HashMap>, - name_serializer_map: HashMap<(MetaString, MetaString), Arc>, - type_id_map: HashMap, - type_name_map: HashMap, - type_info_cache: HashMap, - type_info_map_by_id: HashMap, - type_info_map_by_name: HashMap<(String, String), TypeInfo>, + type_info_map_by_id: HashMap>, + type_info_map: HashMap>, + type_info_map_by_name: HashMap<(String, String), Arc>, + type_info_map_by_ms_name: HashMap<(MetaString, MetaString), Arc>, // Fast lookup by numeric ID for common types type_id_index: Vec, + compatible: bool, } const NO_TYPE_ID: u32 = 1000000000; @@ -216,14 +247,12 @@ const NO_TYPE_ID: u32 = 1000000000; impl Default for TypeResolver { fn default() -> Self { let mut registry = TypeResolver { - serializer_map: HashMap::new(), - name_serializer_map: HashMap::new(), - type_id_map: HashMap::new(), - type_name_map: HashMap::new(), - type_info_cache: HashMap::new(), type_info_map_by_id: HashMap::new(), + type_info_map: HashMap::new(), type_info_map_by_name: HashMap::new(), + type_info_map_by_ms_name: HashMap::new(), type_id_index: Vec::new(), + compatible: false, }; registry.register_builtin_types().unwrap(); registry @@ -231,21 +260,35 @@ impl Default for TypeResolver { } impl TypeResolver { - pub fn get_type_info(&self, type_id: std::any::TypeId) -> Result<&TypeInfo, Error> { - self.type_info_cache.get(&type_id) - .ok_or_else(|| Error::TypeError(format!( - "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", - type_id - ).into())) + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + self.type_info_map.get(type_id) + .ok_or_else(|| { + Error::type_error(format!( + "TypeId {:?} not found in type_info registry, maybe you forgot to register some types", + type_id + )) + }) + .cloned() } - pub fn get_type_info_by_id(&self, id: u32) -> Option<&TypeInfo> { - self.type_info_map_by_id.get(&id) + pub fn get_type_info_by_id(&self, id: u32) -> Option> { + self.type_info_map_by_id.get(&id).cloned() } - pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option<&TypeInfo> { + pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option> { self.type_info_map_by_name .get(&(namespace.to_owned(), type_name.to_owned())) + .cloned() + } + + pub fn get_type_info_by_msname( + &self, + namespace: &MetaString, + type_name: &MetaString, + ) -> Option> { + self.type_info_map_by_ms_name + .get(&(namespace.clone(), type_name.clone())) + .cloned() } /// Fast path for getting type info by numeric ID (avoids HashMap lookup by TypeId) @@ -257,17 +300,16 @@ impl TypeResolver { return Ok(type_id); } } - Err(Error::TypeError( - format!( - "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", - type_id - ) - .into(), - )) + Err(Error::type_error(format!( + "TypeId {:?} not found in type_id_index, maybe you forgot to register some types", + type_id + ))) } pub fn get_harness(&self, id: u32) -> Option> { - self.serializer_map.get(&id).cloned() + self.type_info_map_by_id + .get(&id) + .map(|info| Arc::new(info.get_harness().clone())) } pub fn get_name_harness( @@ -276,14 +318,16 @@ impl TypeResolver { type_name: &MetaString, ) -> Option> { let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map.get(&key).cloned() + self.type_info_map_by_ms_name + .get(&key) + .map(|info| Arc::new(info.get_harness().clone())) } pub fn get_ext_harness(&self, id: u32) -> Result, Error> { - self.serializer_map + self.type_info_map_by_id .get(&id) - .cloned() - .ok_or_else(|| Error::TypeError("ext type must be registered in both peers".into())) + .map(|info| Arc::new(info.get_harness().clone())) + .ok_or_else(|| Error::type_error("ext type must be registered in both peers")) } pub fn get_ext_name_harness( @@ -292,118 +336,124 @@ impl TypeResolver { type_name: &MetaString, ) -> Result, Error> { let key = (namespace.clone(), type_name.clone()); - self.name_serializer_map.get(&key).cloned().ok_or_else(|| { - Error::TypeError("named_ext type must be registered in both peers".into()) - }) + self.type_info_map_by_ms_name + .get(&key) + .map(|info| Arc::new(info.get_harness().clone())) + .ok_or_else(|| Error::type_error("named_ext type must be registered in both peers")) } pub fn get_fory_type_id(&self, rust_type_id: std::any::TypeId) -> Option { - if let Some(type_info) = self.type_info_cache.get(&rust_type_id) { - Some(type_info.get_type_id()) - } else { - self.type_id_map.get(&rust_type_id).copied() - } + self.type_info_map + .get(&rust_type_id) + .map(|info| info.get_type_id()) } fn register_builtin_types(&mut self) -> Result<(), Error> { - use crate::types::TypeId; - let namespace = NAMESPACE_ENCODER.encode_with_encodings("", NAMESPACE_ENCODINGS)?; - let type_name = TYPE_NAME_ENCODER.encode_with_encodings("", TYPE_NAME_ENCODINGS)?; - - macro_rules! register_basic_type { - ($ty:ty, $type_id:expr) => {{ - let type_info = TypeInfo { - type_def: Arc::from(vec![]), - type_meta: Arc::new(TypeMeta::empty()), - type_id: $type_id as u32, - namespace: namespace.clone(), - type_name: type_name.clone(), - register_by_name: false, - }; - self.register_serializer::<$ty>(&type_info)?; - }}; - } - - register_basic_type!(bool, TypeId::BOOL); - register_basic_type!(i8, TypeId::INT8); - register_basic_type!(i16, TypeId::INT16); - register_basic_type!(i32, TypeId::INT32); - register_basic_type!(i64, TypeId::INT64); - register_basic_type!(f32, TypeId::FLOAT32); - register_basic_type!(f64, TypeId::FLOAT64); - register_basic_type!(String, TypeId::STRING); - - register_basic_type!(Vec, TypeId::BOOL_ARRAY); - register_basic_type!(Vec, TypeId::INT8_ARRAY); - register_basic_type!(Vec, TypeId::INT16_ARRAY); - register_basic_type!(Vec, TypeId::INT32_ARRAY); - register_basic_type!(Vec, TypeId::INT64_ARRAY); - register_basic_type!(Vec, TypeId::FLOAT32_ARRAY); - register_basic_type!(Vec, TypeId::FLOAT64_ARRAY); + self.register_internal_serializer::(TypeId::BOOL)?; + self.register_internal_serializer::(TypeId::INT8)?; + self.register_internal_serializer::(TypeId::INT16)?; + self.register_internal_serializer::(TypeId::INT32)?; + self.register_internal_serializer::(TypeId::INT64)?; + self.register_internal_serializer::(TypeId::FLOAT32)?; + self.register_internal_serializer::(TypeId::FLOAT64)?; + self.register_internal_serializer::(TypeId::STRING)?; + + self.register_internal_serializer::>(TypeId::BOOL_ARRAY)?; + self.register_internal_serializer::>(TypeId::INT8_ARRAY)?; + self.register_internal_serializer::>(TypeId::INT16_ARRAY)?; + self.register_internal_serializer::>(TypeId::INT32_ARRAY)?; + self.register_internal_serializer::>(TypeId::INT64_ARRAY)?; + self.register_internal_serializer::>(TypeId::FLOAT32_ARRAY)?; + self.register_internal_serializer::>(TypeId::FLOAT64_ARRAY)?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; + self.register_generic_trait::>()?; Ok(()) } - pub fn register( + pub fn register_by_id( + &mut self, + id: u32, + ) -> Result<(), Error> { + self.register::(id, &EMPTY_STRING, &EMPTY_STRING) + } + + pub fn register_by_namespace( &mut self, - type_info: &TypeInfo, + namespace: &str, + type_name: &str, ) -> Result<(), Error> { + self.register::(0, namespace, type_name) + } + + fn register( + &mut self, + id: u32, + namespace: &str, + type_name: &str, + ) -> Result<(), Error> { + let register_by_name = !type_name.is_empty(); + if !register_by_name && id == 0 { + return Err(Error::not_allowed( + "Either id must be non-zero for ID registration, or type_name must be non-empty for name registration", + )); + } + let actual_type_id = T::fory_actual_type_id(id, register_by_name, self.compatible); + fn write( this: &dyn Any, context: &mut WriteContext, - is_field: bool, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { Some(v) => { - let skip_ref_flag = - crate::serializer::get_skip_ref_flag::(context.get_type_resolver())?; - crate::serializer::write_ref_info_data( - v, - context, - is_field, - skip_ref_flag, - true, - )?; - Ok(()) + T2::fory_write(v, context, write_ref_info, write_type_info, has_generics) } - None => todo!(), + None => Err(Error::type_error(format!( + "Cast type error when writing: {:?}", + T2::fory_static_type_id() + ))), } } fn read( context: &mut ReadContext, - is_field: bool, - skip_ref_flag: bool, + read_ref_info: bool, + read_type_info: bool, ) -> Result, Error> { - match crate::serializer::read_ref_info_data::( + Ok(Box::new(T2::fory_read( context, - is_field, - skip_ref_flag, - true, - ) { - Ok(v) => Ok(Box::new(v)), - Err(e) => Err(e), - } + read_ref_info, + read_type_info, + )?)) } fn write_data( this: &dyn Any, context: &mut WriteContext, - is_field: bool, + has_generics: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => T2::fory_write_data(v, context, is_field), + Some(v) => T2::fory_write_data_generic(v, context, has_generics), None => todo!(), } } fn read_data( context: &mut ReadContext, - is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(context, is_field) { + match T2::fory_read_data(context) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -414,131 +464,172 @@ impl TypeResolver { ) -> Result, Error> { match boxed_any.downcast::() { Ok(concrete) => Ok(Box::new(*concrete) as Box), - Err(_) => Err(Error::TypeError( - "Failed to downcast to concrete type".into(), - )), + Err(_) => Err(Error::type_error("Failed to downcast to concrete type")), } } + let harness = Harness::new( + write::, + read::, + write_data::, + read_data::, + to_serializer::, + ); + + let type_info = TypeInfo::new::( + self, + actual_type_id, + namespace, + type_name, + register_by_name, + harness, + )?; + let rs_type_id = std::any::TypeId::of::(); - if self.type_info_cache.contains_key(&rs_type_id) { - return Err(Error::TypeError( - format!("rs_struct:{:?} already registered", rs_type_id).into(), - )); + if self.type_info_map.contains_key(&rs_type_id) { + return Err(Error::type_error(format!( + "rs_struct:{:?} already registered", + rs_type_id + ))); } - self.type_info_cache.insert(rs_type_id, type_info.clone()); + + // Store in main map + self.type_info_map + .insert(rs_type_id, Arc::new(type_info.clone())); + + // Store by ID self.type_info_map_by_id - .insert(type_info.type_id, type_info.clone()); + .insert(type_info.type_id, Arc::new(type_info.clone())); + + // Update type_id_index for fast lookup let index = T::fory_type_index() as usize; if index >= self.type_id_index.len() { self.type_id_index.resize(index + 1, NO_TYPE_ID); } else if self.type_id_index[index] != NO_TYPE_ID { - return Err(Error::TypeError( - format!("please:{:?} already registered", type_info.type_id).into(), - )); + return Err(Error::type_error(format!( + "Type index {:?} already registered", + index + ))); } self.type_id_index[index] = type_info.type_id; + // Store by name if registered by name if type_info.register_by_name { let namespace = &type_info.namespace; let type_name = &type_info.type_name; - let key = (namespace.clone(), type_name.clone()); - if self.name_serializer_map.contains_key(&key) { - return Err(Error::InvalidData( - format!( - "Namespace:{:?} Name:{:?} already registered_by_name", - namespace, type_name - ) - .into(), - )); + let ms_key = (namespace.clone(), type_name.clone()); + if self.type_info_map_by_ms_name.contains_key(&ms_key) { + return Err(Error::invalid_data(format!( + "Namespace:{:?} Name:{:?} already registered_by_name", + namespace, type_name + ))); } - self.type_name_map.insert(rs_type_id, key.clone()); - self.name_serializer_map.insert( - key, - Arc::from(Harness::new( - write::, - read::, - write_data::, - read_data::, - to_serializer::, - )), - ); + self.type_info_map_by_ms_name + .insert(ms_key, Arc::new(type_info.clone())); let string_key = (namespace.original.clone(), type_name.original.clone()); self.type_info_map_by_name - .insert(string_key, type_info.clone()); - } else { - let type_id = type_info.type_id; - if self.serializer_map.contains_key(&type_id) { - return Err(Error::TypeError( - format!("TypeId {:?} already registered_by_id", type_id).into(), - )); - } - self.type_id_map.insert(rs_type_id, type_id); - self.serializer_map.insert( - type_id, - Arc::from(Harness::new( - write::, - read::, - write_data::, - read_data::, - to_serializer::, - )), - ); + .insert(string_key, Arc::new(type_info)); } Ok(()) } - pub fn register_serializer( + pub fn register_serializer_by_id( &mut self, - type_info: &TypeInfo, + id: u32, ) -> Result<(), Error> { + let actual_type_id = get_ext_actual_type_id(id, false); + let static_type_id = T::fory_static_type_id(); + if static_type_id != TypeId::EXT && static_type_id != TypeId::NAMED_EXT { + return Err(Error::not_allowed( + "register_serializer can only be used for ext and named_ext types", + )); + } + self.register_serializer::(id, actual_type_id, &EMPTY_STRING, &EMPTY_STRING) + } + + pub fn register_serializer_by_namespace( + &mut self, + namespace: &str, + type_name: &str, + ) -> Result<(), Error> { + let actual_type_id = get_ext_actual_type_id(0, true); + let static_type_id = T::fory_static_type_id(); + if static_type_id != TypeId::EXT && static_type_id != TypeId::NAMED_EXT { + return Err(Error::not_allowed( + "register_serializer can only be used for ext and named_ext types", + )); + } + self.register_serializer::(0, actual_type_id, namespace, type_name) + } + + fn register_internal_serializer( + &mut self, + type_id: TypeId, + ) -> Result<(), Error> { + self.register_serializer::(type_id as u32, type_id as u32, &EMPTY_STRING, &EMPTY_STRING) + } + + fn register_serializer( + &mut self, + id: u32, + actual_type_id: u32, + namespace: &str, + type_name: &str, + ) -> Result<(), Error> { + let register_by_name = !type_name.is_empty(); + if !register_by_name && id == 0 { + return Err(Error::not_allowed( + "Either id must be non-zero for ID registration, or type_name must be non-empty for name registration", + )); + } + fn write( this: &dyn Any, context: &mut WriteContext, - is_field: bool, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => Ok(v.fory_write(context, is_field)?), - None => todo!(), + Some(v) => { + Ok(v.fory_write(context, write_ref_info, write_type_info, has_generics)?) + } + None => Err(Error::type_error(format!( + "Cast type error when writing: {:?}", + T2::fory_static_type_id() + ))), } } fn read( context: &mut ReadContext, - is_field: bool, - skip_ref_flag: bool, + read_ref_info: bool, + read_type_info: bool, ) -> Result, Error> { - if skip_ref_flag { - match T2::fory_read_data(context, is_field) { - Ok(v) => Ok(Box::new(v)), - Err(e) => Err(e), - } - } else { - match T2::fory_read(context, is_field) { - Ok(v) => Ok(Box::new(v)), - Err(e) => Err(e), - } - } + Ok(Box::new(T2::fory_read( + context, + read_ref_info, + read_type_info, + )?)) } fn write_data( this: &dyn Any, context: &mut WriteContext, - is_field: bool, + has_generics: bool, ) -> Result<(), Error> { let this = this.downcast_ref::(); match this { - Some(v) => T2::fory_write_data(v, context, is_field), + Some(v) => T2::fory_write_data_generic(v, context, has_generics), None => todo!(), } } fn read_data( context: &mut ReadContext, - is_field: bool, ) -> Result, Error> { - match T2::fory_read_data(context, is_field) { + match T2::fory_read_data(context) { Ok(v) => Ok(Box::new(v)), Err(e) => Err(e), } @@ -549,62 +640,85 @@ impl TypeResolver { ) -> Result, Error> { match boxed_any.downcast::() { Ok(concrete) => Ok(Box::new(*concrete) as Box), - Err(_) => Err(Error::TypeError( - "Failed to downcast to concrete type".into(), - )), + Err(_) => Err(Error::type_error("Failed to downcast to concrete type")), } } + let harness = Harness::new( + write::, + read::, + write_data::, + read_data::, + to_serializer::, + ); + + let type_info = TypeInfo::new_with_empty_fields::( + self, + actual_type_id, + namespace, + type_name, + register_by_name, + harness, + )?; + let rs_type_id = std::any::TypeId::of::(); - if self.type_info_cache.contains_key(&rs_type_id) { - return Err(Error::TypeError( - format!("rs_struct:{:?} already registered", rs_type_id).into(), - )); + if self.type_info_map.contains_key(&rs_type_id) { + return Err(Error::type_error(format!( + "rs_struct:{:?} already registered", + rs_type_id + ))); } - self.type_info_cache.insert(rs_type_id, type_info.clone()); + + // Store in main map + self.type_info_map + .insert(rs_type_id, Arc::new(type_info.clone())); + + // Store by ID + self.type_info_map_by_id + .insert(type_info.type_id, Arc::new(type_info.clone())); + + // Store by name if registered by name if type_info.register_by_name { let namespace = &type_info.namespace; let type_name = &type_info.type_name; - let key = (namespace.clone(), type_name.clone()); - if self.name_serializer_map.contains_key(&key) { - return Err(Error::InvalidData( - format!( - "Namespace:{:?} Name:{:?} already registered_by_name", - namespace, type_name - ) - .into(), - )); + let ms_key = (namespace.clone(), type_name.clone()); + if self.type_info_map_by_ms_name.contains_key(&ms_key) { + return Err(Error::invalid_data(format!( + "Namespace:{:?} Name:{:?} already registered_by_name", + namespace, type_name + ))); } - self.type_name_map.insert(rs_type_id, key.clone()); - self.name_serializer_map.insert( - key, - Arc::from(Harness::new( - write::, - read::, - write_data::, - read_data::, - to_serializer::, - )), - ); - } else { - let type_id = type_info.type_id; - if self.serializer_map.contains_key(&type_id) { - return Err(Error::TypeError( - format!("TypeId {:?} already registered_by_id", type_id).into(), - )); - } - self.type_id_map.insert(rs_type_id, type_id); - self.serializer_map.insert( - type_id, - Arc::from(Harness::new( - write::, - read::, - write_data::, - read_data::, - to_serializer::, - )), - ); + self.type_info_map_by_ms_name + .insert(ms_key, Arc::new(type_info.clone())); + let string_key = (namespace.original.clone(), type_name.original.clone()); + self.type_info_map_by_name + .insert(string_key, Arc::new(type_info)); } Ok(()) } + + /// Register a generic trait type like List, Map, Set + pub fn register_generic_trait( + &mut self, + ) -> Result<(), Error> { + let rs_type_id = std::any::TypeId::of::(); + if self.type_info_map.contains_key(&rs_type_id) { + return Err(Error::type_error(format!( + "Type:{:?} already registered", + rs_type_id + ))); + } + let type_id = T::fory_static_type_id(); + if type_id != TypeId::LIST && type_id != TypeId::MAP && type_id != TypeId::SET { + return Err(Error::not_allowed(format!( + "register_generic_trait can only be used for generic trait types: List, Map, Set, but got type {}", + type_id as u32 + ))); + } + self.register_internal_serializer::(type_id) + } + + pub(crate) fn set_compatible(&mut self, compatible: bool) { + self.compatible = compatible; + } } diff --git a/rust/fory-core/src/row/row.rs b/rust/fory-core/src/row/row.rs index 69795ce52d..c215829c37 100644 --- a/rust/fory-core/src/row/row.rs +++ b/rust/fory-core/src/row/row.rs @@ -101,9 +101,9 @@ impl Row<'_> for NaiveDate { let days = LittleEndian::read_u32(bytes); EPOCH .checked_add_days(Days::new(days.into())) - .ok_or(Error::InvalidData( - format!("Date out of range, {days} days since epoch").into(), - )) + .ok_or(Error::invalid_data(format!( + "Date out of range, {days} days since epoch" + ))) } } @@ -119,9 +119,9 @@ impl Row<'_> for NaiveDateTime { let timestamp = LittleEndian::read_u64(bytes); DateTime::from_timestamp_millis(timestamp as i64) .map(|dt| dt.naive_utc()) - .ok_or(Error::InvalidData( - format!("Date out of range, timestamp:{timestamp}").into(), - )) + .ok_or(Error::invalid_data(format!( + "Date out of range, timestamp:{timestamp}" + ))) } } diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index 1bbfd5d3a8..615d86f4e3 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -15,41 +15,28 @@ // specific language governing permissions and limitations // under the License. +use crate::ensure; use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; +use crate::serializer::util::write_dyn_data_generic; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; +use crate::types::TypeId; use std::any::Any; use std::rc::Rc; use std::sync::Arc; -/// Helper function to serialize a `Box` -pub fn serialize_any_box( - any_box: &Box, - context: &mut WriteContext, - is_field: bool, -) -> Result<(), Error> { - context.writer.write_i8(RefFlag::NotNullValue as i8); - - let concrete_type_id = (**any_box).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id)?; - let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**any_box, context, is_field) -} - /// Helper function to deserialize to `Box` pub fn deserialize_any_box(context: &mut ReadContext) -> Result, Error> { context.inc_depth()?; let ref_flag = context.reader.read_i8()?; if ref_flag != RefFlag::NotNullValue as i8 { - return Err(Error::InvalidRef( - "Expected NotNullValue for Box".into(), - )); + return Err(Error::invalid_ref("Expected NotNullValue for Box")); } - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let result = deserializer_fn(context, true); + let typeinfo = context.read_any_typeinfo()?; + let deserializer_fn = typeinfo.get_harness().get_read_data_fn(); + let result = deserializer_fn(context); context.dec_depth(); result } @@ -61,44 +48,98 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - serialize_any_box(self, context, is_field) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_typeinfo: bool, + has_generics: bool, + ) -> Result<(), Error> { + write_box_any( + self.as_ref(), + context, + write_ref_info, + write_typeinfo, + has_generics, + ) + } + + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + let concrete_type_id = (**self).type_id(); + let typeinfo = context.get_type_info(&concrete_type_id)?; + let serializer_fn = typeinfo.get_harness().get_write_data_fn(); + serializer_fn(&**self, context, has_generics) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - serialize_any_box(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { - deserialize_any_box(context) + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_box_any(context, read_ref_info, read_type_info, None) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - deserialize_any_box(context) + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_box_any(context, read_ref_info, false, Some(type_info)) + } + + fn fory_read_data(_: &mut ReadContext) -> Result { + panic!( + "fory_read_data should not be called directly on polymorphic Rc trait object" + ); } fn fory_get_type_id(_: &TypeResolver) -> Result { - unreachable!("Box has no static type ID - use fory_type_id_dyn") + Err(Error::type_error( + "Box has no static type ID - use fory_type_id_dyn", + )) } fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); type_resolver .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::TypeError("Type not registered".into())) + .ok_or_else(|| Error::type_error("Type not registered")) + } + + fn fory_concrete_type_id(&self) -> std::any::TypeId { + (**self).type_id() } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { - // Rc is polymorphic - type info is written per element + fn fory_is_shared_ref() -> bool { + false + } + + fn fory_static_type_id() -> TypeId { + TypeId::UNKNOWN + } + + fn fory_write_type_info(_context: &mut WriteContext) -> Result<(), Error> { + // Box is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { - // Rc is polymorphic - type info is read per element + fn fory_read_type_info(_context: &mut ReadContext) -> Result<(), Error> { + // Box is polymorphic - type info is read per element Ok(()) } @@ -107,6 +148,58 @@ impl Serializer for Box { } } +pub fn write_box_any( + value: &dyn Any, + context: &mut WriteContext, + write_ref_info: bool, + write_typeinfo: bool, + has_generics: bool, +) -> Result<(), Error> { + if write_ref_info { + context.writer.write_i8(RefFlag::NotNullValue as i8); + } + let concrete_type_id = value.type_id(); + let typeinfo = if write_typeinfo { + context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)? + } else { + context.get_type_info(&concrete_type_id)? + }; + let serializer_fn = typeinfo.get_harness().get_write_data_fn(); + serializer_fn(value, context, has_generics) +} + +pub fn read_box_any( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + context.inc_depth()?; + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + RefFlag::NotNullValue as i8 + }; + if ref_flag != RefFlag::NotNullValue as i8 { + return Err(Error::invalid_data( + "Expected NotNullValue for Box", + )); + } + let typeinfo = if let Some(type_info) = type_info { + type_info + } else { + ensure!( + read_type_info, + Error::invalid_data("Type info must be read for Box") + ); + context.read_any_typeinfo()? + }; + let deserializer_fn = typeinfo.get_harness().get_read_data_fn(); + let result = deserializer_fn(context); + context.dec_depth(); + result +} + impl ForyDefault for Rc { fn fory_default() -> Self { Rc::new(()) @@ -114,85 +207,107 @@ impl ForyDefault for Rc { } impl Serializer for Rc { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - if !context - .ref_writer - .try_write_rc_ref(&mut context.writer, self) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if !write_ref_info + || !context + .ref_writer + .try_write_rc_ref(&mut context.writer, self) { - let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id)?; - let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, context, is_field)? - }; + let concrete_type_id: std::any::TypeId = (**self).type_id(); + let write_data_fn = if write_type_info { + let typeinfo = + context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)?; + typeinfo.get_harness().get_write_data_fn() + } else { + context + .get_type_info(&concrete_type_id)? + .get_harness() + .get_write_data_fn() + }; + write_data_fn(&**self, context, has_generics)?; + } Ok(()) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - self.fory_write(context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_dyn_data_generic(self, context, false) } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_dyn_data_generic(self, context, has_generics) + } - match ref_flag { - RefFlag::Null => Err(Error::InvalidRef("Rc cannot be null".into())), - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - context - .ref_reader - .get_rc_ref::(ref_id) - .ok_or_else(|| { - Error::InvalidData( - format!("Rc reference {} not found", ref_id).into(), - ) - }) - } - RefFlag::NotNullValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; - context.dec_depth(); - Ok(Rc::::from(boxed)) - } - RefFlag::RefValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; - context.dec_depth(); - let rc: Rc = Rc::from(boxed); - context.ref_reader.store_rc_ref(rc.clone()); - Ok(rc) - } - } + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_rc_any(context, read_ref_info, read_type_info, None) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_rc_any(context, read_ref_info, false, Some(type_info)) + } + + fn fory_read_data(_: &mut ReadContext) -> Result { + panic!( + "fory_read_data should not be called directly on polymorphic Rc trait object", + stringify!($trait_name) + ); } fn fory_get_type_id(_: &TypeResolver) -> Result { - unreachable!("Rc has no static type ID - use fory_type_id_dyn") + Err(Error::type_error( + "Rc has no static type ID - use fory_type_id_dyn", + )) } fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); type_resolver .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::TypeError("Type not registered".into())) + .ok_or_else(|| Error::type_error("Type not registered")) + } + + fn fory_concrete_type_id(&self) -> std::any::TypeId { + (**self).type_id() + } + + fn fory_is_shared_ref() -> bool { + true } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_static_type_id() -> TypeId { + TypeId::UNKNOWN + } + + fn fory_write_type_info(_context: &mut WriteContext) -> Result<(), Error> { // Rc is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext) -> Result<(), Error> { // Rc is polymorphic - type info is read per element Ok(()) } @@ -202,6 +317,57 @@ impl Serializer for Rc { } } +pub fn read_rc_any( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Err(Error::invalid_ref("Rc cannot be null")), + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + context + .ref_reader + .get_rc_ref::(ref_id) + .ok_or_else(|| { + Error::invalid_data(format!("Rc reference {} not found", ref_id)) + }) + } + RefFlag::NotNullValue => { + context.inc_depth()?; + let typeinfo = if read_type_info { + context.read_any_typeinfo()? + } else { + type_info.ok_or_else(|| Error::type_error("No type info found for read"))? + }; + let read_data_fn = typeinfo.get_harness().get_read_data_fn(); + let boxed = read_data_fn(context)?; + context.dec_depth(); + Ok(Rc::::from(boxed)) + } + RefFlag::RefValue => { + context.inc_depth()?; + let typeinfo = if read_type_info { + context.read_any_typeinfo()? + } else { + type_info.ok_or_else(|| Error::type_error("No type info found for read"))? + }; + let read_data_fn = typeinfo.get_harness().get_read_data_fn(); + let boxed = read_data_fn(context)?; + context.dec_depth(); + let rc: Rc = Rc::from(boxed); + context.ref_reader.store_rc_ref(rc.clone()); + Ok(rc) + } + } +} + impl ForyDefault for Arc { fn fory_default() -> Self { Arc::new(()) @@ -209,85 +375,108 @@ impl ForyDefault for Arc { } impl Serializer for Arc { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - if !context - .ref_writer - .try_write_arc_ref(&mut context.writer, self) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if !write_ref_info + || !context + .ref_writer + .try_write_arc_ref(&mut context.writer, self) { - let concrete_type_id = (**self).type_id(); - let harness = context.write_any_typeinfo(concrete_type_id)?; - let serializer_fn = harness.get_write_data_fn(); - serializer_fn(&**self, context, is_field)?; + let concrete_type_id: std::any::TypeId = (**self).type_id(); + if write_type_info { + let typeinfo = + context.write_any_typeinfo(TypeId::UNKNOWN as u32, concrete_type_id)?; + let serializer_fn = typeinfo.get_harness().get_write_data_fn(); + serializer_fn(&**self, context, has_generics)?; + } else { + let serializer_fn = context + .get_type_info(&concrete_type_id)? + .get_harness() + .get_write_data_fn(); + serializer_fn(&**self, context, has_generics)?; + } } Ok(()) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - self.fory_write(context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_dyn_data_generic(self, context, false) } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_dyn_data_generic(self, context, has_generics) + } - match ref_flag { - RefFlag::Null => Err(Error::InvalidRef("Arc cannot be null".into())), - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - context - .ref_reader - .get_arc_ref::(ref_id) - .ok_or_else(|| { - Error::InvalidData( - format!("Arc reference {} not found", ref_id).into(), - ) - }) - } - RefFlag::NotNullValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; - context.dec_depth(); - Ok(Arc::::from(boxed)) - } - RefFlag::RefValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed = deserializer_fn(context, true)?; - context.dec_depth(); - let arc: Arc = Arc::from(boxed); - context.ref_reader.store_arc_ref(arc.clone()); - Ok(arc) - } - } + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_arc_any(context, read_ref_info, read_type_info, None) + } + + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_arc_any(context, read_ref_info, false, Some(type_info)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data(_: &mut ReadContext) -> Result { + panic!( + "fory_read_data should not be called directly on polymorphic Rc trait object", + stringify!($trait_name) + ); } fn fory_get_type_id(_type_resolver: &TypeResolver) -> Result { - unreachable!("Arc has no static type ID - use fory_type_id_dyn") + Err(Error::type_error( + "Arc has no static type ID - use fory_type_id_dyn", + )) } fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { let concrete_type_id = (**self).type_id(); type_resolver .get_fory_type_id(concrete_type_id) - .ok_or_else(|| Error::TypeError("Type not registered".into())) + .ok_or_else(|| Error::type_error("Type not registered")) + } + + fn fory_concrete_type_id(&self) -> std::any::TypeId { + (**self).type_id() } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_is_shared_ref() -> bool { + true + } + + fn fory_static_type_id() -> TypeId { + TypeId::UNKNOWN + } + + fn fory_write_type_info(_context: &mut WriteContext) -> Result<(), Error> { // Arc is polymorphic - type info is written per element Ok(()) } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { + fn fory_read_type_info(_context: &mut ReadContext) -> Result<(), Error> { // Arc is polymorphic - type info is read per element Ok(()) } @@ -296,3 +485,56 @@ impl Serializer for Arc { &**self } } + +pub fn read_arc_any( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Err(Error::invalid_ref("Arc cannot be null")), + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + context + .ref_reader + .get_arc_ref::(ref_id) + .ok_or_else(|| { + Error::invalid_data(format!("Arc reference {} not found", ref_id)) + }) + } + RefFlag::NotNullValue => { + context.inc_depth()?; + let typeinfo = if read_type_info { + context.read_any_typeinfo()? + } else { + type_info + .ok_or_else(|| Error::type_error("No type info found for read Arc"))? + }; + let read_data_fn = typeinfo.get_harness().get_read_data_fn(); + let boxed = read_data_fn(context)?; + context.dec_depth(); + Ok(Arc::::from(boxed)) + } + RefFlag::RefValue => { + context.inc_depth()?; + let typeinfo = if read_type_info { + context.read_any_typeinfo()? + } else { + type_info + .ok_or_else(|| Error::type_error("No type info found for read Arc"))? + }; + let read_data_fn = typeinfo.get_harness().get_read_data_fn(); + let boxed = read_data_fn(context)?; + context.dec_depth(); + let arc: Arc = Arc::from(boxed); + context.ref_reader.store_arc_ref(arc.clone()); + Ok(arc) + } + } +} diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index e4cbda85e3..33eddff2c6 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -17,9 +17,10 @@ use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; +use crate::types::TypeId; use std::sync::Arc; impl Serializer for Arc { @@ -27,63 +28,75 @@ impl Serializer for Arc true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - if !context - .ref_writer - .try_write_arc_ref(&mut context.writer, self) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if !write_ref_info + || !context + .ref_writer + .try_write_arc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), context, is_field)? - }; - Ok(()) + if T::fory_is_shared_ref() || T::fory_is_polymorphic() { + let inner_write_ref = T::fory_is_shared_ref(); + return T::fory_write( + &**self, + context, + inner_write_ref, + write_type_info, + has_generics, + ); + } + if write_type_info { + T::fory_write_type_info(context)?; + } + T::fory_write_data_generic(self, context, has_generics) + } else { + Ok(()) + } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - // When Arc is nested inside another shared ref (like Rc>), - // the outer ref calls fory_write_data on the inner Arc. - // We still need to track the Arc's own references here. - self.fory_write(context, is_field) + fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { + panic!("Arc should be written using `fory_write` to handle reference tracking properly"); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { + panic!("Arc should be written using `fory_write` to handle reference tracking properly"); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) + } - Ok(match ref_flag { - RefFlag::Null => Err(Error::InvalidRef("Arc cannot be null".into()))?, - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - context - .ref_reader - .get_arc_ref::(ref_id) - .ok_or(Error::InvalidData( - format!("Arc reference {ref_id} not found").into(), - ))? - } - RefFlag::NotNullValue => { - let inner = T::fory_read_data(context, is_field)?; - Arc::new(inner) - } - RefFlag::RefValue => { - let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(context, is_field)?; - let arc = Arc::new(inner); - context.ref_reader.store_arc_ref_at(ref_id, arc.clone()); - arc - } - }) + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + // if read_ref_info is false, shared refs will be deserialized into new objects each time. + read_arc(context, read_ref_info, read_type_info, None) + } + + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + typeinfo: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_arc(context, read_ref_info, false, Some(typeinfo)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - // When Arc is nested inside another shared ref, fory_read_data is called. - // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(context, is_field) + fn fory_read_data(_: &mut ReadContext) -> Result { + panic!("Arc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly"); } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } fn fory_reserved_space() -> usize { @@ -100,11 +113,68 @@ impl Serializer for Arc (**self).fory_type_id_dyn(type_resolver) } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } } +fn read_arc( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + typeinfo: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Err(Error::invalid_ref("Arc cannot be null")), + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + context + .ref_reader + .get_arc_ref::(ref_id) + .ok_or_else(|| Error::invalid_ref(format!("Arc reference {ref_id} not found"))) + } + RefFlag::NotNullValue => { + let inner = read_arc_inner::(context, read_type_info, typeinfo)?; + Ok(Arc::new(inner)) + } + RefFlag::RefValue => { + let ref_id = context.ref_reader.reserve_ref_id(); + let inner = read_arc_inner::(context, read_type_info, typeinfo)?; + let arc = Arc::new(inner); + context.ref_reader.store_arc_ref_at(ref_id, arc.clone()); + Ok(arc) + } + } +} + +fn read_arc_inner( + context: &mut ReadContext, + read_type_info: bool, + typeinfo: Option>, +) -> Result { + if let Some(typeinfo) = typeinfo { + let inner_read_ref = T::fory_is_shared_ref(); + return T::fory_read_with_type_info(context, inner_read_ref, typeinfo); + } + if T::fory_is_shared_ref() || T::fory_is_polymorphic() { + let inner_read_ref = T::fory_is_shared_ref(); + return T::fory_read(context, inner_read_ref, read_type_info); + } + if read_type_info { + T::fory_read_type_info(context)?; + } + T::fory_read_data(context) +} + impl ForyDefault for Arc { fn fory_default() -> Self { Arc::new(T::fory_default()) diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs index 9c1642bcd4..5dac06404f 100644 --- a/rust/fory-core/src/serializer/bool.rs +++ b/rust/fory-core/src/serializer/bool.rs @@ -19,19 +19,20 @@ use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; -use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; +use crate::serializer::util::read_basic_type_info; +use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; use std::mem; impl Serializer for bool { #[inline(always)] - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { context.writer.write_u8(if *self { 1 } else { 0 }); Ok(()) } #[inline(always)] - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(context.reader.read_u8()? == 1) } @@ -49,19 +50,24 @@ impl Serializer for bool { Ok(TypeId::BOOL as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::BOOL + } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self } #[inline(always)] - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::BOOL as u32); + Ok(()) } #[inline(always)] - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } diff --git a/rust/fory-core/src/serializer/box_.rs b/rust/fory-core/src/serializer/box_.rs index 6cf8730079..79bd3c7292 100644 --- a/rust/fory-core/src/serializer/box_.rs +++ b/rust/fory-core/src/serializer/box_.rs @@ -20,22 +20,26 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; +use crate::types::TypeId; impl Serializer for Box { - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Box::new(T::fory_read_data(context, is_field)?)) + fn fory_read_data(context: &mut ReadContext) -> Result + where + Self: Sized + ForyDefault, + { + Ok(Box::new(T::fory_read_data(context)?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_data(self.as_ref(), context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_data(self.as_ref(), context) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) } fn fory_reserved_space() -> usize { @@ -50,6 +54,10 @@ impl Serializer for Box { (**self).fory_type_id_dyn(type_resolver) } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs index 25469c5898..cad0b5333c 100644 --- a/rust/fory-core/src/serializer/collection.rs +++ b/rust/fory-core/src/serializer/collection.rs @@ -20,9 +20,9 @@ use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::{ForyDefault, Serializer}; -use crate::types::PRIMITIVE_ARRAY_TYPES; +use crate::types::{need_to_write_type_for_field, RefFlag, PRIMITIVE_ARRAY_TYPES}; -// const TRACKING_REF: u8 = 0b1; +const TRACKING_REF: u8 = 0b1; pub const HAS_NULL: u8 = 0b10; @@ -34,21 +34,16 @@ pub const IS_SAME_TYPE: u8 = 0b1000; pub fn write_collection_type_info( context: &mut WriteContext, - is_field: bool, collection_type_id: u32, ) -> Result<(), Error> { - if is_field { - return Ok(()); - } context.writer.write_varuint32(collection_type_id); Ok(()) } -pub fn write_collection<'a, T, I>( +pub fn write_collection_data<'a, T, I>( iter: I, - context: &mut WriteContext, - is_field: bool, + has_generics: bool, ) -> Result<(), Error> where T: Serializer + 'a, @@ -61,9 +56,13 @@ where if len == 0 { return Ok(()); } - let mut header = 0; + if T::fory_is_polymorphic() || T::fory_is_shared_ref() { + return write_collection_data_dyn_ref(iter, context, has_generics); + } + let mut header = IS_SAME_TYPE; let mut has_null = false; - let is_same_type = !T::fory_is_polymorphic(); + let elem_static_type_id = T::fory_static_type_id(); + let is_elem_declared = has_generics && !need_to_write_type_for_field(elem_static_type_id); if T::fory_is_option() { // iter.clone() is zero-copy for item in iter.clone() { @@ -76,56 +75,164 @@ where if has_null { header |= HAS_NULL; } - if is_field { + if is_elem_declared { + header |= DECL_ELEMENT_TYPE; + context.writer.write_u8(header); + } else { + context.writer.write_u8(header); + T::fory_write_type_info(context)?; + } + if !has_null { + for item in iter { + item.fory_write_data_generic(context, has_generics)?; + } + } else { + for item in iter { + if item.fory_is_none() { + context.writer.write_u8(RefFlag::Null as u8); + continue; + } + context.writer.write_u8(RefFlag::NotNullValue as u8); + item.fory_write_data_generic(context, has_generics)?; + } + } + + Ok(()) +} + +/// Slow but versatile collection serialization for dynamic trait object and shared/circular reference. +pub fn write_collection_data_dyn_ref<'a, T, I>( + iter: I, + context: &mut WriteContext, + has_generics: bool, +) -> Result<(), Error> +where + T: Serializer + 'a, + I: IntoIterator, + I::IntoIter: ExactSizeIterator + Clone, +{ + let elem_static_type_id = T::fory_static_type_id(); + let is_elem_declared = has_generics && !need_to_write_type_for_field(elem_static_type_id); + let elem_is_polymorphic = T::fory_is_polymorphic(); + let elem_is_shared_ref = T::fory_is_shared_ref(); + + let iter = iter.into_iter(); + let mut has_null = false; + let mut is_same_type = true; + let mut first_type_id: Option = None; + + for item in iter.clone() { + if item.fory_is_none() { + has_null = true; + } else if elem_is_polymorphic && is_same_type { + let concrete_id = item.fory_concrete_type_id(); + if let Some(first_id) = first_type_id { + if first_id != concrete_id { + is_same_type = false; + } + } else { + first_type_id = Some(concrete_id); + } + } + } + + if elem_is_polymorphic && is_same_type && first_type_id.is_none() { + // All elements are null for a polymorphic collection; fallback to per-element typing. + is_same_type = false; + } + + let mut header = 0u8; + if has_null { + header |= HAS_NULL; + } + if is_elem_declared { header |= DECL_ELEMENT_TYPE; } if is_same_type { header |= IS_SAME_TYPE; } + if elem_is_shared_ref { + header |= TRACKING_REF; + } + context.writer.write_u8(header); - T::fory_write_type_info(context, is_field)?; - // context.writer.reserve((T::reserved_space() + SIZE_OF_REF_AND_TYPE) * len); - if T::fory_is_polymorphic() || T::fory_is_shared_ref() { - // TOTO: make it xlang compatible - for item in iter { - item.fory_write(context, is_field)?; + + if is_same_type && !is_elem_declared { + if elem_is_polymorphic { + let type_id = first_type_id.ok_or_else(|| { + Error::type_error( + "Unable to determine concrete type for polymorphic collection elements", + ) + })?; + context.write_any_typeinfo(T::fory_static_type_id() as u32, type_id)?; + } else { + T::fory_write_type_info(context)?; + } + } + // Write elements data + if is_same_type { + // All elements are same type + if !has_null { + // No null elements + if elem_is_shared_ref { + for item in iter { + item.fory_write(context, true, false, has_generics)?; + } + } else { + for item in iter { + item.fory_write_data_generic(context, has_generics)?; + } + } + } else { + // Has null elements + for item in iter { + item.fory_write(context, true, false, has_generics)?; + } } - Ok(()) } else { - // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); - let skip_ref_flag = is_same_type && !has_null; - for item in iter { - crate::serializer::write_ref_info_data(item, context, is_field, skip_ref_flag, true)?; + // Different types (polymorphic elements with different types) + if !has_null { + // No null elements + if elem_is_shared_ref { + for item in iter { + item.fory_write(context, true, true, has_generics)?; + } + } else { + for item in iter { + item.fory_write(context, false, true, has_generics)?; + } + } + } else { + // Has null elements + for item in iter { + item.fory_write(context, true, true, has_generics)?; + } } - Ok(()) } + + Ok(()) } pub fn read_collection_type_info( context: &mut ReadContext, - is_field: bool, collection_type_id: u32, ) -> Result<(), Error> { - if is_field { - return Ok(()); - } let remote_collection_type_id = context.reader.read_varuint32()?; if PRIMITIVE_ARRAY_TYPES.contains(&remote_collection_type_id) { - return Err(Error::TypeError( + return Err(Error::type_error( "Vec belongs to the `number_array` type, \ and Vec> belongs to the `list` type. \ - You should not read data of type `number_array` as data of type `list`." - .into(), + You should not read data of type `number_array` as data of type `list`.", )); } ensure!( collection_type_id == remote_collection_type_id, - Error::TypeMismatch(collection_type_id, remote_collection_type_id) + Error::type_mismatch(collection_type_id, remote_collection_type_id) ); Ok(()) } -pub fn read_collection(context: &mut ReadContext) -> Result +pub fn read_collection_data(context: &mut ReadContext) -> Result where T: Serializer + ForyDefault, C: FromIterator, @@ -134,20 +241,84 @@ where if len == 0 { return Ok(C::from_iter(std::iter::empty())); } + if T::fory_is_polymorphic() || T::fory_is_shared_ref() { + return read_collection_data_dyn_ref(context, len); + } let header = context.reader.read_u8()?; let declared = (header & DECL_ELEMENT_TYPE) != 0; - T::fory_read_type_info(context, declared)?; + if !declared { + // context.read_any_typeinfo(); + // TODO check whether type info consistent with T + T::fory_read_type_info(context)?; + } let has_null = (header & HAS_NULL) != 0; - let is_same_type = (header & IS_SAME_TYPE) != 0; - if T::fory_is_polymorphic() || T::fory_is_shared_ref() { + ensure!( + (header & IS_SAME_TYPE) != 0, + Error::type_error("Type inconsistent, target type is not polymorphic") + ); + if !has_null { (0..len) - .map(|_| T::fory_read(context, declared)) + .map(|_| T::fory_read_data(context)) .collect::>() } else { - let skip_ref_flag = is_same_type && !has_null; - // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); (0..len) - .map(|_| crate::serializer::read_ref_info_data(context, declared, skip_ref_flag, true)) + .map(|_| { + let flag = context.reader.read_i8()?; + if flag == RefFlag::Null as i8 { + return Ok(T::fory_default()); + } + T::fory_read_data(context) + }) + .collect::>() + } +} + +/// Slow but versatile collection deserialization for dynamic trait object and shared/circular reference. +pub fn read_collection_data_dyn_ref(context: &mut ReadContext, len: u32) -> Result +where + T: Serializer + ForyDefault, + C: FromIterator, +{ + // Read header + let header = context.reader.read_u8()?; + let is_track_ref = (header & TRACKING_REF) != 0; + let is_same_type = (header & IS_SAME_TYPE) != 0; + let has_null = (header & HAS_NULL) != 0; + let is_declared = (header & DECL_ELEMENT_TYPE) != 0; + // Read elements + if is_same_type { + let type_info = if !is_declared { + context.read_any_typeinfo()? + } else { + let rs_type_id = std::any::TypeId::of::(); + context.get_type_resolver().get_type_info(&rs_type_id)? + }; + // All elements are same type + if is_track_ref { + (0..len) + .map(|_| T::fory_read_with_type_info(context, true, type_info.clone())) + .collect::>() + } else if !has_null { + // No null elements + (0..len) + .map(|_| T::fory_read_with_type_info(context, false, type_info.clone())) + .collect::>() + } else { + // Has null elements + (0..len) + .map(|_| { + let flag = context.reader.read_i8()?; + if flag == RefFlag::Null as i8 { + Ok(T::fory_default()) + } else { + T::fory_read_with_type_info(context, false, type_info.clone()) + } + }) + .collect::>() + } + } else { + (0..len) + .map(|_| T::fory_read(context, is_track_ref, true)) .collect::>() } } diff --git a/rust/fory-core/src/serializer/core.rs b/rust/fory-core/src/serializer/core.rs new file mode 100644 index 0000000000..f2ac8f7a6c --- /dev/null +++ b/rust/fory-core/src/serializer/core.rs @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::error::Error; +use crate::meta::FieldInfo; +use crate::resolver::context::{ReadContext, WriteContext}; +use crate::resolver::type_resolver::TypeInfo; +use crate::serializer::{bool, struct_}; +use crate::types::{RefFlag, TypeId}; +use crate::TypeResolver; +use std::any::Any; +use std::sync::Arc; + +pub trait ForyDefault: Sized { + fn fory_default() -> Self; +} + +// We can't add blanket impl for all T: Default because it conflicts with other impls. +// For example, upstream crates may add a new impl of trait `std::default::Default` for +// type `std::rc::Rc<(dyn std::any::Any + 'static)>` in future versions. +// impl ForyDefault for T { +// fn fory_default() -> Self { +// Default::default() +// } +// } + +pub trait Serializer: 'static { + /// Entry point of the serialization. + /// + /// # Parameters + /// + /// * `write_ref_info` - When `true`, WRITES reference flag (null/not-null/ref). When `false`, SKIPS writing ref flag. + /// * `write_type_info` - When `true`, WRITES type information. When `false`, SKIPS writing type info. + /// * `has_generics` - Indicates if the type has generic parameters (used for collection meta). + /// + /// # Notes + /// + /// Serializer for `option/rc/arc/weak` should override this method. + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> + where + Self: Sized, + { + if write_ref_info { + // skip check option/pointer, the Serializer for such types will override `fory_write`. + context.writer.write_i8(RefFlag::NotNullValue as i8); + } + if write_type_info { + // Serializer for dynamic types should override `fory_write` to write actual typeinfo. + Self::fory_write_type_info(context)?; + } + self.fory_write_data_generic(context, has_generics) + } + + /// Write the data into the buffer. Need to be implemented for collection/map. + /// For other types, just forward to `fory_write_data`. + #[allow(unused_variables)] + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + self.fory_write_data(context) + } + + /// Write the data into the buffer. Need to be implemented. + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error>; + + #[inline(always)] + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> + where + Self: Sized, + { + // Serializer for internal types should overwrite this method for faster performance. + let rs_type_id = std::any::TypeId::of::(); + context.write_any_typeinfo(Self::fory_static_type_id() as u32, rs_type_id)?; + Ok(()) + } + + /// Entry point of deserialization. + /// + /// # Parameters + /// + /// * `read_ref_info` - When `true`, READS reference flag from buffer. When `false`, SKIPS reading ref flag. + /// * `read_type_info` - When `true`, READS type information from buffer. When `false`, SKIPS reading type info. + /// + /// # Notes + /// + /// Unlike `fory_write`, read doesn't need `has_generics` - it's only used for writing meta. + /// The meta info is already written in the buffer, so read can parse it directly to decide how to read the data. + /// Serializer for `option/rc/arc/weak` should override this method. + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result + where + Self: Sized + ForyDefault, + { + if read_ref_info { + let ref_flag = context.reader.read_i8()?; + match ref_flag { + flag if flag == RefFlag::Null as i8 => Ok(Self::fory_default()), + flag if flag == RefFlag::NotNullValue as i8 || flag == RefFlag::RefValue as i8 => { + if read_type_info { + Self::fory_read_type_info(context)?; + } + Self::fory_read_data(context) + } + flag if flag == RefFlag::Ref as i8 => { + Err(Error::invalid_ref("Invalid ref, current type is not a ref")) + } + other => Err(Error::invalid_data(format!("Unknown ref flag: {}", other))), + } + } else { + if read_type_info { + Self::fory_read_type_info(context)?; + } + Self::fory_read_data(context) + } + } + + /// Deserialization with pre-read type information. + /// + /// # Parameters + /// + /// * `read_ref_info` - When `true`, READS reference flag from buffer. When `false`, SKIPS reading ref flag. + /// * `type_info` - Type information that has already been read ahead. DO NOT read type info again from buffer. + /// + /// # Notes + /// + /// The type info has already been read and is passed as an argument, so this method should NOT read type info from the buffer. + /// Default implementation ignores the typeinfo, only for morphic types supported by fory directly + /// or ext type registered by user. Dynamic trait types or reference types should override this method. + #[allow(unused_variables)] + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + // Default implementation ignores the provided typeinfo because the static type matches. + // Honor the supplied `read_ref_info` flag so callers can control whether ref metadata is present. + Self::fory_read(context, read_ref_info, false) + } + + fn fory_read_data(context: &mut ReadContext) -> Result + where + Self: Sized + ForyDefault; + + #[inline(always)] + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> + where + Self: Sized, + { + // Serializer for internal types should overwrite this method for faster performance. + context.read_any_typeinfo()?; + Ok(()) + } + + #[inline(always)] + fn fory_is_option() -> bool + where + Self: Sized, + { + false + } + + #[inline(always)] + fn fory_is_none(&self) -> bool { + false + } + + #[inline(always)] + fn fory_is_polymorphic() -> bool + where + Self: Sized, + { + false + } + + #[inline(always)] + fn fory_is_shared_ref() -> bool + where + Self: Sized, + { + false + } + + #[inline(always)] + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + // set to ext to simplify the user defined serializer. + // serializer for other types will override this method. + TypeId::EXT + } + + #[inline(always)] + fn fory_get_type_id(type_resolver: &TypeResolver) -> Result + where + Self: Sized, + { + Ok(type_resolver + .get_type_info(&std::any::TypeId::of::())? + .get_type_id()) + } + + fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result; + + #[inline(always)] + fn fory_concrete_type_id(&self) -> std::any::TypeId { + std::any::TypeId::of::() + } + + /// The possible max memory size of the type. + /// Used to reserve the buffer space to avoid reallocation, which may hurt performance. + #[inline(always)] + fn fory_reserved_space() -> usize + where + Self: Sized, + { + 0 + } + + fn as_any(&self) -> &dyn Any; +} + +pub trait StructSerializer: Serializer + 'static { + #[allow(unused_variables)] + fn fory_fields_info(type_resolver: &TypeResolver) -> Result, Error> { + Ok(Vec::default()) + } + + fn fory_type_index() -> u32 { + unimplemented!() + } + + fn fory_actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { + struct_::actual_type_id(type_id, register_by_name, compatible) + } + + fn fory_get_sorted_field_names() -> &'static [&'static str] { + &[] + } + + // only used by struct + fn fory_read_compatible( + context: &mut ReadContext, + type_info: Arc, + ) -> Result + where + Self: Sized; +} diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs index 7ef84a3055..c5b65476f0 100644 --- a/rust/fory-core/src/serializer/datetime.rs +++ b/rust/fory-core/src/serializer/datetime.rs @@ -19,31 +19,30 @@ use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; +use crate::serializer::util::read_basic_type_info; +use crate::serializer::ForyDefault; use crate::serializer::Serializer; -use crate::serializer::{read_type_info, write_type_info, ForyDefault}; use crate::types::TypeId; use crate::util::EPOCH; -use chrono::{DateTime, Days, NaiveDate, NaiveDateTime}; +use chrono::{NaiveDate, NaiveDateTime}; use std::mem; impl Serializer for NaiveDateTime { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { let dt = self.and_utc(); let micros = dt.timestamp() * 1_000_000 + dt.timestamp_subsec_micros() as i64; context.writer.write_i64(micros); Ok(()) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { let micros = context.reader.read_i64()?; - let seconds = micros / 1_000_000; - let subsec_micros = (micros % 1_000_000) as u32; - let nanos = subsec_micros * 1_000; - DateTime::from_timestamp(seconds, nanos) - .map(|dt| dt.naive_utc()) - .ok_or(Error::InvalidData( - format!("Date out of range, timestamp micros: {micros}").into(), - )) + use chrono::TimeDelta; + let duration = TimeDelta::microseconds(micros); + #[allow(deprecated)] + let epoch_datetime = NaiveDateTime::from_timestamp(0, 0); + let result = epoch_datetime + duration; + Ok(result) } fn fory_reserved_space() -> usize { @@ -58,33 +57,37 @@ impl Serializer for NaiveDateTime { Ok(TypeId::TIMESTAMP as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::TIMESTAMP + } + fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::TIMESTAMP as u32); + Ok(()) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } impl Serializer for NaiveDate { - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { let days_since_epoch = self.signed_duration_since(EPOCH).num_days(); context.writer.write_i32(days_since_epoch as i32); Ok(()) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { let days = context.reader.read_i32()?; - EPOCH - .checked_add_days(Days::new(days as u64)) - .ok_or(Error::InvalidData( - format!("Date out of range, {days} days since epoch").into(), - )) + use chrono::TimeDelta; + let duration = TimeDelta::days(days as i64); + let result = EPOCH + duration; + Ok(result) } fn fory_reserved_space() -> usize { @@ -99,16 +102,21 @@ impl Serializer for NaiveDate { Ok(TypeId::LOCAL_DATE as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::LOCAL_DATE + } + fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::LOCAL_DATE as u32); + Ok(()) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index 3cb3232112..2d07ec109a 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -17,7 +17,6 @@ use crate::ensure; use crate::error::Error; -use crate::meta::{MetaString, TypeMeta}; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::{RefFlag, TypeId}; @@ -32,25 +31,23 @@ pub fn actual_type_id(type_id: u32, register_by_name: bool, _compatible: bool) - } #[inline(always)] -pub fn type_def( - type_id: u32, - namespace: MetaString, - type_name: MetaString, - register_by_name: bool, -) -> (Vec, TypeMeta) { - let meta = TypeMeta::from_fields(type_id, namespace, type_name, register_by_name, vec![]); - let bytes = meta.to_bytes().unwrap(); - (bytes, meta) -} - -#[inline(always)] -pub fn write_type_info( +pub fn write( + this: &T, context: &mut WriteContext, - is_field: bool, + write_ref_info: bool, + write_type_info: bool, ) -> Result<(), Error> { - if is_field { - return Ok(()); + if write_ref_info { + context.writer.write_i8(RefFlag::NotNullValue as i8); } + if write_type_info { + T::fory_write_type_info(context)?; + } + this.fory_write_data(context) +} + +#[inline(always)] +pub fn write_type_info(context: &mut WriteContext) -> Result<(), Error> { let type_id = T::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); let is_named_enum = type_id & 0xff == TypeId::NAMED_ENUM as u32; @@ -62,7 +59,7 @@ pub fn write_type_info( let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; + let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); context.write_meta_string_bytes(&namespace)?; @@ -72,18 +69,40 @@ pub fn write_type_info( } #[inline(always)] -pub fn read_type_info( +pub fn read( context: &mut ReadContext, - is_field: bool, -) -> Result<(), Error> { - if is_field { - return Ok(()); + read_ref_info: bool, + read_type_info: bool, +) -> Result { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + RefFlag::NotNullValue as i8 + }; + if ref_flag == RefFlag::Null as i8 { + Ok(T::fory_default()) + } else if ref_flag == (RefFlag::NotNullValue as i8) || ref_flag == (RefFlag::RefValue as i8) { + if read_type_info { + T::fory_read_type_info(context)?; + } + T::fory_read_data(context) + } else if ref_flag == (RefFlag::Ref as i8) { + Err(Error::invalid_ref("Invalid ref, enum type is not a ref")) + } else { + Err(Error::invalid_data(format!( + "Unknown ref flag: {}", + ref_flag + ))) } +} + +#[inline(always)] +pub fn read_type_info(context: &mut ReadContext) -> Result<(), Error> { let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; let remote_type_id = context.reader.read_varuint32()?; ensure!( local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) + Error::type_mismatch(local_type_id, remote_type_id) ); let is_named_enum = local_type_id & 0xff == TypeId::NAMED_ENUM as u32; if !is_named_enum { @@ -97,37 +116,3 @@ pub fn read_type_info( } Ok(()) } - -#[inline(always)] -pub fn read_compatible(context: &mut ReadContext) -> Result { - T::fory_read_type_info(context, true)?; - T::fory_read_data(context, true) -} - -#[inline(always)] -pub fn write( - this: &T, - - context: &mut WriteContext, - is_field: bool, -) -> Result<(), Error> { - context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, is_field)?; - this.fory_write_data(context, is_field) -} - -#[inline(always)] -pub fn read( - context: &mut ReadContext, - is_field: bool, -) -> Result { - let ref_flag = context.reader.read_i8()?; - if ref_flag == RefFlag::Null as i8 { - Ok(T::fory_default()) - } else if ref_flag == (RefFlag::NotNullValue as i8) { - T::fory_read_type_info(context, false)?; - T::fory_read_data(context, is_field) - } else { - unimplemented!() - } -} diff --git a/rust/fory-core/src/serializer/heap.rs b/rust/fory-core/src/serializer/heap.rs index 7c9af1871a..f74b93997d 100644 --- a/rust/fory-core/src/serializer/heap.rs +++ b/rust/fory-core/src/serializer/heap.rs @@ -20,7 +20,8 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; use crate::serializer::collection::{ - read_collection, read_collection_type_info, write_collection, write_collection_type_info, + read_collection_data, read_collection_type_info, write_collection_data, + write_collection_type_info, }; use crate::serializer::{ForyDefault, Serializer}; @@ -29,20 +30,20 @@ use std::collections::BinaryHeap; use std::mem; impl Serializer for BinaryHeap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_collection_data(self, context, false) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + write_collection_type_info(context, TypeId::SET as u32) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data(context: &mut ReadContext) -> Result { + read_collection_data(context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_collection_type_info(context, TypeId::SET as u32) } fn fory_reserved_space() -> usize { @@ -57,6 +58,10 @@ impl Serializer for BinaryHeap { Ok(TypeId::SET as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::SET + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs index ba3a954aff..b562e5323a 100644 --- a/rust/fory-core/src/serializer/list.rs +++ b/rust/fory-core/src/serializer/list.rs @@ -27,7 +27,8 @@ use std::collections::{LinkedList, VecDeque}; use std::mem; use super::collection::{ - read_collection, read_collection_type_info, write_collection, write_collection_type_info, + read_collection_data, read_collection_type_info, write_collection_data, + write_collection_type_info, }; fn check_primitive() -> Option { @@ -44,31 +45,42 @@ fn check_primitive() -> Option { } impl Serializer for Vec { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { match check_primitive::() { Some(_) => primitive_list::fory_write_data(self, context), - None => write_collection(self, context, is_field), + None => write_collection_data(self, context, false), } } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { match check_primitive::() { - Some(type_id) => primitive_list::fory_write_type_info(context, is_field, type_id), - None => write_collection_type_info(context, is_field, TypeId::LIST as u32), + Some(_) => primitive_list::fory_write_data(self, context), + None => write_collection_data(self, context, has_generics), + } + } + + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + match check_primitive::() { + Some(type_id) => primitive_list::fory_write_type_info(context, type_id), + None => write_collection_type_info(context, TypeId::LIST as u32), } } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { match check_primitive::() { Some(_) => primitive_list::fory_read_data(context), - None => read_collection(context), + None => read_collection_data(context), } } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { match check_primitive::() { - Some(type_id) => primitive_list::fory_read_type_info(context, is_field, type_id), - None => read_collection_type_info(context, is_field, TypeId::LIST as u32), + Some(type_id) => primitive_list::fory_read_type_info(context, type_id), + None => read_collection_type_info(context, TypeId::LIST as u32), } } @@ -96,6 +108,16 @@ impl Serializer for Vec { }) } + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + match check_primitive::() { + Some(type_id) => type_id, + None => TypeId::LIST, + } + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -108,20 +130,28 @@ impl ForyDefault for Vec { } impl Serializer for VecDeque { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_collection_data(self, context, false) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection_type_info(context, is_field, TypeId::LIST as u32) + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_collection_data(self, context, has_generics) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + write_collection_type_info(context, TypeId::LIST as u32) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_collection_type_info(context, is_field, TypeId::LIST as u32) + fn fory_read_data(context: &mut ReadContext) -> Result { + read_collection_data(context) + } + + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_collection_type_info(context, TypeId::LIST as u32) } fn fory_reserved_space() -> usize { @@ -136,6 +166,10 @@ impl Serializer for VecDeque { Ok(TypeId::LIST as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::LIST + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -148,20 +182,28 @@ impl ForyDefault for VecDeque { } impl Serializer for LinkedList { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_collection_data(self, context, false) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection_type_info(context, is_field, TypeId::LIST as u32) + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_collection_data(self, context, has_generics) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + write_collection_type_info(context, TypeId::LIST as u32) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_collection_type_info(context, is_field, TypeId::LIST as u32) + fn fory_read_data(context: &mut ReadContext) -> Result { + read_collection_data(context) + } + + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_collection_type_info(context, TypeId::LIST as u32) } fn fory_reserved_space() -> usize { @@ -176,6 +218,10 @@ impl Serializer for LinkedList { Ok(TypeId::LIST as u32) } + fn fory_static_type_id() -> TypeId { + TypeId::LIST + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index 526ef8927e..37a075b8ea 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -18,13 +18,12 @@ use crate::ensure; use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; -use crate::serializer::{ - read_ref_info_data, read_type_info, write_ref_info_data, write_type_info, ForyDefault, - Serializer, -}; -use crate::types::{TypeId, SIZE_OF_REF_AND_TYPE}; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; +use crate::serializer::util::read_basic_type_info; +use crate::serializer::{ForyDefault, Serializer}; +use crate::types::{need_to_write_type_for_field, TypeId, SIZE_OF_REF_AND_TYPE}; use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; const MAX_CHUNK_SIZE: u8 = 255; @@ -35,63 +34,19 @@ const TRACKING_VALUE_REF: u8 = 0b1000; pub const VALUE_NULL: u8 = 0b10000; const DECL_VALUE_TYPE: u8 = 0b100000; -fn check_and_write_null( - context: &mut WriteContext, - is_field: bool, - key: &K, - value: &V, -) -> Result { - if key.fory_is_none() && value.fory_is_none() { - context.writer.write_u8(KEY_NULL | VALUE_NULL); - return Ok(true); - } - if key.fory_is_none() { - let mut chunk_header = KEY_NULL; - let skip_ref_flag; - if is_field { - skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_type_resolver())?; - chunk_header |= DECL_VALUE_TYPE; - } else { - skip_ref_flag = false; - chunk_header |= TRACKING_VALUE_REF; - } - context.writer.write_u8(chunk_header); - - write_ref_info_data(value, context, is_field, skip_ref_flag, false)?; - return Ok(true); - } - if value.fory_is_none() { - let mut chunk_header = VALUE_NULL; - let skip_ref_flag; - if is_field { - // skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); - skip_ref_flag = true; - chunk_header |= DECL_KEY_TYPE; - } else { - skip_ref_flag = false; - chunk_header |= TRACKING_KEY_REF; - } - context.writer.write_u8(chunk_header); - write_ref_info_data(key, context, is_field, skip_ref_flag, false)?; - return Ok(true); - } - Ok(false) -} - fn write_chunk_size(context: &mut WriteContext, header_offset: usize, size: u8) { context.writer.set_bytes(header_offset + 1, &[size]); } -fn write_map_data<'a, K, V, I>( +pub fn write_map_data<'a, K, V, I>( iter: I, length: usize, - context: &mut WriteContext, - is_field: bool, + has_generics: bool, ) -> Result<(), Error> where - K: Serializer + ForyDefault + 'a + Eq + std::hash::Hash, - V: Serializer + ForyDefault + 'a, + K: Serializer, + V: Serializer, I: Iterator, { context.writer.write_varuint32(length as u32); @@ -102,78 +57,495 @@ where + (V::fory_reserved_space() + SIZE_OF_REF_AND_TYPE) * length; context.writer.reserve(reserved_space); + if K::fory_is_polymorphic() + || K::fory_is_shared_ref() + || V::fory_is_polymorphic() + || V::fory_is_shared_ref() + { + return write_map_data_dyn_ref(iter, context, has_generics); + } let mut header_offset = 0; let mut pair_counter: u8 = 0; let mut need_write_header = true; - let mut skip_key_ref_flag = false; - let mut skip_val_ref_flag = false; + let key_static_type_id = K::fory_static_type_id(); + let val_static_type_id = V::fory_static_type_id(); + let is_key_declared = has_generics && !need_to_write_type_for_field(key_static_type_id); + let is_val_declared = has_generics && !need_to_write_type_for_field(val_static_type_id); for (key, value) in iter { - if need_write_header { - if check_and_write_null(context, is_field, key, value)? { + let key_is_none = key.fory_is_none(); + let value_is_none = value.fory_is_none(); + + if key_is_none || value_is_none { + if !need_write_header && pair_counter > 0 { + write_chunk_size(context, header_offset, pair_counter); + pair_counter = 0; + need_write_header = true; + } + + if key_is_none && value_is_none { + context.writer.write_u8(KEY_NULL | VALUE_NULL); + continue; + } + + if value_is_none { + let mut chunk_header = VALUE_NULL; + if is_key_declared { + chunk_header |= DECL_KEY_TYPE; + context.writer.write_u8(chunk_header); + } else { + context.writer.write_u8(chunk_header); + K::fory_write_type_info(context)?; + } + key.fory_write_data_generic(context, has_generics)?; continue; } + + // key is None, value is not + let mut chunk_header = KEY_NULL; + if is_val_declared { + chunk_header |= DECL_VALUE_TYPE; + context.writer.write_u8(chunk_header); + } else { + context.writer.write_u8(chunk_header); + V::fory_write_type_info(context)?; + } + value.fory_write_data_generic(context, has_generics)?; + continue; + } + + if need_write_header { header_offset = context.writer.len(); context.writer.write_i16(-1); - let mut chunk_header = 0; - if is_field { - chunk_header |= DECL_KEY_TYPE | DECL_VALUE_TYPE; + let mut chunk_header = 0u8; + if is_key_declared { + chunk_header |= DECL_KEY_TYPE; + } else { + K::fory_write_type_info(context)?; } - skip_key_ref_flag = !K::fory_is_polymorphic(); - skip_val_ref_flag = !V::fory_is_polymorphic(); - if !skip_key_ref_flag { - chunk_header |= TRACKING_KEY_REF; - } - if !skip_val_ref_flag { - chunk_header |= TRACKING_VALUE_REF; + if is_val_declared { + chunk_header |= DECL_VALUE_TYPE; + } else { + V::fory_write_type_info(context)?; } - K::fory_write_type_info(context, is_field)?; - V::fory_write_type_info(context, is_field)?; context.writer.set_bytes(header_offset, &[chunk_header]); need_write_header = false; } - if key.fory_is_none() || value.fory_is_none() { + + key.fory_write_data_generic(context, has_generics)?; + value.fory_write_data_generic(context, has_generics)?; + pair_counter += 1; + if pair_counter == MAX_CHUNK_SIZE { write_chunk_size(context, header_offset, pair_counter); pair_counter = 0; need_write_header = true; - check_and_write_null(context, is_field, key, value)?; - continue; } - if K::fory_is_polymorphic() || K::fory_is_shared_ref() { - key.fory_write(context, is_field)?; + } + if pair_counter > 0 { + write_chunk_size(context, header_offset, pair_counter); + } + Ok(()) +} + +/// slow but versatile map serialization for dynamic trait object and shared/circular reference. +fn write_map_data_dyn_ref<'a, K, V, I>( + iter: I, + context: &mut WriteContext, + has_generics: bool, +) -> Result<(), Error> +where + K: Serializer, + V: Serializer, + I: Iterator, +{ + let mut header_offset = 0; + let mut pair_counter: u8 = 0; + let mut need_write_header = true; + let key_static_type_id = K::fory_static_type_id(); + let val_static_type_id = V::fory_static_type_id(); + let is_key_declared = has_generics && !need_to_write_type_for_field(key_static_type_id); + let is_val_declared = has_generics && !need_to_write_type_for_field(val_static_type_id); + let key_is_polymorphic = K::fory_is_polymorphic(); + let val_is_polymorphic = V::fory_is_polymorphic(); + let key_is_shared_ref = K::fory_is_shared_ref(); + let val_is_shared_ref = V::fory_is_shared_ref(); + + // Track the current chunk's key and value types (for polymorphic types) + let mut current_key_type_id: Option = None; + let mut current_val_type_id: Option = None; + + for (key, value) in iter { + // Handle null key/value entries (write as separate single-entry chunks) + if key.fory_is_none() || value.fory_is_none() { + // Finish current chunk if any + if pair_counter > 0 { + write_chunk_size(context, header_offset, pair_counter); + pair_counter = 0; + need_write_header = true; + } + + if key.fory_is_none() && value.fory_is_none() { + context.writer.write_u8(KEY_NULL | VALUE_NULL); + continue; + } else if value.fory_is_none() { + let mut chunk_header = VALUE_NULL; + if key_is_shared_ref { + chunk_header |= TRACKING_KEY_REF; + } + if is_key_declared && !key_is_polymorphic { + chunk_header |= DECL_KEY_TYPE; + context.writer.write_u8(chunk_header); + } else { + context.writer.write_u8(chunk_header); + if key_is_polymorphic { + context.write_any_typeinfo( + K::fory_static_type_id() as u32, + key.fory_concrete_type_id(), + )?; + } else { + K::fory_write_type_info(context)?; + } + } + if key_is_shared_ref { + key.fory_write(context, true, false, has_generics)?; + } else { + key.fory_write_data_generic(context, has_generics)?; + } + continue; + } else { + // key.fory_is_none() + let mut chunk_header = KEY_NULL; + if val_is_shared_ref { + chunk_header |= TRACKING_VALUE_REF; + } + if is_val_declared && !val_is_polymorphic { + chunk_header |= DECL_VALUE_TYPE; + context.writer.write_u8(chunk_header); + } else { + context.writer.write_u8(chunk_header); + if val_is_polymorphic { + context.write_any_typeinfo( + V::fory_static_type_id() as u32, + value.fory_concrete_type_id(), + )?; + } else { + V::fory_write_type_info(context)?; + } + } + if val_is_shared_ref { + value.fory_write(context, true, false, has_generics)?; + } else { + value.fory_write_data_generic(context, has_generics)?; + } + continue; + } + } + + // Get type IDs for polymorphic types + let key_type_id = if key_is_polymorphic { + Some(key.fory_type_id_dyn(context.get_type_resolver())?) + } else { + None + }; + let val_type_id = if val_is_polymorphic { + Some(value.fory_type_id_dyn(context.get_type_resolver())?) + } else { + None + }; + + // Check if we need to start a new chunk due to type changes + let types_changed = if key_is_polymorphic || val_is_polymorphic { + key_type_id != current_key_type_id || val_type_id != current_val_type_id } else { - write_ref_info_data(key, context, is_field, skip_key_ref_flag, true)?; + false + }; + + if need_write_header || types_changed { + // Finish previous chunk if types changed + if types_changed && pair_counter > 0 { + write_chunk_size(context, header_offset, pair_counter); + pair_counter = 0; + } + + // Write new chunk header + header_offset = context.writer.len(); + context.writer.write_i16(-1); // Placeholder for header and size + + let mut chunk_header = 0u8; + + // Set key flags + if key_is_shared_ref { + chunk_header |= TRACKING_KEY_REF; + } + if is_key_declared && !key_is_polymorphic { + chunk_header |= DECL_KEY_TYPE; + } else { + // Write type info for key + if key_is_polymorphic { + context.write_any_typeinfo( + K::fory_static_type_id() as u32, + key.fory_concrete_type_id(), + )?; + } else { + K::fory_write_type_info(context)?; + } + } + + // Set value flags + if val_is_shared_ref { + chunk_header |= TRACKING_VALUE_REF; + } + if is_val_declared && !val_is_polymorphic { + chunk_header |= DECL_VALUE_TYPE; + } else { + // Write type info for value + if val_is_polymorphic { + context.write_any_typeinfo( + V::fory_static_type_id() as u32, + value.fory_concrete_type_id(), + )?; + } else { + V::fory_write_type_info(context)?; + } + } + + context.writer.set_bytes(header_offset, &[chunk_header]); + need_write_header = false; + current_key_type_id = key_type_id; + current_val_type_id = val_type_id; + } + + // Write key-value pair + if key_is_shared_ref { + key.fory_write(context, true, false, has_generics)?; + } else { + key.fory_write_data_generic(context, has_generics)?; } - if V::fory_is_polymorphic() || V::fory_is_shared_ref() { - value.fory_write(context, is_field)?; + if val_is_shared_ref { + value.fory_write(context, true, false, has_generics)?; } else { - write_ref_info_data(value, context, is_field, skip_val_ref_flag, true)?; + value.fory_write_data_generic(context, has_generics)?; } pair_counter += 1; if pair_counter == MAX_CHUNK_SIZE { write_chunk_size(context, header_offset, pair_counter); pair_counter = 0; need_write_header = true; + current_key_type_id = None; + current_val_type_id = None; } } + + // Write final chunk size if any if pair_counter > 0 { write_chunk_size(context, header_offset, pair_counter); } + Ok(()) } +/// Macro to generate read_*_data_dyn_ref functions for HashMap and BTreeMap. +/// This avoids code duplication while maintaining zero runtime cost. +macro_rules! impl_read_map_dyn_ref { + ($fn_name:ident, $map_type:ty, $($extra_trait_bounds:tt)*) => { + fn $fn_name( + context: &mut ReadContext, + mut map: $map_type, + length: u32, + ) -> Result<$map_type, Error> + where + K: Serializer + ForyDefault + $($extra_trait_bounds)*, + V: Serializer + ForyDefault, + { + let key_is_polymorphic = K::fory_is_polymorphic(); + let val_is_polymorphic = V::fory_is_polymorphic(); + let key_is_shared_ref = K::fory_is_shared_ref(); + let val_is_shared_ref = V::fory_is_shared_ref(); + + let mut len_counter = 0u32; + + while len_counter < length { + let header = context.reader.read_u8()?; + + // Handle null key/value entries + if header & KEY_NULL != 0 && header & VALUE_NULL != 0 { + // Both key and value are null + map.insert(K::fory_default(), V::fory_default()); + len_counter += 1; + continue; + } + + if header & KEY_NULL != 0 { + // Null key, non-null value + let value_declared = (header & DECL_VALUE_TYPE) != 0; + let track_value_ref = (header & TRACKING_VALUE_REF) != 0; + + // Determine value type info (if any) + let value_type_info: Option> = if !value_declared { + if val_is_polymorphic { + Some(context.read_any_typeinfo()?) + } else { + V::fory_read_type_info(context)?; + None + } + } else { + None + }; + + // Read value payload + let read_ref = val_is_shared_ref || track_value_ref; + let value = if let Some(type_info) = value_type_info { + V::fory_read_with_type_info(context, read_ref, type_info)? + } else if read_ref { + V::fory_read(context, read_ref, false)? + } else { + V::fory_read_data(context)? + }; + + map.insert(K::fory_default(), value); + len_counter += 1; + continue; + } + + if header & VALUE_NULL != 0 { + // Non-null key, null value + let key_declared = (header & DECL_KEY_TYPE) != 0; + let track_key_ref = (header & TRACKING_KEY_REF) != 0; + + let key_type_info: Option> = if !key_declared { + if key_is_polymorphic { + Some(context.read_any_typeinfo()?) + } else { + K::fory_read_type_info(context)?; + None + } + } else { + None + }; + + let read_ref = key_is_shared_ref || track_key_ref; + let key = if let Some(type_info) = key_type_info { + K::fory_read_with_type_info(context, read_ref, type_info)? + } else if read_ref { + K::fory_read(context, read_ref, false)? + } else { + K::fory_read_data(context)? + }; + + map.insert(key, V::fory_default()); + len_counter += 1; + continue; + } + + // Non-null key and value chunk + let chunk_size = context.reader.read_u8()?; + let key_declared = (header & DECL_KEY_TYPE) != 0; + let value_declared = (header & DECL_VALUE_TYPE) != 0; + let track_key_ref = (header & TRACKING_KEY_REF) != 0; + let track_value_ref = (header & TRACKING_VALUE_REF) != 0; + + let key_type_info: Option> = if !key_declared { + if key_is_polymorphic { + Some(context.read_any_typeinfo()?) + } else { + K::fory_read_type_info(context)?; + None + } + } else { + None + }; + let value_type_info: Option> = if !value_declared { + if val_is_polymorphic { + Some(context.read_any_typeinfo()?) + } else { + V::fory_read_type_info(context)?; + None + } + } else { + None + }; + + let cur_len = len_counter + chunk_size as u32; + ensure!( + cur_len <= length, + Error::invalid_data( + format!("current length {} exceeds total length {}", cur_len, length) + ) + ); + + // Read chunk_size pairs of key-value + let key_read_ref = key_is_shared_ref || track_key_ref; + let val_read_ref = val_is_shared_ref || track_value_ref; + for _ in 0..chunk_size { + let key = if let Some(type_info) = key_type_info.as_ref() { + K::fory_read_with_type_info(context, key_read_ref, type_info.clone())? + } else if key_read_ref { + K::fory_read(context, key_read_ref, false)? + } else { + K::fory_read_data(context)? + }; + + let value = if let Some(type_info) = value_type_info.as_ref() { + V::fory_read_with_type_info(context, val_read_ref, type_info.clone())? + } else if val_read_ref { + V::fory_read(context, val_read_ref, false)? + } else { + V::fory_read_data(context)? + }; + + map.insert(key, value); + } + + len_counter += chunk_size as u32; + } + + Ok(map) + } + }; +} + +// Generate read_hashmap_data_dyn_ref for HashMap +impl_read_map_dyn_ref!( + read_hashmap_data_dyn_ref, + HashMap, + Eq + std::hash::Hash +); + +// Generate read_btreemap_data_dyn_ref for BTreeMap +impl_read_map_dyn_ref!( + read_btreemap_data_dyn_ref, + BTreeMap, + Ord +); + impl Serializer for HashMap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_map_data(self.iter(), self.len(), context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, false) + } + + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, has_generics) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { let len = context.reader.read_varuint32()?; let mut map = HashMap::::with_capacity(len as usize); if len == 0 { return Ok(map); } + if K::fory_is_polymorphic() + || K::fory_is_shared_ref() + || V::fory_is_polymorphic() + || V::fory_is_shared_ref() + { + let map: HashMap = HashMap::with_capacity(len as usize); + return read_hashmap_data_dyn_ref(context, map, len); + } let mut len_counter = 0; loop { if len_counter == len { @@ -188,52 +560,41 @@ impl(context.get_type_resolver())? - } else { - false - }; - let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; + if !value_declared { + V::fory_read_type_info(context)?; + } + let value = V::fory_read_data(context)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { - let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? - } else { - false - }; - let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; + if !key_declared { + K::fory_read_type_info(context)?; + } + let key = K::fory_read_data(context)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8()?; - K::fory_read_type_info(context, key_declared)?; - V::fory_read_type_info(context, value_declared)?; - + if header & DECL_KEY_TYPE == 0 { + K::fory_read_type_info(context)?; + } + if header & DECL_VALUE_TYPE == 0 { + V::fory_read_type_info(context)?; + } let cur_len = len_counter + chunk_size as u32; ensure!( cur_len <= len, - Error::InvalidData( - format!("current length {} exceeds total length {}", cur_len, len).into() - ) + Error::invalid_data(format!( + "current length {} exceeds total length {}", + cur_len, len + )) ); - assert!(len_counter + chunk_size as u32 <= len); for _ in 0..chunk_size { - let key = if K::fory_is_polymorphic() { - K::fory_read(context, key_declared)? - } else { - // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); - read_ref_info_data(context, key_declared, true, true)? - }; - let value = if V::fory_is_polymorphic() { - V::fory_read(context, value_declared)? - } else { - // let skip_ref_flag = crate::serializer::get_skip_ref_flag::(context.get_fory()); - read_ref_info_data(context, value_declared, true, true)? - }; + let key = K::fory_read_data(context)?; + let value = V::fory_read_data(context)?; map.insert(key, value); } len_counter += chunk_size as u32; @@ -253,16 +614,24 @@ impl TypeId + where + Self: Sized, + { + TypeId::MAP + } + fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::MAP as u32); + Ok(()) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } @@ -275,16 +644,32 @@ impl ForyDefault for HashMap { impl Serializer for BTreeMap { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_map_data(self.iter(), self.len(), context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, false) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + write_map_data(self.iter(), self.len(), context, has_generics) + } + + fn fory_read_data(context: &mut ReadContext) -> Result { let len = context.reader.read_varuint32()?; let mut map = BTreeMap::::new(); if len == 0 { return Ok(map); } + if K::fory_is_polymorphic() + || K::fory_is_shared_ref() + || V::fory_is_polymorphic() + || V::fory_is_shared_ref() + { + let map: BTreeMap = BTreeMap::new(); + return read_btreemap_data_dyn_ref(context, map, len); + } let mut len_counter = 0; loop { if len_counter == len { @@ -299,42 +684,41 @@ impl(context.get_type_resolver())? - } else { - false - }; - let value = read_ref_info_data(context, value_declared, skip_ref_flag, false)?; + if !value_declared { + V::fory_read_type_info(context)?; + } + let value = V::fory_read_data(context)?; map.insert(K::fory_default(), value); len_counter += 1; continue; } if header & VALUE_NULL != 0 { - let skip_ref_flag = if key_declared { - crate::serializer::get_skip_ref_flag::(context.get_type_resolver())? - } else { - false - }; - let key = read_ref_info_data(context, key_declared, skip_ref_flag, false)?; + if !key_declared { + K::fory_read_type_info(context)?; + } + let key = K::fory_read_data(context)?; map.insert(key, V::fory_default()); len_counter += 1; continue; } let chunk_size = context.reader.read_u8()?; - K::fory_read_type_info(context, key_declared)?; - V::fory_read_type_info(context, value_declared)?; - assert!(len_counter + chunk_size as u32 <= len); + if header & DECL_KEY_TYPE == 0 { + K::fory_read_type_info(context)?; + } + if header & DECL_VALUE_TYPE == 0 { + V::fory_read_type_info(context)?; + } + let cur_len = len_counter + chunk_size as u32; + ensure!( + cur_len <= len, + Error::invalid_data(format!( + "current length {} exceeds total length {}", + cur_len, len + )) + ); for _ in 0..chunk_size { - let key = if K::fory_is_polymorphic() { - K::fory_read(context, key_declared)? - } else { - read_ref_info_data(context, key_declared, true, true)? - }; - let value = if V::fory_is_polymorphic() { - V::fory_read(context, value_declared)? - } else { - read_ref_info_data(context, value_declared, true, true)? - }; + let key = K::fory_read_data(context)?; + let value = V::fory_read_data(context)?; map.insert(key, value); } len_counter += chunk_size as u32; @@ -354,16 +738,24 @@ impl TypeId + where + Self: Sized, + { + TypeId::MAP + } + fn as_any(&self) -> &dyn std::any::Any { self } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::MAP as u32); + Ok(()) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs index 4e1eca4044..db27390893 100644 --- a/rust/fory-core/src/serializer/mod.rs +++ b/rust/fory-core/src/serializer/mod.rs @@ -15,13 +15,6 @@ // specific language governing permissions and limitations // under the License. -use crate::error::Error; -use crate::meta::{FieldInfo, NAMESPACE_DECODER, TYPE_NAME_DECODER}; -use crate::resolver::context::{ReadContext, WriteContext}; -use crate::types::{RefFlag, TypeId, PRIMITIVE_TYPES}; -use crate::{ensure, TypeResolver}; -use std::any::Any; - pub mod any; mod arc; mod bool; @@ -43,297 +36,9 @@ pub mod skip; mod string; pub mod struct_; pub mod trait_object; +pub mod util; pub mod weak; -#[inline(always)] -pub fn write_ref_info_data( - record: &T, - context: &mut WriteContext, - is_field: bool, - skip_ref_flag: bool, - skip_type_info: bool, -) -> Result<(), Error> { - if record.fory_is_none() { - context.writer.write_i8(RefFlag::Null as i8); - } else { - if !skip_ref_flag { - context.writer.write_i8(RefFlag::NotNullValue as i8); - } - if !skip_type_info { - T::fory_write_type_info(context, is_field)?; - } - record.fory_write_data(context, is_field)?; - } - Ok(()) -} - -#[inline(always)] -pub fn read_ref_info_data( - context: &mut ReadContext, - is_field: bool, - skip_ref_flag: bool, - skip_type_info: bool, -) -> Result { - if !skip_ref_flag { - let ref_flag = context.reader.read_i8()?; - if ref_flag == RefFlag::Null as i8 { - Ok(T::fory_default()) - } else if ref_flag == (RefFlag::NotNullValue as i8) { - if !skip_type_info { - T::fory_read_type_info(context, is_field)?; - } - T::fory_read_data(context, is_field) - } else if ref_flag == (RefFlag::RefValue as i8) { - // First time seeing this referenceable object - if !skip_type_info { - T::fory_read_type_info(context, is_field)?; - } - T::fory_read_data(context, is_field) - } else if ref_flag == (RefFlag::Ref as i8) { - // This is a reference to a previously deserialized object - // For now, just return default - this should be handled by specific types - Ok(T::fory_default()) - } else { - unimplemented!("Unknown ref flag: {}", ref_flag) - } - } else { - if !skip_type_info { - T::fory_read_type_info(context, is_field)?; - } - T::fory_read_data(context, is_field) - } -} - -#[inline(always)] -pub fn write_type_info( - context: &mut WriteContext, - is_field: bool, -) -> Result<(), Error> { - if is_field { - return Ok(()); - } - let type_id = T::fory_get_type_id(context.get_type_resolver())?; - context.writer.write_varuint32(type_id); - Ok(()) -} - -#[inline(always)] -pub fn read_type_info( - context: &mut ReadContext, - is_field: bool, -) -> Result<(), Error> { - if is_field { - return Ok(()); - } - let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; - let remote_type_id = context.reader.read_varuint32()?; - ensure!( - local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) - ); - Ok(()) -} - -#[inline(always)] -pub fn get_skip_ref_flag(type_resolver: &TypeResolver) -> Result { - let elem_type_id = T::fory_get_type_id(type_resolver)?; - Ok(!T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id)) -} - -pub trait ForyDefault: Sized { - fn fory_default() -> Self; -} - -// We can't add blanket impl for all T: Default because it conflicts with other impls. -// For example, upstream crates may add a new impl of trait `std::default::Default` for -// type `std::rc::Rc<(dyn std::any::Any + 'static)>` in future versions. -// impl ForyDefault for T { -// fn fory_default() -> Self { -// Default::default() -// } -// } - -pub trait Serializer: 'static { - /// Entry point of the serialization. - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> - where - Self: Sized, - { - write_ref_info_data(self, context, is_field, false, false) - } - - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result - where - Self: Sized + ForyDefault, - { - read_ref_info_data(context, is_field, false, false) - } - - fn fory_is_option() -> bool - where - Self: Sized, - { - false - } - - fn fory_is_none(&self) -> bool { - false - } - - fn fory_is_polymorphic() -> bool - where - Self: Sized, - { - false - } - - fn fory_is_shared_ref() -> bool - where - Self: Sized, - { - false - } - - fn fory_get_type_id(type_resolver: &TypeResolver) -> Result - where - Self: Sized, - { - Ok(type_resolver - .get_type_info(std::any::TypeId::of::())? - .get_type_id()) - } - - fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result; - - /// The possible max memory size of the type. - /// Used to reserve the buffer space to avoid reallocation, which may hurt performance. - fn fory_reserved_space() -> usize - where - Self: Sized, - { - 0 - } - - fn fory_write_type_info(context: &mut WriteContext, _is_field: bool) -> Result<(), Error> - where - Self: Sized, - { - // default implementation only for ext/named_ext - let type_id = Self::fory_get_type_id(context.get_type_resolver())?; - context.writer.write_varuint32(type_id); - if type_id & 0xff == TypeId::EXT as u32 { - return Ok(()); - } - let rs_type_id = std::any::TypeId::of::(); - if context.is_share_meta() { - let meta_index = context.push_meta(rs_type_id)? as u32; - context.writer.write_varuint32(meta_index); - } else { - let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; - let namespace = type_info.get_namespace().to_owned(); - let type_name = type_info.get_type_name().to_owned(); - context.write_meta_string_bytes(&namespace)?; - context.write_meta_string_bytes(&type_name)?; - } - Ok(()) - } - - fn fory_read_type_info(context: &mut ReadContext, _is_field: bool) -> Result<(), Error> - where - Self: Sized, - { - // default implementation only for ext/named_ext - let local_type_id = Self::fory_get_type_id(context.get_type_resolver())?; - let remote_type_id = context.reader.read_varuint32()?; - ensure!( - local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) - ); - if local_type_id & 0xff == TypeId::EXT as u32 { - return Ok(()); - } - if context.is_share_meta() { - let _meta_index = context.reader.read_varuint32()?; - } else { - let _namespace_msb = context.read_meta_string_bytes()?; - let _type_name_msb = context.read_meta_string_bytes()?; - } - Ok(()) - } - - // only used by struct/enum/ext - fn fory_read_compatible(context: &mut ReadContext) -> Result - where - Self: Sized, - { - // default logic only for ext/named_ext - let remote_type_id = context.reader.read_varuint32()?; - let local_type_id = Self::fory_get_type_id(context.get_type_resolver())?; - ensure!( - local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) - ); - if local_type_id & 0xff == TypeId::EXT as u32 { - context - .get_type_resolver() - .get_ext_harness(local_type_id)? - .get_read_data_fn()(context, true) - .and_then(|b: Box| { - b.downcast::() - .map(|boxed_self| *boxed_self) - .map_err(|_| Error::TypeError("downcast to Self failed".into())) - }) - } else { - let (namespace, type_name) = if context.is_share_meta() { - let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize); - (type_meta.get_namespace(), type_meta.get_type_name()) - } else { - let nsb = context.read_meta_string_bytes()?; - let tsb = context.read_meta_string_bytes()?; - let ns = NAMESPACE_DECODER.decode(&nsb.bytes, nsb.encoding)?; - let ts = TYPE_NAME_DECODER.decode(&tsb.bytes, tsb.encoding)?; - (ns, ts) - }; - context - .get_type_resolver() - .get_ext_name_harness(&namespace, &type_name)? - .get_read_data_fn()(context, true) - .and_then(|b: Box| { - b.downcast::() - .map(|boxed_self| *boxed_self) - .map_err(|_| Error::TypeError("downcast to Self failed".into())) - }) - } - } - - /// Write/Read the data into the buffer. Need to be implemented. - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error>; - - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result - where - Self: Sized + ForyDefault; - - fn fory_concrete_type_id(&self) -> std::any::TypeId { - std::any::TypeId::of::() - } - - fn as_any(&self) -> &dyn Any; -} - -pub trait StructSerializer: Serializer + 'static { - fn fory_fields_info(_: &TypeResolver) -> Result, Error> { - Ok(Vec::default()) - } - - fn fory_type_index() -> u32 { - unimplemented!() - } - fn fory_actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { - struct_::actual_type_id(type_id, register_by_name, compatible) - } - - fn fory_get_sorted_field_names() -> &'static [&'static str] { - &[] - } -} +mod core; +pub use any::{read_box_any, write_box_any}; +pub use core::{ForyDefault, Serializer, StructSerializer}; diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index b3bb0cb45a..f6fcc34527 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -43,8 +43,10 @@ //! during serialization — it is assumed this is a programmer error. use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; +use crate::types::TypeId; +use std::sync::Arc; use std::sync::Mutex; /// `Serializer` impl for `Mutex` @@ -52,21 +54,41 @@ use std::sync::Mutex; /// Simply delegates to the serializer for `T`, allowing thread-safe interior mutable /// containers to be included in serialized graphs. impl Serializer for Mutex { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_data: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { // Don't add ref tracking for Mutex itself, just delegate to inner type // The inner type will handle its own ref tracking let guard = self.lock().unwrap(); - T::fory_write(&*guard, context, is_field) + T::fory_write( + &*guard, + context, + write_ref_data, + write_type_info, + has_generics, + ) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + T::fory_write_data_generic(&*self.lock().unwrap(), context, has_generics) + } + + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { // When called from Rc/Arc, just delegate to inner type's data serialization let guard = self.lock().unwrap(); - T::fory_write_data(&*guard, context, is_field) + T::fory_write_data(&*guard, context) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) } fn fory_reserved_space() -> usize { @@ -74,19 +96,42 @@ impl Serializer for Mutex { T::fory_reserved_space() } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result where Self: Sized + ForyDefault, { - Ok(Mutex::new(T::fory_read(context, is_field)?)) + Ok(Mutex::new(T::fory_read( + context, + read_ref_info, + read_type_info, + )?)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Mutex::new(T::fory_read_data(context, is_field)?)) + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + Ok(Mutex::new(T::fory_read_with_type_info( + context, + read_ref_info, + type_info, + )?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_data(context: &mut ReadContext) -> Result { + Ok(Mutex::new(T::fory_read_data(context)?)) + } + + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } fn fory_get_type_id(type_resolver: &TypeResolver) -> Result { @@ -98,6 +143,10 @@ impl Serializer for Mutex { (*guard).fory_type_id_dyn(type_resolver) } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs index c923bb15f4..f5d53754db 100644 --- a/rust/fory-core/src/serializer/number.rs +++ b/rust/fory-core/src/serializer/number.rs @@ -20,25 +20,21 @@ use crate::error::Error; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; -use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; +use crate::serializer::util::read_basic_type_info; +use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; macro_rules! impl_num_serializer { ($ty:ty, $writer:expr, $reader:expr, $field_type:expr) => { impl Serializer for $ty { #[inline] - fn fory_write_data( - &self, - - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { $writer(&mut context.writer, *self); Ok(()) } #[inline] - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { $reader(&mut context.reader) } @@ -56,22 +52,24 @@ macro_rules! impl_num_serializer { Ok($field_type as u32) } + fn fory_static_type_id() -> TypeId { + $field_type + } + #[inline] fn as_any(&self) -> &dyn std::any::Any { self } #[inline] - fn fory_write_type_info( - context: &mut WriteContext, - is_field: bool, - ) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32($field_type as u32); + Ok(()) } #[inline] - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } impl ForyDefault for $ty { diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 1a6dc0339f..52c0dc138f 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -20,30 +20,102 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; +use crate::types::{RefFlag, TypeId}; impl Serializer for Option { #[inline(always)] - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(Some(T::fory_read_data(context, is_field)?)) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if let Some(v) = self { + // pass has_generics to nested collection/map serializers + T::fory_write(v, context, write_ref_info, write_type_info, has_generics) + } else { + if write_ref_info { + context.writer.write_u8(RefFlag::Null as u8); + } + // no value, skip write type info + Ok(()) + } } #[inline(always)] - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + if let Some(v) = self { + T::fory_write_data(v, context) + } else { + unreachable!("write should be call by serialize") + } } #[inline(always)] - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - if let Some(v) = self { - T::fory_write_data(v, context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) + } + + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result + where + Self: Sized + ForyDefault, + { + if read_ref_info { + let ref_flag = context.reader.read_i8()?; + if ref_flag == RefFlag::Null as i8 { + // null value won't write type info, so we can ignore `read_type_info` + return Ok(None); + } + if T::fory_is_shared_ref() { + // shared ref types always write ref flag, so we can ignore `read_type_info` + context.reader.move_back(1); // rewind to re-read ref flag in nested read + return Ok(Some(T::fory_read(context, true, read_type_info)?)); + } + } + Ok(Some(T::fory_read(context, false, read_type_info)?)) + } + + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: std::sync::Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + if read_ref_info { + let ref_flag = context.reader.read_i8()?; + if ref_flag == RefFlag::Null as i8 { + return Ok(None); + } + } + if T::fory_is_polymorphic() { + // Type info already resolved by caller + Ok(Some(T::fory_read_with_type_info( + context, false, type_info, + )?)) } else { - unreachable!("write should be call by serialize") + Ok(Some(T::fory_read_data(context)?)) } } #[inline(always)] - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_read_data(context: &mut ReadContext) -> Result { + if T::fory_is_polymorphic() { + Ok(Some(T::fory_read(context, false, true)?)) + } else { + Ok(Some(T::fory_read_data(context)?)) + } + } + + #[inline(always)] + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } #[inline(always)] @@ -74,6 +146,11 @@ impl Serializer for Option { self.is_none() } + #[inline(always)] + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self diff --git a/rust/fory-core/src/serializer/primitive_list.rs b/rust/fory-core/src/serializer/primitive_list.rs index 41b19161a9..a8541e2e7c 100644 --- a/rust/fory-core/src/serializer/primitive_list.rs +++ b/rust/fory-core/src/serializer/primitive_list.rs @@ -36,14 +36,7 @@ pub fn fory_write_data(this: &[T], context: &mut WriteContext) -> Result<(), Ok(()) } -pub fn fory_write_type_info( - context: &mut WriteContext, - is_field: bool, - type_id: TypeId, -) -> Result<(), Error> { - if is_field { - return Ok(()); - } +pub fn fory_write_type_info(context: &mut WriteContext, type_id: TypeId) -> Result<(), Error> { context.writer.write_varuint32(type_id as u32); Ok(()) } @@ -51,7 +44,7 @@ pub fn fory_write_type_info( pub fn fory_read_data(context: &mut ReadContext) -> Result, Error> { let size_bytes = context.reader.read_varuint32()? as usize; if size_bytes % std::mem::size_of::() != 0 { - return Err(Error::InvalidData("Invalid data length".into())); + return Err(Error::invalid_data("Invalid data length")); } let len = size_bytes / std::mem::size_of::(); let mut vec: Vec = Vec::with_capacity(len); @@ -64,27 +57,19 @@ pub fn fory_read_data(context: &mut ReadContext) -> Result, Error> { Ok(vec) } -pub fn fory_read_type_info( - context: &mut ReadContext, - is_field: bool, - type_id: TypeId, -) -> Result<(), Error> { - if is_field { - return Ok(()); - } +pub fn fory_read_type_info(context: &mut ReadContext, type_id: TypeId) -> Result<(), Error> { let remote_type_id = context.reader.read_varuint32()?; if remote_type_id == TypeId::LIST as u32 { - return Err(Error::TypeError( + return Err(Error::type_error( "Vec belongs to the `number_array` type, \ and Vec> belongs to the `list` type. \ - You should not read data of type `list` as data of type `number_array`." - .into(), + You should not read data of type `list` as data of type `number_array`.", )); } let local_type_id = type_id as u32; ensure!( local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) + Error::type_mismatch(local_type_id, remote_type_id) ); Ok(()) } diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index c8750b4a97..7114e85bdd 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -17,69 +17,86 @@ use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; +use crate::types::TypeId; use std::rc::Rc; +use std::sync::Arc; impl Serializer for Rc { fn fory_is_shared_ref() -> bool { true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - if !context - .ref_writer - .try_write_rc_ref(&mut context.writer, self) + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if !write_ref_info + || !context + .ref_writer + .try_write_rc_ref(&mut context.writer, self) { - T::fory_write_data(self.as_ref(), context, is_field)? - }; - Ok(()) + if T::fory_is_shared_ref() || T::fory_is_polymorphic() { + let inner_write_ref = T::fory_is_shared_ref(); + return T::fory_write( + &**self, + context, + inner_write_ref, + write_type_info, + has_generics, + ); + } + if write_type_info { + T::fory_write_type_info(context)?; + } + T::fory_write_data_generic(self, context, has_generics) + } else { + Ok(()) + } } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - // When Rc is nested inside another shared ref (like Arc>), - // the outer ref calls fory_write_data on the inner Rc. - // We still need to track the Rc's own references here. - self.fory_write(context, is_field) + fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { + panic!("Rc should be written using `fory_write` to handle reference tracking properly"); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { + panic!("Rc should be written using `fory_write` to handle reference tracking properly"); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; - match ref_flag { - RefFlag::Null => Err(Error::InvalidRef("Rc cannot be null".into())), - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - context.ref_reader.get_rc_ref::(ref_id).ok_or_else(|| { - Error::InvalidRef(format!("Rc reference {ref_id} not found").into()) - }) - } - RefFlag::NotNullValue => { - let inner = T::fory_read_data(context, is_field)?; - Ok(Rc::new(inner)) - } - RefFlag::RefValue => { - let ref_id = context.ref_reader.reserve_ref_id(); - let inner = T::fory_read_data(context, is_field)?; - let rc = Rc::new(inner); - context.ref_reader.store_rc_ref_at(ref_id, rc.clone()); - Ok(rc) - } - } + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) + } + + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_rc(context, read_ref_info, read_type_info, None) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - // When Rc is nested inside another shared ref, fory_read_data is called. - // Delegate to fory_read which handles ref tracking properly. - Self::fory_read(context, is_field) + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + typeinfo: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_rc(context, read_ref_info, false, Some(typeinfo)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_data(_: &mut ReadContext) -> Result { + panic!("Rc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly"); + } + + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } fn fory_reserved_space() -> usize { @@ -96,11 +113,68 @@ impl Serializer for Rc { (**self).fory_type_id_dyn(type_resolver) } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } } +fn read_rc( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + typeinfo: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Err(Error::invalid_ref("Rc cannot be null")), + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + context + .ref_reader + .get_rc_ref::(ref_id) + .ok_or_else(|| Error::invalid_ref(format!("Rc reference {ref_id} not found"))) + } + RefFlag::NotNullValue => { + let inner = read_rc_inner::(context, read_type_info, typeinfo)?; + Ok(Rc::new(inner)) + } + RefFlag::RefValue => { + let ref_id = context.ref_reader.reserve_ref_id(); + let inner = read_rc_inner::(context, read_type_info, typeinfo)?; + let rc = Rc::new(inner); + context.ref_reader.store_rc_ref_at(ref_id, rc.clone()); + Ok(rc) + } + } +} + +fn read_rc_inner( + context: &mut ReadContext, + read_type_info: bool, + typeinfo: Option>, +) -> Result { + if let Some(typeinfo) = typeinfo { + let inner_read_ref = T::fory_is_shared_ref(); + return T::fory_read_with_type_info(context, inner_read_ref, typeinfo); + } + if T::fory_is_shared_ref() || T::fory_is_polymorphic() { + let inner_read_ref = T::fory_is_shared_ref(); + return T::fory_read(context, inner_read_ref, read_type_info); + } + if read_type_info { + T::fory_read_type_info(context)?; + } + T::fory_read_data(context) +} + impl ForyDefault for Rc { fn fory_default() -> Self { Rc::new(T::fory_default()) diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index e1c194d8d8..67d69cb2f3 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -34,43 +34,87 @@ //! ``` use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; +use crate::types::TypeId; use std::cell::RefCell; +use std::sync::Arc; /// `Serializer` impl for `RefCell` /// /// Simply delegates to the serializer for `T`, allowing interior mutable /// containers to be included in serialized graphs. impl Serializer for RefCell { - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result where Self: Sized + ForyDefault, { - Ok(RefCell::new(T::fory_read(context, is_field)?)) + Ok(RefCell::new(T::fory_read( + context, + read_ref_info, + read_type_info, + )?)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Ok(RefCell::new(T::fory_read_data(context, is_field)?)) + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + Ok(RefCell::new(T::fory_read_with_type_info( + context, + read_ref_info, + type_info, + )?)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_data(context: &mut ReadContext) -> Result { + Ok(RefCell::new(T::fory_read_data(context)?)) } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) + } + + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { // Don't add ref tracking for RefCell itself, just delegate to inner type // The inner type will handle its own ref tracking - T::fory_write(&*self.borrow(), context, is_field) + T::fory_write( + &*self.borrow(), + context, + write_ref_info, + write_type_info, + has_generics, + ) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - // When called from Rc, just delegate to inner type's data serialization - T::fory_write_data(&*self.borrow(), context, is_field) + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + T::fory_write_data_generic(&*self.borrow(), context, has_generics) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_data(&*self.borrow(), context) + } + + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) } fn fory_reserved_space() -> usize { @@ -86,6 +130,13 @@ impl Serializer for RefCell { (*self.borrow()).fory_type_id_dyn(type_resolver) } + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs index dd0bfaa0d6..a317f0b362 100644 --- a/rust/fory-core/src/serializer/set.rs +++ b/rust/fory-core/src/serializer/set.rs @@ -20,7 +20,8 @@ use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; use crate::serializer::collection::{ - read_collection, read_collection_type_info, write_collection, write_collection_type_info, + read_collection_data, read_collection_type_info, write_collection_data, + write_collection_type_info, }; use crate::serializer::{ForyDefault, Serializer}; @@ -29,20 +30,20 @@ use std::collections::{BTreeSet, HashSet}; use std::mem; impl Serializer for HashSet { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_collection_data(self, context, false) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + write_collection_type_info(context, TypeId::SET as u32) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data(context: &mut ReadContext) -> Result { + read_collection_data(context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_collection_type_info(context, TypeId::SET as u32) } fn fory_reserved_space() -> usize { @@ -57,6 +58,13 @@ impl Serializer for HashSet< Ok(TypeId::SET as u32) } + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + TypeId::SET + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -69,20 +77,20 @@ impl ForyDefault for HashSet { } impl Serializer for BTreeSet { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection(self, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_collection_data(self, context, false) } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + write_collection_type_info(context, TypeId::SET as u32) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { - read_collection(context) + fn fory_read_data(context: &mut ReadContext) -> Result { + read_collection_data(context) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_collection_type_info(context, is_field, TypeId::SET as u32) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_collection_type_info(context, TypeId::SET as u32) } fn fory_reserved_space() -> usize { @@ -97,6 +105,13 @@ impl Serializer for BTreeSet { Ok(TypeId::SET as u32) } + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + TypeId::SET + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index 8baf7eaaa6..0b9cac2146 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -20,21 +20,16 @@ use crate::error::Error; use crate::meta::FieldType; use crate::resolver::context::ReadContext; use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE}; +use crate::serializer::util; use crate::serializer::Serializer; -use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPES}; +use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES}; use chrono::{NaiveDate, NaiveDateTime}; -pub fn get_read_ref_flag(field_type: &FieldType) -> bool { - let nullable = field_type.nullable; - nullable || !PRIMITIVE_TYPES.contains(&field_type.type_id) -} - macro_rules! basic_type_deserialize { ($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => { $( if $tid == TypeId::$id { - <$ty as Serializer>::fory_read_type_info($context, true)?; - <$ty as Serializer>::fory_read_data($context, true)?; + <$ty as Serializer>::fory_read_data($context)?; return Ok(()); } )+else { @@ -43,16 +38,26 @@ macro_rules! basic_type_deserialize { }; } -// call when is_field && is_compatible_mode #[allow(unreachable_code)] pub fn skip_field_value( context: &mut ReadContext, field_type: &FieldType, read_ref_flag: bool, +) -> Result<(), Error> { + skip_value(context, field_type, read_ref_flag, true) +} + +// call when is_field && is_compatible_mode +#[allow(unreachable_code)] +pub fn skip_value( + context: &mut ReadContext, + field_type: &FieldType, + read_ref_flag: bool, + _is_field: bool, ) -> Result<(), Error> { if read_ref_flag { let ref_flag = context.reader.read_i8()?; - if field_type.nullable && ref_flag == (RefFlag::Null as i8) { + if ref_flag == (RefFlag::Null as i8) { return Ok(()); } } @@ -92,7 +97,7 @@ pub fn skip_field_value( let elem_type = field_type.generics.first().unwrap(); context.inc_depth()?; for _ in 0..length { - skip_field_value(context, elem_type, !skip_ref_flag)?; + skip_value(context, elem_type, !skip_ref_flag, false)?; } context.dec_depth(); } else if type_id == TypeId::MAP { @@ -115,17 +120,17 @@ pub fn skip_field_value( continue; } if header & crate::serializer::map::KEY_NULL != 0 { - // let read_ref_flag = get_read_ref_flag(value_type); + // value_type.nullable determines whether ref flag was written context.inc_depth()?; - skip_field_value(context, value_type, false)?; + skip_value(context, value_type, false, false)?; context.dec_depth(); len_counter += 1; continue; } if header & crate::serializer::map::VALUE_NULL != 0 { - // let read_ref_flag = get_read_ref_flag(key_type); + // key_type.nullable determines whether ref flag was written context.inc_depth()?; - skip_field_value(context, key_type, false)?; + skip_value(context, key_type, false, false)?; context.dec_depth(); len_counter += 1; continue; @@ -133,10 +138,10 @@ pub fn skip_field_value( let chunk_size = context.reader.read_u8()?; context.inc_depth()?; for _ in (0..chunk_size).enumerate() { - // let read_ref_flag = get_read_ref_flag(key_type); - skip_field_value(context, key_type, false)?; - // let read_ref_flag = get_read_ref_flag(value_type); - skip_field_value(context, value_type, false)?; + // key_type.nullable determines whether ref flag was written + skip_value(context, key_type, false, false)?; + // value_type.nullable determines whether ref flag was written + skip_value(context, value_type, false, false)?; } context.dec_depth(); len_counter += chunk_size as u32; @@ -150,15 +155,18 @@ pub fn skip_field_value( let remote_type_id = context.reader.read_varuint32()?; ensure!( type_id_num == remote_type_id, - Error::TypeMismatch(type_id_num, remote_type_id) + Error::type_mismatch(type_id_num, remote_type_id) ); let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize); + let type_meta = context.get_meta(meta_index as usize)?; let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let read_ref_flag = get_read_ref_flag(&field_info.field_type); - skip_field_value(context, &field_info.field_type, read_ref_flag)?; + let read_ref_flag = util::field_requires_ref_flag( + field_info.field_type.type_id, + field_info.field_type.nullable, + ); + skip_value(context, &field_info.field_type, read_ref_flag, true)?; } context.dec_depth(); Ok(()) @@ -166,14 +174,14 @@ pub fn skip_field_value( let remote_type_id = context.reader.read_varuint32()?; ensure!( type_id_num == remote_type_id, - Error::TypeMismatch(type_id_num, remote_type_id) + Error::type_mismatch(type_id_num, remote_type_id) ); let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize); + let type_meta = context.get_meta(meta_index as usize)?; let type_resolver = context.get_type_resolver(); type_resolver .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name())? - .get_read_data_fn()(context, true)?; + .get_read_data_fn()(context)?; Ok(()) } else { unreachable!("unimplemented type: {:?}", type_id); @@ -187,34 +195,42 @@ pub fn skip_field_value( if internal_id == COMPATIBLE_STRUCT_ID { let remote_type_id = context.reader.read_varuint32()?; let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize); + let type_meta = context.get_meta(meta_index as usize)?; ensure!( type_meta.get_type_id() == remote_type_id, - Error::TypeMismatch(type_meta.get_type_id(), remote_type_id) + Error::type_mismatch(type_meta.get_type_id(), remote_type_id) ); let field_infos = type_meta.get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { - let read_ref_flag = get_read_ref_flag(&field_info.field_type); - skip_field_value(context, &field_info.field_type, read_ref_flag)?; + let read_ref_flag = util::field_requires_ref_flag( + field_info.field_type.type_id, + field_info.field_type.nullable, + ); + skip_value(context, &field_info.field_type, read_ref_flag, true)?; } context.dec_depth(); } else if internal_id == ENUM_ID { let _ordinal = context.reader.read_varuint32()?; + let _ordinalx = _ordinal; + println!("skip enum ordinal: {}", _ordinalx); } else if internal_id == EXT_ID { let remote_type_id = context.reader.read_varuint32()?; ensure!( type_id_num == remote_type_id, - Error::TypeMismatch(type_id_num, remote_type_id) + Error::type_mismatch(type_id_num, remote_type_id) ); context.inc_depth()?; let type_resolver = context.get_type_resolver(); type_resolver .get_ext_harness(type_id_num)? - .get_read_data_fn()(context, true)?; + .get_read_data_fn()(context)?; context.dec_depth(); } else { - unreachable!("unimplemented skipped type: {:?}", type_id_num); + return Err(Error::type_error(format!( + "Unknown type id: {}", + type_id_num + ))); } Ok(()) } diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs index ad9b2c25ce..d6b7c2fa6a 100644 --- a/rust/fory-core/src/serializer/string.rs +++ b/rust/fory-core/src/serializer/string.rs @@ -20,7 +20,8 @@ use crate::meta::get_latin1_length; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; -use crate::serializer::{read_type_info, write_type_info, ForyDefault, Serializer}; +use crate::serializer::util::read_basic_type_info; +use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; use std::mem; @@ -32,7 +33,7 @@ enum StrEncoding { impl Serializer for String { #[inline] - fn fory_write_data(&self, context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { let mut len = get_latin1_length(self); if len >= 0 { let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64; @@ -54,7 +55,7 @@ impl Serializer for String { } #[inline] - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { let bitor = context.reader.read_varuint36small()?; let len = bitor >> 2; let encoding = bitor & 0b11; @@ -63,9 +64,10 @@ impl Serializer for String { 1 => StrEncoding::Utf16, 2 => StrEncoding::Utf8, _ => { - return Err(Error::EncodingError( - format!("wrong encoding value: {}", encoding).into(), - )) + return Err(Error::encoding_error(format!( + "wrong encoding value: {}", + encoding + ))) } }; let s = match encoding { @@ -86,23 +88,33 @@ impl Serializer for String { Ok(TypeId::STRING as u32) } + #[inline(always)] fn fory_type_id_dyn(&self, _: &TypeResolver) -> Result { Ok(TypeId::STRING as u32) } + #[inline(always)] + fn fory_static_type_id() -> TypeId + where + Self: Sized, + { + TypeId::STRING + } + #[inline(always)] fn as_any(&self) -> &dyn std::any::Any { self } #[inline(always)] - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_type_info::(context, is_field) + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + context.writer.write_varuint32(TypeId::STRING as u32); + Ok(()) } #[inline(always)] - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - read_type_info::(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + read_basic_type_info::(context) } } diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 5ec0137d6d..98597e60b1 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -20,6 +20,8 @@ use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; use crate::serializer::Serializer; use crate::types::{RefFlag, TypeId}; +use std::any::Any; +use std::sync::OnceLock; #[inline(always)] pub fn actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> u32 { @@ -37,10 +39,7 @@ pub fn actual_type_id(type_id: u32, register_by_name: bool, compatible: bool) -> } #[inline(always)] -pub fn write_type_info( - context: &mut WriteContext, - _is_field: bool, -) -> Result<(), Error> { +pub fn write_type_info(context: &mut WriteContext) -> Result<(), Error> { let type_id = T::fory_get_type_id(context.get_type_resolver())?; context.writer.write_varuint32(type_id); let rs_type_id = std::any::TypeId::of::(); @@ -50,7 +49,7 @@ pub fn write_type_info( let meta_index = context.push_meta(rs_type_id)? as u32; context.writer.write_varuint32(meta_index); } else { - let type_info = context.get_type_resolver().get_type_info(rs_type_id)?; + let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; let namespace = type_info.get_namespace().to_owned(); let type_name = type_info.get_type_name().to_owned(); context.write_meta_string_bytes(&namespace)?; @@ -66,15 +65,12 @@ pub fn write_type_info( } #[inline(always)] -pub fn read_type_info( - context: &mut ReadContext, - _is_field: bool, -) -> Result<(), Error> { +pub fn read_type_info(context: &mut ReadContext) -> Result<(), Error> { let remote_type_id = context.reader.read_varuint32()?; let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; ensure!( local_type_id == remote_type_id, - Error::TypeMismatch(local_type_id, remote_type_id) + Error::type_mismatch(local_type_id, remote_type_id) ); if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { @@ -96,17 +92,151 @@ pub fn read_type_info( pub fn write( this: &T, context: &mut WriteContext, - _is_field: bool, + write_ref_info: bool, + write_type_info: bool, ) -> Result<(), Error> { - if context.is_compatible() { + if write_ref_info { context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, false)?; - this.fory_write_data(context, true)?; - } else { - // currently same - context.writer.write_i8(RefFlag::NotNullValue as i8); - T::fory_write_type_info(context, false)?; - this.fory_write_data(context, true)?; } - Ok(()) + if write_type_info { + T::fory_write_type_info(context)?; + } + this.fory_write_data(context) +} + +/// Global flag to check if ENABLE_FORY_DEBUG_OUTPUT environment variable is set. +static ENABLE_FORY_DEBUG_OUTPUT: OnceLock = OnceLock::new(); + +/// Check if ENABLE_FORY_DEBUG_OUTPUT environment variable is set. +#[inline] +fn enable_debug_output() -> bool { + *ENABLE_FORY_DEBUG_OUTPUT.get_or_init(|| { + std::env::var("ENABLE_FORY_DEBUG_OUTPUT") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(true) + }) +} + +pub type BeforeWriteFieldFunc = + fn(struct_name: &str, field_name: &str, field_value: &dyn Any, context: &mut WriteContext); +pub type AfterWriteFieldFunc = + fn(struct_name: &str, field_name: &str, field_value: &dyn Any, context: &mut WriteContext); +pub type BeforeReadFieldFunc = fn(struct_name: &str, field_name: &str, context: &mut ReadContext); +pub type AfterReadFieldFunc = + fn(struct_name: &str, field_name: &str, field_value: &dyn Any, context: &mut ReadContext); + +fn default_before_write_field( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + context: &mut WriteContext, +) { + if enable_debug_output() { + println!( + "before_write_field:\tstruct={struct_name},\tfield={field_name},\twriter_len={}", + context.writer.len() + ); + } +} + +fn default_after_write_field( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + context: &mut WriteContext, +) { + if enable_debug_output() { + println!( + "after_write_field:\tstruct={struct_name},\tfield={field_name},\twriter_len={}", + context.writer.len() + ); + } +} + +fn default_before_read_field(struct_name: &str, field_name: &str, context: &mut ReadContext) { + if enable_debug_output() { + println!( + "before_read_field:\tstruct={struct_name},\tfield={field_name},\treader_cursor={}", + context.reader.get_cursor() + ); + } +} + +fn default_after_read_field( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + context: &mut ReadContext, +) { + if enable_debug_output() { + println!( + "after_read_field:\tstruct={struct_name},\tfield={field_name},\treader_cursor={}", + context.reader.get_cursor() + ); + } +} + +static mut BEFORE_WRITE_FIELD_FUNC: BeforeWriteFieldFunc = default_before_write_field; +static mut AFTER_WRITE_FIELD_FUNC: AfterWriteFieldFunc = default_after_write_field; +static mut BEFORE_READ_FIELD_FUNC: BeforeReadFieldFunc = default_before_read_field; +static mut AFTER_READ_FIELD_FUNC: AfterReadFieldFunc = default_after_read_field; + +pub fn set_before_write_field_func(func: BeforeWriteFieldFunc) { + unsafe { BEFORE_WRITE_FIELD_FUNC = func } +} + +pub fn set_after_write_field_func(func: AfterWriteFieldFunc) { + unsafe { AFTER_WRITE_FIELD_FUNC = func } +} + +pub fn set_before_read_field_func(func: BeforeReadFieldFunc) { + unsafe { BEFORE_READ_FIELD_FUNC = func } +} + +pub fn set_after_read_field_func(func: AfterReadFieldFunc) { + unsafe { AFTER_READ_FIELD_FUNC = func } +} + +pub fn reset_struct_debug_hooks() { + unsafe { + BEFORE_WRITE_FIELD_FUNC = default_before_write_field; + AFTER_WRITE_FIELD_FUNC = default_after_write_field; + BEFORE_READ_FIELD_FUNC = default_before_read_field; + AFTER_READ_FIELD_FUNC = default_after_read_field; + } +} + +/// Debug method to hook into struct serialization +pub fn struct_before_write_field( + struct_name: &str, + field_name: &str, + field_value: &dyn Any, + context: &mut WriteContext, +) { + unsafe { BEFORE_WRITE_FIELD_FUNC(struct_name, field_name, field_value, context) } +} + +/// Debug method to hook into struct serialization +pub fn struct_after_write_field( + struct_name: &str, + field_name: &str, + field_value: &dyn Any, + context: &mut WriteContext, +) { + unsafe { AFTER_WRITE_FIELD_FUNC(struct_name, field_name, field_value, context) } +} + +/// Debug method to hook into struct deserialization +pub fn struct_before_read_field(struct_name: &str, field_name: &str, context: &mut ReadContext) { + unsafe { BEFORE_READ_FIELD_FUNC(struct_name, field_name, context) } +} + +/// Debug method to hook into struct deserialization +pub fn struct_after_read_field( + struct_name: &str, + field_name: &str, + field_value: &dyn Any, + context: &mut ReadContext, +) { + unsafe { AFTER_READ_FIELD_FUNC(struct_name, field_name, field_value, context) } } diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 7ee6daf929..97195b3ea9 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -15,86 +15,47 @@ // specific language governing permissions and limitations // under the License. +// Re-exports for use in macros - these are needed for macro expansion in user crates +// Even though they appear unused in this file, they are used by the macro-generated code + +use crate::ensure; use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; - -/// Helper functions for trait object serialization to reduce code duplication -/// -/// Writes common trait object headers (ref flag, type ID, compatibility metadata) -pub fn write_trait_object_headers( - context: &mut WriteContext, - fory_type_id: u32, - concrete_type_id: std::any::TypeId, -) -> Result<(), Error> { - use crate::types::{RefFlag, TypeId}; - - context.writer.write_i8(RefFlag::NotNullValue as i8); - context.writer.write_varuint32(fory_type_id); - - if context.is_compatible() - && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 - || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) - { - let meta_index = context.push_meta(concrete_type_id)? as u32; - context.writer.write_varuint32(meta_index); - }; - Ok(()) -} - -/// Reads common trait object headers and returns the type ID -pub fn read_trait_object_headers(context: &mut ReadContext) -> Result { - use crate::types::{RefFlag, TypeId}; - - let ref_flag = context.reader.read_i8()?; - if ref_flag != RefFlag::NotNullValue as i8 { - return Err(Error::InvalidRef( - format!("Expected NotNullValue ref flag, got {ref_flag}").into(), - )); - } - - let fory_type_id = context.reader.read_varuint32()?; - - if context.is_compatible() - && (fory_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 - || fory_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32) - { - let _meta_index = context.reader.read_varuint32()?; - } - Ok(fory_type_id) -} +use crate::RefFlag; +use std::sync::Arc; /// Helper macro for common type resolution and downcasting pattern #[macro_export] macro_rules! downcast_and_serialize { - ($any_ref:expr, $context:expr, $is_field:expr, $trait_name:ident, $($impl_type:ty),+) => {{ + ($any_ref:expr, $context:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( if $any_ref.type_id() == std::any::TypeId::of::<$impl_type>() { if let Some(concrete) = $any_ref.downcast_ref::<$impl_type>() { - concrete.fory_write_data($context, $is_field)?; + concrete.fory_write_data($context)?; return Ok(()); } } )* - return Err($crate::error::Error::TypeError(format!("Failed to downcast to any registered type for trait {}", stringify!($trait_name)).into())); + return Err(fory_core::Error::type_error(format!("Failed to downcast to any registered type for trait {}", stringify!($trait_name)))); }}; } /// Helper macro for common type resolution and deserialization pattern #[macro_export] macro_rules! resolve_and_deserialize { - ($fory_type_id:expr, $context:expr, $is_field:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ + ($fory_type_id:expr, $context:expr, $constructor:expr, $trait_name:ident, $($impl_type:ty),+) => {{ $( if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { if $fory_type_id == registered_type_id { - let concrete_obj = <$impl_type as $crate::serializer::Serializer>::fory_read_data($context, $is_field)?; + let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; return Ok($constructor(concrete_obj)); } } )* - Err($crate::error::Error::TypeError( - format!("Type ID {} not registered for trait {}", $fory_type_id, stringify!($trait_name)).into() + Err(fory_core::Error::type_error( + format!("Type ID {} not registered for trait {}", $fory_type_id, stringify!($trait_name)) )) }}; } @@ -198,66 +159,91 @@ macro_rules! register_trait_type { ); // 4. Serializer implementation for Box (existing functionality) - impl $crate::serializer::Serializer for Box { - fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { - let any_ref = ::as_any(&**self); - let concrete_type_id = any_ref.type_id(); + impl fory_core::Serializer for Box { + fn fory_write(&self, context: &mut fory_core::WriteContext, write_ref_info: bool, write_type_info: bool, has_generics: bool) -> Result<(), fory_core::Error> { + let any_ref = ::as_any(&**self); + fory_core::serializer::write_box_any(any_ref, context, write_ref_info, write_type_info, has_generics) + } - if let Some(fory_type_id) = context.get_type_resolver().get_fory_type_id(concrete_type_id) { - $crate::serializer::trait_object::write_trait_object_headers(context, fory_type_id, concrete_type_id)?; - $crate::downcast_and_serialize!(any_ref, context, is_field, $trait_name, $($impl_type),+); - } else { - return Err($crate::error::Error::TypeError(format!("Type {:?} not registered for Box serialization", concrete_type_id, stringify!($trait_name)).into())); - } + fn fory_write_data(&self, context: &mut fory_core::WriteContext) -> Result<(), fory_core::Error> { + let any_ref = ::as_any(&**self); + fory_core::serializer::write_box_any(any_ref, context, false, false, false) } - fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { - // Delegate to fory_write since this handles the polymorphic dispatch - self.fory_write(context, is_field) + fn fory_write_data_generic(&self, context: &mut fory_core::WriteContext, has_generics: bool) -> Result<(), fory_core::Error> { + let any_ref = ::as_any(&**self); + fory_core::serializer::write_box_any(any_ref, context, false, false, has_generics) } - fn fory_type_id_dyn(&self, type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { - let any_ref = ::as_any(&**self); + fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + let any_ref = ::as_any(&**self); let concrete_type_id = any_ref.type_id(); type_resolver .get_fory_type_id(concrete_type_id) - .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) + .ok_or_else(|| fory_core::Error::type_error("Type not registered for trait object")) } fn fory_is_polymorphic() -> bool { true } - fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { - // Box is polymorphic - type info is written per element - Ok(()) + fn fory_write_type_info(_context: &mut fory_core::WriteContext) -> Result<(), fory_core::Error> { + $crate::not_allowed!("fory_write_type_info should not be called directly on polymorphic Box trait object", stringify!($trait_name)) } - fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { - // Box is polymorphic - type info is read per element - Ok(()) + fn fory_read_type_info(_context: &mut fory_core::ReadContext) -> Result<(), fory_core::Error> { + $crate::not_allowed!("fory_read_type_info should not be called directly on polymorphic Box trait object", stringify!($trait_name)) } - fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { - context.inc_depth()?; - let fory_type_id = $crate::serializer::trait_object::read_trait_object_headers(context)?; - let result = $crate::resolve_and_deserialize!( - fory_type_id, context, is_field, - |obj| Box::new(obj) as Box, - $trait_name, $($impl_type),+ - ); - context.dec_depth(); - result + fn fory_read(context: &mut fory_core::ReadContext, read_ref_info: bool, read_type_info: bool) -> Result { + let boxed_any = fory_core::serializer::read_box_any(context, read_ref_info, read_type_info, None)?; + $( + if boxed_any.is::<$impl_type>() { + let concrete = boxed_any.downcast::<$impl_type>() + .map_err(|_| fory_core::Error::type_error("Downcast failed"))?; + let ptr = Box::new(*concrete); + return Ok(Self::from(ptr)); + } + )* + Err(fory_core::Error::type_error( + format!("Deserialized type does not implement trait {}", stringify!($trait_name)) + )) + } + + fn fory_read_with_type_info( + context: &mut fory_core::ReadContext, + read_ref_info: bool, + type_info: std::sync::Arc, + ) -> Result + where + Self: Sized + fory_core::ForyDefault, + { + let boxed_any = fory_core::serializer::read_box_any(context, read_ref_info, false, Some(type_info))?; + $( + if boxed_any.is::<$impl_type>() { + let concrete = boxed_any.downcast::<$impl_type>() + .map_err(|_| fory_core::Error::type_error("Downcast failed"))?; + let ptr = Box::new(*concrete); + return Ok(Self::from(ptr)); + } + )* + Err(fory_core::Error::type_error( + format!("Deserialized type does not implement trait {}", stringify!($trait_name)) + )) } - fn fory_read_data(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result { + fn fory_read_data(_context: &mut fory_core::ReadContext) -> Result { // This should not be called for polymorphic types like Box // The fory_read method handles the polymorphic dispatch - panic!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)); + $crate::not_allowed!("fory_read_data should not be called directly on polymorphic Box trait object", stringify!($trait_name)) + } + + fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { + $crate::not_allowed!("fory_get_type_id should not be called directly on polymorphic Box trait object", stringify!($trait_name)) } - fn fory_get_type_id(_type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { - Ok($crate::types::TypeId::STRUCT as u32) + fn fory_static_type_id() -> fory_core::TypeId { + fory_core::TypeId::UNKNOWN } fn fory_reserved_space() -> usize { @@ -265,39 +251,11 @@ macro_rules! register_trait_type { } fn fory_concrete_type_id(&self) -> std::any::TypeId { - ::as_any(&**self).type_id() + ::as_any(&**self).type_id() } fn as_any(&self) -> &dyn std::any::Any { - ::as_any(&**self) - } - } - - // Create helper functions for this trait with trait-specific names - $crate::paste::paste! { - #[allow(non_snake_case)] - mod [<__fory_trait_helpers_ $trait_name>] { - use super::*; - - #[allow(dead_code)] - pub fn []( - any_box: Box, - _fory_type_id: u32, - ) -> Result, $crate::error::Error> { - $( - if any_box.is::<$impl_type>() { - let concrete = any_box.downcast::<$impl_type>() - .map_err(|_| $crate::error::Error::TypeError( - format!("Failed to downcast to {}", stringify!($impl_type)).into() - ))?; - return Ok(Box::new(*concrete) as Box); - } - )+ - - Err($crate::error::Error::TypeError( - format!("No matching type found for trait {}", stringify!($trait_name)).into() - )) - } + ::as_any(&**self) } } }; @@ -376,7 +334,7 @@ macro_rules! generate_smart_pointer_wrapper { impl std::fmt::Debug for [<$trait_name $ptr_name>] { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let any_obj = ::as_any(&*self.0); + let any_obj = ::as_any(&*self.0); $( if let Some(concrete) = any_obj.downcast_ref::<$impl_type>() { return write!(f, concat!(stringify!($trait_name), stringify!($ptr_name), "({:?})"), concrete); @@ -400,108 +358,148 @@ macro_rules! generate_smart_pointer_wrapper { }; } +/// Macro to read smart pointer trait objects (Rc, Arc) +/// This macro handles ref tracking and directly constructs the trait object from concrete types +#[macro_export] +macro_rules! read_ptr_trait_object { + ($context:expr, $read_ref_info:expr, $read_type_info:expr, $type_info:expr, $pointer_type:ty, $trait_name:ident, $constructor_expr:expr, $get_ref:ident, $store_ref:ident, $($impl_type:ty),+) => {{ + let ref_flag = if $read_ref_info { + $context.ref_reader.read_ref_flag(&mut $context.reader)? + } else { + fory_core::RefFlag::NotNullValue + }; + match ref_flag { + fory_core::RefFlag::Null => Err(fory_core::Error::invalid_ref(format!("smart pointer to dyn {} cannot be null", stringify!($trait_name)))), + fory_core::RefFlag::Ref => { + let ref_id = $context.ref_reader.read_ref_id(&mut $context.reader)?; + let ptr_ref = $context.ref_reader.$get_ref::(ref_id) + .ok_or_else(|| fory_core::Error::invalid_data(format!("dyn {} reference {} not found", stringify!($trait_name), ref_id)))?; + Ok(Self::from(ptr_ref)) + } + fory_core::RefFlag::NotNullValue => { + $context.inc_depth()?; + let typeinfo = if $read_type_info { + $context.read_any_typeinfo()? + } else { + $type_info.ok_or_else(|| fory_core::Error::type_error("No type info found for read"))? + }; + let fory_type_id = typeinfo.get_type_id(); + $( + if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { + if fory_type_id == registered_type_id { + let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; + $context.dec_depth(); + let ptr = $constructor_expr(concrete_obj) as $pointer_type; + return Ok(Self::from(ptr)); + } + } + )* + $context.dec_depth(); + Err(fory_core::Error::type_error( + format!("Type ID {} not registered for trait {}", fory_type_id, stringify!($trait_name)) + )) + } + fory_core::RefFlag::RefValue => { + $context.inc_depth()?; + let typeinfo = if $read_type_info { + $context.read_any_typeinfo()? + } else { + $type_info.ok_or_else(|| fory_core::Error::type_error("No type info found for read"))? + }; + let fory_type_id = typeinfo.get_type_id(); + $( + if let Some(registered_type_id) = $context.get_type_resolver().get_fory_type_id(std::any::TypeId::of::<$impl_type>()) { + if fory_type_id == registered_type_id { + let concrete_obj = <$impl_type as fory_core::Serializer>::fory_read_data($context)?; + $context.dec_depth(); + let ptr = $constructor_expr(concrete_obj) as $pointer_type; + let wrapper = Self::from(ptr.clone()); + $context.ref_reader.$store_ref(ptr); + return Ok(wrapper); + } + } + )* + $context.dec_depth(); + Err(fory_core::Error::type_error( + format!("Type ID {} not registered for trait {}", fory_type_id, stringify!($trait_name)) + )) + } + } + }}; +} + /// Shared serializer implementation for smart pointer wrappers #[macro_export] macro_rules! impl_smart_pointer_serializer { ($wrapper_name:ident, $pointer_type:ty, $constructor_expr:expr, $trait_name:ident, $try_write_ref:ident, $get_ref:ident, $store_ref:ident, $($impl_type:ty),+) => { - impl $crate::serializer::Serializer for $wrapper_name { - fn fory_write(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { - if !context.ref_writer.$try_write_ref(&mut context.writer, &self.0) { - let any_obj = ::as_any(&*self.0); + impl fory_core::Serializer for $wrapper_name { + fn fory_write(&self, context: &mut fory_core::WriteContext, write_ref_info: bool, write_type_info: bool, has_generics: bool) -> Result<(), fory_core::Error> { + if !write_ref_info || !context.ref_writer.$try_write_ref(&mut context.writer, &self.0) { + let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); - let harness = context.write_any_typeinfo(concrete_type_id)?; - let serializer_fn = harness.get_write_data_fn(); - serializer_fn(any_obj, context, is_field)?; + let typeinfo = if write_type_info { + context.write_any_typeinfo(fory_core::TypeId::UNKNOWN as u32, concrete_type_id)? + } else { + context.get_type_info(&concrete_type_id)? + }; + let serializer_fn = typeinfo.get_harness().get_write_data_fn(); + serializer_fn(any_obj, context, has_generics)?; } Ok(()) } - fn fory_write_data(&self, context: &mut $crate::resolver::context::WriteContext, is_field: bool) -> Result<(), $crate::error::Error> { - let any_obj = ::as_any(&*self.0); - $crate::downcast_and_serialize!(any_obj, context, is_field, $trait_name, $($impl_type),+) + fn fory_write_data(&self, context: &mut fory_core::WriteContext) -> Result<(), fory_core::Error> { + let any_obj = ::as_any(&*self.0); + $crate::downcast_and_serialize!(any_obj, context, $trait_name, $($impl_type),+) } - fn fory_read(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { - use $crate::types::RefFlag; + fn fory_read(context: &mut fory_core::ReadContext, read_ref_info: bool, read_type_info: bool) -> Result { + $crate::read_ptr_trait_object!( + context, + read_ref_info, + read_type_info, + None, + $pointer_type, + $trait_name, + $constructor_expr, + $get_ref, + $store_ref, + $($impl_type),+ + ) + } - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_read_with_type_info(context: &mut fory_core::ReadContext, read_ref_info: bool, type_info: std::sync::Arc) -> Result { + $crate::read_ptr_trait_object!( + context, + read_ref_info, + false, + Some(type_info), + $pointer_type, + $trait_name, + $constructor_expr, + $get_ref, + $store_ref, + $($impl_type),+ + ) + } - match ref_flag { - RefFlag::Null => Err($crate::error::Error::InvalidRef( - format!("{} cannot be null", stringify!($pointer_type), stringify!($trait_name)).into() - )), - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - context.ref_reader.$get_ref::(ref_id) - .map(|ptr| Self::from(ptr)) - .ok_or_else(|| $crate::error::Error::InvalidData( - format!("{} reference {} not found", stringify!($pointer_type), stringify!($trait_name), ref_id).into() - )) - } - RefFlag::NotNullValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(context, is_field)?; - context.dec_depth(); - - $( - if boxed_any.is::<$impl_type>() { - let concrete = boxed_any.downcast::<$impl_type>() - .map_err(|_| $crate::error::Error::TypeError("Downcast failed".into()))?; - let ptr = $constructor_expr(*concrete) as $pointer_type; - return Ok(Self::from(ptr)); - } - )* - - Err($crate::error::Error::TypeError( - format!("Deserialized type does not implement trait {}", stringify!($trait_name)).into() - )) - } - RefFlag::RefValue => { - context.inc_depth()?; - let harness = context.read_any_typeinfo()?; - let deserializer_fn = harness.get_read_data_fn(); - let boxed_any = deserializer_fn(context, is_field)?; - context.dec_depth(); - - $( - if boxed_any.is::<$impl_type>() { - let concrete = boxed_any.downcast::<$impl_type>() - .map_err(|_| $crate::error::Error::TypeError("Downcast failed".into()))?; - let ptr = $constructor_expr(*concrete) as $pointer_type; - context.ref_reader.$store_ref(ptr.clone()); - return Ok(Self::from(ptr)); - } - )* - - Err($crate::error::Error::TypeError( - format!("Deserialized type does not implement trait {}", stringify!($trait_name)).into() - )) - } - } + fn fory_read_data(context: &mut fory_core::ReadContext) -> Result { + $crate::not_allowed!("fory_read_data should not be called directly on polymorphic {} trait object", stringify!($ptr_path), stringify!($trait_name)) } - fn fory_read_data(context: &mut $crate::resolver::context::ReadContext, is_field: bool) -> Result { - let concrete_fory_type_id = context.reader.read_varuint32()?; - $crate::resolve_and_deserialize!( - concrete_fory_type_id, context, is_field, - |obj| { - let pointer = $constructor_expr(obj) as $pointer_type; - Self::from(pointer) - }, - $trait_name, $($impl_type),+ - ) + + fn fory_get_type_id(_type_resolver: &fory_core::TypeResolver) -> Result { + Ok(fory_core::TypeId::STRUCT as u32) } - fn fory_get_type_id(_type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { - Ok($crate::types::TypeId::STRUCT as u32) + fn fory_static_type_id() -> fory_core::TypeId { + fory_core::TypeId::UNKNOWN } - fn fory_write_type_info(_context: &mut $crate::resolver::context::WriteContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_write_type_info(_context: &mut fory_core::WriteContext) -> Result<(), fory_core::Error> { Ok(()) } - fn fory_read_type_info(_context: &mut $crate::resolver::context::ReadContext, _is_field: bool) -> Result<(), $crate::error::Error> { + fn fory_read_type_info(_context: &mut fory_core::ReadContext) -> Result<(), fory_core::Error> { Ok(()) } @@ -509,26 +507,70 @@ macro_rules! impl_smart_pointer_serializer { true } - fn fory_type_id_dyn(&self, type_resolver: &$crate::resolver::type_resolver::TypeResolver) -> Result { - let any_obj = ::as_any(&*self.0); + fn fory_is_shared_ref() -> bool { + true + } + + fn fory_type_id_dyn(&self, type_resolver: &fory_core::TypeResolver) -> Result { + let any_obj = ::as_any(&*self.0); let concrete_type_id = any_obj.type_id(); type_resolver .get_fory_type_id(concrete_type_id) - .ok_or_else(|| $crate::error::Error::TypeError("Type not registered for trait object".into())) + .ok_or_else(|| fory_core::Error::type_error("Type not registered for trait object")) } fn fory_concrete_type_id(&self) -> std::any::TypeId { - ::as_any(&*self.0).type_id() + ::as_any(&*self.0).type_id() } fn as_any(&self) -> &dyn std::any::Any { - ::as_any(&*self.0) + ::as_any(&*self.0) } } }; } -// Keep the existing Box implementation as is +/// Helper macros for automatic conversions in derive code +/// These are used by fory-derive to generate transparent conversions +/// +/// Convert field of type `Rc` to wrapper for serialization +#[macro_export] +macro_rules! wrap_rc { + ($field:expr, $trait_name:ident) => { + $crate::paste::paste! { + [<$trait_name Rc>]::from($field) + } + }; +} + +/// Convert wrapper back to `Rc` for deserialization +#[macro_export] +macro_rules! unwrap_rc { + ($wrapper:expr, $trait_name:ident) => { + std::rc::Rc::::from($wrapper) + }; +} + +/// Convert `Arc` to wrapper for serialization +#[macro_export] +macro_rules! wrap_arc { + ($field:expr, $trait_name:ident) => { + $crate::paste::paste! { + [<$trait_name Arc>]::from($field) + } + }; +} + +/// Convert `Vec>` to `Vec` for serialization +#[macro_export] +macro_rules! wrap_vec_rc { + ($vec:expr, $trait_name:ident) => { + $crate::paste::paste! { + $vec.into_iter().map(|item| [<$trait_name Rc>]::from(item)).collect() + } + }; +} + impl Default for Box { fn default() -> Self { Box::new(0) @@ -542,16 +584,38 @@ impl ForyDefault for Box { } impl Serializer for Box { - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - let fory_type_id = (**self).fory_type_id_dyn(context.get_type_resolver())?; + fn fory_concrete_type_id(&self) -> std::any::TypeId { + (**self).fory_concrete_type_id() + } + + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { + if write_ref_info { + context.writer.write_i8(RefFlag::NotNullValue as i8); + } + let fory_type_id_dyn = self.fory_type_id_dyn(context.get_type_resolver())?; let concrete_type_id = (**self).fory_concrete_type_id(); + if write_type_info { + context.write_any_typeinfo(fory_type_id_dyn, concrete_type_id)?; + }; + self.fory_write_data_generic(context, has_generics) + } - write_trait_object_headers(context, fory_type_id, concrete_type_id)?; - (**self).fory_write_data(context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } - fn fory_write_data(&self, _context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { - panic!("fory_write_data should not be called directly on Box"); + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + (**self).fory_write_data_generic(context, has_generics) } fn fory_type_id_dyn(&self, type_resolver: &TypeResolver) -> Result { @@ -566,98 +630,68 @@ impl Serializer for Box { true } - fn fory_write_type_info(_context: &mut WriteContext, _is_field: bool) -> Result<(), Error> { - // Box is polymorphic - type info is written per element - Ok(()) + fn fory_write_type_info(_context: &mut WriteContext) -> Result<(), Error> { + panic!("Box is polymorphic - can's write type info statically"); } - fn fory_read_type_info(_context: &mut ReadContext, _is_field: bool) -> Result<(), Error> { - // Box is polymorphic - type info is read per element + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + context.read_any_typeinfo()?; Ok(()) } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { - let fory_type_id = read_trait_object_headers(context)?; - context.inc_depth()?; - let type_resolver = context.get_type_resolver(); - - if let Some(harness) = type_resolver.get_harness(fory_type_id) { - let deserializer_fn = harness.get_read_fn(); - let to_serializer_fn = harness.get_to_serializer(); - let boxed_any = deserializer_fn(context, is_field, true)?; - let trait_object = to_serializer_fn(boxed_any)?; - context.dec_depth(); - Ok(trait_object) - } else { - use crate::types::TypeId; - context.dec_depth(); - match fory_type_id { - id if id == TypeId::LIST as u32 => { - Err(Error::TypeError(format!( - "Cannot deserialize LIST type ID {fory_type_id} as Box without knowing concrete type. \ - Use concrete type instead (e.g., Vec)" - ).into())) - } - id if id == TypeId::MAP as u32 => { - Err(Error::TypeError(format!( - "Cannot deserialize MAP type ID {fory_type_id} as Box without knowing concrete type. \ - Use concrete type instead (e.g., HashMap)" - ).into())) - } - id if id == TypeId::SET as u32 => { - Err(Error::TypeError(format!( - "Cannot deserialize SET type ID {fory_type_id} as Box without knowing concrete type. \ - Use concrete type instead (e.g., HashSet)" - ).into())) - } - _ => { - Err(Error::TypeError(format!("Type ID {fory_type_id} not registered").into())) - } - } - } - } - fn fory_read_data(_context: &mut ReadContext, _is_field: bool) -> Result { - panic!("fory_read_data should not be called directly on Box"); + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_box_seralizer(context, read_ref_info, read_type_info, None) } -} -/// Helper macros for automatic conversions in derive code -/// These are used by fory-derive to generate transparent conversions -/// -/// Convert field of type `Rc` to wrapper for serialization -#[macro_export] -macro_rules! wrap_rc { - ($field:expr, $trait_name:ident) => { - $crate::paste::paste! { - [<$trait_name Rc>]::from($field) - } - }; -} + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + type_info: Arc, + ) -> Result + where + Self: Sized + ForyDefault, + { + read_box_seralizer(context, read_ref_info, false, Some(type_info)) + } -/// Convert wrapper back to `Rc` for deserialization -#[macro_export] -macro_rules! unwrap_rc { - ($wrapper:expr, $trait_name:ident) => { - std::rc::Rc::::from($wrapper) - }; + fn fory_read_data(_context: &mut ReadContext) -> Result { + panic!("fory_read_data should not be called directly on Box"); + } } -/// Convert `Arc` to wrapper for serialization -#[macro_export] -macro_rules! wrap_arc { - ($field:expr, $trait_name:ident) => { - $crate::paste::paste! { - [<$trait_name Arc>]::from($field) - } +fn read_box_seralizer( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + context.inc_depth()?; + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + RefFlag::NotNullValue as i8 }; -} - -/// Convert `Vec>` to `Vec` for serialization -#[macro_export] -macro_rules! wrap_vec_rc { - ($vec:expr, $trait_name:ident) => { - $crate::paste::paste! { - $vec.into_iter().map(|item| [<$trait_name Rc>]::from(item)).collect() - } + if ref_flag != RefFlag::NotNullValue as i8 { + return Err(Error::invalid_data( + "Expected NotNullValue for Box", + )); + } + let typeinfo = if let Some(type_info) = type_info { + type_info + } else { + ensure!( + read_type_info, + Error::invalid_data("Type info must be read for Box") + ); + context.read_any_typeinfo()? }; + let harness = typeinfo.get_harness(); + let boxed_any = harness.get_read_data_fn()(context)?; + let trait_object = harness.get_to_serializer()(boxed_any)?; + context.dec_depth(); + Ok(trait_object) } diff --git a/rust/fory-core/src/serializer/util.rs b/rust/fory-core/src/serializer/util.rs new file mode 100644 index 0000000000..a344ce002b --- /dev/null +++ b/rust/fory-core/src/serializer/util.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::ensure; +use crate::error::Error; +use crate::resolver::context::{ReadContext, WriteContext}; +use crate::serializer::{bool, Serializer}; +use crate::types::TypeId; + +const NO_REF_FLAG_TYPE_IDS: [u32; 7] = [ + TypeId::BOOL as u32, + TypeId::INT8 as u32, + TypeId::INT16 as u32, + TypeId::INT32 as u32, + TypeId::INT64 as u32, + TypeId::FLOAT32 as u32, + TypeId::FLOAT64 as u32, +]; + +#[inline(always)] +pub(crate) fn read_basic_type_info(context: &mut ReadContext) -> Result<(), Error> { + let local_type_id = T::fory_get_type_id(context.get_type_resolver())?; + let remote_type_id = context.reader.read_varuint32()?; + ensure!( + local_type_id == remote_type_id, + Error::type_mismatch(local_type_id, remote_type_id) + ); + Ok(()) +} + +/// Check at runtime whether type info should be skipped for a given type id. +/// +/// According to xlang_serialization_spec.md: +/// - For enums (ENUM/NAMED_ENUM), we should skip writing type info +/// - For structs and ext types, we should write type info +#[inline] +pub fn should_skip_type_info_at_runtime(type_id: u32) -> bool { + let internal_type_id = (type_id & 0xff) as i8; + internal_type_id == TypeId::ENUM as i8 || internal_type_id == TypeId::NAMED_ENUM as i8 +} + +#[inline] +pub fn field_requires_ref_flag(type_id: u32, nullable: bool) -> bool { + if nullable { + return true; + } + let internal_type_id = type_id & 0xff; + !NO_REF_FLAG_TYPE_IDS.contains(&internal_type_id) +} + +#[inline(always)] +pub fn write_dyn_data_generic( + value: &T, + context: &mut WriteContext, + has_generics: bool, +) -> Result<(), Error> { + let any_value = value.as_any(); + let concrete_type_id = any_value.type_id(); + let serializer_fn = context + .write_any_typeinfo(T::fory_static_type_id() as u32, concrete_type_id)? + .get_harness() + .get_write_data_fn(); + serializer_fn(any_value, context, has_generics) +} diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index c04735901b..d88f68a876 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -121,11 +121,13 @@ //! - During deserialization, unresolved references will be patched up by `RefReader::add_callback` //! once the strong pointer becomes available. +use crate::ensure; use crate::error::Error; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::resolver::type_resolver::TypeResolver; +use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; +use crate::types::TypeId; use std::cell::UnsafeCell; use std::rc::Rc; use std::sync::Arc; @@ -312,73 +314,70 @@ impl Serializer for RcWeak { true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { if let Some(rc) = self.upgrade() { - if context - .ref_writer - .try_write_rc_ref(&mut context.writer, &rc) + if !write_ref_info + || !context + .ref_writer + .try_write_rc_ref(&mut context.writer, &rc) { - return Ok(()); + if write_type_info { + T::fory_write_type_info(context)?; + } + T::fory_write_data_generic(&*rc, context, has_generics)?; } - T::fory_write_data(&*rc, context, is_field)?; } else { + ensure!(write_ref_info, "Value pointed by RcWeak is null"); + context.writer.write_i8(RefFlag::Null as i8); } Ok(()) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - self.fory_write(context, is_field) + fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { + panic!( + "RcWeak should be written using `fory_write` to handle reference tracking properly" + ); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { + panic!( + "RcWeak should be written using `fory_write` to handle reference tracking properly" + ); } - fn fory_read(context: &mut ReadContext, is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) + } - match ref_flag { - RefFlag::Null => Ok(RcWeak::new()), - RefFlag::RefValue => { - context.inc_depth()?; - let data = T::fory_read_data(context, is_field)?; - context.dec_depth(); - let rc = Rc::new(data); - let ref_id = context.ref_reader.store_rc_ref(rc); - let rc = context.ref_reader.get_rc_ref::(ref_id).unwrap(); - Ok(RcWeak::from(&rc)) - } - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - - if let Some(rc) = context.ref_reader.get_rc_ref::(ref_id) { - Ok(RcWeak::from(&rc)) - } else { - let result_weak = RcWeak::new(); - let callback_weak = result_weak.clone(); - - context.ref_reader.add_callback(Box::new(move |ref_reader| { - if let Some(rc) = ref_reader.get_rc_ref::(ref_id) { - callback_weak.update(Rc::downgrade(&rc)); - } - })); + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_rc_weak::(context, read_ref_info, read_type_info, None) + } - Ok(result_weak) - } - } - _ => Err(Error::InvalidRef( - format!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into(), - )), - } + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + typeinfo: Arc, + ) -> Result { + read_rc_weak::(context, read_ref_info, false, Some(typeinfo)) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + fn fory_read_data(_: &mut ReadContext) -> Result { + Err(Error::not_allowed("RcWeak should be written using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } fn fory_reserved_space() -> usize { @@ -398,11 +397,72 @@ impl Serializer for RcWeak { } } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } } +fn read_rc_weak( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Ok(RcWeak::new()), + RefFlag::RefValue => { + context.inc_depth()?; + let data = if let Some(type_info) = type_info { + T::fory_read_with_type_info(context, false, type_info)? + } else { + if read_type_info { + context.read_any_typeinfo()?; + } + T::fory_read_data(context)? + }; + context.dec_depth(); + let rc = Rc::new(data); + let ref_id = context.ref_reader.store_rc_ref(rc); + let rc = context.ref_reader.get_rc_ref::(ref_id).unwrap(); + Ok(RcWeak::from(&rc)) + } + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + if let Some(rc) = context.ref_reader.get_rc_ref::(ref_id) { + Ok(RcWeak::from(&rc)) + } else { + let result_weak = RcWeak::new(); + let callback_weak = result_weak.clone(); + context.ref_reader.add_callback(Box::new(move |ref_reader| { + if let Some(rc) = ref_reader.get_rc_ref::(ref_id) { + callback_weak.update(Rc::downgrade(&rc)); + } + })); + + Ok(result_weak) + } + } + RefFlag::NotNullValue => { + // let inner = if let Some(typeinfo) = type_info { + // T::fory_read_with_type_info(context, false, typeinfo)? + // } else { + // T::fory_read_data(context)? + // }; + // Ok(RcWeak::from(&Rc::new(inner))) + Err(Error::invalid_ref("RcWeak can't hold a strong ref value")) + } + } +} + impl ForyDefault for RcWeak { fn fory_default() -> Self { RcWeak::new() @@ -414,77 +474,69 @@ impl Serializer for ArcWeak true } - fn fory_write(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { + fn fory_write( + &self, + context: &mut WriteContext, + write_ref_info: bool, + write_type_info: bool, + has_generics: bool, + ) -> Result<(), Error> { if let Some(arc) = self.upgrade() { - // IMPORTANT: If the target Arc was serialized already, just write a ref - if context - .ref_writer - .try_write_arc_ref(&mut context.writer, &arc) + if !write_ref_info + || !context + .ref_writer + .try_write_arc_ref(&mut context.writer, &arc) { - // Already seen, wrote Ref flag + id, we're done - return Ok(()); + if write_type_info { + T::fory_write_type_info(context)?; + } + T::fory_write_data_generic(&*arc, context, has_generics)?; } - // First time seeing this object, write RefValue and then its data - T::fory_write_data(&*arc, context, is_field)?; } else { + ensure!(write_ref_info, "Value pointed by ArcWeak is null"); context.writer.write_i8(RefFlag::Null as i8); } Ok(()) } - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - self.fory_write(context, is_field) + fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { + panic!( + "ArcWeak should be written using `fory_write` to handle reference tracking properly" + ); } - fn fory_write_type_info(context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - T::fory_write_type_info(context, is_field) + fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { + panic!( + "ArcWeak should be written using `fory_write` to handle reference tracking properly" + ); } - fn fory_read(context: &mut ReadContext, _is_field: bool) -> Result { - let ref_flag = context.ref_reader.read_ref_flag(&mut context.reader)?; + fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { + T::fory_write_type_info(context) + } - match ref_flag { - RefFlag::Null => Ok(ArcWeak::new()), - RefFlag::RefValue => { - context.inc_depth()?; - let data = T::fory_read_data(context, _is_field)?; - context.dec_depth(); - let arc = Arc::new(data); - let ref_id = context.ref_reader.store_arc_ref(arc); - let arc = context.ref_reader.get_arc_ref::(ref_id).unwrap(); - let weak = ArcWeak::from(&arc); - Ok(weak) - } - RefFlag::Ref => { - let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; - let weak = ArcWeak::new(); - - if let Some(arc) = context.ref_reader.get_arc_ref::(ref_id) { - weak.update(Arc::downgrade(&arc)); - } else { - // Capture the raw pointer to the UnsafeCell so we can update it in the callback - let weak_ptr = weak.inner.get(); - context.ref_reader.add_callback(Box::new(move |ref_reader| { - if let Some(arc) = ref_reader.get_arc_ref::(ref_id) { - unsafe { - *weak_ptr = Arc::downgrade(&arc); - } - } - })); - } - Ok(weak) - } - _ => Err(Error::InvalidRef( - format!("Weak can only be Null, RefValue or Ref, got {:?}", ref_flag).into(), - )), - } + fn fory_read( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + read_arc_weak(context, read_ref_info, read_type_info, None) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { - Self::fory_read(context, is_field) + + fn fory_read_with_type_info( + context: &mut ReadContext, + read_ref_info: bool, + typeinfo: Arc, + ) -> Result { + read_arc_weak::(context, read_ref_info, false, Some(typeinfo)) } - fn fory_read_type_info(context: &mut ReadContext, is_field: bool) -> Result<(), Error> { - T::fory_read_type_info(context, is_field) + fn fory_read_data(_: &mut ReadContext) -> Result { + Err(Error::not_allowed("ArcWeak should be written using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) + } + + fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { + T::fory_read_type_info(context) } fn fory_reserved_space() -> usize { @@ -504,11 +556,76 @@ impl Serializer for ArcWeak } } + fn fory_static_type_id() -> TypeId { + T::fory_static_type_id() + } + fn as_any(&self) -> &dyn std::any::Any { self } } +fn read_arc_weak( + context: &mut ReadContext, + read_ref_info: bool, + read_type_info: bool, + type_info: Option>, +) -> Result, Error> { + let ref_flag = if read_ref_info { + context.ref_reader.read_ref_flag(&mut context.reader)? + } else { + RefFlag::NotNullValue + }; + match ref_flag { + RefFlag::Null => Ok(ArcWeak::new()), + RefFlag::RefValue => { + context.inc_depth()?; + let data = if let Some(type_info) = type_info { + T::fory_read_with_type_info(context, false, type_info)? + } else { + if read_type_info { + context.read_any_typeinfo()?; + } + T::fory_read_data(context)? + }; + context.dec_depth(); + let arc = Arc::new(data); + let ref_id = context.ref_reader.store_arc_ref(arc); + let arc = context.ref_reader.get_arc_ref::(ref_id).unwrap(); + let weak = ArcWeak::from(&arc); + Ok(weak) + } + RefFlag::Ref => { + let ref_id = context.ref_reader.read_ref_id(&mut context.reader)?; + let weak = ArcWeak::new(); + + if let Some(arc) = context.ref_reader.get_arc_ref::(ref_id) { + weak.update(Arc::downgrade(&arc)); + } else { + // Capture the raw pointer to the UnsafeCell so we can update it in the callback + let weak_ptr = weak.inner.get(); + context.ref_reader.add_callback(Box::new(move |ref_reader| { + if let Some(arc) = ref_reader.get_arc_ref::(ref_id) { + unsafe { + *weak_ptr = Arc::downgrade(&arc); + } + } + })); + } + Ok(weak) + } + RefFlag::NotNullValue => { + // let inner = if let Some(typeinfo) = type_info { + // T::fory_read_with_type_info(context, false, typeinfo)? + // } else { + // T::fory_read_data(context)? + // }; + // Ok(ArcWeak::from(&Arc::new(inner))) + Err(Error::invalid_ref("ArcWeak can't hold a strong ref value")) + } + } +} + impl ForyDefault for ArcWeak { fn fory_default() -> Self { ArcWeak::new() diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index 14fc585349..e4f177a8d3 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -82,10 +82,49 @@ pub enum TypeId { ARROW_RECORD_BATCH = 38, ARROW_TABLE = 39, UNKNOWN = 63, - // only used at receiver peer - ForyNullable = 265, } +pub const BOOL: u32 = TypeId::BOOL as u32; +pub const INT8: u32 = TypeId::INT8 as u32; +pub const INT16: u32 = TypeId::INT16 as u32; +pub const INT32: u32 = TypeId::INT32 as u32; +pub const VAR_INT32: u32 = TypeId::VAR_INT32 as u32; +pub const INT64: u32 = TypeId::INT64 as u32; +pub const VAR_INT64: u32 = TypeId::VAR_INT64 as u32; +pub const SLI_INT64: u32 = TypeId::SLI_INT64 as u32; +pub const FLOAT16: u32 = TypeId::FLOAT16 as u32; +pub const FLOAT32: u32 = TypeId::FLOAT32 as u32; +pub const FLOAT64: u32 = TypeId::FLOAT64 as u32; +pub const STRING: u32 = TypeId::STRING as u32; +pub const ENUM: u32 = TypeId::ENUM as u32; +pub const NAMED_ENUM: u32 = TypeId::NAMED_ENUM as u32; +pub const STRUCT: u32 = TypeId::STRUCT as u32; +pub const COMPATIBLE_STRUCT: u32 = TypeId::COMPATIBLE_STRUCT as u32; +pub const NAMED_STRUCT: u32 = TypeId::NAMED_STRUCT as u32; +pub const NAMED_COMPATIBLE_STRUCT: u32 = TypeId::NAMED_COMPATIBLE_STRUCT as u32; +pub const EXT: u32 = TypeId::EXT as u32; +pub const NAMED_EXT: u32 = TypeId::NAMED_EXT as u32; +pub const LIST: u32 = TypeId::LIST as u32; +pub const SET: u32 = TypeId::SET as u32; +pub const MAP: u32 = TypeId::MAP as u32; +pub const DURATION: u32 = TypeId::DURATION as u32; +pub const TIMESTAMP: u32 = TypeId::TIMESTAMP as u32; +pub const LOCAL_DATE: u32 = TypeId::LOCAL_DATE as u32; +pub const DECIMAL: u32 = TypeId::DECIMAL as u32; +pub const BINARY: u32 = TypeId::BINARY as u32; +pub const ARRAY: u32 = TypeId::ARRAY as u32; +pub const BOOL_ARRAY: u32 = TypeId::BOOL_ARRAY as u32; +pub const INT8_ARRAY: u32 = TypeId::INT8_ARRAY as u32; +pub const INT16_ARRAY: u32 = TypeId::INT16_ARRAY as u32; +pub const INT32_ARRAY: u32 = TypeId::INT32_ARRAY as u32; +pub const INT64_ARRAY: u32 = TypeId::INT64_ARRAY as u32; +pub const FLOAT16_ARRAY: u32 = TypeId::FLOAT16_ARRAY as u32; +pub const FLOAT32_ARRAY: u32 = TypeId::FLOAT32_ARRAY as u32; +pub const FLOAT64_ARRAY: u32 = TypeId::FLOAT64_ARRAY as u32; +pub const ARROW_RECORD_BATCH: u32 = TypeId::ARROW_RECORD_BATCH as u32; +pub const ARROW_TABLE: u32 = TypeId::ARROW_TABLE as u32; +pub const UNKNOWN: u32 = TypeId::UNKNOWN as u32; + const MAX_UNT32: u64 = (1 << 31) - 1; // todo: struct hash @@ -171,6 +210,24 @@ pub static PRIMITIVE_ARRAY_TYPE_MAP: &[(&str, u32, &str)] = &[ ("f64", TypeId::FLOAT64_ARRAY as u32, "Vec"), ]; +#[inline(always)] +pub fn is_primitive_type(type_id: TypeId) -> bool { + matches!( + type_id, + TypeId::BOOL + | TypeId::INT8 + | TypeId::INT16 + | TypeId::INT32 + | TypeId::INT64 + | TypeId::FLOAT32 + | TypeId::FLOAT64 + | TypeId::STRING + | TypeId::LOCAL_DATE + | TypeId::TIMESTAMP + ) +} + +#[inline(always)] pub fn is_internal_type(type_id: u32) -> bool { if type_id == 0 || type_id >= TypeId::UNKNOWN as u32 { return false; @@ -188,6 +245,25 @@ pub fn is_internal_type(type_id: u32) -> bool { !excluded.contains(&type_id) } +#[inline(always)] +pub fn need_to_write_type_for_field(type_id: TypeId) -> bool { + matches!( + type_id, + TypeId::STRUCT + | TypeId::COMPATIBLE_STRUCT + | TypeId::NAMED_STRUCT + | TypeId::NAMED_COMPATIBLE_STRUCT + | TypeId::EXT + | TypeId::NAMED_EXT + | TypeId::UNKNOWN + ) +} + +#[inline(always)] +pub fn is_container_type(type_id: TypeId) -> bool { + type_id == TypeId::LIST || type_id == TypeId::SET || type_id == TypeId::MAP +} + pub fn compute_field_hash(hash: u32, id: i16) -> u32 { let mut new_hash: u64 = (hash as u64) * 31 + (id as u64); while new_hash >= MAX_UNT32 { @@ -252,9 +328,9 @@ impl TryFrom for Language { 4 => Ok(Language::Go), 5 => Ok(Language::Javascript), 6 => Ok(Language::Rust), - _ => Err(Error::InvalidData( - format!("Unsupported language code, value:{num}").into(), - )), + _ => Err(Error::invalid_data(format!( + "Unsupported language code, value:{num}" + ))), } } } diff --git a/rust/fory-derive/src/lib.rs b/rust/fory-derive/src/lib.rs index 059fca4399..0ef903e719 100644 --- a/rust/fory-derive/src/lib.rs +++ b/rust/fory-derive/src/lib.rs @@ -198,13 +198,18 @@ mod util; /// city: String, /// } /// ``` -#[proc_macro_derive(ForyObject)] +#[proc_macro_derive(ForyObject, attributes(fory_debug))] pub fn proc_macro_derive_fory_object(input: proc_macro::TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); // Check if this is being applied to a trait (which is not possible with derive macros) // Derive macros can only be applied to structs, enums, and unions - object::derive_serializer(&input) + let debug_enabled = input + .attrs + .iter() + .any(|attr| attr.path().is_ident("fory_debug")); + + object::derive_serializer(&input, debug_enabled) } /// Derive macro for row-based serialization. diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 61f7c02340..0027646d8c 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -37,15 +37,9 @@ pub fn gen_reserved_space() -> TokenStream { } } -pub fn gen_write_type_info() -> TokenStream { - quote! { - fory_core::serializer::enum_::write_type_info::(context, is_field) - } -} - -pub fn gen_read_type_info() -> TokenStream { +pub fn gen_write(_data_enum: &DataEnum) -> TokenStream { quote! { - fory_core::serializer::enum_::read_type_info::(context, is_field) + fory_core::serializer::enum_::write::(self, context, write_ref_info, write_type_info) } } @@ -62,6 +56,23 @@ pub fn gen_write_data(data_enum: &DataEnum) -> TokenStream { }) } } +pub fn gen_write_type_info() -> TokenStream { + quote! { + fory_core::serializer::enum_::write_type_info::(context) + } +} + +pub fn gen_read(_: &DataEnum) -> TokenStream { + quote! { + fory_core::serializer::enum_::read::(context, read_ref_info, read_type_info) + } +} + +pub fn gen_read_with_type_info(_: &DataEnum) -> TokenStream { + quote! { + fory_core::serializer::enum_::read::(context, read_ref_info, false) + } +} pub fn gen_read_data(data_enum: &DataEnum) -> TokenStream { let variant_idents: Vec<_> = data_enum.variants.iter().map(|v| &v.ident).collect(); @@ -72,25 +83,13 @@ pub fn gen_read_data(data_enum: &DataEnum) -> TokenStream { #( #variant_values => Ok(Self::#variant_idents), )* - _ => return Err(fory_core::error::Error::UnknownEnum("unknown enum value".into())), + _ => return Err(fory_core::error::Error::unknown_enum("unknown enum value")), } } } -pub fn gen_read_compatible() -> TokenStream { - quote! { - fory_core::serializer::enum_::read_compatible::(context) - } -} - -pub fn gen_write(_data_enum: &DataEnum) -> TokenStream { - quote! { - fory_core::serializer::enum_::write::(self, context, is_field) - } -} - -pub fn gen_read(_data_enum: &DataEnum) -> TokenStream { +pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::enum_::read::(context, is_field) + fory_core::serializer::enum_::read_type_info::(context) } } diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index e72ff3eed6..03755e1fee 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -use fory_core::types::RefFlag; use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, quote}; use syn::{Field, Type}; use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, - extract_type_name, is_primitive_type, parse_generic_tree, skip_ref_flag, StructField, + extract_type_name, get_struct_name, is_debug_enabled, is_primitive_type, + should_skip_type_info_for_field, skip_ref_flag, StructField, }; fn create_private_field_name(field: &Field) -> Ident { @@ -41,7 +41,7 @@ fn declare_var(fields: &[&Field]) -> Vec { let ty = &field.ty; let var_name = create_private_field_name(field); match classify_trait_object_field(ty) { - StructField::BoxDyn(_) + StructField::BoxDyn | StructField::RcDyn(_) | StructField::ArcDyn(_) => { quote! { @@ -75,7 +75,7 @@ fn assign_value(fields: &[&Field]) -> Vec { let name = &field.ident; let var_name = create_private_field_name(field); match classify_trait_object_field(&field.ty) { - StructField::BoxDyn(_) | StructField::RcDyn(_) | StructField::ArcDyn(_) => { + StructField::BoxDyn | StructField::RcDyn(_) | StructField::ArcDyn(_) => { quote! { #name: #var_name } @@ -103,28 +103,10 @@ fn assign_value(fields: &[&Field]) -> Vec { fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let ty = &field.ty; - match classify_trait_object_field(ty) { - StructField::BoxDyn(trait_name) => { - let from_any_fn = format_ident!("from_any_internal_{}", trait_name); - let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); + let base = match classify_trait_object_field(ty) { + StructField::BoxDyn => { quote! { - let ref_flag = context.reader.read_i8()?; - if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - return Err(Error::InvalidRef("Expected NotNullValue for trait object field".into())); - } - - let fory_type_id = context.reader.read_varuint32()?; - - let harness = context.get_type_resolver() - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; - - let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(context, true, false)?; - - let base_type_id = fory_type_id >> 8; - let #private_ident = #helper_mod::#from_any_fn(any_box, base_type_id)?; - + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, true, true)?; } } StructField::RcDyn(trait_name) => { @@ -132,7 +114,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::Serializer>::fory_read(context, true, true)?; let #private_ident = std::rc::Rc::::from(wrapper); } } @@ -141,7 +123,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::Serializer>::fory_read(context, true, true)?; let #private_ident = std::sync::Arc::::from(wrapper); } } @@ -150,7 +132,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::Serializer>::fory_read(context, true, false)?; let #private_ident = wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect(); @@ -161,7 +143,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::Serializer>::fory_read(context, true, false)?; let #private_ident = wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect(); @@ -172,7 +154,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::Serializer>::fory_read(context, true, false)?; let #private_ident = wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect(); @@ -183,7 +165,7 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::Serializer>::fory_read(context, true, false)?; let #private_ident = wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect(); @@ -191,21 +173,60 @@ fn gen_read_field(field: &Field, private_ident: &Ident) -> TokenStream { } StructField::Forward => { quote! { - let #private_ident = fory_core::serializer::Serializer::fory_read(context, true)?; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, true, true)?; } } _ => { let skip_ref_flag = skip_ref_flag(ty); - quote! { - let #private_ident = fory_core::serializer::read_ref_info_data::<#ty>(context, true, #skip_ref_flag, false)?; + let skip_type_info = should_skip_type_info_for_field(ty); + if skip_type_info { + // Known types (primitives, strings, collections) - skip type info at compile time + if skip_ref_flag { + quote! { + let #private_ident = <#ty as fory_core::Serializer>::fory_read_data(context)?; + } + } else { + quote! { + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, true, false)?; + } + } + } else { + // Custom types (struct/enum/ext) - need runtime check for enums + quote! { + let is_enum = <#ty as fory_core::Serializer>::fory_static_type_id() == fory_core::types::TypeId::ENUM; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, true, !is_enum)?; + } } } + }; + + if is_debug_enabled() { + let struct_name = get_struct_name().expect("struct context not set"); + let struct_name_lit = syn::LitStr::new(&struct_name, proc_macro2::Span::call_site()); + let field_name = field.ident.as_ref().unwrap().to_string(); + let field_name_lit = syn::LitStr::new(&field_name, proc_macro2::Span::call_site()); + quote! { + fory_core::serializer::struct_::struct_before_read_field( + #struct_name_lit, + #field_name_lit, + context, + ); + #base + fory_core::serializer::struct_::struct_after_read_field( + #struct_name_lit, + #field_name_lit, + (&#private_ident) as &dyn std::any::Any, + context, + ); + } + } else { + base } } pub fn gen_read_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::read_type_info::(context, is_field) + fory_core::serializer::struct_::read_type_info::(context) } } @@ -248,24 +269,12 @@ pub fn gen_read_data(fields: &[&Field]) -> TokenStream { fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenStream { let ty = &field.ty; + let field_kind = classify_trait_object_field(ty); - match classify_trait_object_field(ty) { - StructField::BoxDyn(trait_name) => { - let from_any_fn = format_ident!("from_any_internal_{}", trait_name); - let helper_mod = format_ident!("__fory_trait_helpers_{}", trait_name); + let base = match field_kind { + StructField::BoxDyn => { quote! { - let ref_flag = context.reader.read_i8()?; - if ref_flag != fory_core::types::RefFlag::NotNullValue as i8 { - return Err(Error::InvalidRef("Expected NotNullValue for trait object field".into())); - } - let fory_type_id = context.reader.read_varuint32()?; - let harness = context.get_type_resolver() - .get_harness(fory_type_id) - .ok_or_else(|| Error::TypeError("Type not registered for trait object field".into()))?; - let deserializer_fn = harness.get_read_fn(); - let any_box = deserializer_fn(context, true, false)?; - let base_type_id = fory_type_id >> 8; - #var_name = #helper_mod::#from_any_fn(any_box, base_type_id)?; + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, true, true)?); } } StructField::RcDyn(trait_name) => { @@ -273,7 +282,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::Serializer>::fory_read(context, true, true)?; #var_name = Some(std::rc::Rc::::from(wrapper)); } } @@ -282,7 +291,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper = <#wrapper_ty as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper = <#wrapper_ty as fory_core::Serializer>::fory_read(context, true, true)?; #var_name = Some(std::sync::Arc::::from(wrapper)); } } @@ -291,7 +300,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::Serializer>::fory_read(context, true, false)?; #var_name = Some(wrapper_vec.into_iter() .map(|w| std::rc::Rc::::from(w)) .collect()); @@ -302,7 +311,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_vec = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_vec = as fory_core::Serializer>::fory_read(context, true, false)?; #var_name = Some(wrapper_vec.into_iter() .map(|w| std::sync::Arc::::from(w)) .collect()); @@ -313,7 +322,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::Serializer>::fory_read(context, true, false)?; #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::rc::Rc::::from(v))) .collect()); @@ -324,7 +333,7 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - let wrapper_map = as fory_core::serializer::Serializer>::fory_read(context, true)?; + let wrapper_map = as fory_core::Serializer>::fory_read(context, true, false)?; #var_name = Some(wrapper_map.into_iter() .map(|(k, v)| (k, std::sync::Arc::::from(v))) .collect()); @@ -332,78 +341,155 @@ fn gen_read_compatible_match_arm_body(field: &Field, var_name: &Ident) -> TokenS } StructField::ContainsTraitObject => { quote! { - let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_type_resolver()); - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, skip_ref_flag, false)?); + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, true, true)?); } } StructField::Forward => { quote! { - #var_name = Some(fory_core::serializer::Serializer::fory_read(context, true)?); + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, true, true)?); } } StructField::None => { - let generic_tree = parse_generic_tree(ty); - let local_nullable = generic_tree.name == "Option"; let _base_ty = match &ty { Type::Path(type_path) => &type_path.path.segments.first().unwrap().ident, _ => panic!("Unsupported type"), }; - if local_nullable { - quote! { - if _field.field_type.nullable { - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, false, false)?); - } else { - #var_name = Some( - fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)? - ); - } - } - } else { - let dec_by_option = need_declared_by_option(field); + let skip_type_info = should_skip_type_info_for_field(ty); + let dec_by_option = need_declared_by_option(field); + if skip_type_info { if dec_by_option { quote! { - if !_field.field_type.nullable { - #var_name = Some(fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, true, false)?); } else { - #var_name = fory_core::serializer::read_ref_info_data::>(context, true, false, false)? + #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } } } else { - let null_flag = RefFlag::Null as i8; quote! { - if !_field.field_type.nullable { - #var_name = fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?; + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + #var_name = <#ty as fory_core::Serializer>::fory_read(context, true, false)?; } else { - if context.reader.read_i8()? == #null_flag { - #var_name = <#ty as fory_core::serializer::ForyDefault>::fory_default(); - } else { - #var_name = fory_core::serializer::read_ref_info_data::<#ty>(context, true, true, false)?; - } + #var_name = <#ty as fory_core::Serializer>::fory_read_data(context)?; } } } + } else if dec_by_option { + quote! { + let skip_type_info = fory_core::serializer::util::should_skip_type_info_at_runtime(_field.field_type.type_id); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, true, !skip_type_info)?); + } else { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); + } + } + } else { + quote! { + let skip_type_info = fory_core::serializer::util::should_skip_type_info_at_runtime(_field.field_type.type_id); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + #var_name = <#ty as fory_core::Serializer>::fory_read(context, true, !skip_type_info)?; + } else { + #var_name = <#ty as fory_core::Serializer>::fory_read_data(context)?; + } + } } } + }; + + if is_debug_enabled() { + let struct_name = get_struct_name().expect("struct context not set"); + let struct_name_lit = syn::LitStr::new(&struct_name, proc_macro2::Span::call_site()); + let field_name = field.ident.as_ref().unwrap().to_string(); + let field_name_lit = syn::LitStr::new(&field_name, proc_macro2::Span::call_site()); + quote! { + fory_core::serializer::struct_::struct_before_read_field( + #struct_name_lit, + #field_name_lit, + context, + ); + #base + fory_core::serializer::struct_::struct_after_read_field( + #struct_name_lit, + #field_name_lit, + (&#var_name) as &dyn std::any::Any, + context, + ); + } + } else { + quote! { + #base + } } } pub fn gen_read(struct_ident: &Ident) -> TokenStream { quote! { - let ref_flag = context.reader.read_i8()?; - if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::types::RefFlag::RefValue as i8) { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::RefFlag::RefValue as i8) { + if context.is_compatible() { + let type_info = if read_type_info { + context.read_any_typeinfo()? + } else { + let rs_type_id = std::any::TypeId::of::(); + context.get_type_info(&rs_type_id)? + }; + <#struct_ident as fory_core::StructSerializer>::fory_read_compatible(context, type_info) + } else { + if read_type_info { + ::fory_read_type_info(context)?; + } + ::fory_read_data(context) + } + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) + } else { + Err(fory_core::error::Error::invalid_ref(format!("Unknown ref flag, value:{ref_flag}"))) + } + } +} + +pub fn gen_read_with_type_info(struct_ident: &Ident) -> TokenStream { + // fn fory_read_with_type_info( + // context: &mut ReadContext, + // read_ref_info: bool, + // type_info: Arc, + // ) -> Result + quote! { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::RefFlag::RefValue as i8) { if context.is_compatible() { - <#struct_ident as fory_core::serializer::Serializer>::fory_read_compatible(context) + <#struct_ident as fory_core::StructSerializer>::fory_read_compatible(context, type_info) } else { - ::fory_read_type_info(context, false)?; - ::fory_read_data(context, false) + ::fory_read_data(context) } - } else if ref_flag == (fory_core::types::RefFlag::Null as i8) { - Ok(::fory_default()) - } else if ref_flag == (fory_core::types::RefFlag::Ref as i8) { - // Err(fory_core::error::Error::Ref) - todo!() + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) } else { - Err(fory_core::error::Error::InvalidRef(format!("Unknown ref flag, value:{ref_flag}").into())) + Err(fory_core::error::Error::invalid_ref(format!("Unknown ref flag, value:{ref_flag}"))) } } } @@ -427,28 +513,58 @@ pub fn gen_read_compatible(fields: &[&Field]) -> TokenStream { }) .collect(); + let skip_arm = if is_debug_enabled() { + let struct_name = get_struct_name().expect("struct context not set"); + let struct_name_lit = syn::LitStr::new(&struct_name, proc_macro2::Span::call_site()); + quote! { + _ => { + let field_type = &_field.field_type; + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + field_type.type_id, + field_type.nullable, + ); + let field_name = _field.field_name.as_str(); + fory_core::serializer::struct_::struct_before_read_field( + #struct_name_lit, + field_name, + context, + ); + fory_core::serializer::skip::skip_field_value(context, &field_type, read_ref_flag)?; + let placeholder: &dyn std::any::Any = &(); + fory_core::serializer::struct_::struct_after_read_field( + #struct_name_lit, + field_name, + placeholder, + context, + ); + } + } + } else { + quote! { + _ => { + let field_type = &_field.field_type; + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + field_type.type_id, + field_type.nullable, + ); + fory_core::serializer::skip::skip_field_value(context, field_type, read_ref_flag)?; + } + } + }; + quote! { - let remote_type_id = context.reader.read_varuint32()?; - let meta_index = context.reader.read_varuint32()?; - let meta = context.get_meta(meta_index as usize); - let fields = { - let meta = context.get_meta(meta_index as usize); - meta.get_field_infos().clone() - }; + let fields = type_info.get_type_meta().get_field_infos().clone(); #(#declare_ts)* - - let local_type_hash = context.get_type_resolver().get_type_info(std::any::TypeId::of::())?.get_type_meta().get_hash(); - if meta.get_hash() == local_type_hash { - ::fory_read_data(context, false) + let meta = context.get_type_info(&std::any::TypeId::of::())?.get_type_meta(); + let local_type_hash = meta.get_hash(); + let remote_type_hash = type_info.get_type_meta().get_hash(); + if remote_type_hash == local_type_hash { + ::fory_read_data(context) } else { for _field in fields.iter() { match _field.field_id { #(#match_arms)* - _ => { - let field_type = &_field.field_type; - let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&field_type); - fory_core::serializer::skip::skip_field_value(context, &field_type, read_ref_flag)?; - } + #skip_arm } } Ok(Self { diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 2ba9b9513c..51a1ae1311 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -36,10 +36,10 @@ fn has_existing_default(ast: &syn::DeriveInput, trait_name: &str) -> bool { }) } -pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { +pub fn derive_serializer(ast: &syn::DeriveInput, debug_enabled: bool) -> TokenStream { let name = &ast.ident; use crate::object::util::{clear_struct_context, set_struct_context}; - set_struct_context(&name.to_string()); + set_struct_context(&name.to_string(), debug_enabled); // Check if ForyDefault is already derived/implemented let has_existing_default = has_existing_default(ast, "ForyDefault"); @@ -65,7 +65,10 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { derive_enum::gen_actual_type_id(), quote! { &[] }, derive_enum::gen_field_fields_info(s), - derive_enum::gen_read_compatible(), + quote! { + Err(fory_core::Error::not_allowed("`fory_read_compatible` should only be invoked at struct type" + )) + }, ), syn::Data::Union(_) => { panic!("Union is not supported") @@ -73,34 +76,40 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { }; // Serializer let ( - reserved_space_ts, - write_type_info_ts, - read_type_info_ts, - write_data_ts, - read_data_ts, write_ts, + write_data_ts, + write_type_info_ts, read_ts, + read_with_type_info_ts, + read_data_ts, + read_type_info_ts, + reserved_space_ts, + static_type_id_ts, ) = match &ast.data { syn::Data::Struct(s) => { let fields = sorted_fields(&s.fields); ( - write::gen_reserved_space(&fields), - write::gen_write_type_info(), - read::gen_read_type_info(), - write::gen_write_data(&fields), - read::gen_read_data(&fields), write::gen_write(), + write::gen_write_data(&fields), + write::gen_write_type_info(), read::gen_read(name), + read::gen_read_with_type_info(name), + read::gen_read_data(&fields), + read::gen_read_type_info(), + write::gen_reserved_space(&fields), + quote! { fory_core::TypeId::STRUCT }, ) } syn::Data::Enum(e) => ( - derive_enum::gen_reserved_space(), - derive_enum::gen_write_type_info(), - derive_enum::gen_read_type_info(), - derive_enum::gen_write_data(e), - derive_enum::gen_read_data(e), derive_enum::gen_write(e), + derive_enum::gen_write_data(e), + derive_enum::gen_write_type_info(), derive_enum::gen_read(e), + derive_enum::gen_read_with_type_info(e), + derive_enum::gen_read_data(e), + derive_enum::gen_read_type_info(), + derive_enum::gen_reserved_space(), + quote! { fory_core::TypeId::ENUM }, ), syn::Data::Union(_) => { panic!("Union is not supported") @@ -111,11 +120,11 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { let type_idx = misc::allocate_type_id(); let gen = quote! { - use fory_core::serializer::ForyDefault as _; + use fory_core::ForyDefault as _; #default_impl - impl fory_core::serializer::StructSerializer for #name { + impl fory_core::StructSerializer for #name { fn fory_type_index() -> u32 { #type_idx } @@ -131,8 +140,13 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { fn fory_fields_info(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result, fory_core::error::Error> { #fields_info_ts } + + fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext, type_info: std::sync::Arc) -> Result { + #read_compatible_ts + } } - impl fory_core::serializer::Serializer for #name { + + impl fory_core::Serializer for #name { fn fory_get_type_id(type_resolver: &fory_core::resolver::type_resolver::TypeResolver) -> Result { type_resolver.get_type_id(&std::any::TypeId::of::(), #type_idx) } @@ -145,36 +159,43 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { self } + fn fory_static_type_id() -> fory_core::TypeId + where + Self: Sized, + { + #static_type_id_ts + } + fn fory_reserved_space() -> usize { #reserved_space_ts } - fn fory_write_type_info(context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { - #write_type_info_ts + fn fory_write(&self, context: &mut fory_core::resolver::context::WriteContext, write_ref_info: bool, write_type_info: bool, _: bool) -> Result<(), fory_core::error::Error> { + #write_ts } - fn fory_read_type_info(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result<(), fory_core::error::Error> { - #read_type_info_ts + fn fory_write_data(&self, context: &mut fory_core::resolver::context::WriteContext) -> Result<(), fory_core::error::Error> { + #write_data_ts } - fn fory_write_data(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { - #write_data_ts + fn fory_write_type_info(context: &mut fory_core::resolver::context::WriteContext) -> Result<(), fory_core::error::Error> { + #write_type_info_ts } - fn fory_read_data( context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { - #read_data_ts + fn fory_read(context: &mut fory_core::resolver::context::ReadContext, read_ref_info: bool, read_type_info: bool) -> Result { + #read_ts } - fn fory_write(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) -> Result<(), fory_core::error::Error> { - #write_ts + fn fory_read_with_type_info(context: &mut fory_core::resolver::context::ReadContext, read_ref_info: bool, type_info: std::sync::Arc) -> Result { + #read_with_type_info_ts } - fn fory_read(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result { - #read_ts + fn fory_read_data( context: &mut fory_core::resolver::context::ReadContext) -> Result { + #read_data_ts } - fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext) -> Result { - #read_compatible_ts + fn fory_read_type_info(context: &mut fory_core::resolver::context::ReadContext) -> Result<(), fory_core::error::Error> { + #read_type_info_ts } } }; @@ -225,12 +246,12 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { } StructField::Forward => { quote! { - #ident: <#ty as fory_core::serializer::ForyDefault>::fory_default() + #ident: <#ty as fory_core::ForyDefault>::fory_default() } } _ => { quote! { - #ident: <#ty as fory_core::serializer::ForyDefault>::fory_default() + #ident: <#ty as fory_core::ForyDefault>::fory_default() } } } @@ -238,7 +259,7 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { if has_existing_default { quote! { - impl fory_core::serializer::ForyDefault for #name { + impl fory_core::ForyDefault for #name { fn fory_default() -> Self { Self::default() } @@ -246,7 +267,7 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { } } else { quote! { - impl fory_core::serializer::ForyDefault for #name { + impl fory_core::ForyDefault for #name { fn fory_default() -> Self { Self { #(#field_inits),* @@ -274,7 +295,7 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { if let Some(first_variant) = e.variants.first() { let variant_ident = &first_variant.ident; quote! { - impl fory_core::serializer::ForyDefault for #name { + impl fory_core::ForyDefault for #name { fn fory_default() -> Self { Self::#variant_ident } @@ -287,7 +308,7 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { } } } else { - // impl fory_core::serializer::ForyDefault for #name { + // impl fory_core::ForyDefault for #name { // fn fory_default() -> Self { // panic!("No unit-like variants found in enum {}", stringify!(#name)); // } @@ -296,7 +317,7 @@ fn generate_default_impl(ast: &syn::DeriveInput) -> proc_macro2::TokenStream { } } else { quote! { - impl fory_core::serializer::ForyDefault for #name { + impl fory_core::ForyDefault for #name { fn fory_default() -> Self { Self::default() } diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index ee5e0a5856..0ef41a93e9 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -30,14 +30,17 @@ thread_local! { static MACRO_CONTEXT: RefCell> = const {RefCell::new(None)}; } +#[derive(Clone)] struct MacroContext { struct_name: String, + debug_enabled: bool, } -pub(super) fn set_struct_context(name: &str) { +pub(super) fn set_struct_context(name: &str, debug_enabled: bool) { MACRO_CONTEXT.with(|ctx| { *ctx.borrow_mut() = Some(MacroContext { struct_name: name.to_string(), + debug_enabled, }); }); } @@ -48,10 +51,19 @@ pub(super) fn clear_struct_context() { }); } -fn get_struct_name() -> Option { +pub(super) fn get_struct_name() -> Option { MACRO_CONTEXT.with(|ctx| ctx.borrow().as_ref().map(|c| c.struct_name.clone())) } +pub(super) fn is_debug_enabled() -> bool { + MACRO_CONTEXT.with(|ctx| { + ctx.borrow() + .as_ref() + .map(|c| c.debug_enabled) + .unwrap_or(false) + }) +} + pub(super) fn contains_trait_object(ty: &Type) -> bool { match ty { Type::TraitObject(_) => true, @@ -100,7 +112,8 @@ pub(super) fn create_wrapper_types_arc(trait_name: &str) -> WrapperTypes { } pub(super) enum StructField { - BoxDyn(String), + #[allow(unused_variables)] + BoxDyn, RcDyn(String), ArcDyn(String), VecRc(String), @@ -187,8 +200,8 @@ pub(super) fn classify_trait_object_field(ty: &Type) -> StructField { if is_forward_field(ty) { return StructField::Forward; } - if let Some((_, trait_name)) = is_box_dyn_trait(ty) { - return StructField::BoxDyn(trait_name); + if let Some((_, _)) = is_box_dyn_trait(ty) { + return StructField::BoxDyn; } if let Some((_, trait_name)) = is_rc_dyn_trait(ty) { return StructField::RcDyn(trait_name); @@ -278,6 +291,18 @@ pub(super) fn extract_type_name(ty: &Type) -> String { } } +#[allow(dead_code)] +pub(super) fn is_option(ty: &Type) -> bool { + if let Type::Path(type_path) = ty { + if let Some(seg) = type_path.path.segments.last() { + if seg.ident == "Option" { + return true; + } + } + } + false +} + pub(super) fn parse_generic_tree(ty: &Type) -> TypeNode { // Handle trait objects specially - they can't be parsed as normal types if matches!(ty, Type::TraitObject(_)) { @@ -378,6 +403,10 @@ type FieldGroups = ( FieldGroup, ); +fn extract_option_inner(s: &str) -> Option<&str> { + s.strip_prefix("Option<")?.strip_suffix(">") +} + const PRIMITIVE_TYPE_NAMES: [&str; 7] = ["bool", "i8", "i16", "i32", "i64", "f32", "f64"]; fn get_primitive_type_id(ty: &str) -> u32 { @@ -397,11 +426,121 @@ pub(super) fn is_primitive_type(ty: &str) -> bool { PRIMITIVE_TYPE_NAMES.contains(&ty) } -fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { - fn extract_option_inner(s: &str) -> Option<&str> { - s.strip_prefix("Option<")?.strip_suffix(">") +pub(crate) fn get_type_id_by_type_ast(ty: &Type) -> u32 { + let ty_str: String = ty + .to_token_stream() + .to_string() + .chars() + .filter(|c| !c.is_whitespace()) + .collect::(); + get_type_id_by_name(&ty_str) +} + +/// Get the type ID for a given type string. +/// +/// Returns: +/// - `type_id` for known types (primitives, internal types, collections) +/// - `UNKNOWN` for unknown/user-defined types +pub(crate) fn get_type_id_by_name(ty: &str) -> u32 { + let ty = extract_option_inner(ty).unwrap_or(ty); + // Check primitive types + if PRIMITIVE_TYPE_NAMES.contains(&ty) { + return get_primitive_type_id(ty); + } + + // Check internal types + match ty { + "String" => return TypeId::STRING as u32, + "NaiveDate" => return TypeId::LOCAL_DATE as u32, + "NaiveDateTime" => return TypeId::TIMESTAMP as u32, + "Duration" => return TypeId::DURATION as u32, + "Decimal" => return TypeId::DECIMAL as u32, + "Vec" | "bytes" => return TypeId::BINARY as u32, + _ => {} + } + + // Check primitive arrays + match ty { + "Vec" => return TypeId::BOOL_ARRAY as u32, + "Vec" => return TypeId::INT8_ARRAY as u32, + "Vec" => return TypeId::INT16_ARRAY as u32, + "Vec" => return TypeId::INT32_ARRAY as u32, + "Vec" => return TypeId::INT64_ARRAY as u32, + "Vec" => return TypeId::FLOAT16_ARRAY as u32, + "Vec" => return TypeId::FLOAT32_ARRAY as u32, + "Vec" => return TypeId::FLOAT64_ARRAY as u32, + _ => {} + } + + // Check collection types + if ty.starts_with("Vec<") + || ty.starts_with("VecDeque<") + || ty.starts_with("LinkedList<") + || ty.starts_with("BinaryHeap<") + { + return TypeId::LIST as u32; + } + + if ty.starts_with("HashSet<") || ty.starts_with("BTreeSet<") { + return TypeId::SET as u32; } + if ty.starts_with("HashMap<") || ty.starts_with("BTreeMap<") { + return TypeId::MAP as u32; + } + + // Unknown type + TypeId::UNKNOWN as u32 +} + +fn get_primitive_type_size(type_id_num: u32) -> i32 { + let type_id = TypeId::try_from(type_id_num as i16).unwrap(); + match type_id { + TypeId::BOOL => 1, + TypeId::INT8 => 1, + TypeId::INT16 => 2, + TypeId::INT32 => 4, + TypeId::VAR_INT32 => 4, + TypeId::INT64 => 8, + TypeId::VAR_INT64 => 8, + TypeId::FLOAT16 => 2, + TypeId::FLOAT32 => 4, + TypeId::FLOAT64 => 8, + _ => unreachable!(), + } +} + +fn is_compress(type_id: u32) -> bool { + [ + TypeId::INT32 as u32, + TypeId::INT64 as u32, + TypeId::VAR_INT32 as u32, + TypeId::VAR_INT64 as u32, + ] + .contains(&type_id) +} + +fn is_internal_type_id(type_id: u32) -> bool { + [ + TypeId::STRING as u32, + TypeId::LOCAL_DATE as u32, + TypeId::TIMESTAMP as u32, + TypeId::DURATION as u32, + TypeId::DECIMAL as u32, + TypeId::BINARY as u32, + TypeId::BOOL_ARRAY as u32, + TypeId::INT8_ARRAY as u32, + TypeId::INT16_ARRAY as u32, + TypeId::INT32_ARRAY as u32, + TypeId::INT64_ARRAY as u32, + TypeId::FLOAT16_ARRAY as u32, + TypeId::FLOAT32_ARRAY as u32, + TypeId::FLOAT64_ARRAY as u32, + ] + .contains(&type_id) +} + +fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { let mut primitive_fields = Vec::new(); let mut nullable_primitive_fields = Vec::new(); let mut internal_type_fields = Vec::new(); @@ -418,44 +557,22 @@ fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { } } - fn get_other_internal_type_id(ty: &str) -> u32 { - match ty { - "String" => TypeId::STRING as u32, - "NaiveDate" => TypeId::LOCAL_DATE as u32, - "NaiveDateTime" => TypeId::TIMESTAMP as u32, - "Duration" => TypeId::DURATION as u32, - "Decimal" => TypeId::DECIMAL as u32, - "Vec" | "bytes" => TypeId::BINARY as u32, - "Vec" => TypeId::BOOL_ARRAY as u32, - "Vec" => TypeId::INT8_ARRAY as u32, - "Vec" => TypeId::INT16_ARRAY as u32, - "Vec" => TypeId::INT32_ARRAY as u32, - "Vec" => TypeId::INT64_ARRAY as u32, - "Vec" => TypeId::FLOAT16_ARRAY as u32, - "Vec" => TypeId::FLOAT32_ARRAY as u32, - "Vec" => TypeId::FLOAT64_ARRAY as u32, - _ => 0, - } - } - let mut group_field = |ident: String, ty: &str| { + let type_id = get_type_id_by_name(ty); + // Categorize based on type_id if PRIMITIVE_TYPE_NAMES.contains(&ty) { - primitive_fields.push((ident, ty.to_string(), get_primitive_type_id(ty))); - } else if get_other_internal_type_id(ty) > 0 { - let internal_type_id = get_other_internal_type_id(ty); - internal_type_fields.push((ident, ty.to_string(), internal_type_id)); - } else if ty.starts_with("Vec<") - || ty.starts_with("VecDeque<") - || ty.starts_with("LinkedList<") - || ty.starts_with("BinaryHeap<") - { - list_fields.push((ident, ty.to_string(), TypeId::LIST as u32)); - } else if ty.starts_with("HashSet<") || ty.starts_with("BTreeSet<") { - set_fields.push((ident, ty.to_string(), TypeId::SET as u32)); - } else if ty.starts_with("HashMap<") || ty.starts_with("BTreeMap<") { - map_fields.push((ident, ty.to_string(), TypeId::MAP as u32)); + primitive_fields.push((ident, ty.to_string(), type_id)); + } else if is_internal_type_id(type_id) { + internal_type_fields.push((ident, ty.to_string(), type_id)); + } else if type_id == TypeId::LIST as u32 { + list_fields.push((ident, ty.to_string(), type_id)); + } else if type_id == TypeId::SET as u32 { + set_fields.push((ident, ty.to_string(), type_id)); + } else if type_id == TypeId::MAP as u32 { + map_fields.push((ident, ty.to_string(), type_id)); } else { - other_fields.push((ident, ty.to_string(), TypeId::UNKNOWN as u32)); + // User-defined type + other_fields.push((ident, ty.to_string(), type_id)); } }; @@ -487,33 +604,6 @@ fn group_fields_by_type(fields: &[&Field]) -> FieldGroups { } } - fn get_primitive_type_size(type_id_num: u32) -> i32 { - let type_id = TypeId::try_from(type_id_num as i16).unwrap(); - match type_id { - TypeId::BOOL => 1, - TypeId::INT8 => 1, - TypeId::INT16 => 2, - TypeId::INT32 => 4, - TypeId::VAR_INT32 => 4, - TypeId::INT64 => 8, - TypeId::VAR_INT64 => 8, - TypeId::FLOAT16 => 2, - TypeId::FLOAT32 => 4, - TypeId::FLOAT64 => 8, - _ => unreachable!(), - } - } - - fn is_compress(type_id: u32) -> bool { - [ - TypeId::INT32 as u32, - TypeId::INT64 as u32, - TypeId::VAR_INT32 as u32, - TypeId::VAR_INT64 as u32, - ] - .contains(&type_id) - } - fn numeric_sorter(a: &(String, String, u32), b: &(String, String, u32)) -> std::cmp::Ordering { let compress_a = is_compress(a.2); let compress_b = is_compress(b.2); @@ -592,3 +682,35 @@ pub(crate) fn skip_ref_flag(ty: &Type) -> bool { // !T::fory_is_option() && PRIMITIVE_TYPES.contains(&elem_type_id) PRIMITIVE_TYPE_NAMES.contains(&extract_type_name(ty).as_str()) } + +/// Determine whether to skip writing type info for a struct field based on its type. +/// +/// According to xlang_serialization_spec.md: +/// - Primitive fields: skip type info +/// - Nullable primitive fields (Option): skip type info +/// - Internal type fields (String, Date, arrays): skip type info +/// - List/Set/Map fields: skip type info +/// - Struct fields: WRITE type info +/// - Ext fields: WRITE type info +/// - Enum fields: skip type info (enum is morphic) +/// +/// Returns true if type info should be skipped, false if it should be written. +pub(crate) fn should_skip_type_info_for_field(ty: &Type) -> bool { + let type_id = get_type_id_by_type_ast(ty); + if [ + TypeId::STRUCT as u32, + TypeId::COMPATIBLE_STRUCT as u32, + TypeId::NAMED_STRUCT as u32, + TypeId::NAMED_COMPATIBLE_STRUCT as u32, + TypeId::EXT as u32, + TypeId::NAMED_EXT as u32, + TypeId::UNKNOWN as u32, + ] + .contains(&type_id) + { + // Struct and Ext types need type info + return false; + } + // Primitive, nullable primitive, internal types, List/Set/Map skip type info + true +} diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 035562ecd9..f242da28d6 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -16,9 +16,11 @@ // under the License. use super::util::{ - classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, skip_ref_flag, - StructField, + classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, + get_struct_name, get_type_id_by_type_ast, is_debug_enabled, should_skip_type_info_for_field, + skip_ref_flag, StructField, }; +use fory_core::types::TypeId; use proc_macro2::TokenStream; use quote::quote; use syn::Field; @@ -27,7 +29,7 @@ pub fn gen_reserved_space(fields: &[&Field]) -> TokenStream { let reserved_size_expr: Vec<_> = fields.iter().map(|field| { let ty = &field.ty; match classify_trait_object_field(ty) { - StructField::BoxDyn(_) => { + StructField::BoxDyn => { quote! { fory_core::types::SIZE_OF_REF_AND_TYPE } @@ -36,52 +38,52 @@ pub fn gen_reserved_space(fields: &[&Field]) -> TokenStream { let types = create_wrapper_types_rc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - <#wrapper_ty as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + <#wrapper_ty as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::ArcDyn(trait_name) => { let types = create_wrapper_types_arc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - <#wrapper_ty as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + <#wrapper_ty as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::VecRc(trait_name) => { let types = create_wrapper_types_rc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::VecArc(trait_name) => { let types = create_wrapper_types_arc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::HashMapRc(key_ty, trait_name) => { let types = create_wrapper_types_rc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::HashMapArc(key_ty, trait_name) => { let types = create_wrapper_types_arc(&trait_name); let wrapper_ty = types.wrapper_ty; quote! { - as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } StructField::Forward => { quote! { - <#ty as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + <#ty as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } _ => { quote! { - <#ty as fory_core::serializer::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE + <#ty as fory_core::Serializer>::fory_reserved_space() + fory_core::types::SIZE_OF_REF_AND_TYPE } } } @@ -95,34 +97,17 @@ pub fn gen_reserved_space(fields: &[&Field]) -> TokenStream { pub fn gen_write_type_info() -> TokenStream { quote! { - fory_core::serializer::struct_::write_type_info::(context, is_field) + fory_core::serializer::struct_::write_type_info::(context) } } fn gen_write_field(field: &Field) -> TokenStream { let ty = &field.ty; let ident = &field.ident; - match classify_trait_object_field(ty) { - StructField::BoxDyn(_) => { + let base = match classify_trait_object_field(ty) { + StructField::BoxDyn => { quote! { - { - let any_ref = self.#ident.as_any(); - let concrete_type_id = any_ref.type_id(); - let fory_type_id = context.get_type_resolver() - .get_fory_type_id(concrete_type_id) - .ok_or_else(|| fory_core::error::Error::TypeError("Type not registered for trait object field".into()))?; - - context.writer.write_i8(fory_core::types::RefFlag::NotNullValue as i8)?; - context.writer.write_varuint32(fory_type_id); - - let harness = context - .get_type_resolver() - .get_harness(fory_type_id) - .ok_or_else(|| fory_core::error::Error::TypeError("Harness not found for trait object field".into()))?; - - let serializer_fn = harness.get_write_fn(); - serializer_fn(any_ref, context, true)?; - } + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, true, true, false)?; } } StructField::RcDyn(trait_name) => { @@ -130,10 +115,8 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); - fory_core::serializer::Serializer::fory_write(&wrapper, context, true)?; - } + let wrapper = #wrapper_ty::from(self.#ident.clone() as std::rc::Rc); + <#wrapper_ty as fory_core::Serializer>::fory_write(&wrapper, context, true, true, false)?; } } StructField::ArcDyn(trait_name) => { @@ -141,10 +124,8 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); - fory_core::serializer::Serializer::fory_write(&wrapper, context, true)?; - } + let wrapper = #wrapper_ty::from(self.#ident.clone() as std::sync::Arc); + <#wrapper_ty as fory_core::Serializer>::fory_write(&wrapper, context, true, true, false)?; } } StructField::VecRc(trait_name) => { @@ -152,12 +133,10 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() - .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) - .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true)?; - } + let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() + .map(|item| #wrapper_ty::from(item.clone() as std::rc::Rc)) + .collect(); + as fory_core::Serializer>::fory_write(&wrapper_vec, context, true, false, true)?; } } StructField::VecArc(trait_name) => { @@ -165,12 +144,10 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() - .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) - .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_vec, context, true)?; - } + let wrapper_vec: Vec<#wrapper_ty> = self.#ident.iter() + .map(|item| #wrapper_ty::from(item.clone() as std::sync::Arc)) + .collect(); + as fory_core::Serializer>::fory_write(&wrapper_vec, context, true, false, true)?; } } StructField::HashMapRc(key_ty, trait_name) => { @@ -178,12 +155,10 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() - .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) - .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true)?; - } + let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() + .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::rc::Rc))) + .collect(); + as fory_core::Serializer>::fory_write(&wrapper_map, context, true, false, true)?; } } StructField::HashMapArc(key_ty, trait_name) => { @@ -191,27 +166,79 @@ fn gen_write_field(field: &Field) -> TokenStream { let wrapper_ty = types.wrapper_ty; let trait_ident = types.trait_ident; quote! { - { - let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() - .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) - .collect(); - fory_core::serializer::Serializer::fory_write(&wrapper_map, context, true)?; - } + let wrapper_map: std::collections::HashMap<#key_ty, #wrapper_ty> = self.#ident.iter() + .map(|(k, v)| (k.clone(), #wrapper_ty::from(v.clone() as std::sync::Arc))) + .collect(); + as fory_core::Serializer>::fory_write(&wrapper_map, context, true, false, true)?; } } StructField::Forward => { quote! { - { - fory_core::serializer::Serializer::fory_write(&self.#ident, context, true)?; - } + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, true, true, false)?; } } _ => { let skip_ref_flag = skip_ref_flag(ty); - quote! { - fory_core::serializer::write_ref_info_data::<#ty>(&self.#ident, context, true, #skip_ref_flag, false)?; + let skip_type_info = should_skip_type_info_for_field(ty); + let type_id = get_type_id_by_type_ast(ty); + if type_id == TypeId::LIST as u32 + || type_id == TypeId::SET as u32 + || type_id == TypeId::MAP as u32 + { + quote! { + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, true, false, true)?; + } + } else { + // Known types (primitives, strings, collections) - skip type info at compile time + // For custom types that we can't determine at compile time (like enums), + // we need to check at runtime whether to skip type info + if skip_type_info { + if skip_ref_flag { + quote! { + <#ty as fory_core::Serializer>::fory_write_data(&self.#ident, context)?; + } + } else { + quote! { + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, true, false, false)?; + } + } + } else if skip_ref_flag { + quote! { + let is_enum = <#ty as fory_core::Serializer>::fory_static_type_id() == fory_core::types::TypeId::ENUM; + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, false, !is_enum, false)?; + } + } else { + quote! { + let is_enum = <#ty as fory_core::Serializer>::fory_static_type_id() == fory_core::types::TypeId::ENUM; + <#ty as fory_core::Serializer>::fory_write(&self.#ident, context, true, !is_enum, false)?; + } + } } } + }; + + if is_debug_enabled() { + let struct_name = get_struct_name().expect("struct context not set"); + let struct_name_lit = syn::LitStr::new(&struct_name, proc_macro2::Span::call_site()); + let field_name = field.ident.as_ref().unwrap().to_string(); + let field_name_lit = syn::LitStr::new(&field_name, proc_macro2::Span::call_site()); + quote! { + fory_core::serializer::struct_::struct_before_write_field( + #struct_name_lit, + #field_name_lit, + (&self.#ident) as &dyn std::any::Any, + context, + ); + #base + fory_core::serializer::struct_::struct_after_write_field( + #struct_name_lit, + #field_name_lit, + (&self.#ident) as &dyn std::any::Any, + context, + ); + } + } else { + base } } @@ -225,6 +252,6 @@ pub fn gen_write_data(fields: &[&Field]) -> TokenStream { pub fn gen_write() -> TokenStream { quote! { - fory_core::serializer::struct_::write::(self, context, is_field) + fory_core::serializer::struct_::write::(self, context, write_ref_info, write_type_info) } } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index b8e6c538fe..b5c6d01179 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -717,14 +717,14 @@ //! } //! //! impl Serializer for CustomType { -//! fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { +//! fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { //! context.writer.write_i32(self.value); //! context.writer.write_varuint32(self.name.len() as u32); //! context.writer.write_utf8_string(&self.name); //! Ok(()) //! } //! -//! fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { +//! fn fory_read_data(context: &mut ReadContext) -> Result { //! let value = context.reader.read_i32()?; //! let len = context.reader.read_varuint32()? as usize; //! let name = context.reader.read_utf8_string(len)?; diff --git a/rust/tests/tests/compatible/test_container.rs b/rust/tests/tests/compatible/test_container.rs index 9fa1969a50..0fc43c5ca0 100644 --- a/rust/tests/tests/compatible/test_container.rs +++ b/rust/tests/tests/compatible/test_container.rs @@ -23,6 +23,7 @@ use fory_core::resolver::context::{ReadContext, WriteContext}; use fory_derive::ForyObject; #[derive(ForyObject, PartialEq, Eq, Hash, Debug)] +#[fory_debug] struct Item { id: i32, } diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 46ccf19720..105edb1069 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -142,6 +142,7 @@ fn nonexistent_struct() { #[test] fn option() { #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Animal { f1: Option, f2: Option, @@ -285,6 +286,7 @@ fn nullable_container() { #[test] fn inner_nullable() { #[derive(ForyObject, Debug)] + #[fory_debug] pub struct Item1 { f1: Vec>, f2: HashSet>, @@ -293,6 +295,7 @@ fn inner_nullable() { } #[derive(ForyObject, Debug)] + #[fory_debug] pub struct Item2 { f1: Vec, f2: HashSet, @@ -322,6 +325,7 @@ fn inner_nullable() { #[test] fn nullable_struct() { #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] pub struct Item { name: String, data: Vec>, @@ -329,6 +333,7 @@ fn nullable_struct() { } #[derive(ForyObject, Debug)] + #[fory_debug] pub struct Person1 { f1: Item, f2: Option, @@ -337,6 +342,7 @@ fn nullable_struct() { } #[derive(ForyObject, Debug)] + #[fory_debug] pub struct Person2 { f1: Option, f2: Item, @@ -391,6 +397,7 @@ fn enum_without_payload() { Blue, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Person1 { f1: Color1, f2: Color1, @@ -403,6 +410,7 @@ fn enum_without_payload() { last: i8, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Person2 { // same f1: Color1, @@ -458,6 +466,7 @@ fn named_enum() { White, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Item1 { f1: Color, f2: Color, @@ -472,6 +481,7 @@ fn named_enum() { last: i8, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Item2 { f1: Color, f2: Option, diff --git a/rust/tests/tests/compatible/test_struct_enum.rs b/rust/tests/tests/compatible/test_struct_enum.rs index ca25a0aba8..0d5d92ba6f 100644 --- a/rust/tests/tests/compatible/test_struct_enum.rs +++ b/rust/tests/tests/compatible/test_struct_enum.rs @@ -26,6 +26,7 @@ use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; #[derive(ForyObject, Debug, PartialEq, Eq, Hash)] +#[fory_debug] struct Item { id: i32, } @@ -40,6 +41,7 @@ enum Color { } #[derive(ForyObject, Debug, PartialEq)] +#[fory_debug] struct Person { // primitive f1: bool, @@ -78,6 +80,7 @@ struct Person { } #[derive(ForyObject, Debug, PartialEq)] +#[fory_debug] struct Empty {} #[test] @@ -165,6 +168,7 @@ fn skip_basic() { #[test] fn nested() { #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Element { f1: Vec, f2: HashSet, @@ -174,6 +178,7 @@ fn nested() { f6: HashMap, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Nested { f1: Vec, f2: HashSet, @@ -216,6 +221,7 @@ fn nested() { #[test] fn compatible_nullable() { #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Nonnull { f1: bool, f2: i8, @@ -241,6 +247,7 @@ fn compatible_nullable() { f29: HashMap, } #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct Nullable { f1: Option, f2: Option, @@ -322,6 +329,7 @@ fn compatible_nullable() { #[test] fn name_mismatch() { #[derive(ForyObject, Debug, PartialEq)] + #[fory_debug] struct MismatchPerson { f2: bool, f3: i8, @@ -402,13 +410,12 @@ fn ext() { fn fory_write_data( &self, context: &mut WriteContext, - is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, context, is_field) + write_data(&self.id, context) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(context)?, }) } @@ -457,13 +464,12 @@ fn skip_ext() { fn fory_write_data( &self, context: &mut WriteContext, - is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, context, is_field) + write_data(&self.id, context) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(context)?, }) } @@ -527,13 +533,12 @@ fn compatible_ext() { fn fory_write_data( &self, context: &mut WriteContext, - is_field: bool, ) -> Result<(), fory_core::error::Error> { - write_data(&self.id, context, is_field) + write_data(&self.id, context) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(Self { - id: read_data(context, is_field)?, + id: read_data(context)?, }) } fn fory_type_id_dyn( diff --git a/rust/tests/tests/mod.rs b/rust/tests/tests/mod.rs index bc8c4e4d76..5b26bc5f09 100644 --- a/rust/tests/tests/mod.rs +++ b/rust/tests/tests/mod.rs @@ -16,4 +16,6 @@ // under the License. mod compatible; +mod test_any; +mod test_collection; mod test_max_dyn_depth; diff --git a/rust/tests/tests/test_collection.rs b/rust/tests/tests/test_collection.rs index 116b397072..e8b3c2f72d 100644 --- a/rust/tests/tests/test_collection.rs +++ b/rust/tests/tests/test_collection.rs @@ -21,7 +21,8 @@ use std::collections::{BTreeSet, BinaryHeap, HashSet}; #[test] fn test_btreeset_roundtrip() { - let fory: Fory = Fory::default(); + let mut fory: Fory = Fory::default(); + fory.register_generic_trait::>().unwrap(); let mut original = BTreeSet::new(); original.insert(1); diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index f6bfdc4084..ffd571abbc 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -250,17 +250,14 @@ fn test_string_serializer() { ]; for s in &test_strings { // make is_field=true to skip read/write type_id - assert_eq!(*s, String::fory_read_data(&mut context, true).unwrap()); - assert_eq!( - *s, - String::fory_read_data(&mut context_compress, true).unwrap() - ); + assert_eq!(*s, String::fory_read_data(&mut context).unwrap()); + assert_eq!(*s, String::fory_read_data(&mut context_compress).unwrap()); } let writer = Writer::default(); let fory = Fory::default().compatible(true).xlang(true); let mut context = WriteContext::new_from_fory(writer, &fory); for s in &test_strings { - s.fory_write_data(&mut context, true).unwrap(); + s.fory_write_data(&mut context).unwrap(); } fs::write(&data_file_path, context.writer.dump()).unwrap(); } @@ -614,19 +611,15 @@ struct MyExt { id: i32, } impl Serializer for MyExt { - fn fory_write_data( - &self, - context: &mut WriteContext, - _is_field: bool, - ) -> Result<(), fory_core::error::Error> { + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), fory_core::error::Error> { // set is_field=false to write type_id like in java - write_data(&self.id, context, false) + write_data(&self.id, context) } - fn fory_read_data(context: &mut ReadContext, _is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(Self { // set is_field=false to write type_id like in java - id: read_data(context, false)?, + id: read_data(context)?, }) } diff --git a/rust/tests/tests/test_debug.rs b/rust/tests/tests/test_debug.rs new file mode 100644 index 0000000000..3aa50da918 --- /dev/null +++ b/rust/tests/tests/test_debug.rs @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +use fory_core::buffer::{Reader, Writer}; +use fory_core::fory::Fory; +use fory_core::resolver::context::{ReadContext, WriteContext}; +use fory_core::serializer::struct_::{ + reset_struct_debug_hooks, set_after_read_field_func, set_after_write_field_func, + set_before_read_field_func, set_before_write_field_func, +}; + +#[derive(fory_derive::ForyObject)] +#[fory_debug] +struct DebugSample { + a: i32, + b: String, +} + +fn event_log() -> &'static Mutex> { + static LOG: OnceLock>> = OnceLock::new(); + LOG.get_or_init(|| Mutex::new(Vec::new())) +} + +fn record_event(prefix: &str, struct_name: &str, field_name: &str) { + if struct_name == "DebugSample" { + event_log() + .lock() + .unwrap() + .push(format!("{prefix}:{struct_name}:{field_name}")); + } +} + +fn before_write( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + _context: &mut WriteContext, +) { + record_event("write", struct_name, field_name); +} + +fn after_write( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + _context: &mut WriteContext, +) { + record_event("after_write", struct_name, field_name); +} + +fn before_read(struct_name: &str, field_name: &str, _context: &mut ReadContext) { + record_event("before_read", struct_name, field_name); +} + +fn after_read( + struct_name: &str, + field_name: &str, + _field_value: &dyn Any, + _context: &mut ReadContext, +) { + record_event("after_read", struct_name, field_name); +} + +struct HookGuard { + _lock: MutexGuard<'static, ()>, +} + +impl HookGuard { + fn new() -> Self { + static HOOK_LOCK: OnceLock> = OnceLock::new(); + let lock = HOOK_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap(); + reset_struct_debug_hooks(); + event_log().lock().unwrap().clear(); + HookGuard { _lock: lock } + } +} + +impl Drop for HookGuard { + fn drop(&mut self) { + reset_struct_debug_hooks(); + event_log().lock().unwrap().clear(); + } +} + +#[test] +fn debug_hooks_trigger_for_struct() { + let _guard = HookGuard::new(); + + set_before_write_field_func(before_write); + set_after_write_field_func(after_write); + set_before_read_field_func(before_read); + set_after_read_field_func(after_read); + + let mut fory = Fory::default(); + fory.register::(4001).unwrap(); + let sample = DebugSample { + a: 7, + b: "debug".to_string(), + }; + + let bytes = fory.serialize(&sample).unwrap(); + let _: DebugSample = fory.deserialize(&bytes).unwrap(); + + let entries = event_log().lock().unwrap().clone(); + assert_eq!( + entries.iter().filter(|e| e.starts_with("write:")).count(), + 2 + ); + assert_eq!( + entries + .iter() + .filter(|e| e.starts_with("after_write:")) + .count(), + 2 + ); + assert_eq!( + entries + .iter() + .filter(|e| e.starts_with("before_read:")) + .count(), + 2 + ); + assert_eq!( + entries + .iter() + .filter(|e| e.starts_with("after_read:")) + .count(), + 2 + ); + assert!(entries.contains(&"write:DebugSample:a".to_string())); + assert!(entries.contains(&"after_write:DebugSample:a".to_string())); + assert!(entries.contains(&"before_read:DebugSample:a".to_string())); + assert!(entries.contains(&"after_read:DebugSample:b".to_string())); + + event_log().lock().unwrap().clear(); + + let mut fory_compat = Fory::default().compatible(true); + fory_compat.register::(4001).unwrap(); + let writer = Writer::default(); + let mut write_ctx = WriteContext::new_from_fory(writer, &fory_compat); + fory_compat + .serialize_with_context(&sample, &mut write_ctx) + .unwrap(); + let compat_bytes = write_ctx.writer.dump(); + let reader = Reader::new(compat_bytes.as_slice()); + let mut read_ctx = ReadContext::new_from_fory(reader, &fory_compat); + let _: DebugSample = fory_compat.deserialize_with_context(&mut read_ctx).unwrap(); + + let compat_entries = event_log().lock().unwrap().clone(); + assert!( + compat_entries + .iter() + .filter(|e| e.starts_with("write:")) + .count() + >= 2 + ); + assert!( + compat_entries + .iter() + .filter(|e| e.starts_with("after_write:")) + .count() + >= 2 + ); + assert!( + compat_entries + .iter() + .filter(|e| e.starts_with("before_read:")) + .count() + >= 2 + ); + assert!( + compat_entries + .iter() + .filter(|e| e.starts_with("after_read:")) + .count() + >= 2 + ); +} diff --git a/rust/tests/tests/test_ext.rs b/rust/tests/tests/test_ext.rs index 3e023622a3..625bc43862 100644 --- a/rust/tests/tests/test_ext.rs +++ b/rust/tests/tests/test_ext.rs @@ -60,13 +60,13 @@ fn test_use() { } impl Serializer for Item { - fn fory_write_data(&self, context: &mut WriteContext, is_field: bool) -> Result<(), Error> { - write_data(&self.f1, context, is_field) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + write_data(&self.f1, context) } - fn fory_read_data(context: &mut ReadContext, is_field: bool) -> Result { + fn fory_read_data(context: &mut ReadContext) -> Result { Ok(Self { - f1: read_data(context, is_field)?, + f1: read_data(context)?, f2: 0, }) } diff --git a/rust/tests/tests/test_max_dyn_depth.rs b/rust/tests/tests/test_max_dyn_depth.rs index 2205c95afe..d1e777adf6 100644 --- a/rust/tests/tests/test_max_dyn_depth.rs +++ b/rust/tests/tests/test_max_dyn_depth.rs @@ -20,6 +20,7 @@ use fory_derive::ForyObject; use std::any::Any; #[derive(ForyObject, Debug)] +#[fory_debug] struct Container { value: i32, nested: Option>, @@ -27,6 +28,9 @@ struct Container { #[test] fn test_max_dyn_depth_exceeded_box_dyn_any() { + if fory_core::error::should_panic_on_error() { + return; + } for compatible in [false, true] { let mut fory = Fory::default().max_dyn_depth(2).compatible(compatible); fory.register::(100).unwrap(); @@ -59,6 +63,9 @@ fn test_max_dyn_depth_exceeded_box_dyn_any() { #[test] fn test_max_dyn_depth_within_limit_box_dyn_any() { + if fory_core::error::should_panic_on_error() { + return; + } let mut fory = Fory::default().max_dyn_depth(3); fory.register::(100).unwrap(); @@ -83,6 +90,9 @@ fn test_max_dyn_depth_within_limit_box_dyn_any() { #[test] fn test_max_dyn_depth_default_exceeded() { + if fory_core::error::should_panic_on_error() { + return; + } let mut fory = Fory::default(); fory.register::(100).unwrap(); @@ -111,6 +121,9 @@ fn test_max_dyn_depth_default_exceeded() { #[test] fn test_max_dyn_depth_default_within_limit() { + if fory_core::error::should_panic_on_error() { + return; + } let mut fory = Fory::default(); fory.register::(100).unwrap(); diff --git a/rust/tests/tests/test_meta_string.rs b/rust/tests/tests/test_meta_string.rs index ceab0e6001..cf51600260 100644 --- a/rust/tests/tests/test_meta_string.rs +++ b/rust/tests/tests/test_meta_string.rs @@ -238,6 +238,10 @@ fn test_non_ascii_encoding() { #[test] fn test_non_ascii_encoding_and_non_utf8() { + if fory_core::error::should_panic_on_error() { + println!("Skipping test_non_ascii_encoding_and_non_utf8 when panic on error is enabled"); + return; + } let encoder = &TYPE_NAME_ENCODER; let non_ascii_string = "こんにちは"; diff --git a/rust/tests/tests/test_one_struct.rs b/rust/tests/tests/test_one_struct.rs new file mode 100644 index 0000000000..e50e0eb738 --- /dev/null +++ b/rust/tests/tests/test_one_struct.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use fory_core::fory::Fory; +use fory_derive::ForyObject; +use std::collections::HashMap; + +#[test] +fn test_simple() { + // a single test for cargo expand and analysis: `cargo expand --test test_simple_struct 2>&1 > expanded.rs` + // &["f7", "last", "f2", "f5", "f3", "f6", "f1"] + #[derive(ForyObject, Debug)] + #[fory_debug] + struct Animal1 { + f1: HashMap>, + f2: String, + f3: Vec, + f5: String, + f6: Vec, + f7: i8, + last: i8, + } + + // &["f7", "f5", "last", "f4", "f3", "f6", "f1"] + #[derive(ForyObject, Debug)] + struct Animal2 { + f1: HashMap>, + f3: Vec, + f4: String, + f5: i8, + f6: Vec, + f7: i16, + last: i8, + } + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); + let animal: Animal1 = Animal1 { + f1: HashMap::from([(1, vec![2])]), + f2: String::from("hello"), + f3: vec![1, 2, 3], + f5: String::from("f5"), + f6: vec![42], + f7: 43, + last: 44, + }; + + let bin = fory1.serialize(&animal).unwrap(); + let obj: Animal2 = fory2.deserialize(&bin).unwrap(); + assert_eq!(animal.f1, obj.f1); + assert_eq!(animal.f3, obj.f3); + assert_eq!(obj.f4, String::default()); + assert_eq!(obj.f5, i8::default()); + assert_eq!(obj.f6, Vec::::default()); + assert_eq!(obj.f7, i16::default()); + assert_eq!(animal.last, obj.last); +} diff --git a/rust/tests/tests/test_simple_struct.rs b/rust/tests/tests/test_simple_struct.rs index 1024a6facc..dac10a74cd 100644 --- a/rust/tests/tests/test_simple_struct.rs +++ b/rust/tests/tests/test_simple_struct.rs @@ -15,57 +15,165 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; + use fory_core::fory::Fory; use fory_derive::ForyObject; -use std::collections::HashMap; +// Test 1: Simple struct with one primitive field, non-compatible mode +#[test] +fn test_one_field_primitive_non_compatible() { + #[derive(ForyObject, Debug, PartialEq)] + struct Data { + value: i32, + } + + let mut fory = Fory::default(); + fory.register::(100).unwrap(); + let data = Data { value: 42 }; + let bytes = fory.serialize(&data).unwrap(); + let result: Data = fory.deserialize(&bytes).unwrap(); + assert_eq!(data, result); +} + +// Test 2: Simple struct with one String field, non-compatible mode #[test] -fn test_simple() { - // a single test for cargo expand and analysis: `cargo expand --test test_simple_struct 2>&1 > expanded.rs` - // &["f7", "last", "f2", "f5", "f3", "f6", "f1"] +fn test_one_field_string_non_compatible() { + #[derive(ForyObject, Debug, PartialEq)] + struct Data { + name: String, + } + + let mut fory = Fory::default(); + fory.register::(101).unwrap(); + let data = Data { + name: String::from("hello"), + }; + let bytes = fory.serialize(&data).unwrap(); + let result: Data = fory.deserialize(&bytes).unwrap(); + assert_eq!(data, result); +} + +// Test 3: Compatible mode - serialize with one field, deserialize with different type +#[test] +fn test_compatible_field_type_change() { #[derive(ForyObject, Debug)] - struct Animal1 { - f1: HashMap>, - f2: String, - f3: Vec, - f5: String, - f6: Vec, - f7: i8, - last: i8, + struct Data1 { + value: i32, } - // &["f7", "f5", "last", "f4", "f3", "f6", "f1"] #[derive(ForyObject, Debug)] - struct Animal2 { - f1: HashMap>, - f3: Vec, - f4: String, - f5: i8, - f6: Vec, - f7: i16, - last: i8, + struct Data2 { + value: Option, } + let mut fory1 = Fory::default().compatible(true); let mut fory2 = Fory::default().compatible(true); - fory1.register::(999).unwrap(); - fory2.register::(999).unwrap(); - let animal: Animal1 = Animal1 { - f1: HashMap::from([(1, vec![2])]), - f2: String::from("hello"), - f3: vec![1, 2, 3], - f5: String::from("f5"), - f6: vec![42], - f7: 43, - last: 44, + fory1.register::(100).unwrap(); + fory2.register::(100).unwrap(); + + let data1 = Data1 { value: 42 }; + let bytes = fory1.serialize(&data1).unwrap(); + let result: Data2 = fory2.deserialize(&bytes).unwrap(); + assert_eq!(result.value.unwrap(), 42i32); +} + +// Test 4: Compatible mode - serialize with field, deserialize with empty struct +#[test] +fn test_compatible_to_empty_struct() { + #[derive(ForyObject, Debug)] + struct DataWithField { + value: i32, + name: String, + } + + #[derive(ForyObject, Debug)] + struct EmptyData {} + + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(101).unwrap(); + fory2.register::(101).unwrap(); + + let data1 = DataWithField { + value: 42, + name: String::from("test"), }; + let bytes = fory1.serialize(&data1).unwrap(); + let _result: EmptyData = fory2.deserialize(&bytes).unwrap(); + // If we get here without panic, the test passes +} + +// Test 5: Compatible mode - empty struct to struct with fields (fields get defaults) +#[test] +fn test_compatible_from_empty_struct() { + #[derive(ForyObject, Debug)] + struct EmptyData {} - let bin = fory1.serialize(&animal).unwrap(); - let obj: Animal2 = fory2.deserialize(&bin).unwrap(); - assert_eq!(animal.f1, obj.f1); - assert_eq!(animal.f3, obj.f3); - assert_eq!(obj.f4, String::default()); - assert_eq!(obj.f5, i8::default()); - assert_eq!(obj.f6, Vec::::default()); - assert_eq!(obj.f7, i16::default()); - assert_eq!(animal.last, obj.last); + #[derive(ForyObject, Debug)] + struct DataWithField { + value: i32, + name: String, + } + + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(102).unwrap(); + fory2.register::(102).unwrap(); + + let data1 = EmptyData {}; + let bytes = fory1.serialize(&data1).unwrap(); + let result: DataWithField = fory2.deserialize(&bytes).unwrap(); + assert_eq!(result.value, 0); // Default for i32 + assert_eq!(result.name, String::default()); // Default for String +} + +#[test] +fn test_compatible_vec_to_empty_struct() { + #[derive(ForyObject, Debug)] + struct DataWithField { + value: Vec, + name: String, + } + + #[derive(ForyObject, Debug)] + struct EmptyData {} + + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(101).unwrap(); + fory2.register::(101).unwrap(); + + let data1 = DataWithField { + value: vec![32], + name: String::from("test"), + }; + let bytes = fory1.serialize(&data1).unwrap(); + let _result: EmptyData = fory2.deserialize(&bytes).unwrap(); + // If we get here without panic, the test passes +} + +#[test] +fn test_compatible_map_to_empty_struct() { + #[derive(ForyObject, Debug)] + struct DataWithField { + value: HashMap, + name: String, + } + + #[derive(ForyObject, Debug)] + struct EmptyData {} + + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(101).unwrap(); + fory2.register::(101).unwrap(); + + let data1 = DataWithField { + value: HashMap::from([(String::from("k1"), 1i32), (String::from("k2"), 2i32)]), + name: String::from("test"), + }; + let bytes = fory1.serialize(&data1).unwrap(); + let _result: EmptyData = fory2.deserialize(&bytes).unwrap(); + // If we get here without panic, the test passes } diff --git a/rust/tests/tests/test_trait_object.rs b/rust/tests/tests/test_trait_object.rs index 0919625ac9..03979b27d9 100644 --- a/rust/tests/tests/test_trait_object.rs +++ b/rust/tests/tests/test_trait_object.rs @@ -75,10 +75,7 @@ fn test_option_some_roundtrip() { #[test] fn test_hashmap_roundtrip() { - let mut fory = fory_compatible(); - fory.register_serializer::>(1001) - .unwrap(); - + let fory = fory_compatible(); let mut original = HashMap::new(); original.insert(String::from("one"), 1); original.insert(String::from("two"), 2); @@ -97,9 +94,7 @@ fn test_hashmap_roundtrip() { #[test] fn test_hashset_roundtrip() { - let mut fory = fory_compatible(); - fory.register_serializer::>(1002).unwrap(); - + let fory = fory_compatible(); let mut original = HashSet::new(); original.insert(1); original.insert(2); @@ -118,10 +113,7 @@ fn test_hashset_roundtrip() { #[test] fn test_vec_of_trait_objects() { - let mut fory = fory_compatible(); - fory.register_serializer::>>(3000) - .unwrap(); - + let fory = fory_compatible(); let vec_of_trait_objects: Vec> = vec![ Box::new(42i32), Box::new(String::from("hello")), @@ -136,9 +128,7 @@ fn test_vec_of_trait_objects() { #[test] fn test_hashmap_string_to_trait_objects() { - let mut fory = fory_compatible(); - fory.register_serializer::>>(3002) - .unwrap(); + let fory = fory_compatible(); let mut map: HashMap> = HashMap::new(); map.insert(String::from("int"), Box::new(42i32)); @@ -190,8 +180,6 @@ fn test_vec_of_fory_derived_trait_objects() { let mut fory = fory_compatible(); fory.register::(5000).unwrap(); fory.register::(5001).unwrap(); - fory.register_serializer::>>(3000) - .unwrap(); let vec_of_trait_objects: Vec> = vec![ Box::new(Person { @@ -219,8 +207,6 @@ fn test_hashmap_with_fory_derived_values() { let mut fory = fory_compatible(); fory.register::(5000).unwrap(); fory.register::(5001).unwrap(); - fory.register_serializer::>>(3002) - .unwrap(); let mut map: HashMap> = HashMap::new(); map.insert( From 403fdbe52d7d4e97560d5569ae610d3d4c67d070 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 13:02:33 +0800 Subject: [PATCH 29/37] docs(rust): add rust thread safety and troubleshooting doc (#2781) ## Why? ## What does this PR do? add rust thread safety and troubleshooting doc ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/README.md | 42 +++++++++++++++++++++++++++ rust/fory/src/lib.rs | 67 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/rust/README.md b/rust/README.md index 849afd9355..8f523b305b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -730,6 +730,48 @@ assert_eq!(prefs.values().get(0), "en"); | Memory usage | Full object graph in memory | Only accessed fields in memory | | Suitable for | Small objects, full access | Large objects, selective access | +### 8. Thread-Safe Serialization + +Apache Fory™ Rust is fully thread-safe: `Fory` implements both `Send` and `Sync`, so one configured instance can be shared across threads for concurrent work. The internal read/write context pools are lazily initialized with thread-safe primitives, letting worker threads reuse buffers without coordination. + +```rust +use fory::{Fory, Error}; +use fory::ForyObject; +use std::sync::Arc; +use std::thread; + +#[derive(ForyObject, Clone, Copy, Debug, PartialEq)] +struct Item { + value: i32, +} + +fn main() -> Result<(), Error> { + let mut fory = Fory::default(); + fory.register::(1000)?; + + let fory = Arc::new(fory); + let handles: Vec<_> = (0..8) + .map(|i| { + let shared = Arc::clone(&fory); + thread::spawn(move || { + let item = Item { value: i }; + shared.serialize(&item) + }) + }) + .collect(); + + for handle in handles { + let bytes = handle.join().unwrap()?; + let item: Item = fory.deserialize(&bytes)?; + assert!(item.value >= 0); + } + + Ok(()) +} +``` + +**Tip:** Perform registrations (such as `fory.register::(id)`) before spawning threads so every worker sees the same metadata. Once configured, wrapping the instance in `Arc` is enough to fan out serialization and deserialization tasks safely. + ## 🔧 Supported Types ### Primitive Types diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index b5c6d01179..b0b289aa16 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -1010,25 +1010,45 @@ //! //! ## Thread Safety //! -//! **Important:** `Fory` instances are **not thread-safe**. Use one instance per thread: +//! `Fory` implements `Send` and `Sync`, so a single instance can be shared across threads +//! (for example via `Arc`) for concurrent serialization and deserialization. The +//! internal context pools grow lazily and rely on thread-safe primitives, allowing multiple +//! workers to reuse buffers without additional coordination. //! //! ```rust -//! use std::thread_local; -//! use std::cell::RefCell; +//! use std::sync::Arc; +//! use std::thread; //! use fory::Fory; +//! use fory::ForyObject; //! -//! thread_local! { -//! static FORY: RefCell = RefCell::new(Fory::default()); +//! #[derive(ForyObject, Clone, Copy, Debug)] +//! struct Item { +//! value: i32, //! } //! -//! # fn serialize_data() -> Vec { -//! FORY.with(|fory| { -//! let data = vec![1, 2, 3]; -//! fory.borrow().serialize(&data).unwrap() -//! }) -//! # } +//! let mut fory = Fory::default(); +//! fory.register::(1000).unwrap(); +//! let fory = Arc::new(fory); +//! let handles: Vec<_> = (0..8) +//! .map(|i| { +//! let shared = Arc::clone(&fory); +//! thread::spawn(move || { +//! let item = Item { value: i }; +//! shared.serialize(&item).unwrap() +//! }) +//! }) +//! .collect(); +//! +//! for handle in handles { +//! let bytes = handle.join().unwrap(); +//! let item: Item = fory.deserialize(&bytes).unwrap(); +//! assert!(item.value >= 0); +//! } //! ``` //! +//! **Best practice:** Perform type registration (e.g., `fory.register::(id)`) before +//! spawning worker threads so metadata is ready, then share the configured instance. +//! //! ## Examples //! //! See the `tests/` directory for comprehensive examples: @@ -1038,6 +1058,31 @@ //! - `tests/tests/test_weak.rs` - Circular reference handling //! - `tests/tests/test_cross_language.rs` - Cross-language compatibility //! +//! ## Troubleshooting +//! +//! - **Type registry errors**: Errors such as `TypeId ... not found in type_info registry` mean +//! the type was never registered with the active `Fory` instance. Ensure every serializable +//! struct, enum, or trait implementation calls `register::(type_id)` before use, and reuse +//! the same IDs when deserializing. +//! - **Quick error lookup**: Always prefer the static constructors on +//! [`fory_core::error::Error`]—for example `Error::type_mismatch`, `Error::invalid_data`, or +//! `Error::unknown`. They keep diagnostics consistent and allow optional panic-on-error +//! debugging. +//! - **Panic on error for backtraces**: Set `FORY_PANIC_ON_ERROR=1` (or `true`) together with +//! `RUST_BACKTRACE=1` while running tests or binaries to panic exactly where an error is +//! constructed. Unset the variable afterwards so production paths keep returning `Result`. +//! - **Struct field tracing**: Add the `#[fory_debug]` attribute next to `#[derive(ForyObject)]` +//! when you need per-field instrumentation. Once compiled with debug hooks, call +//! `set_before_write_field_func`, `set_after_write_field_func`, `set_before_read_field_func`, or +//! `set_after_read_field_func` from `fory_core::serializer::struct_` to install custom +//! callbacks, and use `reset_struct_debug_hooks()` to restore defaults. +//! - **Lightweight logging**: If custom callbacks are unnecessary, enable +//! `ENABLE_FORY_DEBUG_OUTPUT=1` to have the default hook handlers print field-level read/write +//! events. This is useful for spotting cursor misalignment or unexpected buffer growth. +//! - **Test-time hygiene**: Some integration tests expect `FORY_PANIC_ON_ERROR` to stay unset. +//! Export it only during focused debugging, and rely on targeted commands such as +//! `cargo test --features tests -p fory-tests --test ` when isolating failures. +//! //! ## Documentation //! //! - **[Protocol Specification](https://fory.apache.org/docs/specification/fory_xlang_serialization_spec)** - Binary protocol details From 63a1145ba78b02f4f2336359aed5f19774bd95a9 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 14:06:38 +0800 Subject: [PATCH 30/37] feat(rust): use rc instead of arc for type meta for faster performance (#2782) ## Why? ## What does this PR do? use rc instead of arc for type meta for faster performance ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/fory-core/src/resolver/context.rs | 33 +++++++-- rust/fory-core/src/resolver/meta_resolver.rs | 12 +-- rust/fory-core/src/resolver/type_resolver.rs | 74 ++++++++++--------- rust/fory-core/src/serializer/any.rs | 12 +-- rust/fory-core/src/serializer/arc.rs | 7 +- rust/fory-core/src/serializer/core.rs | 6 +- rust/fory-core/src/serializer/map.rs | 10 +-- rust/fory-core/src/serializer/mutex.rs | 4 +- rust/fory-core/src/serializer/option.rs | 3 +- rust/fory-core/src/serializer/rc.rs | 7 +- rust/fory-core/src/serializer/refcell.rs | 4 +- rust/fory-core/src/serializer/trait_object.rs | 11 ++- rust/fory-core/src/serializer/weak.rs | 8 +- rust/fory-derive/src/object/read.rs | 2 +- rust/fory-derive/src/object/serializer.rs | 4 +- 15 files changed, 111 insertions(+), 86 deletions(-) diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 714f7dda89..64dca6bd0a 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -27,8 +27,11 @@ use crate::resolver::metastring_resolver::{ use crate::resolver::ref_resolver::{RefReader, RefWriter}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::types; -use std::sync::{Arc, Mutex}; +use std::rc::Rc; +use std::sync::Mutex; +/// Serialization state container used on a single thread at a time. +/// Sharing the same instance across threads simultaneously causes undefined behavior. pub struct WriteContext { // Replicated environment fields (direct access, no Arc indirection for flags) type_resolver: TypeResolver, @@ -87,7 +90,7 @@ impl WriteContext { } #[inline(always)] - pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { self.type_resolver.get_type_info(type_id) } @@ -138,7 +141,7 @@ impl WriteContext { &mut self, fory_type_id: u32, concrete_type_id: std::any::TypeId, - ) -> Result, Error> { + ) -> Result, Error> { if types::is_internal_type(fory_type_id) { self.writer.write_varuint32(fory_type_id); return self @@ -192,6 +195,15 @@ impl WriteContext { } } +// Safety: WriteContext is only shared across threads via higher-level pooling code that +// ensures single-threaded access while the context is in use. Users must never hold the same +// instance on multiple threads simultaneously; that would violate the invariants and result in +// undefined behavior. Under that assumption, marking it Send/Sync is sound. +unsafe impl Send for WriteContext {} +unsafe impl Sync for WriteContext {} + +/// Deserialization state container used on a single thread at a time. +/// Sharing the same instance across threads simultaneously causes undefined behavior. pub struct ReadContext { // Replicated environment fields (direct access, no Arc indirection for flags) type_resolver: TypeResolver, @@ -208,6 +220,13 @@ pub struct ReadContext { current_depth: u32, } +// Safety: ReadContext follows the same invariants as WriteContext—external orchestrators ensure +// single-threaded use. Concurrent access to the same instance across threads is forbidden and +// would result in undefined behavior. With exclusive use guaranteed, the Send/Sync markers are safe +// even though Rc is used internally. +unsafe impl Send for ReadContext {} +unsafe impl Sync for ReadContext {} + impl ReadContext { pub fn new( reader: Reader, @@ -284,7 +303,7 @@ impl ReadContext { } #[inline(always)] - pub fn get_meta(&self, type_index: usize) -> Result<&Arc, Error> { + pub fn get_meta(&self, type_index: usize) -> Result<&Rc, Error> { self.meta_resolver.get(type_index).ok_or_else(|| { Error::type_error(format!("Meta not found for type index: {}", type_index)) }) @@ -298,7 +317,7 @@ impl ReadContext { ) } - pub fn read_any_typeinfo(&mut self) -> Result, Error> { + pub fn read_any_typeinfo(&mut self) -> Result, Error> { let fory_type_id = self.reader.read_varuint32()?; // should be compiled to jump table generation match fory_type_id & 0xff { @@ -311,7 +330,7 @@ impl ReadContext { .ok_or_else(|| Error::type_error("ID harness not found"))?; // Create a new TypeInfo with remote metadata but local harness // TODO: make self.get_meta return type info - Ok(Arc::new(local_type_info.with_remote_meta(remote_meta))) + Ok(Rc::new(local_type_info.with_remote_meta(remote_meta))) } types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { if self.is_share_meta() { @@ -336,7 +355,7 @@ impl ReadContext { } #[inline(always)] - pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { self.type_resolver.get_type_info(type_id) } diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 4cb8b4d90a..525caf702d 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -20,11 +20,11 @@ use crate::error::Error; use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER}; use crate::TypeResolver; use std::collections::HashMap; -use std::sync::Arc; +use std::rc::Rc; #[derive(Default)] pub struct MetaWriterResolver { - type_defs: Vec>>, + type_defs: Vec>>, type_id_index_map: HashMap, } @@ -68,12 +68,12 @@ impl MetaWriterResolver { #[derive(Default)] pub struct MetaReaderResolver { - pub reading_type_defs: Vec>, - parsed_type_defs: HashMap>, + pub reading_type_defs: Vec>, + parsed_type_defs: HashMap>, } impl MetaReaderResolver { - pub fn get(&self, index: usize) -> Option<&Arc> { + pub fn get(&self, index: usize) -> Option<&Rc> { self.reading_type_defs.get(index) } @@ -90,7 +90,7 @@ impl MetaReaderResolver { self.reading_type_defs.push(type_meta.clone()); TypeMeta::skip_bytes(reader, meta_header)?; } else { - let type_meta = Arc::new(TypeMeta::from_bytes_with_header( + let type_meta = Rc::new(TypeMeta::from_bytes_with_header( reader, type_resolver, meta_header, diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index 0fdd8fda54..46c676bcd8 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -25,7 +25,7 @@ use crate::serializer::{ForyDefault, Serializer, StructSerializer}; use crate::util::get_ext_actual_type_id; use crate::{Reader, TypeId}; use std::collections::{HashSet, LinkedList}; -use std::sync::Arc; +use std::rc::Rc; use std::{any::Any, collections::HashMap}; type WriteFn = fn( @@ -92,8 +92,8 @@ impl Harness { #[derive(Clone, Debug)] pub struct TypeInfo { - type_def: Arc>, - type_meta: Arc, + type_def: Rc>, + type_meta: Rc, type_id: u32, namespace: MetaString, type_name: MetaString, @@ -135,7 +135,7 @@ impl TypeInfo { for (i, field_info) in sorted_field_infos.iter_mut().enumerate() { field_info.field_id = i as i16; } - let type_meta = Arc::new(TypeMeta::from_fields( + let type_meta = Rc::new(TypeMeta::from_fields( type_id, namespace_metastring.clone(), type_name_metastring.clone(), @@ -144,7 +144,7 @@ impl TypeInfo { )); let type_def_bytes = type_meta.to_bytes()?; Ok(TypeInfo { - type_def: Arc::from(type_def_bytes), + type_def: Rc::from(type_def_bytes), type_meta, type_id, namespace: namespace_metastring, @@ -176,8 +176,8 @@ impl TypeInfo { let type_def = meta.to_bytes()?; let meta = TypeMeta::from_bytes(&mut Reader::new(&type_def), type_resolver)?; Ok(TypeInfo { - type_def: Arc::from(type_def), - type_meta: Arc::new(meta), + type_def: Rc::from(type_def), + type_meta: Rc::new(meta), type_id, namespace: namespace_metastring, type_name: type_name_metastring, @@ -198,11 +198,11 @@ impl TypeInfo { &self.type_name } - pub fn get_type_def(&self) -> Arc> { + pub fn get_type_def(&self) -> Rc> { self.type_def.clone() } - pub fn get_type_meta(&self) -> Arc { + pub fn get_type_meta(&self) -> Rc { self.type_meta.clone() } @@ -217,7 +217,7 @@ impl TypeInfo { /// Create a new TypeInfo with the same properties but different type_meta. /// This is used during deserialization to create a TypeInfo with remote metadata /// while keeping the local harness for deserialization functions. - pub fn with_remote_meta(&self, remote_meta: Arc) -> TypeInfo { + pub fn with_remote_meta(&self, remote_meta: Rc) -> TypeInfo { TypeInfo { type_def: self.type_def.clone(), type_meta: remote_meta, @@ -233,15 +233,21 @@ impl TypeInfo { /// TypeResolver is a resolver for fast type/serializer dispatch. #[derive(Clone)] pub struct TypeResolver { - type_info_map_by_id: HashMap>, - type_info_map: HashMap>, - type_info_map_by_name: HashMap<(String, String), Arc>, - type_info_map_by_ms_name: HashMap<(MetaString, MetaString), Arc>, + type_info_map_by_id: HashMap>, + type_info_map: HashMap>, + type_info_map_by_name: HashMap<(String, String), Rc>, + type_info_map_by_ms_name: HashMap<(MetaString, MetaString), Rc>, // Fast lookup by numeric ID for common types type_id_index: Vec, compatible: bool, } +// Safety: TypeResolver instances are only shared through higher-level synchronization that +// guarantees thread confinement for mutations, so marking them Send/Sync preserves existing +// invariants despite internal Rc usage. +unsafe impl Send for TypeResolver {} +unsafe impl Sync for TypeResolver {} + const NO_TYPE_ID: u32 = 1000000000; impl Default for TypeResolver { @@ -260,7 +266,7 @@ impl Default for TypeResolver { } impl TypeResolver { - pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { + pub fn get_type_info(&self, type_id: &std::any::TypeId) -> Result, Error> { self.type_info_map.get(type_id) .ok_or_else(|| { Error::type_error(format!( @@ -271,11 +277,11 @@ impl TypeResolver { .cloned() } - pub fn get_type_info_by_id(&self, id: u32) -> Option> { + pub fn get_type_info_by_id(&self, id: u32) -> Option> { self.type_info_map_by_id.get(&id).cloned() } - pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option> { + pub fn get_type_info_by_name(&self, namespace: &str, type_name: &str) -> Option> { self.type_info_map_by_name .get(&(namespace.to_owned(), type_name.to_owned())) .cloned() @@ -285,7 +291,7 @@ impl TypeResolver { &self, namespace: &MetaString, type_name: &MetaString, - ) -> Option> { + ) -> Option> { self.type_info_map_by_ms_name .get(&(namespace.clone(), type_name.clone())) .cloned() @@ -306,27 +312,27 @@ impl TypeResolver { ))) } - pub fn get_harness(&self, id: u32) -> Option> { + pub fn get_harness(&self, id: u32) -> Option> { self.type_info_map_by_id .get(&id) - .map(|info| Arc::new(info.get_harness().clone())) + .map(|info| Rc::new(info.get_harness().clone())) } pub fn get_name_harness( &self, namespace: &MetaString, type_name: &MetaString, - ) -> Option> { + ) -> Option> { let key = (namespace.clone(), type_name.clone()); self.type_info_map_by_ms_name .get(&key) - .map(|info| Arc::new(info.get_harness().clone())) + .map(|info| Rc::new(info.get_harness().clone())) } - pub fn get_ext_harness(&self, id: u32) -> Result, Error> { + pub fn get_ext_harness(&self, id: u32) -> Result, Error> { self.type_info_map_by_id .get(&id) - .map(|info| Arc::new(info.get_harness().clone())) + .map(|info| Rc::new(info.get_harness().clone())) .ok_or_else(|| Error::type_error("ext type must be registered in both peers")) } @@ -334,11 +340,11 @@ impl TypeResolver { &self, namespace: &MetaString, type_name: &MetaString, - ) -> Result, Error> { + ) -> Result, Error> { let key = (namespace.clone(), type_name.clone()); self.type_info_map_by_ms_name .get(&key) - .map(|info| Arc::new(info.get_harness().clone())) + .map(|info| Rc::new(info.get_harness().clone())) .ok_or_else(|| Error::type_error("named_ext type must be registered in both peers")) } @@ -495,11 +501,11 @@ impl TypeResolver { // Store in main map self.type_info_map - .insert(rs_type_id, Arc::new(type_info.clone())); + .insert(rs_type_id, Rc::new(type_info.clone())); // Store by ID self.type_info_map_by_id - .insert(type_info.type_id, Arc::new(type_info.clone())); + .insert(type_info.type_id, Rc::new(type_info.clone())); // Update type_id_index for fast lookup let index = T::fory_type_index() as usize; @@ -525,10 +531,10 @@ impl TypeResolver { ))); } self.type_info_map_by_ms_name - .insert(ms_key, Arc::new(type_info.clone())); + .insert(ms_key, Rc::new(type_info.clone())); let string_key = (namespace.original.clone(), type_name.original.clone()); self.type_info_map_by_name - .insert(string_key, Arc::new(type_info)); + .insert(string_key, Rc::new(type_info)); } Ok(()) } @@ -671,11 +677,11 @@ impl TypeResolver { // Store in main map self.type_info_map - .insert(rs_type_id, Arc::new(type_info.clone())); + .insert(rs_type_id, Rc::new(type_info.clone())); // Store by ID self.type_info_map_by_id - .insert(type_info.type_id, Arc::new(type_info.clone())); + .insert(type_info.type_id, Rc::new(type_info.clone())); // Store by name if registered by name if type_info.register_by_name { @@ -689,10 +695,10 @@ impl TypeResolver { ))); } self.type_info_map_by_ms_name - .insert(ms_key, Arc::new(type_info.clone())); + .insert(ms_key, Rc::new(type_info.clone())); let string_key = (namespace.original.clone(), type_name.original.clone()); self.type_info_map_by_name - .insert(string_key, Arc::new(type_info)); + .insert(string_key, Rc::new(type_info)); } Ok(()) } diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs index 615d86f4e3..80e3f23555 100644 --- a/rust/fory-core/src/serializer/any.rs +++ b/rust/fory-core/src/serializer/any.rs @@ -90,7 +90,7 @@ impl Serializer for Box { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -172,7 +172,7 @@ pub fn read_box_any( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { context.inc_depth()?; let ref_flag = if read_ref_info { @@ -258,7 +258,7 @@ impl Serializer for Rc { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -321,7 +321,7 @@ pub fn read_rc_any( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? @@ -427,7 +427,7 @@ impl Serializer for Arc { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -490,7 +490,7 @@ pub fn read_arc_any( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index 33eddff2c6..ea221dcfab 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -21,6 +21,7 @@ use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use crate::types::TypeId; +use std::rc::Rc; use std::sync::Arc; impl Serializer for Arc { @@ -83,7 +84,7 @@ impl Serializer for Arc fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - typeinfo: Arc, + typeinfo: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -126,7 +127,7 @@ fn read_arc( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - typeinfo: Option>, + typeinfo: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? @@ -159,7 +160,7 @@ fn read_arc( fn read_arc_inner( context: &mut ReadContext, read_type_info: bool, - typeinfo: Option>, + typeinfo: Option>, ) -> Result { if let Some(typeinfo) = typeinfo { let inner_read_ref = T::fory_is_shared_ref(); diff --git a/rust/fory-core/src/serializer/core.rs b/rust/fory-core/src/serializer/core.rs index f2ac8f7a6c..1723b86ba7 100644 --- a/rust/fory-core/src/serializer/core.rs +++ b/rust/fory-core/src/serializer/core.rs @@ -23,7 +23,7 @@ use crate::serializer::{bool, struct_}; use crate::types::{RefFlag, TypeId}; use crate::TypeResolver; use std::any::Any; -use std::sync::Arc; +use std::rc::Rc; pub trait ForyDefault: Sized { fn fory_default() -> Self; @@ -155,7 +155,7 @@ pub trait Serializer: 'static { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -269,7 +269,7 @@ pub trait StructSerializer: Serializer + 'static { // only used by struct fn fory_read_compatible( context: &mut ReadContext, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized; diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index 37a075b8ea..876a0980af 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -23,7 +23,7 @@ use crate::serializer::util::read_basic_type_info; use crate::serializer::{ForyDefault, Serializer}; use crate::types::{need_to_write_type_for_field, TypeId, SIZE_OF_REF_AND_TYPE}; use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; +use std::rc::Rc; const MAX_CHUNK_SIZE: u8 = 255; @@ -380,7 +380,7 @@ macro_rules! impl_read_map_dyn_ref { let track_value_ref = (header & TRACKING_VALUE_REF) != 0; // Determine value type info (if any) - let value_type_info: Option> = if !value_declared { + let value_type_info: Option> = if !value_declared { if val_is_polymorphic { Some(context.read_any_typeinfo()?) } else { @@ -411,7 +411,7 @@ macro_rules! impl_read_map_dyn_ref { let key_declared = (header & DECL_KEY_TYPE) != 0; let track_key_ref = (header & TRACKING_KEY_REF) != 0; - let key_type_info: Option> = if !key_declared { + let key_type_info: Option> = if !key_declared { if key_is_polymorphic { Some(context.read_any_typeinfo()?) } else { @@ -443,7 +443,7 @@ macro_rules! impl_read_map_dyn_ref { let track_key_ref = (header & TRACKING_KEY_REF) != 0; let track_value_ref = (header & TRACKING_VALUE_REF) != 0; - let key_type_info: Option> = if !key_declared { + let key_type_info: Option> = if !key_declared { if key_is_polymorphic { Some(context.read_any_typeinfo()?) } else { @@ -453,7 +453,7 @@ macro_rules! impl_read_map_dyn_ref { } else { None }; - let value_type_info: Option> = if !value_declared { + let value_type_info: Option> = if !value_declared { if val_is_polymorphic { Some(context.read_any_typeinfo()?) } else { diff --git a/rust/fory-core/src/serializer/mutex.rs b/rust/fory-core/src/serializer/mutex.rs index f6fcc34527..fd9c7e7888 100644 --- a/rust/fory-core/src/serializer/mutex.rs +++ b/rust/fory-core/src/serializer/mutex.rs @@ -46,7 +46,7 @@ use crate::resolver::context::{ReadContext, WriteContext}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; -use std::sync::Arc; +use std::rc::Rc; use std::sync::Mutex; /// `Serializer` impl for `Mutex` @@ -114,7 +114,7 @@ impl Serializer for Mutex { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs index 52c0dc138f..ea888e8e8d 100644 --- a/rust/fory-core/src/serializer/option.rs +++ b/rust/fory-core/src/serializer/option.rs @@ -21,6 +21,7 @@ use crate::resolver::context::WriteContext; use crate::resolver::type_resolver::TypeResolver; use crate::serializer::{ForyDefault, Serializer}; use crate::types::{RefFlag, TypeId}; +use std::rc::Rc; impl Serializer for Option { #[inline(always)] @@ -83,7 +84,7 @@ impl Serializer for Option { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: std::sync::Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index 7114e85bdd..2d9ff4ec2c 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -22,7 +22,6 @@ use crate::serializer::{ForyDefault, Serializer}; use crate::types::RefFlag; use crate::types::TypeId; use std::rc::Rc; -use std::sync::Arc; impl Serializer for Rc { fn fory_is_shared_ref() -> bool { @@ -83,7 +82,7 @@ impl Serializer for Rc { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - typeinfo: Arc, + typeinfo: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -126,7 +125,7 @@ fn read_rc( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - typeinfo: Option>, + typeinfo: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? @@ -159,7 +158,7 @@ fn read_rc( fn read_rc_inner( context: &mut ReadContext, read_type_info: bool, - typeinfo: Option>, + typeinfo: Option>, ) -> Result { if let Some(typeinfo) = typeinfo { let inner_read_ref = T::fory_is_shared_ref(); diff --git a/rust/fory-core/src/serializer/refcell.rs b/rust/fory-core/src/serializer/refcell.rs index 67d69cb2f3..0012fffe82 100644 --- a/rust/fory-core/src/serializer/refcell.rs +++ b/rust/fory-core/src/serializer/refcell.rs @@ -38,7 +38,7 @@ use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::types::TypeId; use std::cell::RefCell; -use std::sync::Arc; +use std::rc::Rc; /// `Serializer` impl for `RefCell` /// @@ -63,7 +63,7 @@ impl Serializer for RefCell { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, diff --git a/rust/fory-core/src/serializer/trait_object.rs b/rust/fory-core/src/serializer/trait_object.rs index 97195b3ea9..00aeb975c6 100644 --- a/rust/fory-core/src/serializer/trait_object.rs +++ b/rust/fory-core/src/serializer/trait_object.rs @@ -24,7 +24,7 @@ use crate::resolver::context::{ReadContext, WriteContext}; use crate::resolver::type_resolver::{TypeInfo, TypeResolver}; use crate::serializer::{ForyDefault, Serializer}; use crate::RefFlag; -use std::sync::Arc; +use std::rc::Rc; /// Helper macro for common type resolution and downcasting pattern #[macro_export] @@ -68,7 +68,6 @@ macro_rules! resolve_and_deserialize { /// either you use the wrapper types or use the `Rc` or `Arc` instead if it's not /// inside struct fields. For struct fields, you can use the `Rc`, `Arc` directly, /// fory will generate converters for `Rc` and `Arc` to convert to wrapper for -/// serialization/ deserialization automatically. /// /// The macro generates: /// - `Serializer` implementation for `Box` @@ -213,7 +212,7 @@ macro_rules! register_trait_type { fn fory_read_with_type_info( context: &mut fory_core::ReadContext, read_ref_info: bool, - type_info: std::sync::Arc, + type_info: std::rc::Rc, ) -> Result where Self: Sized + fory_core::ForyDefault, @@ -468,7 +467,7 @@ macro_rules! impl_smart_pointer_serializer { ) } - fn fory_read_with_type_info(context: &mut fory_core::ReadContext, read_ref_info: bool, type_info: std::sync::Arc) -> Result { + fn fory_read_with_type_info(context: &mut fory_core::ReadContext, read_ref_info: bool, type_info: std::rc::Rc) -> Result { $crate::read_ptr_trait_object!( context, read_ref_info, @@ -650,7 +649,7 @@ impl Serializer for Box { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - type_info: Arc, + type_info: Rc, ) -> Result where Self: Sized + ForyDefault, @@ -667,7 +666,7 @@ fn read_box_seralizer( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { context.inc_depth()?; let ref_flag = if read_ref_info { diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index d88f68a876..29d3009a88 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -367,7 +367,7 @@ impl Serializer for RcWeak { fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - typeinfo: Arc, + typeinfo: Rc, ) -> Result { read_rc_weak::(context, read_ref_info, false, Some(typeinfo)) } @@ -410,7 +410,7 @@ fn read_rc_weak( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? @@ -526,7 +526,7 @@ impl Serializer for ArcWeak fn fory_read_with_type_info( context: &mut ReadContext, read_ref_info: bool, - typeinfo: Arc, + typeinfo: Rc, ) -> Result { read_arc_weak::(context, read_ref_info, false, Some(typeinfo)) } @@ -569,7 +569,7 @@ fn read_arc_weak( context: &mut ReadContext, read_ref_info: bool, read_type_info: bool, - type_info: Option>, + type_info: Option>, ) -> Result, Error> { let ref_flag = if read_ref_info { context.ref_reader.read_ref_flag(&mut context.reader)? diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 03755e1fee..b288e0ecd8 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -472,7 +472,7 @@ pub fn gen_read_with_type_info(struct_ident: &Ident) -> TokenStream { // fn fory_read_with_type_info( // context: &mut ReadContext, // read_ref_info: bool, - // type_info: Arc, + // type_info: Rc, // ) -> Result quote! { let ref_flag = if read_ref_info { diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs index 51a1ae1311..20893654cb 100644 --- a/rust/fory-derive/src/object/serializer.rs +++ b/rust/fory-derive/src/object/serializer.rs @@ -141,7 +141,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput, debug_enabled: bool) -> TokenSt #fields_info_ts } - fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext, type_info: std::sync::Arc) -> Result { + fn fory_read_compatible(context: &mut fory_core::resolver::context::ReadContext, type_info: std::rc::Rc) -> Result { #read_compatible_ts } } @@ -186,7 +186,7 @@ pub fn derive_serializer(ast: &syn::DeriveInput, debug_enabled: bool) -> TokenSt #read_ts } - fn fory_read_with_type_info(context: &mut fory_core::resolver::context::ReadContext, read_ref_info: bool, type_info: std::sync::Arc) -> Result { + fn fory_read_with_type_info(context: &mut fory_core::resolver::context::ReadContext, read_ref_info: bool, type_info: std::rc::Rc) -> Result { #read_with_type_info_ts } From 8cc3ad8aec585e2ef273d615ef9c82cedac88128 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 17:00:26 +0800 Subject: [PATCH 31/37] chore(rust): rename fory-tests to tests (#2783) ## Why? tests is more idiomatic to AI coding ## What does this PR do? rename fory-tests to tests ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- AGENTS.md | 4 ++-- CONTRIBUTING.md | 2 +- docs/guide/DEVELOPMENT.md | 2 +- rust/README.md | 4 ++-- rust/fory/src/lib.rs | 2 +- rust/tests/Cargo.toml | 2 +- rust/tests/tests/compatible/test_struct.rs | 2 +- rust/tests/tests/test_complex_struct.rs | 2 +- rust/tests/tests/test_cross_language.rs | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 18786ac256..5d6bb27392 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,7 +140,7 @@ cargo clippy --all-targets --all-features -- -D warnings cargo test --features tests # run specific test -cargo test -p fory-tests --test $test_file $test_method +cargo test -p tests --test $test_file $test_method # run specific test under subdirectory cargo test --test mod $dir$::$test_file::$test_method @@ -356,7 +356,7 @@ Fory serialization for every language is implemented independently to minimize t - **fory-test-core**: Core test utilities and data generators -- **fory-testsuite**: Complex test suite for issues reported by users and hard to reproduce using simple test cases +- **testsuite**: Complex test suite for issues reported by users and hard to reproduce using simple test cases - **benchmark**: Benchmark suite based on jmh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b89a77ca7d..91192cbafd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ go test -v fory_xlang_test.go cd rust cargo test # run test with specific test file and method -cargo test -p fory-tests --test $test_file $test_method +cargo test -p tests --test $test_file $test_method # run specific test under subdirectory cargo test --test mod $dir$::$test_file::$test_method # debug specific test under subdirectory and get backtrace diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md index 921143aaad..8ac96d9cc3 100644 --- a/docs/guide/DEVELOPMENT.md +++ b/docs/guide/DEVELOPMENT.md @@ -97,7 +97,7 @@ cargo build # run test cargo test # run specific test -cargo test -p fory-tests --test $test_file $test_method +cargo test -p tests --test $test_file $test_method # run specific test under subdirectory cargo test --test mod $dir$::$test_file::$test_method # debug specific test under subdirectory and get backtrace diff --git a/rust/README.md b/rust/README.md index 8f523b305b..264c55cd4c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -969,7 +969,7 @@ Note: Static data types (non-dynamic types) are secure by nature and not subject - **Panic on error for backtraces**: Toggle `FORY_PANIC_ON_ERROR=1` (or `true`) alongside `RUST_BACKTRACE=1` when running tests or binaries to panic at the exact site an error is constructed. Reset the variable afterwards to avoid aborting user-facing code paths. - **Struct field tracing**: Add the `#[fory_debug]` attribute alongside `#[derive(ForyObject)]` to tell the macro to emit hook invocations for that type. Once compiled with debug hooks, call `set_before_write_field_func`, `set_after_write_field_func`, `set_before_read_field_func`, or `set_after_read_field_func` (from `fory-core/src/serializer/struct_.rs`) to plug in custom callbacks, and use `reset_struct_debug_hooks()` when you want the defaults back. - **Lightweight logging**: Without custom hooks, enable `ENABLE_FORY_DEBUG_OUTPUT=1` to print field-level read/write events emitted by the default hook functions. This is especially useful when investigating alignment or cursor mismatches. -- **Test-time hygiene**: Some integration tests expect `FORY_PANIC_ON_ERROR` to remain unset. Export it only for focused debugging sessions, and prefer `cargo test --features tests -p fory-tests --test ` when isolating failing scenarios. +- **Test-time hygiene**: Some integration tests expect `FORY_PANIC_ON_ERROR` to remain unset. Export it only for focused debugging sessions, and prefer `cargo test --features tests -p tests --test ` when isolating failing scenarios. ## 🛠️ Development @@ -987,7 +987,7 @@ cargo build cargo test --features tests # Run specific test -cargo test -p fory-tests --test test_complex_struct +cargo test -p tests --test test_complex_struct ``` ### Code Quality diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index b0b289aa16..778c94bbe5 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -1081,7 +1081,7 @@ //! events. This is useful for spotting cursor misalignment or unexpected buffer growth. //! - **Test-time hygiene**: Some integration tests expect `FORY_PANIC_ON_ERROR` to stay unset. //! Export it only during focused debugging, and rely on targeted commands such as -//! `cargo test --features tests -p fory-tests --test ` when isolating failures. +//! `cargo test --features tests -p tests --test ` when isolating failures. //! //! ## Documentation //! diff --git a/rust/tests/Cargo.toml b/rust/tests/Cargo.toml index e7c218d7f3..3f9ee2343a 100644 --- a/rust/tests/Cargo.toml +++ b/rust/tests/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [package] -name = "fory-tests" +name = "tests" version.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/rust/tests/tests/compatible/test_struct.rs b/rust/tests/tests/compatible/test_struct.rs index 105edb1069..cf41c2f717 100644 --- a/rust/tests/tests/compatible/test_struct.rs +++ b/rust/tests/tests/compatible/test_struct.rs @@ -19,7 +19,7 @@ use fory_core::fory::Fory; use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; -// RUSTFLAGS="-Awarnings" cargo expand -p fory-tests --test test_struct +// RUSTFLAGS="-Awarnings" cargo expand -p tests --test test_struct #[test] fn simple() { #[derive(ForyObject, Debug)] diff --git a/rust/tests/tests/test_complex_struct.rs b/rust/tests/tests/test_complex_struct.rs index accc593a71..0cb92de0e4 100644 --- a/rust/tests/tests/test_complex_struct.rs +++ b/rust/tests/tests/test_complex_struct.rs @@ -21,7 +21,7 @@ use fory_derive::ForyObject; use chrono::{DateTime, NaiveDate, NaiveDateTime}; use std::collections::HashMap; -// RUSTFLAGS="-Awarnings" cargo expand -p fory-tests --test test_complex_struct +// RUSTFLAGS="-Awarnings" cargo expand -p tests --test test_complex_struct // #[test] // fn any() { // #[derive(ForyObject, Debug)] diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index ffd571abbc..9a61cc55ce 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -27,7 +27,7 @@ use fory_derive::ForyObject; use std::collections::{HashMap, HashSet}; use std::{fs, vec}; -// RUSTFLAGS="-Awarnings" cargo expand -p fory-tests --test test_cross_language +// RUSTFLAGS="-Awarnings" cargo expand -p tests --test test_cross_language fn get_data_file() -> String { std::env::var("DATA_FILE").expect("DATA_FILE not set") } From c2d4e99cf168f950640671562b0ac4e59649966a Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 18:52:48 +0800 Subject: [PATCH 32/37] feat(python): support dataclass compatible mode for python native mode (#2784) ## Why? support dataclass compatible mode for python native mode ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- python/pyfory/_fory.py | 8 +- python/pyfory/_serialization.pyx | 6 + python/pyfory/meta/typedef.py | 7 +- python/pyfory/serializer.py | 448 ++++++++++++++++++------- python/pyfory/tests/test_serializer.py | 54 ++- python/pyfory/tests/test_struct.py | 193 ++++++++++- python/pyfory/type.py | 8 +- 7 files changed, 577 insertions(+), 147 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index e89f8ffa7f..f283bc4eac 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -121,6 +121,7 @@ class Fory: "_peer_language", "max_depth", "depth", + "field_nullable", ) def __init__( @@ -130,6 +131,7 @@ def __init__( strict: bool = True, compatible: bool = False, max_depth: int = 50, + field_nullable: bool = False, **kwargs, ): """ @@ -157,11 +159,14 @@ def __init__( you disable this option. :param compatible: Whether to enable compatible mode for cross-language serialization. - When enabled, type forward/backward compatibility for struct fields will be enabled. + When enabled, type forward/backward compatibility for dataclass fields will be enabled. :param max_depth: The maximum depth of the deserialization data. If the depth exceeds the maximum depth, an exception will be raised. The default value is 50. + :param field_nullable: + Whether dataclass fields are nullable for python native mode(xlang=False). When enabled, dataclass fields + are always treated as nullable whether or not they are annotated with `Optional`. """ self.language = Language.XLANG if xlang else Language.PYTHON if kwargs.get("language") is not None: @@ -178,6 +183,7 @@ def __init__( strict = kwargs.get("require_type_registration") self.strict = _ENABLE_TYPE_REGISTRATION_FORCIBLY or strict self.compatible = compatible + self.field_nullable = field_nullable if self.is_py else False from pyfory._serialization import MetaStringResolver, SerializationContext from pyfory._registry import TypeResolver diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx index 8be215235e..9b163fdf4b 100644 --- a/python/pyfory/_serialization.pyx +++ b/python/pyfory/_serialization.pyx @@ -804,6 +804,7 @@ cdef class Fory: cdef readonly c_bool strict cdef readonly c_bool is_py cdef readonly c_bool compatible + cdef readonly c_bool field_nullable cdef readonly MapRefResolver ref_resolver cdef readonly TypeResolver type_resolver cdef readonly MetaStringResolver metastring_resolver @@ -824,6 +825,7 @@ cdef class Fory: strict: bool = True, compatible: bool = False, max_depth: int = 50, + field_nullable: bool = False, **kwargs, ): """ @@ -856,6 +858,9 @@ cdef class Fory: The maximum depth of the deserialization data. If the depth exceeds the maximum depth, an exception will be raised. The default value is 50. + :param field_nullable: + Whether dataclass fields are nullable for python native mode(xlang=False). When enabled, dataclass fields + are always treated as nullable whether or not they are annotated with `Optional`. """ self.language = Language.XLANG if xlang else Language.PYTHON if kwargs.get("language") is not None: @@ -872,6 +877,7 @@ cdef class Fory: self.ref_tracking = ref self.ref_resolver = MapRefResolver(ref) self.is_py = self.language == Language.PYTHON + self.field_nullable = field_nullable if self.is_py else False self.metastring_resolver = MetaStringResolver() self.type_resolver = TypeResolver(self, meta_share=compatible) self.serialization_context = SerializationContext(fory=self, scoped_meta_share_enabled=compatible) diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py index 252a5535da..b3108f95bb 100644 --- a/python/pyfory/meta/typedef.py +++ b/python/pyfory/meta/typedef.py @@ -59,7 +59,8 @@ def __init__( self.is_compressed = is_compressed def create_fields_serializer(self, resolver): - field_types = infer_field_types(self.cls) + field_nullable = resolver.fory.field_nullable + field_types = infer_field_types(self.cls, field_nullable=field_nullable) serializers = [field_info.field_type.create_serializer(resolver, field_types.get(field_info.name, None)) for field_info in self.fields] return serializers @@ -252,10 +253,10 @@ def build_field_infos(type_resolver, cls): field_infos = [] nullable_map = {} visitor = StructTypeIdVisitor(type_resolver.fory, cls) - + field_nullable = type_resolver.fory.field_nullable for field_name in field_names: field_type_hint = type_hints.get(field_name, typing.Any) - unwrapped_type, is_nullable = unwrap_optional(field_type_hint) + unwrapped_type, is_nullable = unwrap_optional(field_type_hint, field_nullable=field_nullable) is_nullable = is_nullable or not is_primitive_type(unwrapped_type) nullable_map[field_name] = is_nullable field_type = build_field_type(type_resolver, field_name, unwrapped_type, visitor, is_nullable) diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index 7b66fa5bbe..b009a0bda4 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -377,13 +377,17 @@ def __init__( self._field_names = field_names or self._get_field_names(clz) self._has_slots = hasattr(clz, "__slots__") self._nullable_fields = nullable_fields or {} + field_nullable = fory.field_nullable if self._field_names and not self._nullable_fields: for field_name in self._field_names: if field_name in self._type_hints: - unwrapped_type, is_nullable = unwrap_optional(self._type_hints[field_name]) + unwrapped_type, is_nullable = unwrap_optional(self._type_hints[field_name], field_nullable=field_nullable) is_nullable = is_nullable or not is_primitive_type(unwrapped_type) self._nullable_fields[field_name] = is_nullable + # Cache unwrapped type hints + self._unwrapped_hints = self._compute_unwrapped_hints() + if self._xlang: self._serializers = serializers or [None] * len(self._field_names) if serializers is None: @@ -407,8 +411,20 @@ def __init__( clz, ) else: - # TODO compute hash for non-xlang mode more robustly - self._hash = len(self._field_names) + # For non-xlang mode, use same infrastructure as xlang mode + # Python dataclass serialization follows the same spec as xlang + self._serializers = serializers or [None] * len(self._field_names) + if serializers is None: + visitor = StructFieldSerializerVisitor(fory) + for index, key in enumerate(self._field_names): + unwrapped_type, _ = unwrap_optional(self._type_hints[key]) + serializer = infer_field(key, unwrapped_type, visitor, types_path=[]) + self._serializers[index] = serializer + # In compatible mode, maintain stable field ordering (don't sort) + # In non-compatible mode, sort fields for consistent serialization + if not fory.compatible: + self._field_names, self._serializers = _sort_fields(fory.type_resolver, self._field_names, self._serializers, self._nullable_fields) + self._hash = 0 # Will be computed on first write/read self._generated_write_method = self._gen_write_method() self._generated_read_method = self._gen_read_method() if _ENABLE_FORY_PYTHON_JIT: @@ -419,41 +435,214 @@ def __init__( def _get_field_names(self, clz): if hasattr(clz, "__dict__"): # Regular object with __dict__ - # We can't know the fields without an instance, so we rely on type hints + # For dataclasses, preserve field definition order + # In compatible mode, stable field ordering is critical for schema evolution + if dataclasses.is_dataclass(clz): + # Use dataclasses.fields() to get fields in definition order + return [field.name for field in dataclasses.fields(clz)] + # For non-dataclass objects, sort by key names for consistency return sorted(self._type_hints.keys()) elif hasattr(clz, "__slots__"): # Object with __slots__ return sorted(clz.__slots__) return [] + def _compute_unwrapped_hints(self): + """Compute unwrapped type hints once and cache.""" + from pyfory.type import unwrap_optional + + return {field_name: unwrap_optional(hint)[0] for field_name, hint in self._type_hints.items()} + + def _ensure_hash_computed(self): + """Lazily compute and cache the hash if not already computed.""" + if self._hash == 0: + self._hash = _get_hash(self.fory, self._field_names, self._unwrapped_hints) + return self._hash + + def _write_header(self, buffer): + """Write serialization header (hash or field count based on compatible mode).""" + if not self.fory.compatible: + buffer.write_int32(self._ensure_hash_computed()) + else: + buffer.write_varuint32(len(self._field_names)) + + def _read_header(self, buffer): + """Read serialization header and return number of fields written. + + Returns: + int: Number of fields that were written + + Raises: + TypeNotCompatibleError: If hash doesn't match in non-compatible mode + """ + if not self.fory.compatible: + hash_ = buffer.read_int32() + expected_hash = self._ensure_hash_computed() + if hash_ != expected_hash: + raise TypeNotCompatibleError(f"Hash {hash_} is not consistent with {expected_hash} for type {self.type_}") + return len(self._field_names) + else: + return buffer.read_varuint32() + + def _get_write_stmt_for_codegen(self, serializer, buffer, field_value): + """Generate write statement for code generation based on serializer type.""" + if isinstance(serializer, BooleanSerializer): + return f"{buffer}.write_bool({field_value})" + elif isinstance(serializer, ByteSerializer): + return f"{buffer}.write_int8({field_value})" + elif isinstance(serializer, Int16Serializer): + return f"{buffer}.write_int16({field_value})" + elif isinstance(serializer, Int32Serializer): + return f"{buffer}.write_varint32({field_value})" + elif isinstance(serializer, Int64Serializer): + return f"{buffer}.write_varint64({field_value})" + elif isinstance(serializer, Float32Serializer): + return f"{buffer}.write_float32({field_value})" + elif isinstance(serializer, Float64Serializer): + return f"{buffer}.write_float64({field_value})" + elif isinstance(serializer, StringSerializer): + return f"{buffer}.write_string({field_value})" + else: + return None # Complex type, needs ref handling + + def _get_read_stmt_for_codegen(self, serializer, buffer, field_value): + """Generate read statement for code generation based on serializer type.""" + if isinstance(serializer, BooleanSerializer): + return f"{field_value} = {buffer}.read_bool()" + elif isinstance(serializer, ByteSerializer): + return f"{field_value} = {buffer}.read_int8()" + elif isinstance(serializer, Int16Serializer): + return f"{field_value} = {buffer}.read_int16()" + elif isinstance(serializer, Int32Serializer): + return f"{field_value} = {buffer}.read_varint32()" + elif isinstance(serializer, Int64Serializer): + return f"{field_value} = {buffer}.read_varint64()" + elif isinstance(serializer, Float32Serializer): + return f"{field_value} = {buffer}.read_float32()" + elif isinstance(serializer, Float64Serializer): + return f"{field_value} = {buffer}.read_float64()" + elif isinstance(serializer, StringSerializer): + return f"{field_value} = {buffer}.read_string()" + else: + return None # Complex type, needs ref handling + + def _write_non_nullable_field(self, buffer, field_value, serializer): + """Write a non-nullable field value at runtime.""" + if isinstance(serializer, BooleanSerializer): + buffer.write_bool(field_value) + elif isinstance(serializer, ByteSerializer): + buffer.write_int8(field_value) + elif isinstance(serializer, Int16Serializer): + buffer.write_int16(field_value) + elif isinstance(serializer, Int32Serializer): + buffer.write_varint32(field_value) + elif isinstance(serializer, Int64Serializer): + buffer.write_varint64(field_value) + elif isinstance(serializer, Float32Serializer): + buffer.write_float32(field_value) + elif isinstance(serializer, Float64Serializer): + buffer.write_float64(field_value) + elif isinstance(serializer, StringSerializer): + buffer.write_string(field_value) + else: + self.fory.write_ref_pyobject(buffer, field_value) + + def _read_non_nullable_field(self, buffer, serializer): + """Read a non-nullable field value at runtime.""" + if isinstance(serializer, BooleanSerializer): + return buffer.read_bool() + elif isinstance(serializer, ByteSerializer): + return buffer.read_int8() + elif isinstance(serializer, Int16Serializer): + return buffer.read_int16() + elif isinstance(serializer, Int32Serializer): + return buffer.read_varint32() + elif isinstance(serializer, Int64Serializer): + return buffer.read_varint64() + elif isinstance(serializer, Float32Serializer): + return buffer.read_float32() + elif isinstance(serializer, Float64Serializer): + return buffer.read_float64() + elif isinstance(serializer, StringSerializer): + return buffer.read_string() + else: + return self.fory.read_ref_pyobject(buffer) + + def _write_nullable_field(self, buffer, field_value, serializer): + """Write a nullable field value at runtime.""" + if field_value is None: + buffer.write_int8(NULL_FLAG) + else: + buffer.write_int8(NOT_NULL_VALUE_FLAG) + if isinstance(serializer, StringSerializer): + buffer.write_string(field_value) + else: + self.fory.write_ref_pyobject(buffer, field_value) + + def _read_nullable_field(self, buffer, serializer): + """Read a nullable field value at runtime.""" + flag = buffer.read_int8() + if flag == NULL_FLAG: + return None + else: + if isinstance(serializer, StringSerializer): + return buffer.read_string() + else: + return self.fory.read_ref_pyobject(buffer) + def _gen_write_method(self): context = {} counter = itertools.count(0) buffer, fory, value, value_dict = "buffer", "fory", "value", "value_dict" context[fory] = self.fory + context["_serializers"] = self._serializers + stmts = [ f'"""write method for {self.type_}"""', - f"{buffer}.write_int32({self._hash})", ] + + # Write hash only in non-compatible mode; in compatible mode, write field count + self._ensure_hash_computed() + if not self.fory.compatible: + stmts.append(f"{buffer}.write_int32({self._hash})") + else: + stmts.append(f"{buffer}.write_varuint32({len(self._field_names)})") + if not self._has_slots: stmts.append(f"{value_dict} = {value}.__dict__") - for field_name in self._field_names: - field_type = self._type_hints[field_name] + + # Write field values in order + for index, field_name in enumerate(self._field_names): field_value = f"field_value{next(counter)}" + serializer_var = f"serializer{index}" + serializer = self._serializers[index] + context[serializer_var] = serializer + if not self._has_slots: stmts.append(f"{field_value} = {value_dict}['{field_name}']") else: stmts.append(f"{field_value} = {value}.{field_name}") - if field_type is bool: - stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, bool)) - elif field_type is int: - stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, int)) - elif field_type is float: - stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, float)) - elif field_type is str: - stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, str)) + + is_nullable = self._nullable_fields.get(field_name, False) + if is_nullable: + # Use gen_write_nullable_basic_stmts for nullable basic types + if isinstance(serializer, BooleanSerializer): + stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, bool)) + elif isinstance(serializer, (ByteSerializer, Int16Serializer, Int32Serializer, Int64Serializer)): + stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, int)) + elif isinstance(serializer, (Float32Serializer, Float64Serializer)): + stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, float)) + elif isinstance(serializer, StringSerializer): + stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, str)) + else: + # For complex types, use write_ref_pyobject + stmts.append(f"{fory}.write_ref_pyobject({buffer}, {field_value})") else: - stmts.append(f"{fory}.write_ref_pyobject({buffer}, {field_value})") + stmt = self._get_write_stmt_for_codegen(serializer, buffer, field_value) + if stmt is None: + stmt = f"{fory}.write_ref_pyobject({buffer}, {field_value})" + stmts.append(stmt) + self._write_method_code, func = compile_function( f"write_{self.type_.__module__}_{self.type_.__qualname__}".replace(".", "_"), [buffer, value], @@ -475,36 +664,87 @@ def _gen_read_method(self): context[fory] = self.fory context[obj_class] = self.type_ context[ref_resolver] = self.fory.ref_resolver + context["_serializers"] = self._serializers + current_class_field_names = set(self._get_field_names(self.type_)) + stmts = [ f'"""read method for {self.type_}"""', - f"{obj} = {obj_class}.__new__({obj_class})", - f"{ref_resolver}.reference({obj})", - f"read_hash = {buffer}.read_int32()", - f"if read_hash != {self._hash}:", - f""" raise TypeNotCompatibleError( - "Hash read_hash is not consistent with {self._hash} for {self.type_}")""", ] + + # Read hash only in non-compatible mode; in compatible mode, read field count + self._ensure_hash_computed() + if not self.fory.compatible: + stmts.extend( + [ + f"read_hash = {buffer}.read_int32()", + f"if read_hash != {self._hash}:", + f""" raise TypeNotCompatibleError( + f"Hash {{read_hash}} is not consistent with {self._hash} for type {self.type_}")""", + ] + ) + else: + stmts.append(f"num_fields_written = {buffer}.read_varuint32()") + + stmts.extend( + [ + f"{obj} = {obj_class}.__new__({obj_class})", + f"{ref_resolver}.reference({obj})", + ] + ) + if not self._has_slots: stmts.append(f"{obj_dict} = {obj}.__dict__") - def set_action(value: str): - if not self._has_slots: - return f"{obj_dict}['{field_name}'] = {value}" + # Read field values in order + for index, field_name in enumerate(self._field_names): + serializer_var = f"serializer{index}" + serializer = self._serializers[index] + context[serializer_var] = serializer + field_value = f"field_value{index}" + is_nullable = self._nullable_fields.get(field_name, False) + + # Build field reading statements + field_stmts = [] + + if is_nullable: + # Use gen_read_nullable_basic_stmts for nullable basic types + if isinstance(serializer, BooleanSerializer): + field_stmts.extend(gen_read_nullable_basic_stmts(buffer, bool, lambda v: f"{field_value} = {v}")) + elif isinstance(serializer, (ByteSerializer, Int16Serializer, Int32Serializer, Int64Serializer)): + field_stmts.extend(gen_read_nullable_basic_stmts(buffer, int, lambda v: f"{field_value} = {v}")) + elif isinstance(serializer, (Float32Serializer, Float64Serializer)): + field_stmts.extend(gen_read_nullable_basic_stmts(buffer, float, lambda v: f"{field_value} = {v}")) + elif isinstance(serializer, StringSerializer): + field_stmts.extend(gen_read_nullable_basic_stmts(buffer, str, lambda v: f"{field_value} = {v}")) + else: + # For complex types, use read_ref_pyobject + field_stmts.append(f"{field_value} = {fory}.read_ref_pyobject({buffer})") + else: + stmt = self._get_read_stmt_for_codegen(serializer, buffer, field_value) + if stmt is None: + stmt = f"{field_value} = {fory}.read_ref_pyobject({buffer})" + field_stmts.append(stmt) + + # Set field value if it exists in current class + if field_name not in current_class_field_names: + field_stmts.append(f"# {field_name} is not in {self.type_}") else: - return f"{obj}.{field_name} = {value}" - - for field_name in self._field_names: - field_type = self._type_hints[field_name] - if field_type is bool: - stmts.extend(gen_read_nullable_basic_stmts(buffer, bool, set_action)) - elif field_type is int: - stmts.extend(gen_read_nullable_basic_stmts(buffer, int, set_action)) - elif field_type is float: - stmts.extend(gen_read_nullable_basic_stmts(buffer, float, set_action)) - elif field_type is str: - stmts.extend(gen_read_nullable_basic_stmts(buffer, str, set_action)) + if not self._has_slots: + field_stmts.append(f"{obj_dict}['{field_name}'] = {field_value}") + else: + field_stmts.append(f"{obj}.{field_name} = {field_value}") + + # In compatible mode, wrap field reading in a check + if self.fory.compatible: + stmts.append(f"if {index} < num_fields_written:") + # Indent all field statements + from pyfory.codegen import ident_lines + + field_stmts = ident_lines(field_stmts) + stmts.extend(field_stmts) else: - stmts.append(f"{obj}.{field_name} = {fory}.read_ref_pyobject({buffer})") + stmts.extend(field_stmts) + stmts.append(f"return {obj}") self._read_method_code, func = compile_function( f"read_{self.type_.__module__}_{self.type_.__qualname__}".replace(".", "_"), @@ -518,24 +758,13 @@ def _gen_xwrite_method(self): context = {} counter = itertools.count(0) buffer, fory, value, value_dict = "buffer", "fory", "value", "value_dict" - get_hash_func = "_get_hash" context[fory] = self.fory - context[get_hash_func] = _get_hash - context["_field_names"] = self._field_names - from pyfory.type import unwrap_optional - - unwrapped_hints = {} - for field_name, hint in self._type_hints.items(): - unwrapped, _ = unwrap_optional(hint) - unwrapped_hints[field_name] = unwrapped - context["_type_hints"] = unwrapped_hints context["_serializers"] = self._serializers stmts = [ f'"""xwrite method for {self.type_}"""', ] if not self.fory.compatible: - if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + self._ensure_hash_computed() stmts.append(f"{buffer}.write_int32({self._hash})") if not self._has_slots: stmts.append(f"{value_dict} = {value}.__dict__") @@ -563,21 +792,8 @@ def _gen_xwrite_method(self): else: stmts.append(f"{fory}.xwrite_ref({buffer}, {field_value}, serializer={serializer_var})") else: - if isinstance(serializer, BooleanSerializer): - stmt = f"{buffer}.write_bool({field_value})" - elif isinstance(serializer, ByteSerializer): - stmt = f"{buffer}.write_int8({field_value})" - elif isinstance(serializer, Int16Serializer): - stmt = f"{buffer}.write_int16({field_value})" - elif isinstance(serializer, Int32Serializer): - stmt = f"{buffer}.write_varint32({field_value})" - elif isinstance(serializer, Int64Serializer): - stmt = f"{buffer}.write_varint64({field_value})" - elif isinstance(serializer, Float32Serializer): - stmt = f"{buffer}.write_float32({field_value})" - elif isinstance(serializer, Float64Serializer): - stmt = f"{buffer}.write_float64({field_value})" - else: + stmt = self._get_write_stmt_for_codegen(serializer, buffer, field_value) + if stmt is None: stmt = f"{fory}.xwrite_no_ref({buffer}, {field_value}, serializer={serializer_var})" stmts.append(stmt) self._xwrite_method_code, func = compile_function( @@ -598,27 +814,16 @@ def _gen_xread_method(self): "obj_dict", ) ref_resolver = "ref_resolver" - get_hash_func = "_get_hash" context[fory] = self.fory context[obj_class] = self.type_ context[ref_resolver] = self.fory.ref_resolver - context[get_hash_func] = _get_hash - context["_field_names"] = self._field_names - from pyfory.type import unwrap_optional - - unwrapped_hints = {} - for field_name, hint in self._type_hints.items(): - unwrapped, _ = unwrap_optional(hint) - unwrapped_hints[field_name] = unwrapped - context["_type_hints"] = unwrapped_hints context["_serializers"] = self._serializers current_class_field_names = set(self._get_field_names(self.type_)) stmts = [ f'"""xread method for {self.type_}"""', ] if not self.fory.compatible: - if self._hash == 0: - self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + self._ensure_hash_computed() stmts.extend( [ f"read_hash = {buffer}.read_int32()", @@ -656,21 +861,8 @@ def _gen_xread_method(self): else: stmts.append(f"{field_value} = {fory}.xread_ref({buffer}, serializer={serializer_var})") else: - if isinstance(serializer, BooleanSerializer): - stmt = f"{field_value} = {buffer}.read_bool()" - elif isinstance(serializer, ByteSerializer): - stmt = f"{field_value} = {buffer}.read_int8()" - elif isinstance(serializer, Int16Serializer): - stmt = f"{field_value} = {buffer}.read_int16()" - elif isinstance(serializer, Int32Serializer): - stmt = f"{field_value} = {buffer}.read_varint32()" - elif isinstance(serializer, Int64Serializer): - stmt = f"{field_value} = {buffer}.read_varint64()" - elif isinstance(serializer, Float32Serializer): - stmt = f"{field_value} = {buffer}.read_float32()" - elif isinstance(serializer, Float64Serializer): - stmt = f"{field_value} = {buffer}.read_float64()" - else: + stmt = self._get_read_stmt_for_codegen(serializer, buffer, field_value) + if stmt is None: stmt = f"{field_value} = {fory}.xread_no_ref({buffer}, serializer={serializer_var})" stmts.append(stmt) if field_name not in current_class_field_names: @@ -690,39 +882,49 @@ def _gen_xread_method(self): return func def write(self, buffer, value): - buffer.write_int32(self._hash) - for field_name in self._field_names: + """Write dataclass instance to buffer in Python native format.""" + self._write_header(buffer) + + for index, field_name in enumerate(self._field_names): field_value = getattr(value, field_name) - self.fory.serialize_ref(buffer, field_value) + serializer = self._serializers[index] + is_nullable = self._nullable_fields.get(field_name, False) + + if is_nullable: + self._write_nullable_field(buffer, field_value, serializer) + else: + self._write_non_nullable_field(buffer, field_value, serializer) def read(self, buffer): - hash_ = buffer.read_int32() - if hash_ != self._hash: - raise TypeNotCompatibleError( - f"Hash {hash_} is not consistent with {self._hash} for type {self.type_}", - ) + """Read dataclass instance from buffer in Python native format.""" + num_fields_written = self._read_header(buffer) + obj = self.type_.__new__(self.type_) self.fory.ref_resolver.reference(obj) - for field_name in self._field_names: - field_value = self.fory.read_ref(buffer) - setattr( - obj, - field_name, - field_value, - ) + current_class_field_names = set(self._get_field_names(self.type_)) + + for index, field_name in enumerate(self._field_names): + # Only read if this field was written + if index >= num_fields_written: + break + + serializer = self._serializers[index] + is_nullable = self._nullable_fields.get(field_name, False) + + if is_nullable: + field_value = self._read_nullable_field(buffer, serializer) + else: + field_value = self._read_non_nullable_field(buffer, serializer) + + if field_name in current_class_field_names: + setattr(obj, field_name, field_value) return obj def xwrite(self, buffer: Buffer, value): + """Write dataclass instance to buffer in cross-language format.""" if not self._xlang: raise TypeError("xwrite can only be called when DataClassSerializer is in xlang mode") - if self._hash == 0: - from pyfory.type import unwrap_optional - - unwrapped_hints = {} - for field_name, hint in self._type_hints.items(): - unwrapped, _ = unwrap_optional(hint) - unwrapped_hints[field_name] = unwrapped - self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + self._ensure_hash_computed() if not self.fory.compatible: buffer.write_int32(self._hash) for index, field_name in enumerate(self._field_names): @@ -735,16 +937,10 @@ def xwrite(self, buffer: Buffer, value): self.fory.xwrite_ref(buffer, field_value, serializer=serializer) def xread(self, buffer): + """Read dataclass instance from buffer in cross-language format.""" if not self._xlang: raise TypeError("xread can only be called when DataClassSerializer is in xlang mode") - if self._hash == 0: - from pyfory.type import unwrap_optional - - unwrapped_hints = {} - for field_name, hint in self._type_hints.items(): - unwrapped, _ = unwrap_optional(hint) - unwrapped_hints[field_name] = unwrapped - self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints) + self._ensure_hash_computed() if not self.fory.compatible: hash_ = buffer.read_int32() if hash_ != self._hash: @@ -767,11 +963,7 @@ def xread(self, buffer): else: field_value = self.fory.xread_ref(buffer, serializer=serializer) if field_name in current_class_field_names: - setattr( - obj, - field_name, - field_value, - ) + setattr(obj, field_name, field_value) return obj diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index bcff60272e..7b9f6197fa 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -23,7 +23,7 @@ import pickle import weakref from enum import Enum -from typing import Any, List, Dict +from typing import Any, List, Dict, Optional import numpy as np import pandas as pd @@ -558,13 +558,13 @@ def test_pandas_range_index(): @dataclass(unsafe_hash=True) class PyDataClass1: - f1: int + f1: Optional[int] f2: float f3: str - f4: bool + f4: Optional[bool] f5: Any f6: List - f7: Dict + f7: Optional[Dict] @pytest.mark.parametrize("track_ref", [False, True]) @@ -580,6 +580,52 @@ def test_py_serialize_dataclass(track_ref): assert ser_de(fory, obj2) == obj2 +@dataclass(unsafe_hash=True) +class PyDataClass2: + f1: int + f2: float + f3: str + f4: bool + f5: Any + f6: List + f7: Dict + + +@pytest.mark.parametrize("track_ref", [False, True]) +@pytest.mark.parametrize("compatible", [False, True]) +def test_py_serialize_dataclass_nullable_global(track_ref, compatible): + fory = Fory( + xlang=False, + ref=track_ref, + compatible=compatible, + strict=False, + field_nullable=True, + ) + fory.register(PyDataClass2) + obj1 = PyDataClass2(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"}) + assert ser_de(fory, obj1) == obj1 + obj2 = PyDataClass2(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None) + assert ser_de(fory, obj2) == obj2 + + if compatible: + fory2 = Fory( + xlang=False, + ref=track_ref, + compatible=compatible, + strict=False, + field_nullable=True, + ) + + @dataclass(unsafe_hash=True) + class PyDataClass3: + f1: int + f2: float + + fory2.register(PyDataClass3) + assert fory2.deserialize(fory.serialize(obj1)) == PyDataClass3(f1=1, f2=-2.0) + assert fory2.deserialize(fory.serialize(obj2)) == PyDataClass3(f1=None, f2=-2.0) + + @pytest.mark.parametrize("track_ref", [False, True]) def test_function(track_ref): fory = Fory( diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index cf25365276..093f8396bc 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -414,10 +414,12 @@ class OptionalFieldsObject: f5: str = "" +@pytest.mark.parametrize("xlang", [False, True]) @pytest.mark.parametrize("compatible", [False, True]) -def test_optional_fields(compatible): - fory = Fory(xlang=True, ref=True, compatible=compatible) - fory.register_type(OptionalFieldsObject, typename="example.OptionalFieldsObject") +def test_optional_fields(xlang, compatible): + fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False) + if xlang: + fory.register_type(OptionalFieldsObject, typename="example.OptionalFieldsObject") obj_with_none = OptionalFieldsObject(f1=None, f2=None, f3=None, f4=42, f5="test") result = ser_de(fory, obj_with_none) @@ -451,11 +453,13 @@ class NestedOptionalObject: f3: str = "" +@pytest.mark.parametrize("xlang", [False, True]) @pytest.mark.parametrize("compatible", [False, True]) -def test_nested_optional_fields(compatible): - fory = Fory(xlang=True, ref=True, compatible=compatible) - fory.register_type(ComplexObject, typename="example.ComplexObject") - fory.register_type(NestedOptionalObject, typename="example.NestedOptionalObject") +def test_nested_optional_fields(xlang, compatible): + fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False) + if xlang: + fory.register_type(ComplexObject, typename="example.ComplexObject") + fory.register_type(NestedOptionalObject, typename="example.NestedOptionalObject") obj_with_none = NestedOptionalObject(f1=None, f2=None, f3="test") result = ser_de(fory, obj_with_none) @@ -493,6 +497,181 @@ class OptionalV3: f2: str = "" +@dataclass +class CompatibleV1: + f1: int = 0 + f2: str = "" + f3: float = 0.0 + + +@dataclass +class CompatibleV2: + f1: int = 0 + f2: str = "" + f3: float = 0.0 + f4: bool = False + + +@dataclass +class CompatibleV3: + f1: int = 0 + f2: str = "" + + +@pytest.mark.parametrize("xlang", [False, True]) +def test_compatible_mode_add_field(xlang): + """Test that adding a field with default value works in compatible mode.""" + fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + + fory_v1.register_type(CompatibleV1, typename="example.Compatible") + fory_v2.register_type(CompatibleV2, typename="example.Compatible") + + # V1 object serialized + v1_obj = CompatibleV1(f1=100, f2="test", f3=3.14) + v1_binary = fory_v1.serialize(v1_obj) + + # V2 can read V1 data, new field gets default value + v2_result = fory_v2.deserialize(v1_binary) + assert v2_result.f1 == 100 + assert v2_result.f2 == "test" + assert v2_result.f3 == 3.14 + assert v2_result.f4 is False # Default value + + +@pytest.mark.parametrize("xlang", [False, True]) +def test_compatible_mode_remove_field(xlang): + """Test that removing a field works in compatible mode.""" + fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + fory_v3 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + + fory_v2.register_type(CompatibleV2, typename="example.Compatible") + fory_v3.register_type(CompatibleV3, typename="example.Compatible") + + # V2 object with all fields + v2_obj = CompatibleV2(f1=200, f2="hello", f3=2.71, f4=True) + v2_binary = fory_v2.serialize(v2_obj) + + # V3 can read V2 data, extra fields are ignored + v3_result = fory_v3.deserialize(v2_binary) + assert v3_result.f1 == 200 + assert v3_result.f2 == "hello" + # f3 and f4 from V2 are ignored + + +@pytest.mark.parametrize("xlang", [False, True]) +def test_compatible_mode_bidirectional(xlang): + """Test bidirectional compatible serialization.""" + fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + + fory_v1.register_type(CompatibleV1, typename="example.Compatible") + fory_v2.register_type(CompatibleV2, typename="example.Compatible") + + # V1 -> V2 + v1_obj = CompatibleV1(f1=100, f2="test", f3=3.14) + v1_binary = fory_v1.serialize(v1_obj) + v2_result = fory_v2.deserialize(v1_binary) + assert v2_result.f1 == 100 + assert v2_result.f2 == "test" + assert v2_result.f3 == 3.14 + assert v2_result.f4 is False + + # V2 -> V1 + v2_obj = CompatibleV2(f1=200, f2="hello", f3=2.71, f4=True) + v2_binary = fory_v2.serialize(v2_obj) + v1_result = fory_v1.deserialize(v2_binary) + assert v1_result.f1 == 200 + assert v1_result.f2 == "hello" + assert v1_result.f3 == 2.71 + + +@dataclass +class CompatibleWithOptional: + f1: Optional[int] = None + f2: str = "" + f3: Optional[List[int]] = None + + +@dataclass +class CompatibleWithOptionalV2: + f1: Optional[int] = None + f2: str = "" + f3: Optional[List[int]] = None + f4: Optional[str] = None + + +@pytest.mark.parametrize("xlang", [False, True]) +def test_compatible_mode_with_optional_fields(xlang): + """Test compatible mode with optional fields.""" + fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + + fory_v1.register_type(CompatibleWithOptional, typename="example.CompatibleOptional") + fory_v2.register_type(CompatibleWithOptionalV2, typename="example.CompatibleOptional") + + # V1 with None values + v1_obj = CompatibleWithOptional(f1=None, f2="test", f3=None) + v1_binary = fory_v1.serialize(v1_obj) + v2_result = fory_v2.deserialize(v1_binary) + assert v2_result.f1 is None + assert v2_result.f2 == "test" + assert v2_result.f3 is None + assert v2_result.f4 is None + + # V1 with values + v1_obj2 = CompatibleWithOptional(f1=100, f2="test", f3=[1, 2, 3]) + v1_binary2 = fory_v1.serialize(v1_obj2) + v2_result2 = fory_v2.deserialize(v1_binary2) + assert v2_result2.f1 == 100 + assert v2_result2.f2 == "test" + assert v2_result2.f3 == [1, 2, 3] + assert v2_result2.f4 is None + + +@dataclass +class CompatibleAllTypes: + f_int: int = 0 + f_str: str = "" + f_float: float = 0.0 + f_bool: bool = False + f_list: List[int] = None + f_dict: Dict[str, int] = None + + +@dataclass +class CompatibleAllTypesV2: + f_int: int = 0 + f_str: str = "" + f_float: float = 0.0 + f_bool: bool = False + f_list: List[int] = None + f_dict: Dict[str, int] = None + f_new: str = "default" + + +@pytest.mark.parametrize("xlang", [False, True]) +def test_compatible_mode_all_basic_types(xlang): + """Test compatible mode with all basic types.""" + fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False) + + fory_v1.register_type(CompatibleAllTypes, typename="example.CompatibleAllTypes") + fory_v2.register_type(CompatibleAllTypesV2, typename="example.CompatibleAllTypes") + + v1_obj = CompatibleAllTypes(f_int=42, f_str="hello", f_float=3.14, f_bool=True, f_list=[1, 2, 3], f_dict={"a": 1, "b": 2}) + v1_binary = fory_v1.serialize(v1_obj) + v2_result = fory_v2.deserialize(v1_binary) + + assert v2_result.f_int == 42 + assert v2_result.f_str == "hello" + assert v2_result.f_float == 3.14 + assert v2_result.f_bool is True + assert v2_result.f_list == [1, 2, 3] + assert v2_result.f_dict == {"a": 1, "b": 2} + assert v2_result.f_new == "default" + + def test_optional_compatible_mode_evolution(): fory_v1 = Fory(xlang=True, ref=True, compatible=True) fory_v2 = Fory(xlang=True, ref=True, compatible=True) diff --git a/python/pyfory/type.py b/python/pyfory/type.py index ecd505d136..f41c7841d7 100644 --- a/python/pyfory/type.py +++ b/python/pyfory/type.py @@ -415,14 +415,14 @@ def visit_other(self, field_name, type_, types_path=None): pass -def infer_field_types(type_): +def infer_field_types(type_, field_nullable=False): type_hints = typing.get_type_hints(type_) from pyfory._struct import StructTypeVisitor visitor = StructTypeVisitor(type_) result = {} for name, hint in sorted(type_hints.items()): - unwrapped, _ = unwrap_optional(hint) + unwrapped, _ = unwrap_optional(hint, field_nullable=field_nullable) result[name] = infer_field(name, unwrapped, visitor) return result @@ -435,9 +435,9 @@ def is_optional_type(type_): return False -def unwrap_optional(type_): +def unwrap_optional(type_, field_nullable=False): if not is_optional_type(type_): - return type_, False + return type_, False or field_nullable args = typing.get_args(type_) if hasattr(typing, "get_args") else getattr(type_, "__args__", ()) non_none_types = [arg for arg in args if arg is not type(None)] if len(non_none_types) == 1: From dae1dbd010e03904884e99d6c9cfbc42ac805008 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 21:08:13 +0800 Subject: [PATCH 33/37] feat(java): support deserialize non exist enum variant to default (#2787) ## Why? ## What does this PR do? ## Related issues Closes #2720 https://github.com/apache/fory/pull/2719 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .../java/org/apache/fory/config/Config.java | 10 +- .../org/apache/fory/config/ForyBuilder.java | 16 +++- .../fory/config/UnknownEnumValueStrategy.java | 35 +++++++ .../java/org/apache/fory/meta/ClassDef.java | 5 +- .../fory/serializer/EnumSerializer.java | 30 ++++-- .../fory-core/native-image.properties | 1 + .../java/org/apache/fory/xlang/EnumTest.java | 91 +++++++++++++++++++ 7 files changed, 170 insertions(+), 18 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/config/UnknownEnumValueStrategy.java create mode 100644 java/fory-core/src/test/java/org/apache/fory/xlang/EnumTest.java diff --git a/java/fory-core/src/main/java/org/apache/fory/config/Config.java b/java/fory-core/src/main/java/org/apache/fory/config/Config.java index 4575f3dfc7..3f7c8f8bd3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/Config.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/Config.java @@ -62,7 +62,7 @@ public class Config implements Serializable { private final boolean deserializeNonexistentClass; private final boolean scalaOptimizationEnabled; private transient int configHash; - private final boolean deserializeNonexistentEnumValueAsNull; + private final UnknownEnumValueStrategy unknownEnumValueStrategy; private final boolean serializeEnumByName; private final int bufferSizeLimitBytes; private final int maxDepth; @@ -103,7 +103,7 @@ public Config(ForyBuilder builder) { } asyncCompilationEnabled = builder.asyncCompilationEnabled; scalaOptimizationEnabled = builder.scalaOptimizationEnabled; - deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull; + unknownEnumValueStrategy = builder.unknownEnumValueStrategy; serializeEnumByName = builder.serializeEnumByName; bufferSizeLimitBytes = builder.bufferSizeLimitBytes; maxDepth = builder.maxDepth; @@ -144,7 +144,11 @@ public boolean isStringRefIgnored() { /** ignore Enum Deserialize array out of bounds return null. */ public boolean deserializeNonexistentEnumValueAsNull() { - return deserializeNonexistentEnumValueAsNull; + return unknownEnumValueStrategy == UnknownEnumValueStrategy.RETURN_NULL; + } + + public UnknownEnumValueStrategy getUnknownEnumValueStrategy() { + return unknownEnumValueStrategy; } /** deserialize and serialize enum by name. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index 7184b00ead..38de05e153 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -84,7 +84,7 @@ public final class ForyBuilder { boolean registerGuavaTypes = true; boolean scalaOptimizationEnabled = false; boolean suppressClassRegistrationWarnings = true; - boolean deserializeNonexistentEnumValueAsNull = false; + UnknownEnumValueStrategy unknownEnumValueStrategy = UnknownEnumValueStrategy.NOT_ALLOWED; boolean serializeEnumByName = false; int bufferSizeLimitBytes = 128 * 1024; MetaCompressor metaCompressor = new DeflaterMetaCompressor(); @@ -135,7 +135,19 @@ public ForyBuilder ignoreStringRef(boolean ignoreStringRef) { /** ignore Enum Deserialize array out of bounds. */ public ForyBuilder deserializeNonexistentEnumValueAsNull( boolean deserializeNonexistentEnumValueAsNull) { - this.deserializeNonexistentEnumValueAsNull = deserializeNonexistentEnumValueAsNull; + this.unknownEnumValueStrategy = UnknownEnumValueStrategy.RETURN_NULL; + return this; + } + + /** + * Sets the strategy applied when deserialization encounters an enum value that cannot be + * resolved. + * + * @param action policy to apply for unknown enum values + * @return this builder instance for chaining + */ + public ForyBuilder withUnknownEnumValueStrategy(UnknownEnumValueStrategy action) { + this.unknownEnumValueStrategy = action; return this; } diff --git a/java/fory-core/src/main/java/org/apache/fory/config/UnknownEnumValueStrategy.java b/java/fory-core/src/main/java/org/apache/fory/config/UnknownEnumValueStrategy.java new file mode 100644 index 0000000000..19ffd86d54 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/config/UnknownEnumValueStrategy.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.config; + +/** + * Strategy describing how deserialization handles enum values that cannot be resolved from the + * serialized payload. + */ +public enum UnknownEnumValueStrategy { + /** Throw an exception and stop deserialization. */ + NOT_ALLOWED, + /** Return {@code null} instead of the unknown enum constant. */ + RETURN_NULL, + /** Fallback to the first declared enum constant. */ + RETURN_FIRST_VARIANT, + /** Fallback to the last declared enum constant. */ + RETURN_LAST_VARIANT +} diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java index 2baf75f3b0..9631108d49 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java @@ -414,7 +414,7 @@ public FieldType(int xtypeId, boolean isMonomorphic, boolean nullable, boolean t this.isMonomorphic = isMonomorphic; this.trackingRef = trackingRef; this.nullable = nullable; - this.xtypeId = xtypeId & 0xff; + this.xtypeId = xtypeId; } public boolean isMonomorphic() { @@ -560,8 +560,7 @@ public static FieldType xread( int xtypeId, boolean nullable, boolean trackingRef) { - assert xtypeId <= 0xff; - switch (xtypeId) { + switch (xtypeId & 0xff) { case Types.LIST: case Types.SET: return new CollectionFieldType( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/EnumSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/EnumSerializer.java index f314b91cd5..cc0a691360 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/EnumSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/EnumSerializer.java @@ -115,20 +115,30 @@ public Enum xread(MemoryBuffer buffer) { } private Enum handleNonexistentEnumValue(int value) { - if (fory.getConfig().deserializeNonexistentEnumValueAsNull()) { - return null; - } else { - throw new IllegalArgumentException( - String.format("Enum ordinal %s not in %s", value, Arrays.toString(enumConstants))); + switch (fory.getConfig().getUnknownEnumValueStrategy()) { + case RETURN_NULL: + return null; + case RETURN_FIRST_VARIANT: + return enumConstants[0]; + case RETURN_LAST_VARIANT: + return enumConstants[enumConstants.length - 1]; + default: + throw new IllegalArgumentException( + String.format("Enum ordinal %s not in %s", value, Arrays.toString(enumConstants))); } } private Enum handleNonexistentEnumValue(String value) { - if (fory.getConfig().deserializeNonexistentEnumValueAsNull()) { - return null; - } else { - throw new IllegalArgumentException( - String.format("Enum string %s not in %s", value, Arrays.toString(enumConstants))); + switch (fory.getConfig().getUnknownEnumValueStrategy()) { + case RETURN_NULL: + return null; + case RETURN_FIRST_VARIANT: + return enumConstants[0]; + case RETURN_LAST_VARIANT: + return enumConstants[enumConstants.length - 1]; + default: + throw new IllegalArgumentException( + String.format("Enum string %s not in %s", value, Arrays.toString(enumConstants))); } } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 3f4efe975f..e65de39acd 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -223,6 +223,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.config.ForyBuilder,\ org.apache.fory.config.Language,\ org.apache.fory.config.LongEncoding,\ + org.apache.fory.config.UnknownEnumValueStrategy,\ org.apache.fory.logging.ForyLogger,\ org.apache.fory.logging.LoggerFactory,\ org.apache.fory.memory.BoundsChecking,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/EnumTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/EnumTest.java new file mode 100644 index 0000000000..551d91858c --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/EnumTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.xlang; + +import org.apache.fory.Fory; +import org.apache.fory.config.CompatibleMode; +import org.apache.fory.config.Language; +import org.apache.fory.config.UnknownEnumValueStrategy; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class EnumTest { + enum Color { + Green, + Red, + Blue, + White, + } + + enum Color2 { + Green, + Red, + } + + static class EnumWrapper { + Color color; + } + + static class EnumWrapper2 { + Color2 color; + } + + @Test + public void testEnumEnum() { + Fory fory1 = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(false) + .build(); + fory1.register(Color.class, 101); + fory1.register(Color2.class, 102); + fory1.register(EnumWrapper.class, 103); + Fory fory2 = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withUnknownEnumValueStrategy(UnknownEnumValueStrategy.RETURN_FIRST_VARIANT) + .withCodegen(false) + .build(); + fory2.register(Color.class, 101); + fory2.register(Color2.class, 102); + fory2.register(EnumWrapper2.class, 103); + + EnumWrapper enumWrapper = new EnumWrapper(); + enumWrapper.color = Color.White; + byte[] serialize = fory1.serialize(enumWrapper); + EnumWrapper2 wrapper2 = (EnumWrapper2) fory2.deserialize(serialize); + Assert.assertEquals(wrapper2.color, Color2.Green); + + Fory fory3 = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withUnknownEnumValueStrategy(UnknownEnumValueStrategy.RETURN_LAST_VARIANT) + .withCodegen(false) + .build(); + fory3.register(Color.class, 101); + fory3.register(Color2.class, 102); + fory3.register(EnumWrapper2.class, 103); + EnumWrapper2 wrapper3 = (EnumWrapper2) fory3.deserialize(serialize); + Assert.assertEquals(wrapper3.color, Color2.Red); + } +} From 3e3c6c0d51654983282cfd1d6a702eb8175d5a8d Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 20 Oct 2025 23:15:41 +0800 Subject: [PATCH 34/37] fix(rust): fix rust xlang tests (#2788) ## Why? ## What does this PR do? - fix rust map read - fix java map read - enable rust xlang tests ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .../src/main/java/org/apache/fory/Fory.java | 5 ++ .../collection/MapLikeSerializer.java | 12 +++- .../collection/SerializationBinding.java | 12 ++++ .../java/org/apache/fory/RustXlangTest.java | 2 +- .../src/resolver/metastring_resolver.rs | 64 +++++++++++-------- rust/fory-core/src/serializer/map.rs | 57 ++++++++++------- 6 files changed, 100 insertions(+), 52 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 43b7b74dfb..3ce595398a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -1124,6 +1124,11 @@ public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { } } + public Object xreadNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, classInfoHolder); + return xreadNonRef(buffer, classInfo); + } + public Object xreadNullable(MemoryBuffer buffer, Serializer serializer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.NULL_FLAG) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index c46e1d7fb0..1794317b96 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -699,7 +699,11 @@ public long readJavaNullChunk( fory.decDepth(); } } else { - key = binding.readRef(buffer, keyClassInfoReadCache); + if (trackKeyRef) { + key = binding.readRef(buffer, keyClassInfoReadCache); + } else { + key = binding.readNonRef(buffer, keyClassInfoReadCache); + } } map.put(key, null); } @@ -741,7 +745,11 @@ private void readNullKeyChunk( fory.decDepth(); } } else { - value = binding.readRef(buffer, valueClassInfoReadCache); + if (trackValueRef) { + value = binding.readRef(buffer, valueClassInfoReadCache); + } else { + value = binding.readNonRef(buffer, valueClassInfoReadCache); + } } map.put(null, value); } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java index 6b461f3b16..86dda6849c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SerializationBinding.java @@ -49,6 +49,8 @@ interface SerializationBinding { Object readNonRef(MemoryBuffer buffer); + Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + static SerializationBinding createBinding(Fory fory) { if (fory.isCrossLanguage()) { return new XlangSerializationBinding(fory); @@ -99,6 +101,11 @@ public Object readNonRef(MemoryBuffer buffer) { return fory.readNonRef(buffer); } + @Override + public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + return fory.readNonRef(buffer, classInfoHolder); + } + @Override public void write(MemoryBuffer buffer, Serializer serializer, Object value) { serializer.write(buffer, value); @@ -158,6 +165,11 @@ public Object readNonRef(MemoryBuffer buffer) { return fory.xreadNonRef(buffer); } + @Override + public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + return fory.xreadNonRef(buffer, classInfoHolder); + } + @Override public void write(MemoryBuffer buffer, Serializer serializer, Object value) { serializer.xwrite(buffer, value); diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java index f19367d6f8..eecd52eb05 100644 --- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java @@ -77,7 +77,7 @@ public class RustXlangTest extends ForyTestBase { @BeforeClass public void isRustJavaCIEnabled() { - String enabled = "0"; + String enabled = System.getenv("RUST_TESTCASE_ENABLED"); if (enabled == null || !enabled.equals("1")) { throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); } diff --git a/rust/fory-core/src/resolver/metastring_resolver.rs b/rust/fory-core/src/resolver/metastring_resolver.rs index beb6f4445d..975d27e673 100644 --- a/rust/fory-core/src/resolver/metastring_resolver.rs +++ b/rust/fory-core/src/resolver/metastring_resolver.rs @@ -81,18 +81,38 @@ impl MetaStringBytes { pub(crate) fn from_metastring(meta_string: &MetaString) -> Result { let mut bytes = meta_string.bytes.to_vec(); - let mut hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64; - hash_code = hash_code.abs(); - if hash_code == 0 { - hash_code += 256; + let len = bytes.len(); + // follow python/java implementation: small strings use quick v1/v2 based hash, + // large strings use murmurhash, then mask lower 8 bits for encoding + let encoding_val = meta_string.encoding as i64 & HEADER_MASK; + let hash_code: i64; + if len <= MetaStringReaderResolver::SMALL_STRING_THRESHOLD { + // compute v1 and v2 from bytes little-endian + let mut v1: u64 = 0; + let mut v2: u64 = 0; + for (i, b) in bytes.iter().take(8).enumerate() { + v1 |= (*b as u64) << (8 * i); + } + if bytes.len() > 8 { + for j in 0..usize::min(8, bytes.len() - 8) { + v2 |= (bytes[8 + j] as u64) << (8 * j); + } + } + // ((v1 * 31 + v2) >> 8 << 8) | encoding + let tmp: u128 = (v1 as u128).wrapping_mul(31u128).wrapping_add(v2 as u128); + let masked = ((tmp >> 8) << 8) as u64; + hash_code = masked as i64 | encoding_val; + } else { + let mut hc = murmurhash3_x64_128(&bytes, 47).0 as i64; + hc = hc.abs(); + if hc == 0 { + hc += 256; + } + hc = (hc as u64 & 0xffffffffffffff00) as i64; + hash_code = hc | encoding_val; } - hash_code = (hash_code as u64 & 0xffffffffffffff00) as i64; - let encoding = meta_string.encoding; - let header = encoding as i64 & HEADER_MASK; - hash_code |= header; - let header = (hash_code & HEADER_MASK) as u8; - let encoding = byte_to_encoding(header); + // ensure we have 16 bytes to extract first8/second8 if bytes.len() < 16 { bytes.resize(16, 0); } @@ -107,7 +127,13 @@ impl MetaStringBytes { })?; let second8 = u64::from_le_bytes(second8); - Ok(Self::new(bytes, hash_code, encoding, first8, second8)) + Ok(Self::new( + bytes, + hash_code, + byte_to_encoding((hash_code & HEADER_MASK) as u8), + first8, + second8, + )) } } @@ -134,20 +160,8 @@ impl MetaStringWriterResolver { if let Some(b) = self.meta_string_to_bytes.get(m) { return b.clone(); } - let bytes = m.bytes.to_vec(); - let hash_code = murmurhash3_x64_128(&bytes, 47).0 as i64; - let encoding = m.encoding; - let mut first8: u64 = 0; - let mut second8: u64 = 0; - for (i, b) in bytes.iter().take(8).enumerate() { - first8 |= (*b as u64) << (8 * i); - } - if bytes.len() > 8 { - for j in 0..usize::min(8, bytes.len() - 8) { - second8 |= (bytes[8 + j] as u64) << (8 * j); - } - } - let msb = MetaStringBytes::new(bytes, hash_code, encoding, first8, second8); + // create via from_metastring to keep same hash/encoding rules + let msb = MetaStringBytes::from_metastring(m).expect("from_metastring"); self.meta_string_to_bytes.insert(m.clone(), msb.clone()); msb } diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs index 876a0980af..f3b2fd8187 100644 --- a/rust/fory-core/src/serializer/map.rs +++ b/rust/fory-core/src/serializer/map.rs @@ -559,20 +559,16 @@ impl Date: Mon, 20 Oct 2025 23:42:14 +0800 Subject: [PATCH 35/37] feat(go): update codegen field sorting to generate smaller and faster code (#2779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? The previous field ordering algorithm mixed different field types (strings, lists, maps, etc.) into broad groups that were difficult to maintain and reason about at compile time. The new specification provides a clearer categorization that: 1. Improves compile-time predictability: Separates fields into distinct groups (primitives, other internal types, lists, sets, maps, others) with clear sorting rules for each group 2. Enhances cross-language compatibility: Aligns with the updated xlang serialization specification for consistent behavior across all Fory language implementations 3. Simplifies code generation: Each field group has its own simple sorting rule, making the generated code more maintainable and easier to verify ## What does this PR do? Modified `fory/codegen/utils.go`: * Rewrote sortFields() function to implement the new 6-group categorization * Added getFieldGroup() function to categorize fields into their respective sorting groups * Added field group constants (groupPrimitive, groupOtherInternalType, groupList, groupSet, groupMap, groupOther) * Each group now has its own clear sorting logic as specified in the xlang serialization spec ## Related issues More details see in pr https://github.com/apache/fory/pull/2749 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark Co-authored-by: Shawn Yang Co-authored-by: Pan Li <1162953505@qq.com> --- ci/run_ci.sh | 10 ++-- go/fory/codegen/utils.go | 115 +++++++++++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 23 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 5a3e01ed1f..f27835341f 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -372,11 +372,11 @@ case $1 in ;; go) echo "Executing fory go tests for go" -# cd "$ROOT/go/fory" -# go install ./cmd/fory -# cd "$ROOT/go/fory/tests" -# go generate -# go test -v + cd "$ROOT/go/fory" + go install ./cmd/fory + cd "$ROOT/go/fory/tests" + go generate + go test -v cd "$ROOT/go/fory" go test -v echo "Executing fory go tests succeeds" diff --git a/go/fory/codegen/utils.go b/go/fory/codegen/utils.go index 18c5de0551..06915952d3 100644 --- a/go/fory/codegen/utils.go +++ b/go/fory/codegen/utils.go @@ -273,22 +273,28 @@ func getTypeIDValue(typeID string) int { } // sortFields sorts fields according to Fory protocol specification -// This matches the reflection-based sorting exactly for cross-language compatibility +// This matches the new field ordering specification for cross-language compatibility func sortFields(fields []*FieldInfo) { sort.Slice(fields, func(i, j int) bool { f1, f2 := fields[i], fields[j] - // Group primitives first (matching reflection's boxed group) - if f1.IsPrimitive && !f2.IsPrimitive { - return true - } - if !f1.IsPrimitive && f2.IsPrimitive { - return false + // Categorize fields into groups + group1 := getFieldGroup(f1) + group2 := getFieldGroup(f2) + + // Sort by group first + if group1 != group2 { + return group1 < group2 } - if f1.IsPrimitive && f2.IsPrimitive { - // Match reflection's boxed sorting logic exactly - // First: handle compression types (INT32/INT64/VAR_INT32/VAR_INT64) + // Within same group, apply group-specific sorting + switch group1 { + case groupPrimitive: + // Primitive fields: larger size first, smaller later, variable size last + // When same size, sort by type id + // When same size and type id, sort by snake case field name + + // Handle compression types (INT32/INT64/VAR_INT32/VAR_INT64) compressI := f1.TypeID == "INT32" || f1.TypeID == "INT64" || f1.TypeID == "VAR_INT32" || f1.TypeID == "VAR_INT64" compressJ := f2.TypeID == "INT32" || f2.TypeID == "INT64" || @@ -298,24 +304,97 @@ func sortFields(fields []*FieldInfo) { return !compressI && compressJ // non-compress comes first } - // Then: by size (descending) + // Sort by size (descending) if f1.PrimitiveSize != f2.PrimitiveSize { return f1.PrimitiveSize > f2.PrimitiveSize } - // Finally: by name (ascending) + // Sort by type ID + if f1.TypeID != f2.TypeID { + return getTypeIDValue(f1.TypeID) < getTypeIDValue(f2.TypeID) + } + + // Finally by name return f1.SnakeName < f2.SnakeName - } - // For non-primitives: STRING comes in final group, others in others group - // All sorted by type ID, then by name (matching reflection) - if f1.TypeID != f2.TypeID { - return getTypeIDValue(f1.TypeID) < getTypeIDValue(f2.TypeID) + case groupOtherInternalType: + // Other internal type fields: sort by type id then snake case field name + if f1.TypeID != f2.TypeID { + return getTypeIDValue(f1.TypeID) < getTypeIDValue(f2.TypeID) + } + return f1.SnakeName < f2.SnakeName + + case groupList, groupSet, groupMap, groupOther: + // List/Set/Map/Other fields: sort by snake case field name only + return f1.SnakeName < f2.SnakeName + + default: + // Fallback: sort by name + return f1.SnakeName < f2.SnakeName } - return f1.SnakeName < f2.SnakeName }) } +// Field group constants for sorting +const ( + groupPrimitive = 0 // primitive and nullable primitive fields + groupOtherInternalType = 1 // other internal type fields (string, timestamp, etc.) + groupList = 2 // list fields + groupSet = 3 // set fields + groupMap = 4 // map fields + groupOther = 5 // other fields +) + +// getFieldGroup categorizes a field into its sorting group +func getFieldGroup(field *FieldInfo) int { + typeID := field.TypeID + + // Primitive fields (including nullable primitives) + // types: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64 + if field.IsPrimitive { + return groupPrimitive + } + + // List fields + if typeID == "LIST" { + return groupList + } + + // Set fields + if typeID == "SET" { + return groupSet + } + + // Map fields + if typeID == "MAP" { + return groupMap + } + + // Other internal type fields + // These are fory internal types that are not primitives/lists/sets/maps + // Examples: STRING, TIMESTAMP, LOCAL_DATE, NAMED_STRUCT, etc. + internalTypes := map[string]bool{ + "STRING": true, + "TIMESTAMP": true, + "LOCAL_DATE": true, + "NAMED_STRUCT": true, + "STRUCT": true, + "BINARY": true, + "ENUM": true, + "NAMED_ENUM": true, + "EXT": true, + "NAMED_EXT": true, + "INTERFACE": true, // for interface{} types + } + + if internalTypes[typeID] { + return groupOtherInternalType + } + + // Everything else goes to "other fields" + return groupOther +} + // computeStructHash computes a hash for struct schema compatibility // This implementation aligns with the reflection-based hash calculation func computeStructHash(s *StructInfo) int32 { From b883644f4178e38bac6c5088f48a56662b536983 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Tue, 21 Oct 2025 00:02:13 +0800 Subject: [PATCH 36/37] feat(rust): make type meta resolve return type info directly (#2789) ## Why? ## What does this PR do? make type meta resolve return type info directly to avoid uncesssary copy ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- rust/fory-core/src/fory.rs | 2 +- rust/fory-core/src/resolver/context.rs | 24 ++---- rust/fory-core/src/resolver/meta_resolver.rs | 59 ++++++++++++--- rust/fory-core/src/resolver/type_resolver.rs | 78 +++++++++++++++++--- rust/fory-core/src/serializer/skip.rs | 10 ++- 5 files changed, 130 insertions(+), 43 deletions(-) diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index bb1983ee3a..ca3f500964 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -420,7 +420,7 @@ impl Fory { if context.is_compatible() { let meta_offset = context.reader.read_i32()?; if meta_offset != -1 { - bytes_to_skip = context.load_meta(meta_offset as usize)?; + bytes_to_skip = context.load_type_meta(meta_offset as usize)?; } } let result = ::fory_read(context, true, true); diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 64dca6bd0a..a347d54a93 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -19,7 +19,7 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; use crate::fory::Fory; -use crate::meta::{MetaString, TypeMeta}; +use crate::meta::MetaString; use crate::resolver::meta_resolver::{MetaReaderResolver, MetaWriterResolver}; use crate::resolver::metastring_resolver::{ MetaStringBytes, MetaStringReaderResolver, MetaStringWriterResolver, @@ -303,14 +303,14 @@ impl ReadContext { } #[inline(always)] - pub fn get_meta(&self, type_index: usize) -> Result<&Rc, Error> { + pub fn get_type_info_by_index(&self, type_index: usize) -> Result<&Rc, Error> { self.meta_resolver.get(type_index).ok_or_else(|| { - Error::type_error(format!("Meta not found for type index: {}", type_index)) + Error::type_error(format!("TypeInfo not found for type index: {}", type_index)) }) } #[inline(always)] - pub fn load_meta(&mut self, offset: usize) -> Result { + pub fn load_type_meta(&mut self, offset: usize) -> Result { self.meta_resolver.load( &self.type_resolver, &mut Reader::new(&self.reader.slice_after_cursor()[offset..]), @@ -323,22 +323,14 @@ impl ReadContext { match fory_type_id & 0xff { types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { let meta_index = self.reader.read_varuint32()? as usize; - let remote_meta = self.get_meta(meta_index)?.clone(); - let local_type_info = self - .type_resolver - .get_type_info_by_id(fory_type_id) - .ok_or_else(|| Error::type_error("ID harness not found"))?; - // Create a new TypeInfo with remote metadata but local harness - // TODO: make self.get_meta return type info - Ok(Rc::new(local_type_info.with_remote_meta(remote_meta))) + let type_info = self.get_type_info_by_index(meta_index)?.clone(); + Ok(type_info) } types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { if self.is_share_meta() { let meta_index = self.reader.read_varuint32()? as usize; - self.get_meta(meta_index)?; - self.type_resolver - .get_type_info_by_id(fory_type_id) - .ok_or_else(|| Error::type_error("ID harness not found")) + let type_info = self.get_type_info_by_index(meta_index)?.clone(); + Ok(type_info) } else { let namespace = self.meta_resolver.read_metastring(&mut self.reader)?; let type_name = self.meta_resolver.read_metastring(&mut self.reader)?; diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 525caf702d..dbd4db05cb 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -18,6 +18,7 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; use crate::meta::{Encoding, MetaString, TypeMeta, NAMESPACE_DECODER}; +use crate::resolver::type_resolver::TypeInfo; use crate::TypeResolver; use std::collections::HashMap; use std::rc::Rc; @@ -68,13 +69,13 @@ impl MetaWriterResolver { #[derive(Default)] pub struct MetaReaderResolver { - pub reading_type_defs: Vec>, - parsed_type_defs: HashMap>, + pub reading_type_infos: Vec>, + parsed_type_infos: HashMap>, } impl MetaReaderResolver { - pub fn get(&self, index: usize) -> Option<&Rc> { - self.reading_type_defs.get(index) + pub fn get(&self, index: usize) -> Option<&Rc> { + self.reading_type_infos.get(index) } pub fn load( @@ -83,11 +84,11 @@ impl MetaReaderResolver { reader: &mut Reader, ) -> Result { let meta_size = reader.read_varuint32()?; - // self.reading_type_defs.reserve(meta_size as usize); + // self.reading_type_infos.reserve(meta_size as usize); for _ in 0..meta_size { let meta_header = reader.read_i64()?; - if let Some(type_meta) = self.parsed_type_defs.get(&meta_header) { - self.reading_type_defs.push(type_meta.clone()); + if let Some(type_info) = self.parsed_type_infos.get(&meta_header) { + self.reading_type_infos.push(type_info.clone()); TypeMeta::skip_bytes(reader, meta_header)?; } else { let type_meta = Rc::new(TypeMeta::from_bytes_with_header( @@ -95,11 +96,45 @@ impl MetaReaderResolver { type_resolver, meta_header, )?); - if self.parsed_type_defs.len() < MAX_PARSED_NUM_TYPE_DEFS { - // avoid malicious type defs to OOM parsed_type_defs - self.parsed_type_defs.insert(meta_header, type_meta.clone()); + + // Try to find local type info + let type_info = if type_meta.get_namespace().original.is_empty() { + // Registered by ID + let type_id = type_meta.get_type_id(); + if let Some(local_type_info) = type_resolver.get_type_info_by_id(type_id) { + // Use local harness with remote metadata + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + Some(local_type_info.get_harness()), + )) + } else { + // No local type found, use stub harness + Rc::new(TypeInfo::from_remote_meta(type_meta.clone(), None)) + } + } else { + // Registered by name + let namespace = &type_meta.get_namespace().original; + let type_name = &type_meta.get_type_name().original; + if let Some(local_type_info) = + type_resolver.get_type_info_by_name(namespace, type_name) + { + // Use local harness with remote metadata + Rc::new(TypeInfo::from_remote_meta( + type_meta.clone(), + Some(local_type_info.get_harness()), + )) + } else { + // No local type found, use stub harness + Rc::new(TypeInfo::from_remote_meta(type_meta.clone(), None)) + } + }; + + if self.parsed_type_infos.len() < MAX_PARSED_NUM_TYPE_DEFS { + // avoid malicious type defs to OOM parsed_type_infos + self.parsed_type_infos + .insert(meta_header, type_info.clone()); } - self.reading_type_defs.push(type_meta); + self.reading_type_infos.push(type_info); } } Ok(reader.get_cursor()) @@ -131,6 +166,6 @@ impl MetaReaderResolver { } pub fn reset(&mut self) { - self.reading_type_defs.clear(); + self.reading_type_infos.clear(); } } diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs index 46c676bcd8..7da594172a 100644 --- a/rust/fory-core/src/resolver/type_resolver.rs +++ b/rust/fory-core/src/resolver/type_resolver.rs @@ -214,22 +214,80 @@ impl TypeInfo { &self.harness } - /// Create a new TypeInfo with the same properties but different type_meta. - /// This is used during deserialization to create a TypeInfo with remote metadata - /// while keeping the local harness for deserialization functions. - pub fn with_remote_meta(&self, remote_meta: Rc) -> TypeInfo { + /// Create a TypeInfo from remote TypeMeta with a stub harness + /// Used when the type doesn't exist locally during deserialization + pub fn from_remote_meta( + remote_meta: Rc, + local_harness: Option<&Harness>, + ) -> TypeInfo { + let type_id = remote_meta.get_type_id(); + let namespace = remote_meta.get_namespace(); + let type_name = remote_meta.get_type_name(); + let type_def_bytes = remote_meta.to_bytes().unwrap_or_default(); + let register_by_name = !namespace.original.is_empty() || !type_name.original.is_empty(); + + let harness = if let Some(h) = local_harness { + h.clone() + } else { + // Create a stub harness that returns errors when called + Harness::new( + stub_write_fn, + stub_read_fn, + stub_write_data_fn, + stub_read_data_fn, + stub_to_serializer_fn, + ) + }; + TypeInfo { - type_def: self.type_def.clone(), + type_def: Rc::from(type_def_bytes), type_meta: remote_meta, - type_id: self.type_id, - namespace: self.namespace.clone(), - type_name: self.type_name.clone(), - register_by_name: self.register_by_name, - harness: self.harness.clone(), + type_id, + namespace, + type_name, + register_by_name, + harness, } } } +// Stub functions for when a type doesn't exist locally +fn stub_write_fn( + _: &dyn Any, + _: &mut WriteContext, + _: bool, + _: bool, + _: bool, +) -> Result<(), Error> { + Err(Error::type_error( + "Cannot serialize unknown remote type - type not registered locally", + )) +} + +fn stub_read_fn(_: &mut ReadContext, _: bool, _: bool) -> Result, Error> { + Err(Error::type_error( + "Cannot deserialize unknown remote type - type not registered locally", + )) +} + +fn stub_write_data_fn(_: &dyn Any, _: &mut WriteContext, _: bool) -> Result<(), Error> { + Err(Error::type_error( + "Cannot serialize unknown remote type - type not registered locally", + )) +} + +fn stub_read_data_fn(_: &mut ReadContext) -> Result, Error> { + Err(Error::type_error( + "Cannot deserialize unknown remote type - type not registered locally", + )) +} + +fn stub_to_serializer_fn(_: Box) -> Result, Error> { + Err(Error::type_error( + "Cannot convert unknown remote type to serializer", + )) +} + /// TypeResolver is a resolver for fast type/serializer dispatch. #[derive(Clone)] pub struct TypeResolver { diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index 0b9cac2146..8cb0d421a7 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -158,8 +158,8 @@ pub fn skip_value( Error::type_mismatch(type_id_num, remote_type_id) ); let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize)?; - let field_infos = type_meta.get_field_infos().to_vec(); + let type_info = context.get_type_info_by_index(meta_index as usize)?; + let field_infos = type_info.get_type_meta().get_field_infos().to_vec(); context.inc_depth()?; for field_info in field_infos.iter() { let read_ref_flag = util::field_requires_ref_flag( @@ -177,8 +177,9 @@ pub fn skip_value( Error::type_mismatch(type_id_num, remote_type_id) ); let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize)?; + let type_info = context.get_type_info_by_index(meta_index as usize)?; let type_resolver = context.get_type_resolver(); + let type_meta = type_info.get_type_meta(); type_resolver .get_ext_name_harness(&type_meta.get_namespace(), &type_meta.get_type_name())? .get_read_data_fn()(context)?; @@ -195,7 +196,8 @@ pub fn skip_value( if internal_id == COMPATIBLE_STRUCT_ID { let remote_type_id = context.reader.read_varuint32()?; let meta_index = context.reader.read_varuint32()?; - let type_meta = context.get_meta(meta_index as usize)?; + let type_info = context.get_type_info_by_index(meta_index as usize)?; + let type_meta = type_info.get_type_meta(); ensure!( type_meta.get_type_id() == remote_type_id, Error::type_mismatch(type_meta.get_type_id(), remote_type_id) From 580b9e66aa02be59808b1a64ef8dc13582ec719d Mon Sep 17 00:00:00 2001 From: rajuyadav03 <162599391+rajuyadav03@users.noreply.github.com> Date: Tue, 21 Oct 2025 05:11:12 +0530 Subject: [PATCH 37/37] Testing PR in forked repo --- scripts/remove-commented-commit.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/remove-commented-commit.sh b/scripts/remove-commented-commit.sh index 960d3988a4..ef060940d9 100755 --- a/scripts/remove-commented-commit.sh +++ b/scripts/remove-commented-commit.sh @@ -39,5 +39,6 @@ grep -v '^\s*#' "$COMMIT_MSG_FILE" | \ # Replace original file with cleaned content mv "$TEMP_FILE" "$COMMIT_MSG_FILE" +#test exit 0