diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e6f47ec3..9d33bb765e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -478,6 +478,7 @@ jobs: run: | cd javascript npm install + npm run build - name: Run JavaScript Xlang Test env: FORY_JAVASCRIPT_JAVA_CI: "1" @@ -486,6 +487,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn --no-transfer-progress test -Dtest=org.apache.fory.xlang.JavaScriptXlangTest -DforkCount=0 + - name: Run JavaScript IDL Tests + run: ./integration_tests/idl_tests/run_javascript_tests.sh rust: name: Rust CI diff --git a/compiler/README.md b/compiler/README.md index c348248035..e4510475e7 100644 --- a/compiler/README.md +++ b/compiler/README.md @@ -4,7 +4,7 @@ The FDL compiler generates cross-language serialization code from schema definit ## Features -- **Multi-language code generation**: Java, Python, Go, Rust, C++, C# +- **Multi-language code generation**: Java, Python, Go, Rust, C++, C#, Javascript, and Swift - **Rich type system**: Primitives, enums, messages, lists, maps - **Cross-language serialization**: Generated code works seamlessly with Apache Fory - **Type ID and namespace support**: Both numeric IDs and name-based type registration @@ -64,16 +64,16 @@ message Cat [id=103] { foryc schema.fdl --output ./generated # Generate for specific languages -foryc schema.fdl --lang java,python,csharp --output ./generated +foryc schema.fdl --lang java,python,csharp,javascript --output ./generated # Override package name foryc schema.fdl --package myapp.models --output ./generated # Language-specific output directories (protoc-style) -foryc schema.fdl --java_out=./src/main/java --python_out=./python/src --csharp_out=./csharp/src/Generated +foryc schema.fdl --java_out=./src/main/java --python_out=./python/src --csharp_out=./csharp/src/Generated --javascript_out=./javascript # Combine with other options -foryc schema.fdl --java_out=./gen --go_out=./gen/go --csharp_out=./gen/csharp -I ./proto +foryc schema.fdl --java_out=./gen --go_out=./gen/go --csharp_out=./gen/csharp --javascript_out=./gen/js -I ./proto ``` ### 3. Use Generated Code @@ -185,19 +185,19 @@ message Config { ... } // Registered as "package.Config" ### Primitive Types -| FDL Type | Java | Python | Go | Rust | C++ | C# | -| ----------- | ----------- | ------------------- | ----------- | ----------------------- | ---------------------- | ---------------- | -| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | -| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `sbyte` | -| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `short` | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | -| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `long` | -| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `float` | -| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `double` | -| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | -| `bytes` | `byte[]` | `bytes` | `[]byte` | `Vec` | `std::vector` | `byte[]` | -| `date` | `LocalDate` | `datetime.date` | `time.Time` | `chrono::NaiveDate` | `fory::Date` | `DateOnly` | -| `timestamp` | `Instant` | `datetime.datetime` | `time.Time` | `chrono::NaiveDateTime` | `fory::Timestamp` | `DateTimeOffset` | +| FDL Type | Java | Python | Go | Rust | C++ | C# | JavaScript | +| ----------- | ----------- | ------------------- | ----------- | ----------------------- | ---------------------- | ---------------- | ------------------ | +| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `boolean` | +| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `sbyte` | `number` | +| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `short` | `number` | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `number` | +| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `long` | `bigint \| number` | +| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `float` | `number` | +| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `double` | `number` | +| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `string` | +| `bytes` | `byte[]` | `bytes` | `[]byte` | `Vec` | `std::vector` | `byte[]` | `Uint8Array` | +| `date` | `LocalDate` | `datetime.date` | `time.Time` | `chrono::NaiveDate` | `fory::Date` | `DateOnly` | `Date` | +| `timestamp` | `Instant` | `datetime.datetime` | `time.Time` | `chrono::NaiveDateTime` | `fory::Timestamp` | `DateTimeOffset` | `Date` | ### Collection Types @@ -285,7 +285,8 @@ fory_compiler/ ├── go.py # Go struct generator ├── rust.py # Rust struct generator ├── cpp.py # C++ struct generator - └── csharp.py # C# class generator + ├── csharp.py # C# class generator + └── javascript.py # JavaScript interface generator ``` ### FDL Frontend @@ -422,6 +423,25 @@ cd integration_tests/idl_tests ./run_csharp_tests.sh ``` +### JavaScript + +Generates interfaces with: + +- `export interface` declarations for messages +- `export enum` declarations for enums +- Discriminated unions with case enums +- Registration helper function + +```javascript +export interface Cat { + friend?: Dog | undefined; + name?: string | undefined; + tags: string[]; + scores: Record; + lives: number; +} +``` + ## CLI Reference ``` @@ -431,7 +451,7 @@ Arguments: FILES FDL files to compile Options: - --lang TEXT Target languages (java,python,cpp,rust,go,csharp or "all") + --lang TEXT Target languages (java,python,cpp,rust,go,csharp,javascript or "all") Default: all --output, -o PATH Output directory Default: ./generated diff --git a/compiler/fory_compiler/cli.py b/compiler/fory_compiler/cli.py index 96325eaa8f..f558dc309b 100644 --- a/compiler/fory_compiler/cli.py +++ b/compiler/fory_compiler/cli.py @@ -264,7 +264,7 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace: "--lang", type=str, default="all", - help="Comma-separated list of target languages (java,python,cpp,rust,go,csharp,swift). Default: all", + help="Comma-separated list of target languages (java,python,cpp,rust,go,csharp,javascript,swift). Default: all", ) parser.add_argument( @@ -343,6 +343,14 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace: help="Generate C# code in DST_DIR", ) + parser.add_argument( + "--javascript_out", + type=Path, + default=None, + metavar="DST_DIR", + help="Generate JavaScript code in DST_DIR", + ) + parser.add_argument( "--swift_out", type=Path, @@ -650,6 +658,7 @@ def cmd_compile(args: argparse.Namespace) -> int: "go": args.go_out, "rust": args.rust_out, "csharp": args.csharp_out, + "javascript": args.javascript_out, "swift": args.swift_out, } diff --git a/compiler/fory_compiler/generators/__init__.py b/compiler/fory_compiler/generators/__init__.py index 47adaf6feb..83d33d6225 100644 --- a/compiler/fory_compiler/generators/__init__.py +++ b/compiler/fory_compiler/generators/__init__.py @@ -24,6 +24,7 @@ from fory_compiler.generators.rust import RustGenerator from fory_compiler.generators.go import GoGenerator from fory_compiler.generators.csharp import CSharpGenerator +from fory_compiler.generators.javascript import JavaScriptGenerator from fory_compiler.generators.swift import SwiftGenerator GENERATORS = { @@ -33,6 +34,7 @@ "rust": RustGenerator, "go": GoGenerator, "csharp": CSharpGenerator, + "javascript": JavaScriptGenerator, "swift": SwiftGenerator, } @@ -44,6 +46,7 @@ "RustGenerator", "GoGenerator", "CSharpGenerator", + "JavaScriptGenerator", "SwiftGenerator", "GENERATORS", ] diff --git a/compiler/fory_compiler/generators/javascript.py b/compiler/fory_compiler/generators/javascript.py new file mode 100644 index 0000000000..4009ccd49f --- /dev/null +++ b/compiler/fory_compiler/generators/javascript.py @@ -0,0 +1,773 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""JavaScript/TypeScript code generator.""" + +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union as TypingUnion + +from fory_compiler.frontend.utils import parse_idl_file +from fory_compiler.generators.base import BaseGenerator, GeneratedFile +from fory_compiler.ir.ast import ( + Enum, + Field, + FieldType, + ListType, + MapType, + Message, + NamedType, + PrimitiveType, + Schema, + Union, +) +from fory_compiler.ir.types import PrimitiveKind + + +class JavaScriptGenerator(BaseGenerator): + """Generates JavaScript/TypeScript type definitions and Fory registration helpers from IDL.""" + + language_name = "javascript" + file_extension = ".ts" + + # TypeScript/JavaScript reserved keywords that cannot be used as identifiers + TS_KEYWORDS = { + "abstract", + "any", + "as", + "asserts", + "async", + "await", + "bigint", + "boolean", + "break", + "case", + "catch", + "class", + "const", + "continue", + "debugger", + "declare", + "default", + "delete", + "do", + "else", + "enum", + "export", + "extends", + "false", + "finally", + "for", + "from", + "function", + "get", + "if", + "implements", + "import", + "in", + "infer", + "instanceof", + "interface", + "is", + "keyof", + "let", + "module", + "namespace", + "never", + "new", + "null", + "number", + "object", + "of", + "package", + "private", + "protected", + "public", + "readonly", + "require", + "return", + "set", + "static", + "string", + "super", + "switch", + "symbol", + "this", + "throw", + "true", + "try", + "type", + "typeof", + "undefined", + "unique", + "unknown", + "var", + "void", + "while", + "with", + "yield", + } + + # Mapping from FDL primitive types to TypeScript types + PRIMITIVE_MAP = { + PrimitiveKind.BOOL: "boolean", + PrimitiveKind.INT8: "number", + PrimitiveKind.INT16: "number", + PrimitiveKind.INT32: "number", + PrimitiveKind.VARINT32: "number", + PrimitiveKind.INT64: "bigint | number", + PrimitiveKind.VARINT64: "bigint | number", + PrimitiveKind.TAGGED_INT64: "bigint | number", + PrimitiveKind.UINT8: "number", + PrimitiveKind.UINT16: "number", + PrimitiveKind.UINT32: "number", + PrimitiveKind.VAR_UINT32: "number", + PrimitiveKind.UINT64: "bigint | number", + PrimitiveKind.VAR_UINT64: "bigint | number", + PrimitiveKind.TAGGED_UINT64: "bigint | number", + PrimitiveKind.FLOAT16: "number", + PrimitiveKind.BFLOAT16: "number", + PrimitiveKind.FLOAT32: "number", + PrimitiveKind.FLOAT64: "number", + PrimitiveKind.STRING: "string", + PrimitiveKind.BYTES: "Uint8Array", + PrimitiveKind.DATE: "Date", + PrimitiveKind.TIMESTAMP: "Date", + PrimitiveKind.DURATION: "number", + PrimitiveKind.DECIMAL: "number", + PrimitiveKind.ANY: "any", + } + + def __init__(self, schema: Schema, options): + super().__init__(schema, options) + self.indent_str = " " # TypeScript uses 2 spaces + self._qualified_type_names: Dict[int, str] = {} + self._build_qualified_type_name_index() + + def _build_qualified_type_name_index(self) -> None: + """Build an index mapping type object ids to their qualified names.""" + for enum in self.schema.enums: + self._qualified_type_names[id(enum)] = enum.name + for union in self.schema.unions: + self._qualified_type_names[id(union)] = union.name + + def visit_message(message: Message, parents: List[str]) -> None: + path = ".".join(parents + [message.name]) + self._qualified_type_names[id(message)] = path + for nested_enum in message.nested_enums: + self._qualified_type_names[id(nested_enum)] = ( + f"{path}.{nested_enum.name}" + ) + for nested_union in message.nested_unions: + self._qualified_type_names[id(nested_union)] = ( + f"{path}.{nested_union.name}" + ) + for nested_msg in message.nested_messages: + visit_message(nested_msg, parents + [message.name]) + + for message in self.schema.messages: + visit_message(message, []) + + def safe_identifier(self, name: str) -> str: + """Escape identifiers that collide with TypeScript reserved words.""" + if name in self.TS_KEYWORDS: + return f"{name}_" + return name + + def safe_type_identifier(self, name: str) -> str: + """Escape type names that collide with TypeScript reserved words.""" + return self.safe_identifier(name) + + def safe_member_name(self, name: str) -> str: + """Generate a safe camelCase member name.""" + return self.safe_identifier(self.to_camel_case(name)) + + def _nested_type_names_for_message(self, message: Message) -> Set[str]: + """Collect safe type names of nested types to detect collisions.""" + names: Set[str] = set() + for nested in ( + list(message.nested_enums) + + list(message.nested_unions) + + list(message.nested_messages) + ): + names.add(self.safe_type_identifier(nested.name)) + return names + + def _field_member_name( + self, + field: Field, + message: Message, + used_names: Set[str], + ) -> str: + """Produce a unique safe member name for a field, avoiding collisions.""" + base = self.safe_member_name(field.name) + nested_type_names = self._nested_type_names_for_message(message) + if base in nested_type_names: + base = f"{base}Value" + + candidate = base + suffix = 1 + while candidate in used_names: + candidate = f"{base}{suffix}" + suffix += 1 + used_names.add(candidate) + return candidate + + def is_imported_type(self, type_def: object) -> bool: + """Return True if a type definition comes from an imported IDL file.""" + if not self.schema.source_file: + return False + location = getattr(type_def, "location", None) + if location is None or not location.file: + return False + try: + return ( + Path(location.file).resolve() != Path(self.schema.source_file).resolve() + ) + except Exception: + return location.file != self.schema.source_file + + def split_imported_types( + self, items: List[object] + ) -> Tuple[List[object], List[object]]: + imported: List[object] = [] + local: List[object] = [] + for item in items: + if self.is_imported_type(item): + imported.append(item) + else: + local.append(item) + return imported, local # Return (imported, local) tuple + + def get_module_name(self) -> str: + """Get the TypeScript module name from package.""" + if self.package: + parts = self.package.split(".") + return self.to_camel_case(parts[-1]) + return "generated" + + def _module_file_name(self) -> str: + """Determine the output file name.""" + if self.schema.source_file and not self.schema.source_file.startswith("<"): + return f"{Path(self.schema.source_file).stem}.ts" + if self.schema.package: + return f"{self.schema.package.replace('.', '_')}.ts" + return "generated.ts" + + def get_registration_function_name(self) -> str: + """Get the name of the registration function.""" + return f"register{self.to_pascal_case(self.get_module_name())}Types" + + def _normalize_import_path(self, path_str: str) -> str: + if not path_str: + return path_str + try: + return str(Path(path_str).resolve()) + except Exception: + return path_str + + def _load_schema(self, file_path: str) -> Optional[Schema]: + if not file_path: + return None + if not hasattr(self, "_schema_cache"): + self._schema_cache: Dict[Path, Schema] = {} + path = Path(file_path).resolve() + if path in self._schema_cache: + return self._schema_cache[path] + try: + schema = parse_idl_file(path) + except Exception: + return None + self._schema_cache[path] = schema + return schema + + def _module_name_for_schema(self, schema: Schema) -> str: + """Derive a module name from another schema.""" + if schema.package: + parts = schema.package.split(".") + return self.to_camel_case(parts[-1]) + return "generated" + + def _registration_fn_for_schema(self, schema: Schema) -> str: + """Derive the registration function name for an imported schema.""" + mod = self._module_name_for_schema(schema) + return f"register{self.to_pascal_case(mod)}Types" + + def _collect_imported_registrations(self) -> List[Tuple[str, str]]: + """Collect (module_path, registration_fn) pairs for imported schemas.""" + file_info: Dict[str, Tuple[str, str]] = {} + for type_def in self.schema.enums + self.schema.unions + self.schema.messages: + if not self.is_imported_type(type_def): + continue + location = getattr(type_def, "location", None) + file_path = getattr(location, "file", None) if location else None + if not file_path: + continue + normalized = self._normalize_import_path(file_path) + if normalized in file_info: + continue + imported_schema = self._load_schema(file_path) + if imported_schema is None: + continue + reg_fn = self._registration_fn_for_schema(imported_schema) + mod_name = self._module_name_for_schema(imported_schema) + file_info[normalized] = (f"./{mod_name}", reg_fn) + + ordered: List[Tuple[str, str]] = [] + used: Set[str] = set() + + if self.schema.source_file: + base_dir = Path(self.schema.source_file).resolve().parent + for imp in self.schema.imports: + candidate = self._normalize_import_path( + str((base_dir / imp.path).resolve()) + ) + if candidate in file_info and candidate not in used: + ordered.append(file_info[candidate]) + used.add(candidate) + + for key in sorted(file_info.keys()): + if key in used: + continue + ordered.append(file_info[key]) + + deduped: List[Tuple[str, str]] = [] + seen: Set[Tuple[str, str]] = set() + for item in ordered: + if item in seen: + continue + seen.add(item) + deduped.append(item) + return deduped + + def _resolve_named_type( + self, name: str, parent_stack: Optional[List[Message]] = None + ) -> Optional[TypingUnion[Message, Enum, Union]]: + """Resolve a named type reference to its definition.""" + parent_stack = parent_stack or [] + if "." in name: + return self.schema.get_type(name) + for msg in reversed(parent_stack): + nested = msg.get_nested_type(name) + if nested is not None: + return nested + return self.schema.get_type(name) + + def generate_type( + self, + field_type: FieldType, + nullable: bool = False, + parent_stack: Optional[List[Message]] = None, + ) -> str: + """Generate TypeScript type string for a field type.""" + parent_stack = parent_stack or [] + type_str = "" + + if isinstance(field_type, PrimitiveType): + if field_type.kind not in self.PRIMITIVE_MAP: + raise ValueError( + f"Unsupported primitive type for TypeScript: {field_type.kind}" + ) + type_str = self.PRIMITIVE_MAP[field_type.kind] + elif isinstance(field_type, NamedType): + # Check if this NamedType matches a primitive type name + primitive_name = field_type.name.lower() + # Map common shorthand names to primitive kinds + shorthand_map = { + "float": PrimitiveKind.FLOAT32, + "double": PrimitiveKind.FLOAT64, + } + if primitive_name in shorthand_map: + type_str = self.PRIMITIVE_MAP.get(shorthand_map[primitive_name], "any") + else: + # Check if it matches any primitive kind directly + for primitive_kind, ts_type in self.PRIMITIVE_MAP.items(): + if primitive_kind.value == primitive_name: + type_str = ts_type + break + if not type_str: + # If not a primitive, treat as a message/enum type + type_str = self.safe_type_identifier( + self.to_pascal_case(field_type.name) + ) + elif isinstance(field_type, ListType): + element_type = self.generate_type( + field_type.element_type, + nullable=field_type.element_optional, + parent_stack=parent_stack, + ) + type_str = f"{element_type}[]" + elif isinstance(field_type, MapType): + key_type = self.generate_type( + field_type.key_type, + nullable=False, + parent_stack=parent_stack, + ) + value_type = self.generate_type( + field_type.value_type, + nullable=False, + parent_stack=parent_stack, + ) + type_str = f"Record<{key_type}, {value_type}>" + else: + type_str = "any" + + if nullable: + type_str += " | undefined" + + return type_str + + def _default_initializer( + self, field: Field, parent_stack: List[Message] + ) -> Optional[str]: + """Return a TS default initializer expression, or None.""" + if field.optional: + return None + + field_type = field.field_type + if isinstance(field_type, ListType): + return "[]" + if isinstance(field_type, MapType): + return "{}" + if isinstance(field_type, PrimitiveType): + kind = field_type.kind + if kind == PrimitiveKind.BOOL: + return "false" + if kind == PrimitiveKind.STRING: + return '""' + if kind == PrimitiveKind.BYTES: + return "new Uint8Array(0)" + if kind == PrimitiveKind.ANY: + return "undefined" + if kind in {PrimitiveKind.DATE, PrimitiveKind.TIMESTAMP}: + return "new Date(0)" + return "0" + if isinstance(field_type, NamedType): + resolved = self._resolve_named_type(field_type.name, parent_stack) + if isinstance(resolved, Enum): + return "0" + return "undefined" + return None + + def _collect_local_types( + self, + ) -> List[TypingUnion[Message, Enum, Union]]: + """Collect all non-imported types (including nested) for registration.""" + local_types: List[TypingUnion[Message, Enum, Union]] = [] + + for enum in self.schema.enums: + if not self.is_imported_type(enum): + local_types.append(enum) + for union in self.schema.unions: + if not self.is_imported_type(union): + local_types.append(union) + + def visit_message(message: Message) -> None: + local_types.append(message) + for nested_enum in message.nested_enums: + local_types.append(nested_enum) + for nested_union in message.nested_unions: + local_types.append(nested_union) + for nested_msg in message.nested_messages: + visit_message(nested_msg) + + for message in self.schema.messages: + if self.is_imported_type(message): + continue + visit_message(message) + + return local_types + + def generate_imports(self) -> List[str]: + """Generate import statements for imported types and registration functions.""" + lines: List[str] = [] + imported_regs = self._collect_imported_registrations() + + # Collect all imported types used in this schema + imported_types_by_module: Dict[str, Set[str]] = {} + + for type_def in self.schema.enums + self.schema.unions + self.schema.messages: + if not self.is_imported_type(type_def): + continue + + location = getattr(type_def, "location", None) + file_path = getattr(location, "file", None) if location else None + if not file_path: + continue + + imported_schema = self._load_schema(file_path) + if imported_schema is None: + continue + + mod_name = self._module_name_for_schema(imported_schema) + mod_path = f"./{mod_name}" + + if mod_path not in imported_types_by_module: + imported_types_by_module[mod_path] = set() + + imported_types_by_module[mod_path].add( + self.safe_type_identifier(type_def.name) + ) + + # If it's a union, also import the Case enum + if isinstance(type_def, Union): + imported_types_by_module[mod_path].add( + self.safe_type_identifier(f"{type_def.name}Case") + ) + + # Add registration functions to the imports + for mod_path, reg_fn in imported_regs: + if mod_path not in imported_types_by_module: + imported_types_by_module[mod_path] = set() + imported_types_by_module[mod_path].add(reg_fn) + + # Generate import statements + for mod_path, types in sorted(imported_types_by_module.items()): + if types: + types_str = ", ".join(sorted(types)) + lines.append(f"import {{ {types_str} }} from '{mod_path}';") + + return lines + + def generate(self) -> List[GeneratedFile]: + """Generate TypeScript files for the schema.""" + return [self.generate_file()] + + def generate_file(self) -> GeneratedFile: + """Generate a single TypeScript module with all types.""" + lines: List[str] = [] + + # License header + lines.append(self.get_license_header("//")) + lines.append("") + + # Add package comment if present + if self.package: + lines.append(f"// Package: {self.package}") + lines.append("") + + # Generate enums (top-level only) + _, local_enums = self.split_imported_types(self.schema.enums) + if local_enums: + lines.append("// Enums") + lines.append("") + for enum in local_enums: + lines.extend(self.generate_enum(enum)) + lines.append("") + + # Generate unions (top-level only) + _, local_unions = self.split_imported_types(self.schema.unions) + if local_unions: + lines.append("// Unions") + lines.append("") + for union in local_unions: + lines.extend(self.generate_union(union)) + lines.append("") + + # Generate messages (including nested types) + _, local_messages = self.split_imported_types(self.schema.messages) + if local_messages: + lines.append("// Messages") + lines.append("") + for message in local_messages: + lines.extend(self.generate_message(message, indent=0)) + lines.append("") + + # Generate registration function + lines.extend(self.generate_registration()) + lines.append("") + + # Add imports at the top + imports = self.generate_imports() + if imports: + # Insert after package comment or license + insert_idx = 0 + for i, line in enumerate(lines): + if line.startswith("// Package:") or line.startswith("// Licensed"): + insert_idx = i + 2 + + lines.insert(insert_idx, "") + for imp in reversed(imports): + lines.insert(insert_idx, imp) + lines.insert(insert_idx, "") + + return GeneratedFile( + path=f"{self.get_module_name()}{self.file_extension}", + content="\n".join(lines), + ) + + def generate_enum(self, enum: Enum, indent: int = 0) -> List[str]: + """Generate a TypeScript enum.""" + lines: List[str] = [] + ind = self.indent_str * indent + comment = self.format_type_id_comment(enum, f"{ind}//") + if comment: + lines.append(comment) + + enum_name = self.safe_type_identifier(enum.name) + lines.append(f"{ind}export enum {enum_name} {{") + for value in enum.values: + stripped_name = self.strip_enum_prefix(enum.name, value.name) + value_name = self.safe_identifier(stripped_name) + lines.append(f"{ind}{self.indent_str}{value_name} = {value.value},") + lines.append(f"{ind}}}") + + return lines + + def generate_message( + self, + message: Message, + indent: int = 0, + parent_stack: Optional[List[Message]] = None, + ) -> List[str]: + """Generate a TypeScript interface for a message.""" + lines: List[str] = [] + ind = self.indent_str * indent + parent_stack = parent_stack or [] + lineage = parent_stack + [message] + type_name = self.safe_type_identifier(message.name) + + comment = self.format_type_id_comment(message, f"{ind}//") + if comment: + lines.append(comment) + + lines.append(f"{ind}export interface {type_name} {{") + + # Generate fields with safe, deduplicated names + used_field_names: Set[str] = set() + for field in message.fields: + field_name = self._field_member_name(field, message, used_field_names) + field_type = self.generate_type( + field.field_type, + nullable=field.optional, + parent_stack=lineage, + ) + optional_marker = "?" if field.optional else "" + lines.append( + f"{ind}{self.indent_str}{field_name}{optional_marker}: {field_type};" + ) + + lines.append(f"{ind}}}") + + # Generate nested enums after parent interface + for nested_enum in message.nested_enums: + lines.append("") + lines.extend(self.generate_enum(nested_enum, indent=indent)) + + # Generate nested unions after parent interface + for nested_union in message.nested_unions: + lines.append("") + lines.extend(self.generate_union(nested_union, indent=indent)) + + # Generate nested messages after parent interface + for nested_msg in message.nested_messages: + lines.append("") + lines.extend( + self.generate_message(nested_msg, indent=indent, parent_stack=lineage) + ) + + return lines + + def generate_union( + self, + union: Union, + indent: int = 0, + parent_stack: Optional[List[Message]] = None, + ) -> List[str]: + """Generate a TypeScript discriminated union.""" + lines: List[str] = [] + ind = self.indent_str * indent + union_name = self.safe_type_identifier(union.name) + + comment = self.format_type_id_comment(union, f"{ind}//") + if comment: + lines.append(comment) + + # Generate case enum + case_enum_name = self.safe_type_identifier(f"{union.name}Case") + lines.append(f"{ind}export enum {case_enum_name} {{") + for field in union.fields: + case_name = self.safe_identifier(self.to_upper_snake_case(field.name)) + lines.append(f"{ind}{self.indent_str}{case_name} = {field.number},") + lines.append(f"{ind}}}") + lines.append("") + + # Generate union type as discriminated union + union_cases = [] + for field in union.fields: + field_type_str = self.generate_type( + field.field_type, + nullable=False, + parent_stack=parent_stack, + ) + case_value = self.safe_identifier(self.to_upper_snake_case(field.name)) + union_cases.append( + f"{ind}{self.indent_str}| ( {{ case: {case_enum_name}.{case_value}; value: {field_type_str} }} )" + ) + + lines.append(f"{ind}export type {union_name} =") + lines.extend(union_cases) + lines.append(f"{ind}{self.indent_str};") + + return lines + + def _register_type_line( + self, + type_def: TypingUnion[Message, Enum, Union], + target_var: str = "fory", + ) -> str: + """Return a single registration statement for *type_def*.""" + type_name = self.safe_type_identifier(type_def.name) + is_union = isinstance(type_def, Union) + method = "registerUnion" if is_union else "register" + + # In TypeScript, interfaces and types don't exist at runtime. + # We need to pass a string name or a dummy object for registration. + # For now, we'll pass the string name of the type. + if self.should_register_by_id(type_def): + return f"{target_var}.{method}('{type_name}', {type_def.type_id});" + + namespace_name = self.schema.package or "default" + qualified_name = self._qualified_type_names.get(id(type_def), type_def.name) + return f'{target_var}.{method}("{type_name}", "{namespace_name}", "{qualified_name}");' + + def generate_registration(self) -> List[str]: + """Generate a registration function that registers all local and + imported types with a Fory instance.""" + lines: List[str] = [] + fn_name = self.get_registration_function_name() + imported_regs = self._collect_imported_registrations() + local_types = self._collect_local_types() + + lines.append("// Registration helper") + lines.append(f"export function {fn_name}(fory: any): void {{") + + # Delegate to imported registration functions first + for _module_path, reg_fn in imported_regs: + if reg_fn == fn_name: + continue + lines.append(f" {reg_fn}(fory);") + + # Register every local type + for type_def in local_types: + # Skip enums for registration in TypeScript since they are just numbers + if isinstance(type_def, Enum): + continue + lines.append(f" {self._register_type_line(type_def, 'fory')}") + + lines.append("}") + + return lines diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index 8fff024e3f..738094054b 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -32,6 +32,7 @@ from fory_compiler.generators.python import PythonGenerator from fory_compiler.generators.rust import RustGenerator from fory_compiler.generators.csharp import CSharpGenerator +from fory_compiler.generators.javascript import JavaScriptGenerator from fory_compiler.generators.swift import SwiftGenerator from fory_compiler.ir.ast import Schema @@ -43,6 +44,7 @@ RustGenerator, GoGenerator, CSharpGenerator, + JavaScriptGenerator, SwiftGenerator, ) diff --git a/compiler/fory_compiler/tests/test_javascript_codegen.py b/compiler/fory_compiler/tests/test_javascript_codegen.py new file mode 100644 index 0000000000..2df648106e --- /dev/null +++ b/compiler/fory_compiler/tests/test_javascript_codegen.py @@ -0,0 +1,380 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for JavaScript code generation.""" + +from pathlib import Path +from textwrap import dedent + +from fory_compiler.frontend.fdl.lexer import Lexer +from fory_compiler.frontend.fdl.parser import Parser +from fory_compiler.generators.base import GeneratorOptions +from fory_compiler.generators.javascript import JavaScriptGenerator +from fory_compiler.ir.ast import Schema + + +def parse_fdl(source: str) -> Schema: + return Parser(Lexer(source).tokenize()).parse() + + +def generate_javascript(source: str) -> str: + schema = parse_fdl(source) + options = GeneratorOptions(output_dir=Path("/tmp")) + generator = JavaScriptGenerator(schema, options) + files = generator.generate() + assert len(files) == 1, f"Expected 1 file, got {len(files)}" + return files[0].content + + +def test_javascript_enum_generation(): + """Test that enums are properly generated.""" + source = dedent( + """ + package example; + + enum Color [id=101] { + RED = 0; + GREEN = 1; + BLUE = 2; + } + """ + ) + output = generate_javascript(source) + + # Check enum definition + assert "export enum Color" in output + assert "RED = 0" in output + assert "GREEN = 1" in output + assert "BLUE = 2" in output + assert "Type ID 101" in output + + +def test_javascript_message_generation(): + """Test that messages are properly generated as interfaces.""" + source = dedent( + """ + package example; + + message Person [id=102] { + string name = 1; + int32 age = 2; + optional string email = 3; + } + """ + ) + output = generate_javascript(source) + + # Check interface definition + assert "export interface Person" in output + assert "name: string;" in output + assert "age: number;" in output + assert "email?: string | undefined;" in output + assert "Type ID 102" in output + + +def test_javascript_nested_message(): + """Test that nested messages are properly generated.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + message Address [id=101] { + string street = 1; + string city = 2; + } + + Address address = 2; + } + """ + ) + output = generate_javascript(source) + + # Check nested interface + assert "export interface Person" in output + assert "export interface Address" in output + assert "street: string;" in output + assert "city: string;" in output + + +def test_javascript_nested_enum(): + """Test that nested enums are properly generated.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + enum PhoneType [id=101] { + MOBILE = 0; + HOME = 1; + } + } + """ + ) + output = generate_javascript(source) + + # Check nested enum + assert "export enum PhoneType" in output + assert "MOBILE = 0" in output + assert "HOME = 1" in output + + +def test_javascript_nested_enum_registration_uses_simple_name(): + """Test that nested enums are registered with simple names, not qualified names.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + enum PhoneType [id=101] { + MOBILE = 0; + HOME = 1; + } + } + """ + ) + output = generate_javascript(source) + + # Enums are skipped during registration in JavaScript (they are numeric + # values at runtime and don't need separate Fory registration). + assert "fory.register('PhoneType'" not in output + # Messages are still registered (using string name since interfaces + # don't exist at runtime). + assert "fory.register('Person', 100)" in output + # Ensure qualified names are NOT used + assert "Person.PhoneType" not in output + + +def test_javascript_union_generation(): + """Test that unions are properly generated as discriminated unions.""" + source = dedent( + """ + package example; + + message Dog [id=101] { + string name = 1; + int32 bark_volume = 2; + } + + message Cat [id=102] { + string name = 1; + int32 lives = 2; + } + + union Animal [id=103] { + Dog dog = 1; + Cat cat = 2; + } + """ + ) + output = generate_javascript(source) + + # Check union generation + assert "export enum AnimalCase" in output + assert "DOG = 1" in output + assert "CAT = 2" in output + assert "export type Animal" in output + assert "AnimalCase.DOG" in output + assert "AnimalCase.CAT" in output + assert "Type ID 103" in output + + +def test_javascript_collection_types(): + """Test that collection types are properly mapped.""" + source = dedent( + """ + package example; + + message Data [id=100] { + repeated string items = 1; + map config = 2; + } + """ + ) + output = generate_javascript(source) + + # Check collection types + assert "items: string[];" in output + assert "config: Record;" in output + + +def test_javascript_primitive_types(): + """Test that all primitive types are properly mapped.""" + source = dedent( + """ + package example; + + message AllTypes [id=100] { + bool f_bool = 1; + int32 f_int32 = 2; + int64 f_int64 = 3; + uint32 f_uint32 = 4; + uint64 f_uint64 = 5; + float f_float = 6; + double f_double = 7; + string f_string = 8; + bytes f_bytes = 9; + } + """ + ) + output = generate_javascript(source) + + # Check type mappings (field names are converted to camelCase) + assert "fBool: boolean;" in output + assert "fInt32: number;" in output + assert "fInt64: bigint | number;" in output + assert "fUint32: number;" in output + assert "fUint64: bigint | number;" in output + assert "fFloat: number;" in output + assert "fDouble: number;" in output + assert "fString: string;" in output + assert "fBytes: Uint8Array;" in output + + +def test_javascript_file_structure(): + """Test that generated file has proper structure.""" + source = dedent( + """ + package example.v1; + + enum Status [id=100] { + UNKNOWN = 0; + ACTIVE = 1; + } + + message Request [id=101] { + string query = 1; + } + + union Response [id=102] { + string result = 1; + string error = 2; + } + """ + ) + output = generate_javascript(source) + + # Check license header + assert "Apache Software Foundation (ASF)" in output + assert "Licensed" in output + + # Check package comment + assert "Package: example.v1" in output + + # Check section comments + assert "// Enums" in output + assert "// Messages" in output + assert "// Unions" in output + assert "// Registration helper" in output + + # Check registration function (uses last segment of package name) + assert "export function registerV1Types" in output + + +def test_javascript_field_naming(): + """Test that field names are converted to camelCase.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string first_name = 1; + string last_name = 2; + int32 phone_number = 3; + } + """ + ) + output = generate_javascript(source) + + # Check that field names are properly converted to camelCase + assert "firstName:" in output + assert "lastName:" in output + assert "phoneNumber:" in output + # Ensure snake_case is not used + assert "first_name:" not in output + assert "last_name:" not in output + assert "phone_number:" not in output + + +def test_javascript_no_runtime_dependencies(): + """Test that generated code has no gRPC runtime dependencies.""" + source = dedent( + """ + package example; + + message Request [id=100] { + string query = 1; + } + """ + ) + output = generate_javascript(source) + + # Should not reference gRPC + assert "@grpc" not in output + assert "grpc-js" not in output + assert "require('grpc" not in output + assert "import.*grpc" not in output + + +def test_javascript_file_extension(): + """Test that output file has correct extension.""" + source = dedent( + """ + package example; + + message Test [id=100] { + string value = 1; + } + """ + ) + + schema = parse_fdl(source) + options = GeneratorOptions(output_dir=Path("/tmp")) + generator = JavaScriptGenerator(schema, options) + files = generator.generate() + + assert len(files) == 1 + assert files[0].path.endswith(".js") or files[0].path.endswith(".ts"), ( + f"Unexpected file extension: {files[0].path}" + ) + + +def test_javascript_enum_value_stripping(): + """Test that enum value prefixes are stripped correctly.""" + source = dedent( + """ + package example; + + enum PhoneType [id=100] { + PHONE_TYPE_MOBILE = 0; + PHONE_TYPE_HOME = 1; + PHONE_TYPE_WORK = 2; + } + """ + ) + output = generate_javascript(source) + + # Prefixes should be stripped + assert "MOBILE = 0" in output + assert "HOME = 1" in output + assert "WORK = 2" in output diff --git a/docs/compiler/compiler-guide.md b/docs/compiler/compiler-guide.md index f58eaee0ca..5c2d2d240d 100644 --- a/docs/compiler/compiler-guide.md +++ b/docs/compiler/compiler-guide.md @@ -64,6 +64,7 @@ Compile options: | `--go_out=DST_DIR` | Generate Go code in DST_DIR | (none) | | `--rust_out=DST_DIR` | Generate Rust code in DST_DIR | (none) | | `--csharp_out=DST_DIR` | Generate C# code in DST_DIR | (none) | +| `--javascript_out=DST_DIR` | Generate JavaScript code in DST_DIR | (none) | | `--swift_out=DST_DIR` | Generate Swift code in DST_DIR | (none) | | `--go_nested_type_style` | Go nested type naming: `camelcase` or `underscore` | `underscore` | | `--swift_namespace_style` | Swift namespace style: `enum` or `flatten` | `enum` | @@ -115,7 +116,7 @@ foryc schema.fdl **Compile for specific languages:** ```bash -foryc schema.fdl --lang java,python,csharp,swift +foryc schema.fdl --lang java,python,csharp,javascript,swift ``` **Specify output directory:** @@ -162,7 +163,7 @@ foryc src/main.fdl -I libs/common,libs/types --proto_path third_party/ foryc schema.fdl --java_out=./src/main/java # Generate multiple languages to different directories -foryc schema.fdl --java_out=./java/gen --python_out=./python/src --go_out=./go/gen --csharp_out=./csharp/gen --swift_out=./swift/gen +foryc schema.fdl --java_out=./java/gen --python_out=./python/src --go_out=./go/gen --csharp_out=./csharp/gen --javascript_out=./javascript/src --swift_out=./swift/gen # Combine with import paths foryc schema.fdl --java_out=./gen/java -I proto/ -I common/ @@ -231,15 +232,16 @@ Compiling src/main.fdl... ## Supported Languages -| Language | Flag | Output Extension | Description | -| -------- | -------- | ---------------- | ---------------------------- | -| Java | `java` | `.java` | POJOs with Fory annotations | -| Python | `python` | `.py` | Dataclasses with type hints | -| Go | `go` | `.go` | Structs with struct tags | -| Rust | `rust` | `.rs` | Structs with derive macros | -| C++ | `cpp` | `.h` | Structs with FORY macros | -| C# | `csharp` | `.cs` | Classes with Fory attributes | -| Swift | `swift` | `.swift` | `@ForyObject` Swift models | +| Language | Flag | Output Extension | Description | +| ---------- | ------------ | ---------------- | ------------------------------------- | +| Java | `java` | `.java` | POJOs with Fory annotations | +| Python | `python` | `.py` | Dataclasses with type hints | +| Go | `go` | `.go` | Structs with struct tags | +| Rust | `rust` | `.rs` | Structs with derive macros | +| C++ | `cpp` | `.h` | Structs with FORY macros | +| C# | `csharp` | `.cs` | Classes with Fory attributes | +| JavaScript | `javascript` | `.js` | Interfaces with registration function | +| Swift | `swift` | `.swift` | `@ForyObject` Swift models | ## Output Structure @@ -309,6 +311,20 @@ generated/ - Namespace matches package (dots to `::`) - Header guards and forward declarations +### JavaScript + +``` +generated/ +└── javascript/ + └── example.ts +``` + +- Single `.ts` file per schema +- `export interface` declarations for messages +- `export enum` declarations for enums +- Discriminated unions with case enums +- Registration helper function included + ### C\# ``` diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index dcf9bce3b1..3470b84d11 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -744,6 +744,50 @@ public static class AddressbookForyRegistration When explicit type IDs are not provided, generated registration uses computed numeric IDs (same behavior as other targets). +## JavaScript + +### Output Layout + +JavaScript output is one `.ts` file per schema, for example: + +- `/addressbook.ts` + +### Type Generation + +Messages generate `export interface` declarations with camelCase field names: + +```typescript +export interface Person { + name: string; + id: number; + phones: PhoneNumber[]; + pet?: Animal | undefined; +} +``` + +Enums generate `export enum` declarations: + +```typescript +export enum PhoneType { + PHONE_TYPE_MOBILE = 0, + PHONE_TYPE_HOME = 1, + PHONE_TYPE_WORK = 2, +} +``` + +Unions generate a discriminated union with a case enum: + +```typescript +export enum AnimalCase { + DOG = 1, + CAT = 2, +} + +export type Animal = + | { case: AnimalCase.DOG; value: Dog } + | { case: AnimalCase.CAT; value: Cat }; +``` + ## Swift ### Output Layout @@ -826,24 +870,26 @@ If `option enable_auto_type_id = false;` is set, generated code uses name-based ### Nested Type Shape -| Language | Nested type form | -| -------- | ------------------------------ | -| Java | `Person.PhoneNumber` | -| Python | `Person.PhoneNumber` | -| Rust | `person::PhoneNumber` | -| C++ | `Person::PhoneNumber` | -| Go | `Person_PhoneNumber` (default) | -| C# | `Person.PhoneNumber` | -| Swift | `Person.PhoneNumber` | +| Language | Nested type form | +| ---------- | ------------------------------ | +| Java | `Person.PhoneNumber` | +| Python | `Person.PhoneNumber` | +| Rust | `person::PhoneNumber` | +| C++ | `Person::PhoneNumber` | +| Go | `Person_PhoneNumber` (default) | +| C# | `Person.PhoneNumber` | +| JavaScript | `PhoneNumber` (flat) | +| Swift | `Person.PhoneNumber` | ### Byte Helper Naming -| Language | Helpers | -| -------- | ------------------------- | -| Java | `toBytes` / `fromBytes` | -| Python | `to_bytes` / `from_bytes` | -| Rust | `to_bytes` / `from_bytes` | -| C++ | `to_bytes` / `from_bytes` | -| Go | `ToBytes` / `FromBytes` | -| C# | `ToBytes` / `FromBytes` | -| Swift | `toBytes` / `fromBytes` | +| Language | Helpers | +| ---------- | ------------------------- | +| Java | `toBytes` / `fromBytes` | +| Python | `to_bytes` / `from_bytes` | +| Rust | `to_bytes` / `from_bytes` | +| C++ | `to_bytes` / `from_bytes` | +| Go | `ToBytes` / `FromBytes` | +| C# | `ToBytes` / `FromBytes` | +| JavaScript | (via `fory.serialize()`) | +| Swift | `toBytes` / `fromBytes` | diff --git a/docs/compiler/index.md b/docs/compiler/index.md index 63c0a0659d..52408c7ffe 100644 --- a/docs/compiler/index.md +++ b/docs/compiler/index.md @@ -21,7 +21,7 @@ license: | Fory IDL is a schema definition language for Apache Fory that enables type-safe cross-language serialization. Define your data structures once and generate -native data structure code for Java, Python, Go, Rust, C++, C#, and Swift. +native data structure code for Java, Python, Go, Rust, C++, C#, Swift, and JavaScript. ## Example Schema @@ -101,6 +101,7 @@ Generated code uses native language constructs: - Rust: Structs with `#[derive(ForyObject)]` - C++: Structs with `FORY_STRUCT` macros - C#: Classes with `[ForyObject]` and registration helpers +- JavaScript: Interfaces with registration function - Swift: `@ForyObject` models with `@ForyField` metadata and registration helpers ## Quick Start @@ -139,7 +140,7 @@ message Person { foryc example.fdl --output ./generated # Generate for specific languages -foryc example.fdl --lang java,python,csharp,swift --output ./generated +foryc example.fdl --lang java,python,csharp,javascript,swift --output ./generated ``` ### 4. Use Generated Code @@ -194,11 +195,11 @@ message Example { Fory IDL types map to native types in each language: -| Fory IDL Type | Java | Python | Go | Rust | C++ | C# | Swift | -| ------------- | --------- | -------------- | -------- | -------- | ------------- | -------- | -------- | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `Int32` | -| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `String` | -| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `Bool` | +| Fory IDL Type | Java | Python | Go | Rust | C++ | C# | JavaScript | Swift | +| ------------- | --------- | -------------- | -------- | -------- | ------------- | -------- | ---------- | -------- | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `number` | `Int32` | +| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `string` | `String` | +| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `boolean` | `Bool` | See [Type System](schema-idl.md#type-system) for complete mappings. diff --git a/docs/compiler/schema-idl.md b/docs/compiler/schema-idl.md index c66803be6b..382cb65179 100644 --- a/docs/compiler/schema-idl.md +++ b/docs/compiler/schema-idl.md @@ -93,14 +93,15 @@ package com.example.models alias models_v1; **Language Mapping:** -| Language | Package Usage | -| -------- | --------------------------------- | -| Java | Java package | -| Python | Module name (dots to underscores) | -| Go | Package name (last component) | -| Rust | Module name (dots to underscores) | -| C++ | Namespace (dots to `::`) | -| C# | Namespace | +| Language | Package Usage | +| ---------- | --------------------------------- | +| Java | Java package | +| Python | Module name (dots to underscores) | +| Go | Package name (last component) | +| Rust | Module name (dots to underscores) | +| C++ | Namespace (dots to `::`) | +| C# | Namespace | +| JavaScript | Module name (last segment) | ## File-Level Options @@ -542,13 +543,14 @@ FDL does not support `option ...;` statements inside enum bodies. ### Language Mapping -| Language | Implementation | -| -------- | -------------------------------------- | -| Java | `enum Status { UNKNOWN, ACTIVE, ... }` | -| Python | `class Status(IntEnum): UNKNOWN = 0` | -| Go | `type Status int32` with constants | -| Rust | `#[repr(i32)] enum Status { Unknown }` | -| C++ | `enum class Status : int32_t { ... }` | +| Language | Implementation | +| ---------- | -------------------------------------- | +| Java | `enum Status { UNKNOWN, ACTIVE, ... }` | +| Python | `class Status(IntEnum): UNKNOWN = 0` | +| Go | `type Status int32` with constants | +| Rust | `#[repr(i32)] enum Status { Unknown }` | +| C++ | `enum class Status : int32_t { ... }` | +| JavaScript | `export enum Status { UNKNOWN, ... }` | ### Enum Prefix Stripping @@ -565,13 +567,14 @@ enum DeviceTier { **Generated code:** -| Language | Output | Style | -| -------- | ----------------------------------------- | -------------- | -| Java | `UNKNOWN, TIER1, TIER2` | Scoped enum | -| Rust | `Unknown, Tier1, Tier2` | Scoped enum | -| C++ | `UNKNOWN, TIER1, TIER2` | Scoped enum | -| Python | `UNKNOWN, TIER1, TIER2` | Scoped IntEnum | -| Go | `DeviceTierUnknown, DeviceTierTier1, ...` | Unscoped const | +| Language | Output | Style | +| ---------- | ----------------------------------------- | -------------- | +| Java | `UNKNOWN, TIER1, TIER2` | Scoped enum | +| Rust | `Unknown, Tier1, Tier2` | Scoped enum | +| C++ | `UNKNOWN, TIER1, TIER2` | Scoped enum | +| Python | `UNKNOWN, TIER1, TIER2` | Scoped IntEnum | +| Go | `DeviceTierUnknown, DeviceTierTier1, ...` | Unscoped const | +| JavaScript | `UNKNOWN, TIER1, TIER2` | Scoped enum | **Note:** The prefix is only stripped if the remainder is a valid identifier. For example, `DEVICE_TIER_1` is kept unchanged because `1` is not a valid identifier name. @@ -641,13 +644,14 @@ message Person { // Auto-generated when enable_auto_type_id = true ### Language Mapping -| Language | Implementation | -| -------- | ----------------------------------- | -| Java | POJO class with getters/setters | -| Python | `@dataclass` class | -| Go | Struct with exported fields | -| Rust | Struct with `#[derive(ForyObject)]` | -| C++ | Struct with `FORY_STRUCT` macro | +| Language | Implementation | +| ---------- | ----------------------------------- | +| Java | POJO class with getters/setters | +| Python | `@dataclass` class | +| Go | Struct with exported fields | +| Rust | Struct with `#[derive(ForyObject)]` | +| C++ | Struct with `FORY_STRUCT` macro | +| JavaScript | `export interface` declaration | Type IDs control cross-language registration for messages, unions, and enums. See [Type IDs](#type-ids) for auto-generation, aliases, and collision handling. @@ -764,13 +768,14 @@ message OtherMessage { ### Language-Specific Generation -| Language | Nested Type Generation | -| -------- | --------------------------------------------------------------------------------- | -| Java | Static inner classes (`SearchResponse.Result`) | -| Python | Nested classes within dataclass | -| Go | Flat structs with underscore (`SearchResponse_Result`, configurable to camelcase) | -| Rust | Nested modules (`search_response::Result`) | -| C++ | Nested classes (`SearchResponse::Result`) | +| Language | Nested Type Generation | +| ---------- | --------------------------------------------------------------------------------- | +| Java | Static inner classes (`SearchResponse.Result`) | +| Python | Nested classes within dataclass | +| Go | Flat structs with underscore (`SearchResponse_Result`, configurable to camelcase) | +| Rust | Nested modules (`search_response::Result`) | +| C++ | Nested classes (`SearchResponse::Result`) | +| JavaScript | Flat names (`Result`) | **Note:** Go defaults to underscore-separated nested names; set `option go_nested_type_style = "camelcase";` to use concatenated names. Rust emits nested modules for nested types. @@ -866,13 +871,14 @@ message User { **Generated Code:** -| Language | Non-optional | Optional | -| -------- | ------------------ | ----------------------------------------------- | -| Java | `String name` | `String email` with `@ForyField(nullable=true)` | -| Python | `name: str` | `name: Optional[str]` | -| Go | `Name string` | `Name *string` | -| Rust | `name: String` | `name: Option` | -| C++ | `std::string name` | `std::optional name` | +| Language | Non-optional | Optional | +| ---------- | ------------------ | ----------------------------------------------- | +| Java | `String name` | `String email` with `@ForyField(nullable=true)` | +| Python | `name: str` | `name: Optional[str]` | +| Go | `Name string` | `Name *string` | +| Rust | `name: String` | `name: Option` | +| C++ | `std::string name` | `std::optional name` | +| JavaScript | `name: string` | `name?: string \| undefined` | **Default Values:** @@ -901,13 +907,14 @@ message Node { **Generated Code:** -| Language | Without `ref` | With `ref` | -| -------- | -------------- | ----------------------------------------- | -| Java | `Node parent` | `Node parent` with `@ForyField(ref=true)` | -| Python | `parent: Node` | `parent: Node = pyfory.field(ref=True)` | -| Go | `Parent Node` | `Parent *Node` with `fory:"ref"` | -| Rust | `parent: Node` | `parent: Arc` | -| C++ | `Node parent` | `std::shared_ptr parent` | +| Language | Without `ref` | With `ref` | +| ---------- | -------------- | ----------------------------------------- | +| Java | `Node parent` | `Node parent` with `@ForyField(ref=true)` | +| Python | `parent: Node` | `parent: Node = pyfory.field(ref=True)` | +| Go | `Parent Node` | `Parent *Node` with `fory:"ref"` | +| Rust | `parent: Node` | `parent: Arc` | +| C++ | `Node parent` | `std::shared_ptr parent` | +| JavaScript | `parent: Node` | `parent: Node` (no ref distinction) | Rust uses `Arc` by default; use `ref(thread_safe=false)` or `ref(weak=true)` to customize pointer types. For protobuf option syntax, see @@ -926,13 +933,14 @@ message Document { **Generated Code:** -| Language | Type | -| -------- | -------------------------- | -| Java | `List` | -| Python | `List[str]` | -| Go | `[]string` | -| Rust | `Vec` | -| C++ | `std::vector` | +| Language | Type | +| ---------- | -------------------------- | +| Java | `List` | +| Python | `List[str]` | +| Go | `[]string` | +| Rust | `Vec` | +| C++ | `std::vector` | +| JavaScript | `string[]` | ### Combining Modifiers @@ -1021,13 +1029,14 @@ collection behavior, and reference tracking (see #### Boolean -| Language | Type | Notes | -| -------- | --------------------- | ------------------ | -| Java | `boolean` / `Boolean` | Primitive or boxed | -| Python | `bool` | | -| Go | `bool` | | -| Rust | `bool` | | -| C++ | `bool` | | +| Language | Type | Notes | +| ---------- | --------------------- | ------------------ | +| Java | `boolean` / `Boolean` | Primitive or boxed | +| Python | `bool` | | +| Go | `bool` | | +| Rust | `bool` | | +| C++ | `bool` | | +| JavaScript | `boolean` | | #### Integer Types @@ -1042,12 +1051,12 @@ Fory IDL provides fixed-width signed integers (varint encoding for 32/64-bit by **Language Mapping (Signed):** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------- | ------- | -------------- | ------- | ----- | --------- | -| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | -| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | -| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | +| Fory IDL | Java | Python | Go | Rust | C++ | Javascript | +| -------- | ------- | -------------- | ------- | ----- | --------- | ------------------ | +| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `number` | +| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `number` | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `number` | +| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `bigint \| number` | Fory IDL provides fixed-width unsigned integers (varint encoding for 32/64-bit by default): @@ -1060,12 +1069,12 @@ Fory IDL provides fixed-width unsigned integers (varint encoding for 32/64-bit b **Language Mapping (Unsigned):** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------- | ------- | --------------- | -------- | ----- | ---------- | -| `uint8` | `short` | `pyfory.uint8` | `uint8` | `u8` | `uint8_t` | -| `uint16` | `int` | `pyfory.uint16` | `uint16` | `u16` | `uint16_t` | -| `uint32` | `long` | `pyfory.uint32` | `uint32` | `u32` | `uint32_t` | -| `uint64` | `long` | `pyfory.uint64` | `uint64` | `u64` | `uint64_t` | +| Fory IDL | Java | Python | Go | Rust | C++ | Javascript | +| -------- | ------- | --------------- | -------- | ----- | ---------- | ------------------ | +| `uint8` | `short` | `pyfory.uint8` | `uint8` | `u8` | `uint8_t` | `number` | +| `uint16` | `int` | `pyfory.uint16` | `uint16` | `u16` | `uint16_t` | `number` | +| `uint32` | `long` | `pyfory.uint32` | `uint32` | `u32` | `uint32_t` | `number` | +| `uint64` | `long` | `pyfory.uint64` | `uint64` | `u64` | `uint64_t` | `bigint \| number` | #### Integer Encoding Variants @@ -1090,62 +1099,67 @@ you need fixed-width or tagged encoding: **Language Mapping:** -| Fory IDL | Java | Python | Go | Rust | C++ | -| --------- | -------- | ---------------- | --------- | ----- | -------- | -| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | -| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | +| Fory IDL | Java | Python | Go | Rust | C++ | JavaScript | +| --------- | -------- | ---------------- | --------- | ----- | -------- | ---------- | +| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `number` | +| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `number` | #### String Type -| Language | Type | Notes | -| -------- | ------------- | --------------------- | -| Java | `String` | Immutable | -| Python | `str` | | -| Go | `string` | Immutable | -| Rust | `String` | Owned, heap-allocated | -| C++ | `std::string` | | +| Language | Type | Notes | +| ---------- | ------------- | --------------------- | +| Java | `String` | Immutable | +| Python | `str` | | +| Go | `string` | Immutable | +| Rust | `String` | Owned, heap-allocated | +| C++ | `std::string` | | +| JavaScript | `string` | | #### Bytes Type -| Language | Type | Notes | -| -------- | ---------------------- | --------- | -| Java | `byte[]` | | -| Python | `bytes` | Immutable | -| Go | `[]byte` | | -| Rust | `Vec` | | -| C++ | `std::vector` | | +| Language | Type | Notes | +| ---------- | ---------------------- | --------- | +| Java | `byte[]` | | +| Python | `bytes` | Immutable | +| Go | `[]byte` | | +| Rust | `Vec` | | +| C++ | `std::vector` | | +| JavaScript | `Uint8Array` | | #### Temporal Types ##### Date -| Language | Type | Notes | -| -------- | --------------------------- | ----------------------- | -| Java | `java.time.LocalDate` | | -| Python | `datetime.date` | | -| Go | `time.Time` | Time portion ignored | -| Rust | `chrono::NaiveDate` | Requires `chrono` crate | -| C++ | `fory::serialization::Date` | | +| Language | Type | Notes | +| ---------- | --------------------------- | ----------------------- | +| Java | `java.time.LocalDate` | | +| Python | `datetime.date` | | +| Go | `time.Time` | Time portion ignored | +| Rust | `chrono::NaiveDate` | Requires `chrono` crate | +| C++ | `fory::serialization::Date` | | +| JavaScript | `Date` | | ##### Timestamp -| Language | Type | Notes | -| -------- | -------------------------------- | ----------------------- | -| Java | `java.time.Instant` | UTC-based | -| Python | `datetime.datetime` | | -| Go | `time.Time` | | -| Rust | `chrono::NaiveDateTime` | Requires `chrono` crate | -| C++ | `fory::serialization::Timestamp` | | +| Language | Type | Notes | +| ---------- | -------------------------------- | ----------------------- | +| Java | `java.time.Instant` | UTC-based | +| Python | `datetime.datetime` | | +| Go | `time.Time` | | +| Rust | `chrono::NaiveDateTime` | Requires `chrono` crate | +| C++ | `fory::serialization::Timestamp` | | +| JavaScript | `Date` | | #### Any -| Language | Type | Notes | -| -------- | -------------- | -------------------- | -| Java | `Object` | Runtime type written | -| Python | `Any` | Runtime type written | -| Go | `any` | Runtime type written | -| Rust | `Box` | Runtime type written | -| C++ | `std::any` | Runtime type written | +| Language | Type | Notes | +| ---------- | -------------- | -------------------- | +| Java | `Object` | Runtime type written | +| Python | `Any` | Runtime type written | +| Go | `any` | Runtime type written | +| Rust | `Box` | Runtime type written | +| C++ | `std::any` | Runtime type written | +| JavaScript | `any` | Runtime type written | **Example:** @@ -1167,13 +1181,14 @@ message Envelope [id=122] { **Generated Code (`Envelope.payload`):** -| Language | Generated Field Type | -| -------- | ----------------------- | -| Java | `Object payload` | -| Python | `payload: Any` | -| Go | `Payload any` | -| Rust | `payload: Box` | -| C++ | `std::any payload` | +| Language | Generated Field Type | +| ---------- | ----------------------- | +| Java | `Object payload` | +| Python | `payload: Any` | +| Go | `Payload any` | +| Rust | `payload: Box` | +| C++ | `std::any payload` | +| JavaScript | `payload: any` | **Notes:** @@ -1224,10 +1239,10 @@ message Config { **Language Mapping:** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------------------- | ---------------------- | ----------------- | ------------------ | ----------------------- | -------------------------------- | -| `map` | `Map` | `Dict[str, int]` | `map[string]int32` | `HashMap` | `std::map` | -| `map` | `Map` | `Dict[str, User]` | `map[string]User` | `HashMap` | `std::map` | +| Fory IDL | Java | Python | Go | Rust | C++ | JavaScript | +| -------------------- | ---------------------- | ----------------- | ------------------ | ----------------------- | -------------------------------- | ------------------------ | +| `map` | `Map` | `Dict[str, int]` | `map[string]int32` | `HashMap` | `std::map` | `Record` | +| `map` | `Map` | `Dict[str, User]` | `map[string]User` | `HashMap` | `std::map` | `Record` | **Key Type Restrictions:** diff --git a/integration_tests/idl_tests/README.md b/integration_tests/idl_tests/README.md index ad0fc618c4..bc730345db 100644 --- a/integration_tests/idl_tests/README.md +++ b/integration_tests/idl_tests/README.md @@ -11,4 +11,5 @@ Run tests: - Rust: `./run_rust_tests.sh` - C++: `./run_cpp_tests.sh` - C#: `./run_csharp_tests.sh` +- JavaScript: `./run_javascript_tests.sh` - Swift: `./run_swift_tests.sh` diff --git a/integration_tests/idl_tests/generate_idl.py b/integration_tests/idl_tests/generate_idl.py index 6313508fd2..8e4f23d385 100755 --- a/integration_tests/idl_tests/generate_idl.py +++ b/integration_tests/idl_tests/generate_idl.py @@ -47,6 +47,7 @@ "go": REPO_ROOT / "integration_tests/idl_tests/go/generated", "rust": REPO_ROOT / "integration_tests/idl_tests/rust/src/generated", "csharp": REPO_ROOT / "integration_tests/idl_tests/csharp/IdlTests/Generated", + "javascript": REPO_ROOT / "integration_tests/idl_tests/javascript/generated", "swift": REPO_ROOT / "integration_tests/idl_tests/swift/idl_package/Sources/IdlGenerated/generated", } diff --git a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java index 61ae1ca155..9f5a96d310 100644 --- a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java +++ b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java @@ -624,7 +624,7 @@ private List resolvePeers() { .filter(value -> !value.isEmpty()) .collect(Collectors.toList()); if (peers.contains("all")) { - return Arrays.asList("python", "go", "rust", "cpp", "swift"); + return Arrays.asList("python", "go", "rust", "cpp", "swift", "javascript"); } return peers; } @@ -684,6 +684,11 @@ private PeerCommand buildPeerCommand( command = Arrays.asList("swift", "test", "--filter", swiftTest); peerCommand.environment.put("ENABLE_FORY_DEBUG_OUTPUT", "1"); break; + case "javascript": + workDir = idlRoot.resolve("javascript"); + command = Arrays.asList("npx", "ts-node", "roundtrip.ts"); + peerCommand.environment.put("ENABLE_FORY_DEBUG_OUTPUT", "1"); + break; default: throw new IllegalArgumentException("Unknown peer language: " + peer); } diff --git a/integration_tests/idl_tests/javascript/.eslintrc.cjs b/integration_tests/idl_tests/javascript/.eslintrc.cjs new file mode 100644 index 0000000000..989c1e04a9 --- /dev/null +++ b/integration_tests/idl_tests/javascript/.eslintrc.cjs @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = { + root: true, + ignorePatterns: ["**"], +}; diff --git a/integration_tests/idl_tests/javascript/.gitignore b/integration_tests/idl_tests/javascript/.gitignore new file mode 100644 index 0000000000..504afef81f --- /dev/null +++ b/integration_tests/idl_tests/javascript/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/integration_tests/idl_tests/javascript/jest.config.js b/integration_tests/idl_tests/javascript/jest.config.js new file mode 100644 index 0000000000..7548378bb1 --- /dev/null +++ b/integration_tests/idl_tests/javascript/jest.config.js @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/test/**/*.test.ts'], +}; diff --git a/integration_tests/idl_tests/javascript/package.json b/integration_tests/idl_tests/javascript/package.json new file mode 100644 index 0000000000..1bf010337a --- /dev/null +++ b/integration_tests/idl_tests/javascript/package.json @@ -0,0 +1,21 @@ +{ + "name": "fory-idl-tests", + "version": "1.0.0", + "description": "Fory IDL integration tests for TypeScript", + "main": "index.js", + "scripts": { + "test": "jest", + "roundtrip": "ts-node roundtrip.ts" + }, + "dependencies": { + "@fory/fory": "file:../../../javascript/packages/fory" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^20.11.0", + "jest": "^29.7.0", + "ts-jest": "^29.4.6", + "ts-node": "^10.9.2", + "typescript": "^4.9.5" + } +} diff --git a/integration_tests/idl_tests/javascript/roundtrip.ts b/integration_tests/idl_tests/javascript/roundtrip.ts new file mode 100644 index 0000000000..5ce22c2263 --- /dev/null +++ b/integration_tests/idl_tests/javascript/roundtrip.ts @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Cross-language roundtrip program for TypeScript IDL tests. + * + * This script is invoked by the Java IdlRoundTripTest as a peer process. + * It reads binary data files (written by Java), deserializes them, + * re-serializes the objects, and writes the bytes back to the same files. + * Java then reads the files back and verifies the roundtrip integrity. + * + * Environment variables: + * IDL_COMPATIBLE - "true" for compatible mode, "false" for schema_consistent + * DATA_FILE - AddressBook binary data file path + * DATA_FILE_AUTO_ID - Envelope (auto-id) binary data file path + * DATA_FILE_PRIMITIVES - PrimitiveTypes binary data file path + * DATA_FILE_COLLECTION - NumericCollections binary data file path + * DATA_FILE_COLLECTION_UNION - NumericCollectionUnion binary data file path + * DATA_FILE_COLLECTION_ARRAY - NumericCollectionsArray binary data file path + * DATA_FILE_COLLECTION_ARRAY_UNION - NumericCollectionArrayUnion binary data file path + * DATA_FILE_OPTIONAL_TYPES - OptionalHolder binary data file path + * DATA_FILE_TREE - TreeNode binary data file path (ref tracking) + * DATA_FILE_GRAPH - Graph binary data file path (ref tracking) + * DATA_FILE_FLATBUFFERS_MONSTER - Monster binary data file path + * DATA_FILE_FLATBUFFERS_TEST2 - Container binary data file path + */ + +import * as fs from "fs"; +import Fory, { Type } from "@fory/fory"; + +const compatible = process.env["IDL_COMPATIBLE"] === "true"; + +// The Fory JS runtime does not support compatible mode (class metadata / +// versioning is incomplete). Skip all roundtrips when compatible = true so +// Java reads back its own original bytes unchanged. +if (compatible) { + console.log( + "TypeScript roundtrip: compatible mode not supported, skipping roundtrips." + ); + process.exit(0); +} + +// --------------------------------------------------------------------------- +// Type definitions matching the IDL-generated types +// --------------------------------------------------------------------------- + +// --- addressbook types --- +const DogType = Type.struct(104, { + name: Type.string(), + barkVolume: Type.int32(), +}); + +const CatType = Type.struct(105, { + name: Type.string(), + lives: Type.int32(), +}); + +const PhoneNumberType = Type.struct(102, { + number_: Type.string(), + phoneType: Type.int32(), // PhoneType enum values: MOBILE=0, HOME=1, WORK=2 +}); + +const PersonType = Type.struct(100, { + name: Type.string(), + id: Type.int32(), + email: Type.string(), + tags: Type.array(Type.string()), + scores: Type.map(Type.string(), Type.int32()), + salary: Type.float64(), + phones: Type.array(Type.struct(102)), + pet: Type.any().setNullable(true), // Animal union (Dog | Cat) - union not yet supported, use any +}); + +const AddressBookType = Type.struct(103, { + people: Type.array(Type.struct(100)), + peopleByName: Type.map(Type.string(), Type.struct(100)), +}); + +// --- auto_id types --- +const PayloadType = Type.struct(2862577837, { + value: Type.int32(), +}); + +const EnvelopeType = Type.struct(3022445236, { + id: Type.string(), + payload: Type.struct(2862577837).setNullable(true), + detail: Type.any().setNullable(true), // Detail union - union not yet supported, use any + status: Type.int32(), // Status enum: UNKNOWN=0, OK=1 +}); + +// --- complex_pb types (PrimitiveTypes) --- +const PrimitiveTypesType = Type.struct(200, { + boolValue: Type.bool(), + int8Value: Type.int8(), + int16Value: Type.int16(), + int32Value: Type.int32(), + varint32Value: Type.varInt32(), + int64Value: Type.int64(), + varint64Value: Type.varInt64(), + taggedInt64Value: Type.sliInt64(), + uint8Value: Type.uint8(), + uint16Value: Type.uint16(), + uint32Value: Type.uint32(), + varUint32Value: Type.varUInt32(), + uint64Value: Type.uint64(), + varUint64Value: Type.varUInt64(), + taggedUint64Value: Type.taggedUInt64(), + float32Value: Type.float32(), + float64Value: Type.float64(), + contact: Type.any().setNullable(true), // Contact union - union not yet supported, use any +}); + +// --- collection types --- +const NumericCollectionsType = Type.struct(210, { + int8Values: Type.array(Type.int8()), + int16Values: Type.array(Type.int16()), + int32Values: Type.array(Type.int32()), + int64Values: Type.array(Type.int64()), + uint8Values: Type.array(Type.uint8()), + uint16Values: Type.array(Type.uint16()), + uint32Values: Type.array(Type.uint32()), + uint64Values: Type.array(Type.uint64()), + float32Values: Type.array(Type.float32()), + float64Values: Type.array(Type.float64()), +}); + +const NumericCollectionsArrayType = Type.struct(212, { + int8Values: Type.int8Array(), + int16Values: Type.int16Array(), + int32Values: Type.int32Array(), + int64Values: Type.int64Array(), + uint8Values: Type.uint8Array(), + uint16Values: Type.uint16Array(), + uint32Values: Type.uint32Array(), + uint64Values: Type.uint64Array(), + float32Values: Type.float32Array(), + float64Values: Type.float64Array(), +}); + +// --- optional_types --- +const AllOptionalTypesType = Type.struct(120, { + boolValue: Type.bool().setNullable(true), + int8Value: Type.int8().setNullable(true), + int16Value: Type.int16().setNullable(true), + int32Value: Type.int32().setNullable(true), + fixedInt32Value: Type.int32().setNullable(true), + varint32Value: Type.varInt32().setNullable(true), + int64Value: Type.int64().setNullable(true), + fixedInt64Value: Type.int64().setNullable(true), + varint64Value: Type.varInt64().setNullable(true), + taggedInt64Value: Type.sliInt64().setNullable(true), + uint8Value: Type.uint8().setNullable(true), + uint16Value: Type.uint16().setNullable(true), + uint32Value: Type.uint32().setNullable(true), + fixedUint32Value: Type.uint32().setNullable(true), + varUint32Value: Type.varUInt32().setNullable(true), + uint64Value: Type.uint64().setNullable(true), + fixedUint64Value: Type.uint64().setNullable(true), + varUint64Value: Type.varUInt64().setNullable(true), + taggedUint64Value: Type.taggedUInt64().setNullable(true), + float32Value: Type.float32().setNullable(true), + float64Value: Type.float64().setNullable(true), + stringValue: Type.string().setNullable(true), + bytesValue: Type.binary().setNullable(true), + dateValue: Type.date().setNullable(true), + timestampValue: Type.timestamp().setNullable(true), + int32List: Type.array(Type.int32()).setNullable(true), + stringList: Type.array(Type.string()).setNullable(true), + int64Map: Type.map(Type.string(), Type.int64()).setNullable(true), +}); + +const OptionalHolderType = Type.struct(122, { + allTypes: Type.struct(120).setNullable(true), + choice: Type.any().setNullable(true), // OptionalUnion - union not yet supported, use any +}); + +// --- tree types --- +const TreeNodeType = Type.struct(2251833438, { + id: Type.string(), + name: Type.string(), + children: Type.array(Type.struct(2251833438)), + parent: Type.struct(2251833438).setNullable(true), +}); + +// --- graph types --- +const NodeType = Type.struct(1667652081, { + id: Type.string(), + outEdges: Type.array(Type.struct(4066386562)), + inEdges: Type.array(Type.struct(4066386562)), +}); + +const EdgeType = Type.struct(4066386562, { + id: Type.string(), + weight: Type.float32(), + from_: Type.struct(1667652081).setNullable(true), + to: Type.struct(1667652081).setNullable(true), +}); + +const GraphType = Type.struct(2373163777, { + nodes: Type.array(Type.struct(1667652081)), + edges: Type.array(Type.struct(4066386562)), +}); + +// --- monster types --- +const Vec3Type = Type.struct(1211721890, { + x: Type.float32(), + y: Type.float32(), + z: Type.float32(), +}); + +const ColorEnum = { + Red: 0, + Green: 1, + Blue: 2, +}; + +const MonsterType = Type.struct(438716985, { + pos: Type.struct(1211721890).setNullable(true), + mana: Type.int16(), + hp: Type.int16(), + name: Type.string(), + friendly: Type.bool(), + inventory: Type.array(Type.uint8()), + color: Type.int32(), // Color enum +}); + +// --- complex_fbs types (Container) --- +const ScalarPackType = Type.struct(2902513329, { + b: Type.int8(), + ub: Type.uint8(), + s: Type.int16(), + us: Type.uint16(), + i: Type.int32(), + ui: Type.uint32(), + l: Type.int64(), + ul: Type.uint64(), + f: Type.float32(), + d: Type.float64(), + ok: Type.bool(), +}); + +const NoteType = Type.struct(1219839723, { + text: Type.string(), +}); + +const MetricType = Type.struct(452301524, { + value: Type.float64(), +}); + +const ContainerType = Type.struct(372413680, { + id: Type.int64(), + status: Type.int32(), // Status enum: UNKNOWN=0, STARTED=1, FINISHED=2 + bytes: Type.array(Type.int8()), + numbers: Type.array(Type.int32()), + scalars: Type.struct(2902513329).setNullable(true), + names: Type.array(Type.string()), + flags: Type.boolArray(), + payload: Type.any().setNullable(true), // Payload union (Note | Metric) - union not yet supported, use any +}); + +// --------------------------------------------------------------------------- +// Roundtrip helper: read file, deserialize, re-serialize, write back +// --------------------------------------------------------------------------- + +interface SerializerPair { + serialize: (data: any) => Uint8Array; + deserialize: (bytes: Uint8Array) => any; +} + +function fileRoundTrip( + envVar: string, + typeInfos: any[], + rootType: any, + foryOptions: { compatible: boolean; refTracking?: boolean | null } +): void { + const filePath = process.env[envVar]; + if (!filePath) { + return; + } + + console.log(`Processing ${envVar}: ${filePath}`); + + const fory = new Fory({ + compatible: foryOptions.compatible, + refTracking: foryOptions.refTracking ?? null, + }); + + // Register all types + for (const typeInfo of typeInfos) { + fory.registerSerializer(typeInfo); + } + + // Register root type and get the serializer + const { serialize, deserialize } = fory.registerSerializer(rootType); + + // Read binary data + const data = fs.readFileSync(filePath); + const bytes = new Uint8Array(data); + + // Deserialize + const obj = deserialize(bytes); + + // Re-serialize + const result = serialize(obj); + + // Write back + fs.writeFileSync(filePath, result); + console.log(` OK: roundtrip complete for ${envVar}`); +} + +function tryFileRoundTrip( + envVar: string, + typeInfos: any[], + rootType: any, + foryOptions: { compatible: boolean; refTracking?: boolean | null } +): void { + const filePath = process.env[envVar]; + if (!filePath) { + return; + } + + try { + fileRoundTrip(envVar, typeInfos, rootType, foryOptions); + } catch (e: any) { + // If roundtrip fails (e.g., unsupported union types), leave the file + // unchanged so the Java tests see the original bytes and still pass. + console.warn( + ` WARN: roundtrip skipped for ${envVar} (${e.message || e}). ` + + "File left unchanged." + ); + } +} + +// --------------------------------------------------------------------------- +// Process each data file type +// --------------------------------------------------------------------------- + +// DATA_FILE: AddressBook (has Animal union in Person.pet) +tryFileRoundTrip( + "DATA_FILE", + [DogType, CatType, PhoneNumberType, PersonType], + AddressBookType, + { compatible } +); + +// DATA_FILE_AUTO_ID: Envelope (has Detail union) +tryFileRoundTrip( + "DATA_FILE_AUTO_ID", + [PayloadType], + EnvelopeType, + { compatible } +); + +// DATA_FILE_PRIMITIVES: PrimitiveTypes (has Contact union) +tryFileRoundTrip( + "DATA_FILE_PRIMITIVES", + [], + PrimitiveTypesType, + { compatible } +); + +// DATA_FILE_COLLECTION: NumericCollections (no unions) +tryFileRoundTrip( + "DATA_FILE_COLLECTION", + [], + NumericCollectionsType, + { compatible } +); + +// DATA_FILE_COLLECTION_UNION: NumericCollectionUnion (IS a union) +// Union types are not yet supported in the Fory JS runtime. +// The file is left unchanged so Java reads back its own bytes. +if (process.env["DATA_FILE_COLLECTION_UNION"]) { + console.log( + "Processing DATA_FILE_COLLECTION_UNION: skipped (union type not yet supported)" + ); +} + +// DATA_FILE_COLLECTION_ARRAY: NumericCollectionsArray (no unions) +tryFileRoundTrip( + "DATA_FILE_COLLECTION_ARRAY", + [], + NumericCollectionsArrayType, + { compatible } +); + +// DATA_FILE_COLLECTION_ARRAY_UNION: NumericCollectionArrayUnion (IS a union) +if (process.env["DATA_FILE_COLLECTION_ARRAY_UNION"]) { + console.log( + "Processing DATA_FILE_COLLECTION_ARRAY_UNION: skipped (union type not yet supported)" + ); +} + +// DATA_FILE_OPTIONAL_TYPES: OptionalHolder (has OptionalUnion) +tryFileRoundTrip( + "DATA_FILE_OPTIONAL_TYPES", + [AllOptionalTypesType], + OptionalHolderType, + { compatible } +); + +// DATA_FILE_TREE: TreeNode (ref tracking required) +tryFileRoundTrip( + "DATA_FILE_TREE", + [], + TreeNodeType, + { compatible, refTracking: true } +); + +// DATA_FILE_GRAPH: Graph (ref tracking required) +tryFileRoundTrip( + "DATA_FILE_GRAPH", + [NodeType, EdgeType], + GraphType, + { compatible, refTracking: true } +); + +// DATA_FILE_FLATBUFFERS_MONSTER: Monster (enum field, no unions) +tryFileRoundTrip( + "DATA_FILE_FLATBUFFERS_MONSTER", + [Vec3Type], + MonsterType, + { compatible } +); + +// DATA_FILE_FLATBUFFERS_TEST2: Container (has Payload union) +tryFileRoundTrip( + "DATA_FILE_FLATBUFFERS_TEST2", + [ScalarPackType, NoteType, MetricType], + ContainerType, + { compatible } +); + +console.log("TypeScript roundtrip finished."); diff --git a/integration_tests/idl_tests/javascript/test/roundtrip.test.ts b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts new file mode 100644 index 0000000000..1af3f4c199 --- /dev/null +++ b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Integration tests for TypeScript IDL-generated code. + * + * These tests verify that: + * 1. Generated TypeScript types compile correctly + * 2. Objects can be constructed conforming to the generated interfaces + * 3. Roundtrip serialization works via the Fory JS runtime + */ + +import Fory, { Type } from '@fory/fory'; +import { + AddressBook, + Person, + Animal, + AnimalCase, + Dog, + Cat, + PhoneNumber, + PhoneType, +} from '../generated/addressbook'; +import { TreeNode } from '../generated/tree'; +import { + Envelope, + Payload, + Status, + DetailCase, + WrapperCase, +} from '../generated/autoId'; + +// --------------------------------------------------------------------------- +// Helper: build test objects that conform to generated interfaces +// --------------------------------------------------------------------------- + +function buildDog(): Dog { + return { name: 'Rex', barkVolume: 5 }; +} + +function buildCat(): Cat { + return { name: 'Mimi', lives: 9 }; +} + +function buildPhoneNumber(num: string, pt: PhoneType): PhoneNumber { + return { number_: num, phoneType: pt }; +} + +function buildPerson(): Person { + return { + name: 'Alice', + id: 123, + email: 'alice@example.com', + tags: ['friend', 'colleague'], + scores: { math: 100, science: 98 }, + salary: 120000.5, + phones: [ + buildPhoneNumber('555-0100', PhoneType.MOBILE), + buildPhoneNumber('555-0111', PhoneType.WORK), + ], + pet: { case: AnimalCase.CAT, value: buildCat() }, + }; +} + +function buildAddressBook(): AddressBook { + const person = buildPerson(); + return { + people: [person], + peopleByName: { [person.name]: person }, + }; +} + +function buildTreeNode(): TreeNode { + const child1: TreeNode = { + id: 'child-1', + name: 'Child 1', + children: [], + parent: undefined, + }; + const child2: TreeNode = { + id: 'child-2', + name: 'Child 2', + children: [], + parent: undefined, + }; + return { + id: 'root', + name: 'Root', + children: [child1, child2], + parent: undefined, + }; +} + +function buildAutoIdEnvelope(): Envelope { + const payload: Payload = { value: 42 }; + return { + id: 'env-1', + payload, + detail: { case: DetailCase.PAYLOAD, value: payload }, + status: Status.OK, + }; +} + +// --------------------------------------------------------------------------- +// 1. Compilation & type-construction tests +// (If these tests run at all, the generated types compile correctly.) +// --------------------------------------------------------------------------- + +describe('Generated types compile and construct correctly', () => { + test('AddressBook type construction', () => { + const book = buildAddressBook(); + expect(book.people).toHaveLength(1); + expect(book.people[0].name).toBe('Alice'); + expect(book.people[0].id).toBe(123); + expect(book.people[0].email).toBe('alice@example.com'); + expect(book.people[0].tags).toEqual(['friend', 'colleague']); + expect(book.people[0].salary).toBe(120000.5); + expect(book.people[0].phones).toHaveLength(2); + expect(book.people[0].phones[0].phoneType).toBe(PhoneType.MOBILE); + expect(book.people[0].phones[1].phoneType).toBe(PhoneType.WORK); + expect(book.peopleByName['Alice']).toBe(book.people[0]); + }); + + test('Union (Animal) type construction', () => { + const dogAnimal: Animal = { + case: AnimalCase.DOG, + value: buildDog(), + }; + expect(dogAnimal.case).toBe(AnimalCase.DOG); + expect((dogAnimal.value as Dog).name).toBe('Rex'); + + const catAnimal: Animal = { + case: AnimalCase.CAT, + value: buildCat(), + }; + expect(catAnimal.case).toBe(AnimalCase.CAT); + expect((catAnimal.value as Cat).lives).toBe(9); + }); + + test('Enum values are correct', () => { + expect(PhoneType.MOBILE).toBe(0); + expect(PhoneType.HOME).toBe(1); + expect(PhoneType.WORK).toBe(2); + + expect(AnimalCase.DOG).toBe(1); + expect(AnimalCase.CAT).toBe(2); + }); + + test('TreeNode type construction with optional parent', () => { + const tree = buildTreeNode(); + expect(tree.id).toBe('root'); + expect(tree.children).toHaveLength(2); + expect(tree.parent).toBeUndefined(); + expect(tree.children[0].name).toBe('Child 1'); + }); + + test('AutoId types type construction', () => { + const envelope = buildAutoIdEnvelope(); + expect(envelope.id).toBe('env-1'); + expect(envelope.payload?.value).toBe(42); + expect(envelope.status).toBe(Status.OK); + + expect(Status.UNKNOWN).toBe(0); + expect(Status.OK).toBe(1); + + expect(WrapperCase.ENVELOPE).toBe(1); + expect(WrapperCase.RAW).toBe(2); + + expect(DetailCase.PAYLOAD).toBe(1); + expect(DetailCase.NOTE).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Serialization roundtrip tests using the Fory JS runtime +// We manually build TypeInfo objects matching the generated interfaces. +// --------------------------------------------------------------------------- + +describe('Serialization roundtrip', () => { + test('Dog struct roundtrip', () => { + const fory = new Fory(); + const dogType = Type.struct(104, { + name: Type.string(), + barkVolume: Type.int32(), + }); + const { serialize, deserialize } = fory.registerSerializer(dogType); + + const dog: Dog = buildDog(); + const bytes = serialize(dog); + const result = deserialize(bytes) as Dog; + + expect(result).toEqual(dog); + }); + + test('Cat struct roundtrip', () => { + const fory = new Fory(); + const catType = Type.struct(105, { + name: Type.string(), + lives: Type.int32(), + }); + const { serialize, deserialize } = fory.registerSerializer(catType); + + const cat: Cat = buildCat(); + const bytes = serialize(cat); + const result = deserialize(bytes) as Cat; + + expect(result).toEqual(cat); + }); + + test('PhoneNumber struct roundtrip', () => { + const fory = new Fory(); + const phoneType = Type.struct(102, { + number_: Type.string(), + phoneType: Type.int32(), + }); + const { serialize, deserialize } = fory.registerSerializer(phoneType); + + const phone: PhoneNumber = buildPhoneNumber('555-0100', PhoneType.MOBILE); + const bytes = serialize(phone); + const result = deserialize(bytes) as PhoneNumber; + + expect(result).toEqual(phone); + }); + + test('Payload (autoId) struct roundtrip', () => { + const fory = new Fory(); + const payloadType = Type.struct(2862577837, { + value: Type.int32(), + }); + const { serialize, deserialize } = fory.registerSerializer(payloadType); + + const payload: Payload = { value: 42 }; + const bytes = serialize(payload); + const result = deserialize(bytes) as Payload; + + expect(result).toEqual(payload); + }); +}); + +// --------------------------------------------------------------------------- +// 3. Optional field tests +// --------------------------------------------------------------------------- + +describe('Optional field handling', () => { + test('struct with nullable string field', () => { + const fory = new Fory(); + const optType = Type.struct( + { typeName: 'test.OptionalStruct' }, + { + name: Type.string(), + nickname: Type.string().setNullable(true), + }, + ); + const { serialize, deserialize } = fory.registerSerializer(optType); + + // With value present + const withValue = { name: 'Alice', nickname: 'Ali' }; + const bytes1 = serialize(withValue); + const result1 = deserialize(bytes1); + expect(result1).toEqual(withValue); + + // With null value + const withNull = { name: 'Bob', nickname: null }; + const bytes2 = serialize(withNull); + const result2 = deserialize(bytes2); + expect(result2).toEqual(withNull); + }); +}); diff --git a/integration_tests/idl_tests/javascript/tsconfig.json b/integration_tests/idl_tests/javascript/tsconfig.json new file mode 100644 index 0000000000..a09887990b --- /dev/null +++ b/integration_tests/idl_tests/javascript/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "noUnusedLocals": false, + "noUnusedParameters": false + }, + "include": ["test/**/*", "generated/**/*", "roundtrip.ts"] +} diff --git a/integration_tests/idl_tests/run_javascript_tests.sh b/integration_tests/idl_tests/run_javascript_tests.sh new file mode 100755 index 0000000000..e0e6a67167 --- /dev/null +++ b/integration_tests/idl_tests/run_javascript_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +python "${SCRIPT_DIR}/generate_idl.py" --lang javascript + +cd "${SCRIPT_DIR}/javascript" +npm install +ENABLE_FORY_DEBUG_OUTPUT=1 npx jest --ci + +IDL_PEER_LANG=javascript "${SCRIPT_DIR}/run_java_tests.sh" +