From c9286ebfa7f46123885bb9969d21dbb57246902d Mon Sep 17 00:00:00 2001 From: Sheel Nidhan Date: Mon, 17 Nov 2025 15:27:06 -0800 Subject: [PATCH 01/23] (feat) Add data preprocessing pipeline for HLPW --- .../config/external_aero_etl_hlpw.yaml | 61 +++++++++++++ .../config/variables/surface/hlpw.yaml | 22 +++++ .../config/variables/volume/hlpw.yaml | 20 ++++ .../external_aero_surface_data_processors.py | 91 +++++++++++++++++++ .../external_aero_volume_data_processors.py | 14 +++ 5 files changed, 208 insertions(+) create mode 100644 examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml create mode 100644 examples/external_aerodynamics/config/variables/surface/hlpw.yaml create mode 100644 examples/external_aerodynamics/config/variables/volume/hlpw.yaml diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml new file mode 100644 index 0000000..d5bdc12 --- /dev/null +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defaults: + - /external_aero_etl_drivaerml + - override /variables: hlpw + - _self_ + +etl: + common: + kind: hlpw + model_type: combined # produce data for which model? surface, volume, combined + + processing: + num_processes: 1 + + transformations: + surface_preprocessing: + surface_processors: + + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.decimate_mesh + _partial_: true + algo: decimate_pro + reduction: 0.0 + preserve_topology: false + + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.normalize_surface_normals + _partial_: true + + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw + _partial_: true + PREF: 176.352 #update to default HLPW value + UREF: 2679.505 + + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 + _partial_: true + + volume_preprocessing: + volume_processors: + + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw + _partial_: true + + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 + _partial_: true + + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data + _partial_: true diff --git a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml new file mode 100644 index 0000000..0f4dc68 --- /dev/null +++ b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml @@ -0,0 +1,22 @@ +# @package etl.transformations.surface_preprocessing +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +surface_variables: + PROJ(AVG(P)): scalar + AVG(TAU_WALL(0)): scalar + AVG(TAU_WALL(1)): scalar + AVG(TAU_WALL(2)): scalar \ No newline at end of file diff --git a/examples/external_aerodynamics/config/variables/volume/hlpw.yaml b/examples/external_aerodynamics/config/variables/volume/hlpw.yaml new file mode 100644 index 0000000..5eb653d --- /dev/null +++ b/examples/external_aerodynamics/config/variables/volume/hlpw.yaml @@ -0,0 +1,20 @@ +# @package etl.transformations.volume_preprocessing +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +volume_variables: + avg(P): scalar + avg(u): vector diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 9514b55..6691293 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -58,6 +58,69 @@ def default_surface_processing_for_external_aerodynamics( return data +def default_surface_processing_for_external_aerodynamics_hlpw( + data: ExternalAerodynamicsExtractedDataInMemory, + surface_variables: list[str], + nbf_field_name: str = "N_BF", +) -> ExternalAerodynamicsExtractedDataInMemory: + """ + Default surface processing for HLPW dataset. + + Uses the N_BF (Normal Boundary Faces) field for computing normals and areas, + which is more accurate than computing them separately with PyVista. + + Important: Converts point data to cell data before processing, as HLPW + data may be stored at vertices rather than cell centers. + + Args: + data: External aerodynamics data with surface polydata + surface_variables: List of variable names to extract from surface data + nbf_field_name: Name of the area-weighted normal field (default: "N_BF") + + Returns: + Data with surface fields, mesh centers, normals, and areas extracted + """ + + # Convert point data to cell data (important for HLPW!) + # Data might be stored at vertices, need to move to cell centers + data.surface_polydata = data.surface_polydata.point_data_to_cell_data() + + # Extract surface fields (pressure, wall shear stress, etc.) + cell_data = (data.surface_polydata.cell_data[k] for k in surface_variables) + data.surface_fields = np.concatenate( + [d if d.ndim > 1 else d[:, np.newaxis] for d in cell_data], axis=-1 + ) + + # Extract mesh centers + data.surface_mesh_centers = np.array(data.surface_polydata.cell_centers().points) + + # Check if N_BF field exists - REQUIRED for HLPW + if nbf_field_name not in data.surface_polydata.cell_data: + logger.error( + f"Field '{nbf_field_name}' not found in surface cell_data. " + f"Available fields: {list(data.surface_polydata.cell_data.keys())}" + ) + raise ValueError( + f"Required field '{nbf_field_name}' not found in surface data. " + f"HLPW processing requires N_BF field for accurate normal and area computation." + ) + + # Use N_BF (area-weighted normal vectors) - HLPW-specific + surface_normals_area = np.array( + data.surface_polydata.cell_data[nbf_field_name] + ).astype(np.float32) + + # Compute areas as magnitude of N_BF + data.surface_areas = np.linalg.norm(surface_normals_area, axis=1).astype(np.float32) + + # Compute unit normals by normalizing N_BF + data.surface_normals = surface_normals_area / np.reshape( + data.surface_areas, (-1, 1) + ) + + return data + + def filter_invalid_surface_cells( data: ExternalAerodynamicsExtractedDataInMemory, tolerance: float = 1e-6, @@ -172,6 +235,34 @@ def non_dimensionalize_surface_fields( return data +def non_dimensionalize_surface_fields_hlpw( + data: ExternalAerodynamicsExtractedDataInMemory, + PREF: float = 176.352, # HLPW reference pressure (Pa) +) -> ExternalAerodynamicsExtractedDataInMemory: + """ + Non-dimensionalize surface fields using HLPW reference values. + + This follows the HLPW convention: + - Both pressure and WSS fields: divided by PREF + + Args: + data: External aerodynamics data with surface fields + PREF: Reference pressure for non-dimensionalization + + Returns: + Data with non-dimensionalized surface fields + """ + + if data.surface_fields is None or len(data.surface_fields) == 0: + logger.error(f"Surface fields are empty: {data.surface_fields}") + return data + + # Non-dimensionalize pressure (first column) by PREF + data.surface_fields /= PREF + + return data + + def update_surface_data_to_float32( data: ExternalAerodynamicsExtractedDataInMemory, ) -> ExternalAerodynamicsExtractedDataInMemory: diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index c7a3e2f..07e8c17 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -141,6 +141,20 @@ def non_dimensionalize_volume_fields( ) return data +def non_dimensionalize_volume_fields_hlpw( + data: ExternalAerodynamicsExtractedDataInMemory, + pref: float = PhysicsConstants.PREF, + uref: float = PhysicsConstants.UREF, +) -> ExternalAerodynamicsExtractedDataInMemory: + """Non-dimensionalize volume fields.""" + + # Pressure + data.volume_fields[:, :1] = data.volume_fields[:, :1] / pref + # Velocity + data.volume_fields[:, 1:] = data.volume_fields[:, 1:] / uref + + return data + def update_volume_data_to_float32( data: ExternalAerodynamicsExtractedDataInMemory, From f251b34147464e68b41e657f63c2afe0530474a1 Mon Sep 17 00:00:00 2001 From: Sheel Nidhan Date: Mon, 17 Nov 2025 16:10:04 -0800 Subject: [PATCH 02/23] (chore) Fix errors in the pipeline --- .../config/external_aero_etl_hlpw.yaml | 24 +++++---- examples/external_aerodynamics/constants.py | 1 + .../data_transformations.py | 54 +++++++++++++++++++ .../external_aero_surface_data_processors.py | 4 +- .../external_aero_volume_data_processors.py | 4 +- examples/external_aerodynamics/paths.py | 31 +++++++++++ 6 files changed, 105 insertions(+), 13 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index d5bdc12..c6328af 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -16,7 +16,8 @@ defaults: - /external_aero_etl_drivaerml - - override /variables: hlpw + - override /variables/surface: hlpw + - override /variables/volume: hlpw - _self_ etl: @@ -29,33 +30,38 @@ etl: transformations: surface_preprocessing: + # Use HLPW-specific transformation that handles N_BF field + _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsSurfaceTransformationHLPW + _convert_: all + nbf_field_name: "N_BF" + surface_processors: - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.decimate_mesh _partial_: true algo: decimate_pro reduction: 0.0 preserve_topology: false - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.normalize_surface_normals - _partial_: true + # Note: normalize_surface_normals not needed - already done via N_BF normalization + # HLPW-specific non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true - PREF: 176.352 #update to default HLPW value - UREF: 2679.505 + PREF: 176.352 # HLPW reference pressure (Pa) - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true volume_preprocessing: volume_processors: - + # HLPW-specific volume non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true - + PREF: 176.352 # HLPW reference pressure (Pa) + UREF: 2679.505 # HLPW reference velocity (m/s) - update to actual value + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true diff --git a/examples/external_aerodynamics/constants.py b/examples/external_aerodynamics/constants.py index bba1c9b..3e7595c 100644 --- a/examples/external_aerodynamics/constants.py +++ b/examples/external_aerodynamics/constants.py @@ -44,6 +44,7 @@ class DatasetKind(str, Enum): DRIVESIM = "drivesim" DRIVAERML = "drivaerml" AHMEDML = "ahmedml" + HLPW = "hlpw" @dataclass(frozen=True) diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index ae5c9aa..924e275 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -26,6 +26,7 @@ ) from examples.external_aerodynamics.external_aero_surface_data_processors import ( default_surface_processing_for_external_aerodynamics, + default_surface_processing_for_external_aerodynamics_hlpw, ) from examples.external_aerodynamics.external_aero_volume_data_processors import ( default_volume_processing_for_external_aerodynamics, @@ -173,6 +174,59 @@ def transform( return data +class ExternalAerodynamicsSurfaceTransformationHLPW(DataTransformation): + """Transforms surface data for HLPW using N_BF field.""" + + def __init__( + self, + cfg: ProcessingConfig, + surface_variables: Optional[dict[str, str]] = None, + surface_processors: Optional[tuple[Callable, ...]] = None, + nbf_field_name: str = "N_BF", + ): + super().__init__(cfg) + self.logger = logging.getLogger(__name__) + + self.surface_variables = surface_variables + self.surface_processors = surface_processors + self.nbf_field_name = nbf_field_name + self.constants = PhysicsConstants() + + if surface_variables is None: + self.logger.error("Surface variables are empty!") + raise ValueError("Surface variables are empty!") + + self.logger.info( + f"Initializing ExternalAerodynamicsSurfaceTransformationHLPW with " + f"surface_variables: {surface_variables}, nbf_field_name: {nbf_field_name}, " + f"and surface_processors: {surface_processors}" + ) + self.logger.info( + "This will only be processed if the model_type is surface/combined." + ) + + def transform( + self, data: ExternalAerodynamicsExtractedDataInMemory + ) -> ExternalAerodynamicsExtractedDataInMemory: + """Transform surface data for HLPW using N_BF field.""" + + if data.surface_polydata is not None: + + # Use HLPW-specific default processing (with N_BF field) + data = default_surface_processing_for_external_aerodynamics_hlpw( + data, self.surface_variables, self.nbf_field_name + ) + + if self.surface_processors is not None: + for processor in self.surface_processors: + data = processor(data) + + # Delete raw surface data to save memory + data.surface_polydata = None + + return data + + class ExternalAerodynamicsVolumeTransformation(DataTransformation): """Transforms volume data for External Aerodynamics model.""" diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 6691293..85f0ff3 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -237,7 +237,7 @@ def non_dimensionalize_surface_fields( def non_dimensionalize_surface_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - PREF: float = 176.352, # HLPW reference pressure (Pa) + pref: float = 176.352, # HLPW reference pressure (Pa) ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using HLPW reference values. @@ -258,7 +258,7 @@ def non_dimensionalize_surface_fields_hlpw( return data # Non-dimensionalize pressure (first column) by PREF - data.surface_fields /= PREF + data.surface_fields /= pref return data diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index 07e8c17..3f55538 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -143,8 +143,8 @@ def non_dimensionalize_volume_fields( def non_dimensionalize_volume_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - pref: float = PhysicsConstants.PREF, - uref: float = PhysicsConstants.UREF, + pref: float, + uref: float, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" diff --git a/examples/external_aerodynamics/paths.py b/examples/external_aerodynamics/paths.py index 1f25b29..45a88d3 100644 --- a/examples/external_aerodynamics/paths.py +++ b/examples/external_aerodynamics/paths.py @@ -140,6 +140,35 @@ def geometry_path(car_dir: Path) -> Path: return car_dir / f"ahmed_{index}.stl" +class HLPWPaths: + """Utility class for handling HLPW dataset file paths. + + HLPW file naming pattern: + - Directory: geo_LHC001_AoA_16 + - Geometry: geo_LHC001_AoA_16.stl + - Surface: boundary_geo_LHC001_AoA_16.vtu + - Volume: volume_geo_LHC001_AoA_16.vtu (NOT *_coarse.vtu) + """ + + @staticmethod + def geometry_path(car_dir: Path) -> Path: + """Returns geometry path for HLPW dataset.""" + dirname = car_dir.name + return car_dir / f"{dirname}.stl" + + @staticmethod + def surface_path(car_dir: Path) -> Path: + """Returns surface data path for HLPW dataset.""" + dirname = car_dir.name + return car_dir / f"boundary_{dirname}.vtu" + + @staticmethod + def volume_path(car_dir: Path) -> Path: + """Returns volume data path for HLPW dataset (NOT the coarse version).""" + dirname = car_dir.name + return car_dir / f"volume_{dirname}.vtu" + + def get_path_getter(kind: DatasetKind): """Returns path getter for a given dataset type.""" @@ -150,3 +179,5 @@ def get_path_getter(kind: DatasetKind): return DrivAerMLPaths case DatasetKind.DRIVESIM: return DriveSimPaths + case DatasetKind.HLPW: + return HLPWPaths From d65adcb2aa603bca2fc50d55b9211172470131bc Mon Sep 17 00:00:00 2001 From: Sheel Nidhan Date: Tue, 18 Nov 2025 21:45:09 -0800 Subject: [PATCH 03/23] (feat) Pipeline for global params addition similar to volume/surface/STL processing --- ...External_Aero_Data_Processing_Reference.md | 2 + .../config/external_aero_etl_drivaerml.yaml | 11 + .../config/external_aero_etl_hlpw.yaml | 7 + .../config/variables/global/ahmedml.yaml | 28 +++ .../config/variables/global/drivaerml.yaml | 28 +++ .../config/variables/global/hlpw.yaml | 21 ++ .../external_aerodynamics/data_sources.py | 2 + .../data_transformations.py | 68 ++++-- ...rnal_aero_global_params_data_processors.py | 216 ++++++++++++++++++ examples/external_aerodynamics/paths.py | 25 ++ examples/external_aerodynamics/schemas.py | 14 +- 11 files changed, 403 insertions(+), 19 deletions(-) create mode 100644 examples/external_aerodynamics/config/variables/global/ahmedml.yaml create mode 100644 examples/external_aerodynamics/config/variables/global/drivaerml.yaml create mode 100644 examples/external_aerodynamics/config/variables/global/hlpw.yaml create mode 100644 examples/external_aerodynamics/external_aero_global_params_data_processors.py diff --git a/examples/external_aerodynamics/External_Aero_Data_Processing_Reference.md b/examples/external_aerodynamics/External_Aero_Data_Processing_Reference.md index 70c2dcf..e556926 100644 --- a/examples/external_aerodynamics/External_Aero_Data_Processing_Reference.md +++ b/examples/external_aerodynamics/External_Aero_Data_Processing_Reference.md @@ -95,6 +95,8 @@ For more information, see the [Zarr project website](https://zarr.dev/). ├── stl_centers # Cell centers ├── stl_coordinates # Vertex coordinates ├── stl_faces # Face connectivity + ├── global_params_values # Simulation-specific global parameters (optional) + ├── global_params_reference # Reference values for global parameters (optional) ├── surface_areas # Cell areas (surface, optional) ├── surface_fields # Field data (surface, optional) ├── surface_mesh_centers # Cell centers (surface, optional) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index bd01564..6cb5a44 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -17,6 +17,7 @@ defaults: - /variables/surface: drivaerml - /variables/volume: drivaerml + - /variables/global: drivaerml - /serialization_format: zarr # Default value, can be overridden via CLI - /override_transformations: ${serialization_format} - _self_ @@ -97,6 +98,16 @@ etl: _partial_: true - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true + + global_params_preprocessing: + _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsGlobalParamsTransformation + _convert_: all + global_parameters: ${global_parameters} + # Regardless of whether there are any additional global params processors, + # we always apply the default global params processing. This ensure that there are global params references present. + global_params_processors: + - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params + _partial_: true write_ready_transformation: ${override_transformations.write_ready_transformation} diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index c6328af..9a2c84a 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -18,6 +18,7 @@ defaults: - /external_aero_etl_drivaerml - override /variables/surface: hlpw - override /variables/volume: hlpw + - override /variables/global: hlpw - _self_ etl: @@ -65,3 +66,9 @@ etl: - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true + + global_params_preprocessing: + global_parameters: ${global_parameters} + global_params_processors: + - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params_hlpw + _partial_: true diff --git a/examples/external_aerodynamics/config/variables/global/ahmedml.yaml b/examples/external_aerodynamics/config/variables/global/ahmedml.yaml new file mode 100644 index 0000000..69bf1f9 --- /dev/null +++ b/examples/external_aerodynamics/config/variables/global/ahmedml.yaml @@ -0,0 +1,28 @@ +# @package etl +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global_parameters: + inlet_velocity: + type: vector + reference: [30.0] # Inlet velocity vector in m/s + air_density: + type: scalar + reference: 1.205 # Air density in kg/m³ + pressure: + type: scalar + reference: 101325.0 # Reference pressure in Pa + diff --git a/examples/external_aerodynamics/config/variables/global/drivaerml.yaml b/examples/external_aerodynamics/config/variables/global/drivaerml.yaml new file mode 100644 index 0000000..69bf1f9 --- /dev/null +++ b/examples/external_aerodynamics/config/variables/global/drivaerml.yaml @@ -0,0 +1,28 @@ +# @package etl +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global_parameters: + inlet_velocity: + type: vector + reference: [30.0] # Inlet velocity vector in m/s + air_density: + type: scalar + reference: 1.205 # Air density in kg/m³ + pressure: + type: scalar + reference: 101325.0 # Reference pressure in Pa + diff --git a/examples/external_aerodynamics/config/variables/global/hlpw.yaml b/examples/external_aerodynamics/config/variables/global/hlpw.yaml new file mode 100644 index 0000000..b5e0091 --- /dev/null +++ b/examples/external_aerodynamics/config/variables/global/hlpw.yaml @@ -0,0 +1,21 @@ +# @package etl +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global_parameters: + AoA: + type: scalar + reference: 22.0 # Angle of Attack in degrees diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 39f1888..4417351 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -250,6 +250,8 @@ def _write_zarr( # Write optional arrays if present for field in [ + "global_params_values", + "global_params_reference", "surface_mesh_centers", "surface_normals", "surface_areas", diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 924e275..5c09c60 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -21,23 +21,24 @@ import numpy as np from numcodecs import Blosc -from examples.external_aerodynamics.external_aero_geometry_data_processors import ( +from physicsnemo_curator.etl.data_transformations import DataTransformation +from physicsnemo_curator.etl.processing_config import ProcessingConfig + +from .constants import PhysicsConstants +from .external_aero_geometry_data_processors import ( default_geometry_processing_for_external_aerodynamics, ) -from examples.external_aerodynamics.external_aero_surface_data_processors import ( +from .external_aero_global_params_data_processors import ( + default_global_params_processing_for_external_aerodynamics, +) +from .external_aero_surface_data_processors import ( default_surface_processing_for_external_aerodynamics, default_surface_processing_for_external_aerodynamics_hlpw, ) -from examples.external_aerodynamics.external_aero_volume_data_processors import ( +from .external_aero_utils import to_float32 +from .external_aero_volume_data_processors import ( default_volume_processing_for_external_aerodynamics, ) -from physicsnemo_curator.etl.data_transformations import DataTransformation -from physicsnemo_curator.etl.processing_config import ProcessingConfig - -from .constants import PhysicsConstants -from .external_aero_utils import ( - to_float32, -) from .schemas import ( ExternalAerodynamicsExtractedDataInMemory, ExternalAerodynamicsNumpyDataInMemory, @@ -155,7 +156,6 @@ def transform( """Transform surface data for External Aerodynamics model.""" if data.surface_polydata is not None: - # Regardless of whether there are any additional surface processors, # we always apply the default surface processing. # This will ensure that the bare minimum criteria for surface data is met. @@ -175,7 +175,7 @@ def transform( class ExternalAerodynamicsSurfaceTransformationHLPW(DataTransformation): - """Transforms surface data for HLPW using N_BF field.""" + """Transforms surface data for HLPW and uses N_BF field for surface_area calculation""" def __init__( self, @@ -211,7 +211,6 @@ def transform( """Transform surface data for HLPW using N_BF field.""" if data.surface_polydata is not None: - # Use HLPW-specific default processing (with N_BF field) data = default_surface_processing_for_external_aerodynamics_hlpw( data, self.surface_variables, self.nbf_field_name @@ -256,9 +255,7 @@ def __init__( def transform( self, data: ExternalAerodynamicsExtractedDataInMemory ) -> ExternalAerodynamicsExtractedDataInMemory: - if data.volume_unstructured_grid is not None: - # Regardless of whether there are any additional volume processors, # we always apply the default volume processing. # This will ensure that the bare minimum criteria for volume data is met. @@ -277,6 +274,45 @@ def transform( return data +class ExternalAerodynamicsGlobalParamsTransformation(DataTransformation): + """Transforms global parameters values and references for External Aerodynamics model.""" + def __init__( + self, + cfg: ProcessingConfig, + global_parameters: Optional[dict] = None, + global_params_processors: Optional[tuple[Callable, ...]] = None, + ): + super().__init__(cfg) + self.global_parameters = global_parameters + self.global_params_processors = global_params_processors + self.logger = logging.getLogger(__name__) + + if global_parameters is None: + self.logger.error("No global_parameters provided. Please provide global parameters") + + def transform( + self, data: ExternalAerodynamicsExtractedDataInMemory + ) -> ExternalAerodynamicsExtractedDataInMemory: + """Transform global_params data for External Aerodynamics model. + + Processes global parameter references from config and extracts values from simulation data. + """ + + if self.global_parameters is not None: + # Apply default processing to set up reference arrays from config + data = default_global_params_processing_for_external_aerodynamics( + data, self.global_parameters + ) + + # Apply any custom processors (e.g., extract values from simulation files) + # Pass global_parameters so processors know the types (vector vs scalar) + if self.global_params_processors is not None: + for processor in self.global_params_processors: + data = processor(data, self.global_parameters) + + return data + + class ExternalAerodynamicsZarrTransformation(DataTransformation): """Transforms External Aerodynamics data for Zarr storage format.""" @@ -357,6 +393,8 @@ def transform( stl_faces=self._prepare_array(data.stl_faces), stl_areas=self._prepare_array(data.stl_areas), metadata=data.metadata, + global_params_values=self._prepare_array(data.metadata.global_params_values), + global_params_reference=self._prepare_array(data.metadata.global_params_reference), surface_mesh_centers=self._prepare_array(data.surface_mesh_centers), surface_normals=self._prepare_array(data.surface_normals), surface_areas=self._prepare_array(data.surface_areas), diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py new file mode 100644 index 0000000..7020c30 --- /dev/null +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import numpy as np + +from examples.external_aerodynamics.schemas import ( + ExternalAerodynamicsExtractedDataInMemory, +) + +logging.basicConfig( + format="%(asctime)s - Process %(process)d - %(levelname)s - %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +def default_global_params_processing_for_external_aerodynamics( + data: ExternalAerodynamicsExtractedDataInMemory, + global_parameters: dict, +) -> ExternalAerodynamicsExtractedDataInMemory: + """Default global parameters processing for External Aerodynamics. + + Extracts and flattens global parameter references from config into a 1D numpy array. + Handles both vector and scalar parameter types. + + Args: + data: Container with simulation data and metadata + global_parameters: Dict from config with structure: + { + "param_name": { + "type": "vector" or "scalar", + "reference": value or list + } + } + + Returns: + Updated data with global_params_reference set in metadata + """ + # Build dictionaries for types and reference values + global_params_types = { + name: params["type"] + for name, params in global_parameters.items() + } + + global_params_reference_dict = { + name: params["reference"] + for name, params in global_parameters.items() + } + + # Arrange global parameters reference in a list based on the type of the parameter + global_params_reference_list = [] + for name, param_type in global_params_types.items(): + if param_type == "vector": + # Vector parameters should be lists - extend to flatten + global_params_reference_list.extend(global_params_reference_dict[name]) + elif param_type == "scalar": + # Scalar parameters are single values - append directly + global_params_reference_list.append(global_params_reference_dict[name]) + else: + raise ValueError( + f"Global parameter '{name}' has unsupported type '{param_type}'. " + f"Must be 'vector' or 'scalar'." + ) + + # Convert to numpy array and store in metadata + data.metadata.global_params_reference = np.array( + global_params_reference_list, dtype=np.float32 + ) + + logger.info( + f"Processed global_params_reference: {data.metadata.global_params_reference} " + f"with shape {data.metadata.global_params_reference.shape}" + ) + + return data + + +def process_global_params( + data: ExternalAerodynamicsExtractedDataInMemory, + global_parameters: dict, +) -> ExternalAerodynamicsExtractedDataInMemory: + """Base processor for global parameters - to be overridden for specific datasets. + + This is a placeholder that should be replaced by dataset-specific implementations + (e.g., process_global_params_hlpw). + + By default, sets global_params_values equal to global_params_reference, + assuming simulation conditions match reference conditions. + + Args: + data: Container with simulation data and metadata + global_parameters: Dict from config with parameter definitions + + Returns: + Updated data with global_params_values set in metadata + """ + if data.metadata.global_params_reference is None: + logger.warning( + "global_params_reference not set. Skipping global_params_values processing." + ) + return data + + # Default behavior: assume simulation values match reference + data.metadata.global_params_values = data.metadata.global_params_reference.copy() + + logger.info( + f"Set global_params_values to reference values: {data.metadata.global_params_values}" + ) + + return data + + +# ============================================================================ +# Case-Specific Processors +# ============================================================================ +# These functions demonstrate how to extract global_params_values from +# simulation data for specific datasets. Replace process_global_params above +# with these in your config for case-specific processing. + + +def process_global_params_hlpw( + data: ExternalAerodynamicsExtractedDataInMemory, + global_parameters: dict, +) -> ExternalAerodynamicsExtractedDataInMemory: + """Extract global parameters from HLPW simulation data. + + For HLPW, typically: + - AoA (Angle of Attack) varies per simulation and can be extracted from filename or metadata + + Args: + data: Container with simulation data and metadata + global_parameters: Dict from config with parameter definitions + + Returns: + Updated data with global_params_values extracted from simulation + """ + if data.metadata.global_params_reference is None: + logger.warning( + "global_params_reference not set. Skipping global_params_values processing." + ) + return data + + # Build a dict of extracted values keyed by parameter name + extracted_values = {} + + # Extract AoA from filename (e.g., "geo_LHC001_AoA_16" -> 16.0) + filename = data.metadata.filename + if "AoA_" in filename: + try: + # Extract string after "AoA_" + # Example: "geo_LHC001_AoA_16" -> "16" + # Example: "geo_LHC001_AoA_16_something" -> "16" + after_aoa = filename.split("AoA_")[1] + # Take everything up to next underscore or end of string + aoa_str = after_aoa.split("_")[0] if "_" in after_aoa else after_aoa + aoa = float(aoa_str) + extracted_values["AoA"] = aoa + logger.info(f"Extracted AoA={aoa} from filename: {filename}") + except (IndexError, ValueError) as e: + logger.warning( + f"Could not extract AoA from filename '{filename}': {e}. " + f"Using reference value." + ) + extracted_values["AoA"] = global_parameters["AoA"]["reference"] + else: + # Fallback to reference if not in filename + logger.error( + f"AoA pattern not found in filename '{filename}'." + ) + extracted_values["AoA"] = global_parameters["AoA"]["reference"] + + # Build the flattened array using the same logic as reference processing + global_params_values_list = [] + for name, params in global_parameters.items(): + param_type = params["type"] + value = extracted_values.get(name, params["reference"]) + + if param_type == "vector": + # For vectors, ensure value is a list and extend + if not isinstance(value, (list, tuple)): + value = [value] + global_params_values_list.extend(value) + elif param_type == "scalar": + # For scalars, append directly + global_params_values_list.append(value) + else: + raise ValueError( + f"Global parameter '{name}' has unsupported type '{param_type}'. " + f"Must be 'vector' or 'scalar'." + ) + + data.metadata.global_params_values = np.array( + global_params_values_list, dtype=np.float32 + ) + + logger.info( + f"Processed global_params_values for HLPW: {data.metadata.global_params_values}" + ) + + return data \ No newline at end of file diff --git a/examples/external_aerodynamics/paths.py b/examples/external_aerodynamics/paths.py index 45a88d3..e8f0cea 100644 --- a/examples/external_aerodynamics/paths.py +++ b/examples/external_aerodynamics/paths.py @@ -168,6 +168,31 @@ def volume_path(car_dir: Path) -> Path: dirname = car_dir.name return car_dir / f"volume_{dirname}.vtu" + @staticmethod + def extract_aoa(dirname: str) -> float: + """Extract Angle of Attack (AoA) from HLPW directory name. + + Args: + dirname: Directory name in format 'geo_LHC001_AoA_16' + + Returns: + Angle of Attack in degrees as a float + + Raises: + ValueError: If AoA cannot be extracted from the filename + """ + parts = dirname.split("_") + # Look for 'AoA' followed by the angle value + for i, part in enumerate(parts): + if part == "AoA" and i + 1 < len(parts): + try: + return float(parts[i + 1]) + except ValueError: + raise ValueError( + f"Could not parse AoA value from '{parts[i + 1]}' in directory: {dirname}" + ) + raise ValueError(f"AoA not found in directory name: {dirname}") + def get_path_getter(kind: DatasetKind): """Returns path getter for a given dataset type.""" diff --git a/examples/external_aerodynamics/schemas.py b/examples/external_aerodynamics/schemas.py index 363d44a..c186fa1 100644 --- a/examples/external_aerodynamics/schemas.py +++ b/examples/external_aerodynamics/schemas.py @@ -31,6 +31,7 @@ class ExternalAerodynamicsMetadata: Version history: - 1.0: Initial version with expected metadata fields. + - 1.1: Added AoA (Angle of Attack) field. """ # Simulation identifiers @@ -38,8 +39,8 @@ class ExternalAerodynamicsMetadata: dataset_type: ModelType # Physical parameters - stream_velocity: Optional[float] = None - air_density: Optional[float] = None + global_params_values: Optional[np.ndarray] = None + global_params_reference: Optional[np.ndarray] = None # Geometry bounds x_bound: Optional[tuple[float, float]] = None # xmin, xmax @@ -107,6 +108,7 @@ class ExternalAerodynamicsZarrDataInMemory: Version history: - 1.0: Initial version with prepared arrays for Zarr storage + - 1.1: Added global_params_values and global_params_reference as top-level datasets """ # Metadata @@ -118,6 +120,10 @@ class ExternalAerodynamicsZarrDataInMemory: stl_faces: PreparedZarrArrayInfo stl_areas: PreparedZarrArrayInfo + # Global parameters + global_params_values: Optional[PreparedZarrArrayInfo] = None + global_params_reference: Optional[PreparedZarrArrayInfo] = None + # Surface data surface_mesh_centers: Optional[PreparedZarrArrayInfo] = None surface_normals: Optional[PreparedZarrArrayInfo] = None @@ -137,8 +143,8 @@ class ExternalAerodynamicsNumpyMetadata: """ filename: str - stream_velocity: float - air_density: float + global_params_values: np.ndarray + global_params_reference: np.ndarray @dataclass(frozen=True) From 6b70451976301ec06412db6383583ff6da153752 Mon Sep 17 00:00:00 2001 From: Sheel Nidhan Date: Sat, 22 Nov 2025 19:44:21 -0800 Subject: [PATCH 04/23] (chore) fix error and clean implementation --- .../config/external_aero_etl_drivaerml.yaml | 2 +- .../config/external_aero_etl_hlpw.yaml | 14 +++++++----- .../config/variables/surface/hlpw.yaml | 1 + .../config/variables/volume/hlpw.yaml | 1 + .../external_aerodynamics/data_sources.py | 19 ++++++++++++++-- .../data_transformations.py | 22 +++++++++++++++++-- ...rnal_aero_global_params_data_processors.py | 1 - .../external_aero_surface_data_processors.py | 9 +++++--- .../external_aero_volume_data_processors.py | 11 +++++++--- 9 files changed, 62 insertions(+), 18 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index 6cb5a44..cd02e96 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -102,7 +102,7 @@ etl: global_params_preprocessing: _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsGlobalParamsTransformation _convert_: all - global_parameters: ${global_parameters} + global_parameters: ${etl.global_parameters} # Regardless of whether there are any additional global params processors, # we always apply the default global params processing. This ensure that there are global params references present. global_params_processors: diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index 9a2c84a..017f2dd 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -48,7 +48,8 @@ etl: # HLPW-specific non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true - PREF: 176.352 # HLPW reference pressure (Pa) + pref: 176.352 # HLPW reference pressure (Pa) + tref: 518.67 # HLPW reference temperature (K) - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -58,9 +59,10 @@ etl: # HLPW-specific volume non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true - PREF: 176.352 # HLPW reference pressure (Pa) - UREF: 2679.505 # HLPW reference velocity (m/s) - update to actual value - + pref: 176.352 # HLPW reference pressure (Pa) + uref: 2679.505 # HLPW reference velocity (m/s) + tref: 518.67 # HLPW reference temperature (K) + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true @@ -68,7 +70,7 @@ etl: _partial_: true global_params_preprocessing: - global_parameters: ${global_parameters} + global_parameters: ${etl.global_parameters} global_params_processors: - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params_hlpw - _partial_: true + _partial_: true \ No newline at end of file diff --git a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml index 0f4dc68..54fd09e 100644 --- a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml +++ b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml @@ -17,6 +17,7 @@ surface_variables: PROJ(AVG(P)): scalar + PROJ(AVG(T)): scalar AVG(TAU_WALL(0)): scalar AVG(TAU_WALL(1)): scalar AVG(TAU_WALL(2)): scalar \ No newline at end of file diff --git a/examples/external_aerodynamics/config/variables/volume/hlpw.yaml b/examples/external_aerodynamics/config/variables/volume/hlpw.yaml index 5eb653d..d05dec1 100644 --- a/examples/external_aerodynamics/config/variables/volume/hlpw.yaml +++ b/examples/external_aerodynamics/config/variables/volume/hlpw.yaml @@ -17,4 +17,5 @@ volume_variables: avg(P): scalar + avg(T): scalar avg(u): vector diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 4417351..9120d29 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -235,18 +235,29 @@ def _write_zarr( zarr_store = zarr.DirectoryStore(output_path) root = zarr.group(store=zarr_store) - # Write metadata as attributes - root.attrs.update(asdict(data.metadata)) + self.logger.info(f"Created zarr store at {output_path}") + + # Write metadata as attributes (exclude numpy arrays that will be stored as datasets) + metadata_dict = asdict(data.metadata) + # Remove numpy array fields - these are stored as separate zarr datasets below + metadata_dict.pop('global_params_values', None) + metadata_dict.pop('global_params_reference', None) + root.attrs.update(metadata_dict) + self.logger.info(f"Wrote metadata attributes at {output_path}") # Write required arrays for field in ["stl_coordinates", "stl_centers", "stl_faces", "stl_areas"]: array_info = getattr(data, field) + self.logger.info(f"Writing required field '{field}': array_info={array_info is not None}, type={type(array_info)}") + if array_info is None: + raise ValueError(f"Required field '{field}' is None - cannot write zarr dataset") root.create_dataset( field, data=array_info.data, chunks=array_info.chunks, compressor=array_info.compressor, ) + self.logger.info(f"Successfully wrote field '{field}' with shape {array_info.data.shape}") # Write optional arrays if present for field in [ @@ -267,6 +278,10 @@ def _write_zarr( chunks=array_info.chunks, compressor=array_info.compressor, ) + self.logger.info(f"Successfully wrote field '{field}' with shape {array_info.data.shape}") + else: + self.logger.warning(f"{array_info} is absent in the dataset") + def should_skip(self, filename: str) -> bool: """Checks whether the file should be skipped. diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 5c09c60..820a836 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -371,6 +371,24 @@ def _prepare_array(self, array: np.ndarray) -> PreparedZarrArrayInfo: compressor=self.compressor, ) + def _prepare_array_no_compression(self, array: np.ndarray) -> PreparedZarrArrayInfo: + """Prepare small array for Zarr storage without compression. + + Used for small metadata arrays like global parameters where compression + overhead exceeds any space savings. + """ + if array is None: + return None + + # Store entire array in a single chunk (no chunking for small arrays) + chunks = array.shape + + return PreparedZarrArrayInfo( + data=np.float32(array), + chunks=chunks, + compressor=None, # No compression + ) + def transform( self, data: ExternalAerodynamicsExtractedDataInMemory ) -> ExternalAerodynamicsZarrDataInMemory: @@ -393,8 +411,8 @@ def transform( stl_faces=self._prepare_array(data.stl_faces), stl_areas=self._prepare_array(data.stl_areas), metadata=data.metadata, - global_params_values=self._prepare_array(data.metadata.global_params_values), - global_params_reference=self._prepare_array(data.metadata.global_params_reference), + global_params_values=self._prepare_array_no_compression(data.metadata.global_params_values), + global_params_reference=self._prepare_array_no_compression(data.metadata.global_params_reference), surface_mesh_centers=self._prepare_array(data.surface_mesh_centers), surface_normals=self._prepare_array(data.surface_normals), surface_areas=self._prepare_array(data.surface_areas), diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index 7020c30..d37d366 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -133,7 +133,6 @@ def process_global_params( # simulation data for specific datasets. Replace process_global_params above # with these in your config for case-specific processing. - def process_global_params_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, global_parameters: dict, diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 5c27158..16de675 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -242,6 +242,7 @@ def non_dimensionalize_surface_fields( def non_dimensionalize_surface_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, pref: float = 176.352, # HLPW reference pressure (Pa) + tref: float = 518.67, ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using HLPW reference values. @@ -260,9 +261,11 @@ def non_dimensionalize_surface_fields_hlpw( if data.surface_fields is None or len(data.surface_fields) == 0: logger.error(f"Surface fields are empty: {data.surface_fields}") return data - - # Non-dimensionalize pressure (first column) by PREF - data.surface_fields /= pref + # Non-dimensionalize temperature by TREF + data.surface_fields[0:1] /= tref + + # Non-dimensionalize pressure and shear stress by PREF + data.surface_fields[1:] /= pref return data diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index f43c264..3edfb0b 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -157,15 +157,20 @@ def non_dimensionalize_volume_fields( def non_dimensionalize_volume_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - pref: float, - uref: float, + pref: float = 176.352, + tref: float = 518.67, + uref: float = 2679.505, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" # Pressure data.volume_fields[:, :1] = data.volume_fields[:, :1] / pref + + # Temperature + data.volume_fields[:, 1:2] = data.volume_fields[:, 1:2] / tref + # Velocity - data.volume_fields[:, 1:] = data.volume_fields[:, 1:] / uref + data.volume_fields[:, 2:] = data.volume_fields[:, 2:] / uref return data From a72d1ade909df4802ac582a962c0612ea8619ac8 Mon Sep 17 00:00:00 2001 From: snidhan Date: Sat, 22 Nov 2025 20:30:32 -0800 Subject: [PATCH 05/23] (chore) clean code for PR --- .../external_aerodynamics/data_sources.py | 8 ++--- .../data_transformations.py | 4 +-- ...rnal_aero_global_params_data_processors.py | 35 +++---------------- examples/external_aerodynamics/paths.py | 28 +-------------- 4 files changed, 10 insertions(+), 65 deletions(-) diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 9120d29..43247c7 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -235,15 +235,12 @@ def _write_zarr( zarr_store = zarr.DirectoryStore(output_path) root = zarr.group(store=zarr_store) - self.logger.info(f"Created zarr store at {output_path}") - - # Write metadata as attributes (exclude numpy arrays that will be stored as datasets) + # Remove 'global_params_values' and `global_params_reference` that are written + # as part of the data itself metadata_dict = asdict(data.metadata) - # Remove numpy array fields - these are stored as separate zarr datasets below metadata_dict.pop('global_params_values', None) metadata_dict.pop('global_params_reference', None) root.attrs.update(metadata_dict) - self.logger.info(f"Wrote metadata attributes at {output_path}") # Write required arrays for field in ["stl_coordinates", "stl_centers", "stl_faces", "stl_areas"]: @@ -257,7 +254,6 @@ def _write_zarr( chunks=array_info.chunks, compressor=array_info.compressor, ) - self.logger.info(f"Successfully wrote field '{field}' with shape {array_info.data.shape}") # Write optional arrays if present for field in [ diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 820a836..7e67f14 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -374,8 +374,7 @@ def _prepare_array(self, array: np.ndarray) -> PreparedZarrArrayInfo: def _prepare_array_no_compression(self, array: np.ndarray) -> PreparedZarrArrayInfo: """Prepare small array for Zarr storage without compression. - Used for small metadata arrays like global parameters where compression - overhead exceeds any space savings. + Used for small arrays like `global_params_reference` and `global_params_values` """ if array is None: return None @@ -411,6 +410,7 @@ def transform( stl_faces=self._prepare_array(data.stl_faces), stl_areas=self._prepare_array(data.stl_areas), metadata=data.metadata, + # `global_params_values` and `global_params_reference` are saved without compression global_params_values=self._prepare_array_no_compression(data.metadata.global_params_values), global_params_reference=self._prepare_array_no_compression(data.metadata.global_params_reference), surface_mesh_centers=self._prepare_array(data.surface_mesh_centers), diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index d37d366..c74836c 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -67,10 +67,8 @@ def default_global_params_processing_for_external_aerodynamics( global_params_reference_list = [] for name, param_type in global_params_types.items(): if param_type == "vector": - # Vector parameters should be lists - extend to flatten global_params_reference_list.extend(global_params_reference_dict[name]) elif param_type == "scalar": - # Scalar parameters are single values - append directly global_params_reference_list.append(global_params_reference_dict[name]) else: raise ValueError( @@ -83,11 +81,6 @@ def default_global_params_processing_for_external_aerodynamics( global_params_reference_list, dtype=np.float32 ) - logger.info( - f"Processed global_params_reference: {data.metadata.global_params_reference} " - f"with shape {data.metadata.global_params_reference.shape}" - ) - return data @@ -114,15 +107,12 @@ def process_global_params( logger.warning( "global_params_reference not set. Skipping global_params_values processing." ) + raise ValueError("global_params_reference are absent in the configuration") return data # Default behavior: assume simulation values match reference data.metadata.global_params_values = data.metadata.global_params_reference.copy() - logger.info( - f"Set global_params_values to reference values: {data.metadata.global_params_values}" - ) - return data @@ -140,7 +130,7 @@ def process_global_params_hlpw( """Extract global parameters from HLPW simulation data. For HLPW, typically: - - AoA (Angle of Attack) varies per simulation and can be extracted from filename or metadata + - AoA (Angle of Attack) varies per simulation and can be extracted from filename Args: data: Container with simulation data and metadata @@ -153,6 +143,7 @@ def process_global_params_hlpw( logger.warning( "global_params_reference not set. Skipping global_params_values processing." ) + raise ValueError("global_params_reference are absent in the configuration") return data # Build a dict of extracted values keyed by parameter name @@ -161,7 +152,6 @@ def process_global_params_hlpw( # Extract AoA from filename (e.g., "geo_LHC001_AoA_16" -> 16.0) filename = data.metadata.filename if "AoA_" in filename: - try: # Extract string after "AoA_" # Example: "geo_LHC001_AoA_16" -> "16" # Example: "geo_LHC001_AoA_16_something" -> "16" @@ -171,18 +161,11 @@ def process_global_params_hlpw( aoa = float(aoa_str) extracted_values["AoA"] = aoa logger.info(f"Extracted AoA={aoa} from filename: {filename}") - except (IndexError, ValueError) as e: - logger.warning( - f"Could not extract AoA from filename '{filename}': {e}. " - f"Using reference value." - ) - extracted_values["AoA"] = global_parameters["AoA"]["reference"] else: # Fallback to reference if not in filename - logger.error( + raise ValueError( f"AoA pattern not found in filename '{filename}'." ) - extracted_values["AoA"] = global_parameters["AoA"]["reference"] # Build the flattened array using the same logic as reference processing global_params_values_list = [] @@ -191,12 +174,8 @@ def process_global_params_hlpw( value = extracted_values.get(name, params["reference"]) if param_type == "vector": - # For vectors, ensure value is a list and extend - if not isinstance(value, (list, tuple)): - value = [value] global_params_values_list.extend(value) elif param_type == "scalar": - # For scalars, append directly global_params_values_list.append(value) else: raise ValueError( @@ -207,9 +186,5 @@ def process_global_params_hlpw( data.metadata.global_params_values = np.array( global_params_values_list, dtype=np.float32 ) - - logger.info( - f"Processed global_params_values for HLPW: {data.metadata.global_params_values}" - ) - + return data \ No newline at end of file diff --git a/examples/external_aerodynamics/paths.py b/examples/external_aerodynamics/paths.py index e8f0cea..e2d1765 100644 --- a/examples/external_aerodynamics/paths.py +++ b/examples/external_aerodynamics/paths.py @@ -168,32 +168,6 @@ def volume_path(car_dir: Path) -> Path: dirname = car_dir.name return car_dir / f"volume_{dirname}.vtu" - @staticmethod - def extract_aoa(dirname: str) -> float: - """Extract Angle of Attack (AoA) from HLPW directory name. - - Args: - dirname: Directory name in format 'geo_LHC001_AoA_16' - - Returns: - Angle of Attack in degrees as a float - - Raises: - ValueError: If AoA cannot be extracted from the filename - """ - parts = dirname.split("_") - # Look for 'AoA' followed by the angle value - for i, part in enumerate(parts): - if part == "AoA" and i + 1 < len(parts): - try: - return float(parts[i + 1]) - except ValueError: - raise ValueError( - f"Could not parse AoA value from '{parts[i + 1]}' in directory: {dirname}" - ) - raise ValueError(f"AoA not found in directory name: {dirname}") - - def get_path_getter(kind: DatasetKind): """Returns path getter for a given dataset type.""" @@ -205,4 +179,4 @@ def get_path_getter(kind: DatasetKind): case DatasetKind.DRIVESIM: return DriveSimPaths case DatasetKind.HLPW: - return HLPWPaths + return HLPWPaths \ No newline at end of file From 88362dc1cd97106cf8932a53afe53bb69881503a Mon Sep 17 00:00:00 2001 From: snidhan Date: Tue, 25 Nov 2025 19:24:18 -0800 Subject: [PATCH 06/23] (chore) refactor constants.py and its propagation through config --- .gitignore | 1 + .vscode/settings.json | 22 ++++++ .../config/external_aero_etl_drivaerml.yaml | 13 +++- .../config/external_aero_etl_hlpw.yaml | 11 ++- examples/external_aerodynamics/constants.py | 13 +++- ...rnal_aero_global_params_data_processors.py | 75 +++++++++---------- .../external_aero_surface_data_processors.py | 64 ++++++++-------- .../external_aero_volume_data_processors.py | 25 +++++-- 8 files changed, 134 insertions(+), 90 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 25c7789..8e35bb0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .coverage* .pytest_cache* .ruff_cache* +.vscode/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..56d6da2 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,22 @@ +{ + "workbench.colorCustomizations": { + "activityBar.activeBackground": "#93e6fc", + "activityBar.background": "#93e6fc", + "activityBar.foreground": "#15202b", + "activityBar.inactiveForeground": "#15202b99", + "activityBarBadge.background": "#fa45d4", + "activityBarBadge.foreground": "#15202b", + "commandCenter.border": "#15202b99", + "sash.hoverBorder": "#93e6fc", + "statusBar.background": "#61dafb", + "statusBar.foreground": "#15202b", + "statusBarItem.hoverBackground": "#2fcefa", + "statusBarItem.remoteBackground": "#61dafb", + "statusBarItem.remoteForeground": "#15202b", + "titleBar.activeBackground": "#61dafb", + "titleBar.activeForeground": "#15202b", + "titleBar.inactiveBackground": "#61dafb99", + "titleBar.inactiveForeground": "#15202b99" + }, + "peacock.remoteColor": "#61dafb" +} \ No newline at end of file diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index cd02e96..bebe075 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -75,12 +75,15 @@ etl: algo: decimate_pro # can be one of {decimate_pro, decimate} reduction: 0.0 # 0 means no decimation. preserve_topology: false + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.normalize_surface_normals _partial_: true + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields _partial_: true - air_density: 1.205 # kg/m³ - stream_velocity: 30.00 # m/s + physical_constants: + _target_: examples.external_aerodynamics.constants.PhysicsConstantsCarAerodynamics + - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -94,8 +97,12 @@ etl: volume_processors: - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields _partial_: true + physical_constants: + _target_: examples.external_aerodynamics.constants.PhysicsConstantsCarAerodynamics + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true @@ -104,7 +111,7 @@ etl: _convert_: all global_parameters: ${etl.global_parameters} # Regardless of whether there are any additional global params processors, - # we always apply the default global params processing. This ensure that there are global params references present. + # We always apply the default global params processing. This ensure that there are global params references present. global_params_processors: - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params _partial_: true diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index 017f2dd..29a9944 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -48,20 +48,19 @@ etl: # HLPW-specific non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true - pref: 176.352 # HLPW reference pressure (Pa) - tref: 518.67 # HLPW reference temperature (K) + physical_constants: + _target_: examples.external_aerodynamics.constants.PhysicsConstantsHLPW - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true volume_preprocessing: volume_processors: - # HLPW-specific volume non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true - pref: 176.352 # HLPW reference pressure (Pa) - uref: 2679.505 # HLPW reference velocity (m/s) - tref: 518.67 # HLPW reference temperature (K) + physical_constants: + _target_: examples.external_aerodynamics.constants.PhysicsConstantsHLPW + - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true diff --git a/examples/external_aerodynamics/constants.py b/examples/external_aerodynamics/constants.py index 3e7595c..c08735d 100644 --- a/examples/external_aerodynamics/constants.py +++ b/examples/external_aerodynamics/constants.py @@ -23,13 +23,22 @@ @dataclass(frozen=True) -class PhysicsConstants: - """Physical constants used in the simulation.""" +class PhysicsConstantsCarAerodynamics: + """Physical constants used in the simulation in DriveAerML.""" AIR_DENSITY: float = 1.205 # kg/m³ STREAM_VELOCITY: float = 30.00 # m/s +@dataclass(frozen=True) +class PhysicsConstantsHLPW: + """Physical constants used in the simulation for HLPW dataset.""" + + PREF: float = 176.352 # HLPW reference pressure + UREF: float = 2679.505 # HLPW reference velocity + TREF: float = 518.67 # HLPW reference temperature + + class ModelType(str, Enum): """Types of models that can be processed.""" diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index c74836c..9865422 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -35,10 +35,10 @@ def default_global_params_processing_for_external_aerodynamics( global_parameters: dict, ) -> ExternalAerodynamicsExtractedDataInMemory: """Default global parameters processing for External Aerodynamics. - + Extracts and flattens global parameter references from config into a 1D numpy array. Handles both vector and scalar parameter types. - + Args: data: Container with simulation data and metadata global_parameters: Dict from config with structure: @@ -48,21 +48,19 @@ def default_global_params_processing_for_external_aerodynamics( "reference": value or list } } - + Returns: Updated data with global_params_reference set in metadata """ # Build dictionaries for types and reference values global_params_types = { - name: params["type"] - for name, params in global_parameters.items() + name: params["type"] for name, params in global_parameters.items() } global_params_reference_dict = { - name: params["reference"] - for name, params in global_parameters.items() + name: params["reference"] for name, params in global_parameters.items() } - + # Arrange global parameters reference in a list based on the type of the parameter global_params_reference_list = [] for name, param_type in global_params_types.items(): @@ -75,12 +73,12 @@ def default_global_params_processing_for_external_aerodynamics( f"Global parameter '{name}' has unsupported type '{param_type}'. " f"Must be 'vector' or 'scalar'." ) - + # Convert to numpy array and store in metadata data.metadata.global_params_reference = np.array( global_params_reference_list, dtype=np.float32 ) - + return data @@ -89,17 +87,17 @@ def process_global_params( global_parameters: dict, ) -> ExternalAerodynamicsExtractedDataInMemory: """Base processor for global parameters - to be overridden for specific datasets. - + This is a placeholder that should be replaced by dataset-specific implementations (e.g., process_global_params_hlpw). - + By default, sets global_params_values equal to global_params_reference, assuming simulation conditions match reference conditions. - + Args: data: Container with simulation data and metadata global_parameters: Dict from config with parameter definitions - + Returns: Updated data with global_params_values set in metadata """ @@ -109,33 +107,34 @@ def process_global_params( ) raise ValueError("global_params_reference are absent in the configuration") return data - + # Default behavior: assume simulation values match reference data.metadata.global_params_values = data.metadata.global_params_reference.copy() - + return data # ============================================================================ # Case-Specific Processors # ============================================================================ -# These functions demonstrate how to extract global_params_values from +# These functions demonstrate how to extract global_params_values from # simulation data for specific datasets. Replace process_global_params above # with these in your config for case-specific processing. + def process_global_params_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, global_parameters: dict, ) -> ExternalAerodynamicsExtractedDataInMemory: """Extract global parameters from HLPW simulation data. - + For HLPW, typically: - AoA (Angle of Attack) varies per simulation and can be extracted from filename - + Args: data: Container with simulation data and metadata global_parameters: Dict from config with parameter definitions - + Returns: Updated data with global_params_values extracted from simulation """ @@ -145,34 +144,32 @@ def process_global_params_hlpw( ) raise ValueError("global_params_reference are absent in the configuration") return data - + # Build a dict of extracted values keyed by parameter name extracted_values = {} - + # Extract AoA from filename (e.g., "geo_LHC001_AoA_16" -> 16.0) filename = data.metadata.filename if "AoA_" in filename: - # Extract string after "AoA_" - # Example: "geo_LHC001_AoA_16" -> "16" - # Example: "geo_LHC001_AoA_16_something" -> "16" - after_aoa = filename.split("AoA_")[1] - # Take everything up to next underscore or end of string - aoa_str = after_aoa.split("_")[0] if "_" in after_aoa else after_aoa - aoa = float(aoa_str) - extracted_values["AoA"] = aoa - logger.info(f"Extracted AoA={aoa} from filename: {filename}") + # Extract string after "AoA_" + # Example: "geo_LHC001_AoA_16" -> "16" + # Example: "geo_LHC001_AoA_16_something" -> "16" + after_aoa = filename.split("AoA_")[1] + # Take everything up to next underscore or end of string + aoa_str = after_aoa.split("_")[0] if "_" in after_aoa else after_aoa + aoa = float(aoa_str) + extracted_values["AoA"] = aoa + logger.info(f"Extracted AoA={aoa} from filename: {filename}") else: # Fallback to reference if not in filename - raise ValueError( - f"AoA pattern not found in filename '{filename}'." - ) - + raise ValueError(f"AoA pattern not found in filename '{filename}'.") + # Build the flattened array using the same logic as reference processing global_params_values_list = [] for name, params in global_parameters.items(): param_type = params["type"] value = extracted_values.get(name, params["reference"]) - + if param_type == "vector": global_params_values_list.extend(value) elif param_type == "scalar": @@ -182,9 +179,9 @@ def process_global_params_hlpw( f"Global parameter '{name}' has unsupported type '{param_type}'. " f"Must be 'vector' or 'scalar'." ) - + data.metadata.global_params_values = np.array( global_params_values_list, dtype=np.float32 ) - - return data \ No newline at end of file + + return data diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 16de675..6b92354 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -20,7 +20,10 @@ import numpy as np -from examples.external_aerodynamics.constants import PhysicsConstants +from examples.external_aerodynamics.constants import ( + PhysicsConstantsCarAerodynamics, + PhysicsConstantsHLPW, +) from examples.external_aerodynamics.external_aero_utils import to_float32 from examples.external_aerodynamics.external_aero_validation_utils import ( check_field_statistics, @@ -65,35 +68,35 @@ def default_surface_processing_for_external_aerodynamics_hlpw( ) -> ExternalAerodynamicsExtractedDataInMemory: """ Default surface processing for HLPW dataset. - + Uses the N_BF (Normal Boundary Faces) field for computing normals and areas, which is more accurate than computing them separately with PyVista. - - Important: Converts point data to cell data before processing, as HLPW + + Important: Converts point data to cell data before processing, as HLPW data may be stored at vertices rather than cell centers. - + Args: data: External aerodynamics data with surface polydata surface_variables: List of variable names to extract from surface data nbf_field_name: Name of the area-weighted normal field (default: "N_BF") - + Returns: Data with surface fields, mesh centers, normals, and areas extracted """ - + # Convert point data to cell data (important for HLPW!) # Data might be stored at vertices, need to move to cell centers data.surface_polydata = data.surface_polydata.point_data_to_cell_data() - + # Extract surface fields (pressure, wall shear stress, etc.) cell_data = (data.surface_polydata.cell_data[k] for k in surface_variables) data.surface_fields = np.concatenate( [d if d.ndim > 1 else d[:, np.newaxis] for d in cell_data], axis=-1 ) - + # Extract mesh centers data.surface_mesh_centers = np.array(data.surface_polydata.cell_centers().points) - + # Check if N_BF field exists - REQUIRED for HLPW if nbf_field_name not in data.surface_polydata.cell_data: logger.error( @@ -104,15 +107,15 @@ def default_surface_processing_for_external_aerodynamics_hlpw( f"Required field '{nbf_field_name}' not found in surface data. " f"HLPW processing requires N_BF field for accurate normal and area computation." ) - + # Use N_BF (area-weighted normal vectors) - HLPW-specific surface_normals_area = np.array( data.surface_polydata.cell_data[nbf_field_name] ).astype(np.float32) - + # Compute areas as magnitude of N_BF data.surface_areas = np.linalg.norm(surface_normals_area, axis=1).astype(np.float32) - + # Compute unit normals by normalizing N_BF data.surface_normals = surface_normals_area / np.reshape( data.surface_areas, (-1, 1) @@ -180,7 +183,7 @@ def filter_invalid_surface_cells( logger.info( f"Filtered {n_filtered} invalid surface cells " - f"({n_filtered/n_total*100:.2f}% of {n_total} total cells):" + f"({n_filtered / n_total * 100:.2f}% of {n_total} total cells):" ) logger.info(f" - {n_area_filtered} cells with area <= {tolerance}") logger.info(f" - {n_normal_filtered} cells with normal L2-norm <= {tolerance}") @@ -215,10 +218,15 @@ def normalize_surface_normals( def non_dimensionalize_surface_fields( data: ExternalAerodynamicsExtractedDataInMemory, - air_density: float = PhysicsConstants.AIR_DENSITY, - stream_velocity: float = PhysicsConstants.STREAM_VELOCITY, + physics_constants: PhysicsConstantsCarAerodynamics, ) -> ExternalAerodynamicsExtractedDataInMemory: - """Non-dimensionalize surface fields.""" + """ + Non-dimensionalize surface fields using PhysicsConstantsCarAerodynamics. + Note: Both DriveAerML and AhmedML use the same non-dimensional constants + """ + + air_density = physics_constants.AIR_DENSITY + stream_velocity = physics_constants.STREAM_VELOCITY if data.surface_fields.shape[0] == 0: logger.error(f"Surface fields are empty: {data.surface_fields}") @@ -241,23 +249,15 @@ def non_dimensionalize_surface_fields( def non_dimensionalize_surface_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - pref: float = 176.352, # HLPW reference pressure (Pa) - tref: float = 518.67, + physics_constants: PhysicsConstantsHLPW, ) -> ExternalAerodynamicsExtractedDataInMemory: """ - Non-dimensionalize surface fields using HLPW reference values. - - This follows the HLPW convention: - - Both pressure and WSS fields: divided by PREF - - Args: - data: External aerodynamics data with surface fields - PREF: Reference pressure for non-dimensionalization - - Returns: - Data with non-dimensionalized surface fields + Non-dimensionalize surface fields using PhysicsConstantsHLPW. """ - + + pref = physics_constants.PREF + tref = physics_constants.TREF + if data.surface_fields is None or len(data.surface_fields) == 0: logger.error(f"Surface fields are empty: {data.surface_fields}") return data @@ -266,7 +266,7 @@ def non_dimensionalize_surface_fields_hlpw( # Non-dimensionalize pressure and shear stress by PREF data.surface_fields[1:] /= pref - + return data diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index 3edfb0b..ea47c2f 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -19,7 +19,11 @@ import numpy as np -from examples.external_aerodynamics.constants import PhysicsConstants +from examples.external_aerodynamics.constants import ( + PhysicsConstantsCarAerodynamics, + PhysicsConstantsHLPW, +) + from examples.external_aerodynamics.external_aero_utils import ( get_volume_data, to_float32, @@ -110,7 +114,7 @@ def filter_volume_invalid_cells( logger.info( f"Filtered {n_filtered} invalid volume cells " - f"({n_filtered/n_total*100:.2f}% of {n_total} total cells):" + f"({n_filtered / n_total * 100:.2f}% of {n_total} total cells):" ) logger.info(f" - {n_coords_filtered} cells with NaN in coordinates") logger.info(f" - {n_fields_filtered} cells with NaN/inf in fields") @@ -125,11 +129,13 @@ def filter_volume_invalid_cells( def non_dimensionalize_volume_fields( data: ExternalAerodynamicsExtractedDataInMemory, - air_density: float = PhysicsConstants.AIR_DENSITY, - stream_velocity: float = PhysicsConstants.STREAM_VELOCITY, + physics_constants: PhysicsConstantsCarAerodynamics, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" + air_density = physics_constants.AIR_DENSITY + stream_velocity = physics_constants.STREAM_VELOCITY + if data.volume_fields.shape[0] == 0: logger.error(f"Volume fields are empty: {data.volume_fields}") return data @@ -155,14 +161,17 @@ def non_dimensionalize_volume_fields( return data + def non_dimensionalize_volume_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - pref: float = 176.352, - tref: float = 518.67, - uref: float = 2679.505, + physics_constants: PhysicsConstantsHLPW, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" + pref = physics_constants.PREF + tref = physics_constants.TREF + uref = physics_constants.UREF + # Pressure data.volume_fields[:, :1] = data.volume_fields[:, :1] / pref @@ -171,7 +180,7 @@ def non_dimensionalize_volume_fields_hlpw( # Velocity data.volume_fields[:, 2:] = data.volume_fields[:, 2:] / uref - + return data From fe19370ca530b7eaf7403aded544420b49723e90 Mon Sep 17 00:00:00 2001 From: snidhan Date: Tue, 25 Nov 2025 19:57:19 -0800 Subject: [PATCH 07/23] (chore) remove VSCode settings file --- .vscode/settings.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 56d6da2..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "workbench.colorCustomizations": { - "activityBar.activeBackground": "#93e6fc", - "activityBar.background": "#93e6fc", - "activityBar.foreground": "#15202b", - "activityBar.inactiveForeground": "#15202b99", - "activityBarBadge.background": "#fa45d4", - "activityBarBadge.foreground": "#15202b", - "commandCenter.border": "#15202b99", - "sash.hoverBorder": "#93e6fc", - "statusBar.background": "#61dafb", - "statusBar.foreground": "#15202b", - "statusBarItem.hoverBackground": "#2fcefa", - "statusBarItem.remoteBackground": "#61dafb", - "statusBarItem.remoteForeground": "#15202b", - "titleBar.activeBackground": "#61dafb", - "titleBar.activeForeground": "#15202b", - "titleBar.inactiveBackground": "#61dafb99", - "titleBar.inactiveForeground": "#15202b99" - }, - "peacock.remoteColor": "#61dafb" -} \ No newline at end of file From d380280179dea5f07c902b2e518afc128ddf0f6d Mon Sep 17 00:00:00 2001 From: snidhan Date: Tue, 25 Nov 2025 19:58:15 -0800 Subject: [PATCH 08/23] (chore) update external aerodynamics data processing and logging --- .../config/external_aero_etl_hlpw.yaml | 4 +++ .../config/variables/surface/hlpw.yaml | 2 +- .../external_aerodynamics/data_sources.py | 2 +- .../data_transformations.py | 29 ++++++++++++------- ...rnal_aero_global_params_data_processors.py | 2 -- .../external_aero_surface_data_processors.py | 7 ++--- .../external_aero_volume_data_processors.py | 3 -- 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index 29a9944..7dded5b 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -55,6 +55,8 @@ etl: _partial_: true volume_preprocessing: + _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsVolumeTransformation + _convert_: all volume_processors: - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true @@ -69,6 +71,8 @@ etl: _partial_: true global_params_preprocessing: + _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsGlobalParamsTransformation + _convert_: all global_parameters: ${etl.global_parameters} global_params_processors: - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params_hlpw diff --git a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml index 54fd09e..c4cc4aa 100644 --- a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml +++ b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml @@ -16,8 +16,8 @@ # limitations under the License. surface_variables: - PROJ(AVG(P)): scalar PROJ(AVG(T)): scalar + PROJ(AVG(P)): scalar AVG(TAU_WALL(0)): scalar AVG(TAU_WALL(1)): scalar AVG(TAU_WALL(2)): scalar \ No newline at end of file diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 43247c7..b2b5c87 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -276,7 +276,7 @@ def _write_zarr( ) self.logger.info(f"Successfully wrote field '{field}' with shape {array_info.data.shape}") else: - self.logger.warning(f"{array_info} is absent in the dataset") + self.logger.error(f"{field} is absent in the dataset") def should_skip(self, filename: str) -> bool: diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 7e67f14..03505c4 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -24,7 +24,7 @@ from physicsnemo_curator.etl.data_transformations import DataTransformation from physicsnemo_curator.etl.processing_config import ProcessingConfig -from .constants import PhysicsConstants +from .constants import PhysicsConstantsCarAerodynamics from .external_aero_geometry_data_processors import ( default_geometry_processing_for_external_aerodynamics, ) @@ -71,8 +71,8 @@ def transform( # Create minimal metadata numpy_metadata = ExternalAerodynamicsNumpyMetadata( filename=data.metadata.filename, - stream_velocity=data.metadata.stream_velocity, - air_density=data.metadata.air_density, + global_params_values=data.metadata.global_params_values, + global_params_reference=data.metadata.global_params_reference, ) return ExternalAerodynamicsNumpyDataInMemory( @@ -137,7 +137,7 @@ def __init__( self.surface_variables = surface_variables self.surface_processors = surface_processors - self.constants = PhysicsConstants() + self.constants = PhysicsConstantsCarAerodynamics() if surface_variables is None: self.logger.error("Surface variables are empty!") @@ -190,7 +190,7 @@ def __init__( self.surface_variables = surface_variables self.surface_processors = surface_processors self.nbf_field_name = nbf_field_name - self.constants = PhysicsConstants() + self.constants = PhysicsConstantsCarAerodynamics() if surface_variables is None: self.logger.error("Surface variables are empty!") @@ -238,7 +238,7 @@ def __init__( super().__init__(cfg) self.volume_variables = volume_variables self.volume_processors = volume_processors - self.constants = PhysicsConstants() + self.constants = PhysicsConstantsCarAerodynamics() self.logger = logging.getLogger(__name__) if volume_variables is None: @@ -276,6 +276,7 @@ def transform( class ExternalAerodynamicsGlobalParamsTransformation(DataTransformation): """Transforms global parameters values and references for External Aerodynamics model.""" + def __init__( self, cfg: ProcessingConfig, @@ -288,13 +289,15 @@ def __init__( self.logger = logging.getLogger(__name__) if global_parameters is None: - self.logger.error("No global_parameters provided. Please provide global parameters") + self.logger.error( + "No global_parameters provided. Please provide global parameters" + ) def transform( self, data: ExternalAerodynamicsExtractedDataInMemory ) -> ExternalAerodynamicsExtractedDataInMemory: """Transform global_params data for External Aerodynamics model. - + Processes global parameter references from config and extracts values from simulation data. """ @@ -373,7 +376,7 @@ def _prepare_array(self, array: np.ndarray) -> PreparedZarrArrayInfo: def _prepare_array_no_compression(self, array: np.ndarray) -> PreparedZarrArrayInfo: """Prepare small array for Zarr storage without compression. - + Used for small arrays like `global_params_reference` and `global_params_values` """ if array is None: @@ -411,8 +414,12 @@ def transform( stl_areas=self._prepare_array(data.stl_areas), metadata=data.metadata, # `global_params_values` and `global_params_reference` are saved without compression - global_params_values=self._prepare_array_no_compression(data.metadata.global_params_values), - global_params_reference=self._prepare_array_no_compression(data.metadata.global_params_reference), + global_params_values=self._prepare_array_no_compression( + data.metadata.global_params_values + ), + global_params_reference=self._prepare_array_no_compression( + data.metadata.global_params_reference + ), surface_mesh_centers=self._prepare_array(data.surface_mesh_centers), surface_normals=self._prepare_array(data.surface_normals), surface_areas=self._prepare_array(data.surface_areas), diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index 9865422..616173b 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -106,7 +106,6 @@ def process_global_params( "global_params_reference not set. Skipping global_params_values processing." ) raise ValueError("global_params_reference are absent in the configuration") - return data # Default behavior: assume simulation values match reference data.metadata.global_params_values = data.metadata.global_params_reference.copy() @@ -143,7 +142,6 @@ def process_global_params_hlpw( "global_params_reference not set. Skipping global_params_values processing." ) raise ValueError("global_params_reference are absent in the configuration") - return data # Build a dict of extracted values keyed by parameter name extracted_values = {} diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 6b92354..1282a4c 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -240,9 +240,6 @@ def non_dimensionalize_surface_fields( # Non-dimensionalize surface fields data.surface_fields = data.surface_fields / (air_density * stream_velocity**2.0) - # Update metadata - data.metadata.air_density = air_density - data.metadata.stream_velocity = stream_velocity return data @@ -262,10 +259,10 @@ def non_dimensionalize_surface_fields_hlpw( logger.error(f"Surface fields are empty: {data.surface_fields}") return data # Non-dimensionalize temperature by TREF - data.surface_fields[0:1] /= tref + data.surface_fields[:, 0:1] /= tref # Non-dimensionalize pressure and shear stress by PREF - data.surface_fields[1:] /= pref + data.surface_fields[:, 1:] /= pref return data diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index ea47c2f..0957ec2 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -155,9 +155,6 @@ def non_dimensionalize_volume_fields( stream_velocity * length_scale ) - # Update metadata - data.metadata.air_density = air_density - data.metadata.stream_velocity = stream_velocity return data From 6e6712aea9d50ca3ea29bd0024c2b6a88ba28d4f Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 15:53:01 -0800 Subject: [PATCH 09/23] (chore) set data_processors inputs to previous state --- .../config/external_aero_etl_drivaerml.yaml | 8 ++++---- examples/external_aerodynamics/data_sources.py | 17 +++++++++++------ .../external_aero_surface_data_processors.py | 13 ++++--------- .../external_aero_volume_data_processors.py | 15 +++++---------- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index bebe075..ea77900 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -81,8 +81,8 @@ etl: - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields _partial_: true - physical_constants: - _target_: examples.external_aerodynamics.constants.PhysicsConstantsCarAerodynamics + air_density: 1.205 + stream_velocity: 30.00 - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -97,8 +97,8 @@ etl: volume_processors: - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields _partial_: true - physical_constants: - _target_: examples.external_aerodynamics.constants.PhysicsConstantsCarAerodynamics + air_density: 1.205 + stream_velocity: 30.00 - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index b2b5c87..d997502 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -238,16 +238,20 @@ def _write_zarr( # Remove 'global_params_values' and `global_params_reference` that are written # as part of the data itself metadata_dict = asdict(data.metadata) - metadata_dict.pop('global_params_values', None) - metadata_dict.pop('global_params_reference', None) + metadata_dict.pop("global_params_values", None) + metadata_dict.pop("global_params_reference", None) root.attrs.update(metadata_dict) # Write required arrays for field in ["stl_coordinates", "stl_centers", "stl_faces", "stl_areas"]: array_info = getattr(data, field) - self.logger.info(f"Writing required field '{field}': array_info={array_info is not None}, type={type(array_info)}") + self.logger.info( + f"Writing required field '{field}': array_info={array_info is not None}, type={type(array_info)}" + ) if array_info is None: - raise ValueError(f"Required field '{field}' is None - cannot write zarr dataset") + raise ValueError( + f"Required field '{field}' is None - cannot write zarr dataset" + ) root.create_dataset( field, data=array_info.data, @@ -274,11 +278,12 @@ def _write_zarr( chunks=array_info.chunks, compressor=array_info.compressor, ) - self.logger.info(f"Successfully wrote field '{field}' with shape {array_info.data.shape}") + self.logger.info( + f"Successfully wrote field '{field}' with shape {array_info.data.shape}" + ) else: self.logger.error(f"{field} is absent in the dataset") - def should_skip(self, filename: str) -> bool: """Checks whether the file should be skipped. diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 1282a4c..309d746 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -218,16 +218,14 @@ def normalize_surface_normals( def non_dimensionalize_surface_fields( data: ExternalAerodynamicsExtractedDataInMemory, - physics_constants: PhysicsConstantsCarAerodynamics, + air_density: PhysicsConstantsCarAerodynamics.AIR_DENSITY, + stream_velocity: PhysicsConstantsCarAerodynamics.STREAM_VELOCITY, ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using PhysicsConstantsCarAerodynamics. Note: Both DriveAerML and AhmedML use the same non-dimensional constants """ - air_density = physics_constants.AIR_DENSITY - stream_velocity = physics_constants.STREAM_VELOCITY - if data.surface_fields.shape[0] == 0: logger.error(f"Surface fields are empty: {data.surface_fields}") return data @@ -240,21 +238,18 @@ def non_dimensionalize_surface_fields( # Non-dimensionalize surface fields data.surface_fields = data.surface_fields / (air_density * stream_velocity**2.0) - return data def non_dimensionalize_surface_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - physics_constants: PhysicsConstantsHLPW, + pref: PhysicsConstantsHLPW.PREF, + tref: PhysicsConstantsHLPW.TREF, ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using PhysicsConstantsHLPW. """ - pref = physics_constants.PREF - tref = physics_constants.TREF - if data.surface_fields is None or len(data.surface_fields) == 0: logger.error(f"Surface fields are empty: {data.surface_fields}") return data diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index 0957ec2..23292f7 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -129,13 +129,11 @@ def filter_volume_invalid_cells( def non_dimensionalize_volume_fields( data: ExternalAerodynamicsExtractedDataInMemory, - physics_constants: PhysicsConstantsCarAerodynamics, + air_density: PhysicsConstantsCarAerodynamics.AIR_DENSITY, + stream_velocity: PhysicsConstantsCarAerodynamics.STREAM_VELOCITY, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" - air_density = physics_constants.AIR_DENSITY - stream_velocity = physics_constants.STREAM_VELOCITY - if data.volume_fields.shape[0] == 0: logger.error(f"Volume fields are empty: {data.volume_fields}") return data @@ -155,20 +153,17 @@ def non_dimensionalize_volume_fields( stream_velocity * length_scale ) - return data def non_dimensionalize_volume_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - physics_constants: PhysicsConstantsHLPW, + pref: PhysicsConstantsHLPW.PREF, + tref: PhysicsConstantsHLPW.TREF, + uref: PhysicsConstantsHLPW.UREF, ) -> ExternalAerodynamicsExtractedDataInMemory: """Non-dimensionalize volume fields.""" - pref = physics_constants.PREF - tref = physics_constants.TREF - uref = physics_constants.UREF - # Pressure data.volume_fields[:, :1] = data.volume_fields[:, :1] / pref From 2eb368ae50ecddfa14763176e5dfd80e0c2bb49f Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 15:59:38 -0800 Subject: [PATCH 10/23] (chore) remove whitespace --- .../config/external_aero_etl_drivaerml.yaml | 4 ---- .../config/external_aero_etl_hlpw.yaml | 13 +++++-------- .../external_aero_surface_data_processors.py | 2 +- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index fa035dc..ff7990b 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -78,12 +78,10 @@ etl: - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.normalize_surface_normals _partial_: true - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields _partial_: true air_density: 1.205 stream_velocity: 30.00 - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -99,10 +97,8 @@ etl: _partial_: true air_density: 1.205 stream_velocity: 30.00 - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index 7dded5b..35f161e 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -48,9 +48,8 @@ etl: # HLPW-specific non-dimensionalization - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true - physical_constants: - _target_: examples.external_aerodynamics.constants.PhysicsConstantsHLPW - + pref: 176.352 + tref: 518.67 - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -60,13 +59,11 @@ etl: volume_processors: - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true - physical_constants: - _target_: examples.external_aerodynamics.constants.PhysicsConstantsHLPW - - + pref: 176.352 + tref: 518.67 + uref: 2679.505 - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data _partial_: true diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 394ce60..912c0f4 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -68,7 +68,7 @@ def default_surface_processing_for_external_aerodynamics_hlpw( Default surface processing for HLPW dataset. Uses the N_BF (Normal Boundary Faces) field for computing normals and areas, - which is more accurate than computing them separately with PyVista. + which is faster than computing them separately with PyVista for HLPW dataset. Important: Converts point data to cell data before processing, as HLPW data may be stored at vertices rather than cell centers. From 5751d6bf97cca0010bfc7769ff8be21e0d6afd78 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 16:03:54 -0800 Subject: [PATCH 11/23] (chore) refactor drivaerml and hlpw yaml files to comply with new location of run_etl.py --- .../config/external_aero_etl_drivaerml.yaml | 18 ++++++++--------- .../config/external_aero_etl_hlpw.yaml | 20 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index ff7990b..c7779cc 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -76,13 +76,13 @@ etl: reduction: 0.0 # 0 means no decimation. preserve_topology: false - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.normalize_surface_normals + - _target_: external_aero_surface_data_processors.normalize_surface_normals _partial_: true - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields + - _target_: external_aero_surface_data_processors.non_dimensionalize_surface_fields _partial_: true air_density: 1.205 stream_velocity: 30.00 - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 + - _target_: external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true volume_preprocessing: @@ -95,21 +95,21 @@ etl: volume_processors: - _target_: external_aero_volume_data_processors.non_dimensionalize_volume_fields _partial_: true - air_density: 1.205 - stream_velocity: 30.00 - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 + air_density: 1.205 # kg/m³ + stream_velocity: 30.00 # m/s + - _target_: external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data + - _target_: external_aero_volume_data_processors.shuffle_volume_data _partial_: true global_params_preprocessing: - _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsGlobalParamsTransformation + _target_: data_transformations.ExternalAerodynamicsGlobalParamsTransformation _convert_: all global_parameters: ${etl.global_parameters} # Regardless of whether there are any additional global params processors, # We always apply the default global params processing. This ensure that there are global params references present. global_params_processors: - - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params + - _target_: external_aero_global_params_data_processors.process_global_params _partial_: true write_ready_transformation: ${override_transformations.write_ready_transformation} diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index 35f161e..cbbc5c6 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -32,12 +32,12 @@ etl: transformations: surface_preprocessing: # Use HLPW-specific transformation that handles N_BF field - _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsSurfaceTransformationHLPW + _target_: data_transformations.ExternalAerodynamicsSurfaceTransformationHLPW _convert_: all nbf_field_name: "N_BF" surface_processors: - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.decimate_mesh + - _target_: external_aero_surface_data_processors.decimate_mesh _partial_: true algo: decimate_pro reduction: 0.0 @@ -46,31 +46,31 @@ etl: # Note: normalize_surface_normals not needed - already done via N_BF normalization # HLPW-specific non-dimensionalization - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw + - _target_: external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true pref: 176.352 tref: 518.67 - - _target_: examples.external_aerodynamics.external_aero_surface_data_processors.update_surface_data_to_float32 + - _target_: external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true volume_preprocessing: - _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsVolumeTransformation + _target_: data_transformations.ExternalAerodynamicsVolumeTransformation _convert_: all volume_processors: - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw + - _target_: external_aero_volume_data_processors.non_dimensionalize_volume_fields_hlpw _partial_: true pref: 176.352 tref: 518.67 uref: 2679.505 - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.update_volume_data_to_float32 + - _target_: external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - - _target_: examples.external_aerodynamics.external_aero_volume_data_processors.shuffle_volume_data + - _target_: external_aero_volume_data_processors.shuffle_volume_data _partial_: true global_params_preprocessing: - _target_: examples.external_aerodynamics.data_transformations.ExternalAerodynamicsGlobalParamsTransformation + _target_: data_transformations.ExternalAerodynamicsGlobalParamsTransformation _convert_: all global_parameters: ${etl.global_parameters} global_params_processors: - - _target_: examples.external_aerodynamics.external_aero_global_params_data_processors.process_global_params_hlpw + - _target_: external_aero_global_params_data_processors.process_global_params_hlpw _partial_: true \ No newline at end of file From ea52c3aa23c86e2eba282f5b4b40960b593f28d9 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 16:06:30 -0800 Subject: [PATCH 12/23] (chore) further clean up drivaerml yaml --- .../config/external_aero_etl_drivaerml.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index c7779cc..304cab8 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -75,13 +75,12 @@ etl: algo: decimate_pro # can be one of {decimate_pro, decimate} reduction: 0.0 # 0 means no decimation. preserve_topology: false - - _target_: external_aero_surface_data_processors.normalize_surface_normals _partial_: true - _target_: external_aero_surface_data_processors.non_dimensionalize_surface_fields _partial_: true - air_density: 1.205 - stream_velocity: 30.00 + air_density: 1.205 # kg/m³ + stream_velocity: 30.00 # m/s - _target_: external_aero_surface_data_processors.update_surface_data_to_float32 _partial_: true @@ -95,8 +94,6 @@ etl: volume_processors: - _target_: external_aero_volume_data_processors.non_dimensionalize_volume_fields _partial_: true - air_density: 1.205 # kg/m³ - stream_velocity: 30.00 # m/s - _target_: external_aero_volume_data_processors.update_volume_data_to_float32 _partial_: true - _target_: external_aero_volume_data_processors.shuffle_volume_data From 240dd21230ef85673f24cfccaab664a24336fbd7 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 16:07:11 -0800 Subject: [PATCH 13/23] (chore) minor comment fix --- examples/external_aerodynamics/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/external_aerodynamics/constants.py b/examples/external_aerodynamics/constants.py index c08735d..de3ac7b 100644 --- a/examples/external_aerodynamics/constants.py +++ b/examples/external_aerodynamics/constants.py @@ -24,7 +24,7 @@ @dataclass(frozen=True) class PhysicsConstantsCarAerodynamics: - """Physical constants used in the simulation in DriveAerML.""" + """Physical constants used in the simulation in DriveAerML or AhmedML""" AIR_DENSITY: float = 1.205 # kg/m³ STREAM_VELOCITY: float = 30.00 # m/s From 552da5b6dcfe809f5e0b267a2279350a41e89d24 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 17:01:32 -0800 Subject: [PATCH 14/23] (chore) update imports in data processors --- .../external_aero_surface_data_processors.py | 7 +++---- .../external_aero_volume_data_processors.py | 11 +++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 912c0f4..4dd62ef 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -19,13 +19,12 @@ from typing import Optional import numpy as np - -from examples.external_aerodynamics.constants import ( +from constants import ( PhysicsConstantsCarAerodynamics, PhysicsConstantsHLPW, ) -from examples.external_aerodynamics.external_aero_utils import to_float32 -from examples.external_aerodynamics.external_aero_validation_utils import ( +from external_aero_utils import to_float32 +from external_aero_validation_utils import ( check_field_statistics, check_surface_physics_bounds, ) diff --git a/examples/external_aerodynamics/external_aero_volume_data_processors.py b/examples/external_aerodynamics/external_aero_volume_data_processors.py index 2003a50..9c71aa7 100644 --- a/examples/external_aerodynamics/external_aero_volume_data_processors.py +++ b/examples/external_aerodynamics/external_aero_volume_data_processors.py @@ -18,17 +18,12 @@ from typing import Optional import numpy as np - -from examples.external_aerodynamics.constants import ( +from constants import ( PhysicsConstantsCarAerodynamics, PhysicsConstantsHLPW, ) - -from examples.external_aerodynamics.external_aero_utils import ( - get_volume_data, - to_float32, -) -from examples.external_aerodynamics.external_aero_validation_utils import ( +from external_aero_utils import get_volume_data, to_float32 +from external_aero_validation_utils import ( check_field_statistics, check_volume_physics_bounds, ) From 437dc883e25fdd389b09cdb98b5b77ac086ded0a Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 17:15:35 -0800 Subject: [PATCH 15/23] (chore) fix import in data_transformations --- .../data_transformations.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 5343348..0e01660 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -19,27 +19,23 @@ from typing import Callable, Optional import numpy as np -from numcodecs import Blosc - -from physicsnemo_curator.etl.data_transformations import DataTransformation -from physicsnemo_curator.etl.processing_config import ProcessingConfig - -from .constants import PhysicsConstantsCarAerodynamics -from .external_aero_geometry_data_processors import ( +from constants import PhysicsConstantsCarAerodynamics +from external_aero_geometry_data_processors import ( default_geometry_processing_for_external_aerodynamics, ) -from .external_aero_global_params_data_processors import ( +from external_aero_global_params_data_processors import ( default_global_params_processing_for_external_aerodynamics, ) -from .external_aero_surface_data_processors import ( +from external_aero_surface_data_processors import ( default_surface_processing_for_external_aerodynamics, default_surface_processing_for_external_aerodynamics_hlpw, ) -from .external_aero_utils import to_float32 -from .external_aero_volume_data_processors import ( +from external_aero_utils import to_float32 +from external_aero_volume_data_processors import ( default_volume_processing_for_external_aerodynamics, ) -from .schemas import ( +from numcodecs import Blosc +from schemas import ( ExternalAerodynamicsExtractedDataInMemory, ExternalAerodynamicsNumpyDataInMemory, ExternalAerodynamicsNumpyMetadata, From 1a0619e772125080a20fc6198aaefc4eef2c43a8 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 18:29:38 -0800 Subject: [PATCH 16/23] (feat) improve metadata processing and global_params processing --- examples/external_aerodynamics/constants.py | 24 ++ .../external_aerodynamics/data_analysis.ipynb | 271 +++++++++++------- .../external_aerodynamics/data_sources.py | 11 +- .../data_transformations.py | 8 +- .../external_aero_geometry_data_processors.py | 2 +- ...rnal_aero_global_params_data_processors.py | 23 +- examples/external_aerodynamics/paths.py | 3 +- examples/external_aerodynamics/schemas.py | 20 +- 8 files changed, 221 insertions(+), 141 deletions(-) diff --git a/examples/external_aerodynamics/constants.py b/examples/external_aerodynamics/constants.py index de3ac7b..7f905eb 100644 --- a/examples/external_aerodynamics/constants.py +++ b/examples/external_aerodynamics/constants.py @@ -62,3 +62,27 @@ class DefaultVariables: SURFACE: tuple[str, ...] = ("pMean", "wallShearStress") VOLUME: tuple[str, ...] = ("UMean", "pMean") + + +def get_physics_constants(kind: DatasetKind) -> dict[str, float]: + """Get physics constants dict based on dataset kind. Add a branch + to the if-elif pipeline below to populate metadata with values + used for non-dimensionalization + + Args: + kind: The dataset kind (from config etl.common.kind) + + Returns: + Dictionary of physics constant names to values. + + Raises: + ValueError: If dataset kind is unknown. + """ + if kind in (DatasetKind.DRIVAERML, DatasetKind.AHMEDML, DatasetKind.DRIVESIM): + c = PhysicsConstantsCarAerodynamics() + return {"air_density": c.AIR_DENSITY, "stream_velocity": c.STREAM_VELOCITY} + elif kind == DatasetKind.HLPW: + c = PhysicsConstantsHLPW() + return {"pref": c.PREF, "uref": c.UREF, "tref": c.TREF} + else: + raise ValueError(f"Unknown dataset kind: {kind}") diff --git a/examples/external_aerodynamics/data_analysis.ipynb b/examples/external_aerodynamics/data_analysis.ipynb index 19ac185..9cf016e 100644 --- a/examples/external_aerodynamics/data_analysis.ipynb +++ b/examples/external_aerodynamics/data_analysis.ipynb @@ -56,19 +56,15 @@ "import numpy as np\n", "\n", "\n", - "def compute_feature_descriptor(\n", - " areas_zarr,\n", - " coords_zarr,\n", - " centers_zarr\n", - " ):\n", + "def compute_feature_descriptor(areas_zarr, coords_zarr, centers_zarr):\n", " \"\"\"\n", " Compute a statistical feature descriptor vector using CuPy.\n", - " \n", + "\n", " Parameters:\n", " areas_zarr: Zarr array of triangle areas\n", " coords_zarr: Zarr array of shape (N, 3) with all surface point coordinates\n", " centers_zarr: Zarr array of shape (M, 3) with triangle center coordinates\n", - " \n", + "\n", " Returns:\n", " descriptor (np.ndarray): shape (10,)\n", " \"\"\"\n", @@ -95,14 +91,19 @@ "\n", " pca_eigvals = eigvals_norm.get() # Convert to NumPy\n", "\n", - "\n", " # Final descriptor vector\n", - " descriptor = np.array([\n", - " surface_area_mean,\n", - " surface_area_std,\n", - " centroid_dist_mean, centroid_dist_std,\n", - " pca_eigvals[0], pca_eigvals[1], pca_eigvals[2],\n", - " ], dtype=np.float32)\n", + " descriptor = np.array(\n", + " [\n", + " surface_area_mean,\n", + " surface_area_std,\n", + " centroid_dist_mean,\n", + " centroid_dist_std,\n", + " pca_eigvals[0],\n", + " pca_eigvals[1],\n", + " pca_eigvals[2],\n", + " ],\n", + " dtype=np.float32,\n", + " )\n", "\n", " return descriptor" ] @@ -140,7 +141,7 @@ " n_neighbors=n_neighbors,\n", " min_dist=min_dist,\n", " random_state=42,\n", - " ) \n", + " )\n", " embedding_cp = umap_model.fit_transform(descriptors_cp)\n", "\n", " embedding_np = cp.asnumpy(embedding_cp)\n", @@ -168,7 +169,7 @@ "source": [ "clusterer = cuml.cluster.HDBSCAN(\n", " min_cluster_size=10,\n", - " metric='euclidean',\n", + " metric=\"euclidean\",\n", " prediction_data=True,\n", " cluster_selection_epsilon=1.5,\n", " allow_single_cluster=True,\n", @@ -206,35 +207,36 @@ "import zarr\n", "\n", "train_cluster_color_map = {\n", - " -1: 'blue',\n", - " 0: 'green',\n", - " 1: 'red',\n", + " -1: \"blue\",\n", + " 0: \"green\",\n", + " 1: \"red\",\n", "}\n", "test_cluster_color_map = {\n", - " -1: 'black',\n", - " 0: 'red',\n", - " 1: 'green',\n", - " 2: 'purple',\n", - " 3: 'orange',\n", - " 4: 'yellow',\n", - " 5: 'pink',\n", + " -1: \"black\",\n", + " 0: \"red\",\n", + " 1: \"green\",\n", + " 2: \"purple\",\n", + " 3: \"orange\",\n", + " 4: \"yellow\",\n", + " 5: \"pink\",\n", "}\n", "\n", + "\n", "def plot_umap_embeddings_with_probabilities(\n", - " training_dataset_embeddings,\n", - " training_dataset_name = \"Train dataset\",\n", - " test_dataset_embeddings = None,\n", - " test_dataset_name = \"Test dataset\",\n", - " train_dataset_cluster_labels = None,\n", - " test_dataset_cluster_labels = None,\n", - " train_dataset_probabilities = None,\n", - " test_dataset_probabilities = None,\n", - " title=\"UMAP Projection\", \n", - " outlier_threshold=0.5,\n", - " ):\n", + " training_dataset_embeddings,\n", + " training_dataset_name=\"Train dataset\",\n", + " test_dataset_embeddings=None,\n", + " test_dataset_name=\"Test dataset\",\n", + " train_dataset_cluster_labels=None,\n", + " test_dataset_cluster_labels=None,\n", + " train_dataset_probabilities=None,\n", + " test_dataset_probabilities=None,\n", + " title=\"UMAP Projection\",\n", + " outlier_threshold=0.5,\n", + "):\n", " \"\"\"\n", " Plot UMAP embeddings with probability-based outlier detection.\n", - " \n", + "\n", " Args:\n", " training_dataset_embeddings: Array of training dataset embeddings\n", " training_dataset_name: Name of the training dataset\n", @@ -248,10 +250,10 @@ " outlier_threshold: Probability threshold below which points are considered outliers\n", " \"\"\"\n", " plt.figure(figsize=(12, 10))\n", - " \n", + "\n", " # Define default colors for different datasets\n", - " train_dataset_color = 'blue'\n", - " test_dataset_color = 'red'\n", + " train_dataset_color = \"blue\"\n", + " test_dataset_color = \"red\"\n", "\n", " # Plot training dataset\n", " if train_dataset_cluster_labels is not None:\n", @@ -262,27 +264,46 @@ " training_dataset_embeddings[mask, 0],\n", " training_dataset_embeddings[mask, 1],\n", " c=[train_cluster_color_map[cluster_label]],\n", - " marker='o',\n", - " s=(30 + 70 * train_dataset_probabilities[mask]).astype(int), \n", - " edgecolor='k', alpha=0.8,\n", - " label=f'{training_dataset_name} (cluster {cluster_label})')\n", + " marker=\"o\",\n", + " s=(30 + 70 * train_dataset_probabilities[mask]).astype(int),\n", + " edgecolor=\"k\",\n", + " alpha=0.8,\n", + " label=f\"{training_dataset_name} (cluster {cluster_label})\",\n", + " )\n", " else:\n", " plt.scatter(\n", " training_dataset_embeddings[:, 0],\n", " training_dataset_embeddings[:, 1],\n", - " c=train_dataset_color, marker='o', s=80, \n", - " edgecolor='k', alpha=0.8, \n", - " label=f'{training_dataset_name}'\n", + " c=train_dataset_color,\n", + " marker=\"o\",\n", + " s=80,\n", + " edgecolor=\"k\",\n", + " alpha=0.8,\n", + " label=f\"{training_dataset_name}\",\n", " )\n", " # Add text annotations every 10 points\n", " for i, (x, y) in enumerate(training_dataset_embeddings):\n", " if i % 10 == 0:\n", - " plt.annotate(f'{i}', (x, y), \n", - " xytext=(5, 5), textcoords='offset points',\n", - " fontsize=8, fontweight='bold',\n", - " bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.8, edgecolor='black'),\n", - " arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0', \n", - " color=train_dataset_color, alpha=0.7))\n", + " plt.annotate(\n", + " f\"{i}\",\n", + " (x, y),\n", + " xytext=(5, 5),\n", + " textcoords=\"offset points\",\n", + " fontsize=8,\n", + " fontweight=\"bold\",\n", + " bbox=dict(\n", + " boxstyle=\"round,pad=0.2\",\n", + " facecolor=\"white\",\n", + " alpha=0.8,\n", + " edgecolor=\"black\",\n", + " ),\n", + " arrowprops=dict(\n", + " arrowstyle=\"->\",\n", + " connectionstyle=\"arc3,rad=0\",\n", + " color=train_dataset_color,\n", + " alpha=0.7,\n", + " ),\n", + " )\n", "\n", " # Plot test dataset\n", " if test_dataset_cluster_labels is not None:\n", @@ -292,104 +313,129 @@ " test_dataset_embeddings[:, 0],\n", " test_dataset_embeddings[:, 1],\n", " c=test_cluster_color_map[cluster_label],\n", - " marker='o',\n", - " s=(30 + 70 * test_dataset_probabilities).astype(int), \n", - " edgecolor='k', alpha=0.8, \n", - " label=f'{test_dataset_name} (cluster {cluster_label})'\n", - " ) \n", + " marker=\"o\",\n", + " s=(30 + 70 * test_dataset_probabilities).astype(int),\n", + " edgecolor=\"k\",\n", + " alpha=0.8,\n", + " label=f\"{test_dataset_name} (cluster {cluster_label})\",\n", + " )\n", "\n", " # Outlier detection\n", " mask = test_dataset_cluster_labels == cluster_label\n", " is_outlier = test_dataset_probabilities[mask] < outlier_threshold\n", " if is_outlier.sum() > 0:\n", " masked_outlier_points = test_dataset_embeddings[mask][is_outlier]\n", - " masked_test_dataset_probabilities = test_dataset_probabilities[mask][is_outlier]\n", + " masked_test_dataset_probabilities = test_dataset_probabilities[mask][\n", + " is_outlier\n", + " ]\n", " plt.scatter(\n", " masked_outlier_points[:, 0],\n", " masked_outlier_points[:, 1],\n", " c=test_cluster_color_map[-1],\n", - " marker='x',\n", - " s=(30 + 70 * masked_test_dataset_probabilities).astype(int), \n", - " linewidth=2, alpha=0.8, \n", - " label=f'{test_dataset_name} (outlier)'\n", + " marker=\"x\",\n", + " s=(30 + 70 * masked_test_dataset_probabilities).astype(int),\n", + " linewidth=2,\n", + " alpha=0.8,\n", + " label=f\"{test_dataset_name} (outlier)\",\n", " )\n", " if is_outlier.sum() < len(mask):\n", " masked_inlier_points = test_dataset_embeddings[mask][~is_outlier]\n", - " masked_inlier_probabilities = test_dataset_probabilities[mask][~is_outlier]\n", + " masked_inlier_probabilities = test_dataset_probabilities[mask][\n", + " ~is_outlier\n", + " ]\n", " plt.scatter(\n", " masked_inlier_points[:, 0],\n", " masked_inlier_points[:, 1],\n", " c=test_cluster_color_map[cluster_label],\n", - " marker='o', s=(30 + 70 * masked_inlier_probabilities).astype(int), \n", - " edgecolor='k', alpha=0.8, \n", - " label=f'{test_dataset_name} (cluster {cluster_label})'\n", + " marker=\"o\",\n", + " s=(30 + 70 * masked_inlier_probabilities).astype(int),\n", + " edgecolor=\"k\",\n", + " alpha=0.8,\n", + " label=f\"{test_dataset_name} (cluster {cluster_label})\",\n", " )\n", " else:\n", " plt.scatter(\n", " test_dataset_embeddings[:, 0],\n", " test_dataset_embeddings[:, 1],\n", - " c=test_dataset_color, marker='o', s=80, \n", - " edgecolor='k', alpha=0.8, \n", - " label=f'{test_dataset_name}'\n", + " c=test_dataset_color,\n", + " marker=\"o\",\n", + " s=80,\n", + " edgecolor=\"k\",\n", + " alpha=0.8,\n", + " label=f\"{test_dataset_name}\",\n", " )\n", "\n", " # Add text annotations every 10 points\n", " for i, (x, y) in enumerate(test_dataset_embeddings):\n", " if i % 10 == 0:\n", - " plt.annotate(f'{i}', (x, y), \n", - " xytext=(5, 5), textcoords='offset points',\n", - " fontsize=8, fontweight='bold',\n", - " bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.8, edgecolor='black'),\n", - " arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0', \n", - " color=test_dataset_color, alpha=0.7))\n", + " plt.annotate(\n", + " f\"{i}\",\n", + " (x, y),\n", + " xytext=(5, 5),\n", + " textcoords=\"offset points\",\n", + " fontsize=8,\n", + " fontweight=\"bold\",\n", + " bbox=dict(\n", + " boxstyle=\"round,pad=0.2\",\n", + " facecolor=\"white\",\n", + " alpha=0.8,\n", + " edgecolor=\"black\",\n", + " ),\n", + " arrowprops=dict(\n", + " arrowstyle=\"->\",\n", + " connectionstyle=\"arc3,rad=0\",\n", + " color=test_dataset_color,\n", + " alpha=0.7,\n", + " ),\n", + " )\n", "\n", " plt.title(title)\n", " plt.xlabel(\"UMAP 1\")\n", " plt.ylabel(\"UMAP 2\")\n", " plt.grid(True, alpha=0.3)\n", - " \n", + "\n", " # Create custom legend to avoid duplicates\n", " handles, labels = plt.gca().get_legend_handles_labels()\n", " by_label = dict(zip(labels, handles))\n", - " plt.legend(by_label.values(), by_label.keys(), loc='best', bbox_to_anchor=(1.05, 1))\n", - " \n", + " plt.legend(by_label.values(), by_label.keys(), loc=\"best\", bbox_to_anchor=(1.05, 1))\n", + "\n", " plt.tight_layout()\n", " plt.show()\n", "\n", + "\n", "# Load the data\n", "def extract_key_data(zarr_path, required_keys):\n", - "\n", - " stores = [s for s in os.listdir(zarr_path) if s.endswith('.zarr')]\n", + " stores = [s for s in os.listdir(zarr_path) if s.endswith(\".zarr\")]\n", " stores = sorted(stores)\n", "\n", " data = {}\n", " for key in required_keys:\n", " data[key] = []\n", - " data['valid_stores'] = []\n", + " data[\"valid_stores\"] = []\n", "\n", " for store_name in stores:\n", " try:\n", " store_path = os.path.join(zarr_path, store_name)\n", - " store = zarr.open(store_path, mode='r')\n", - " \n", + " store = zarr.open(store_path, mode=\"r\")\n", + "\n", " # Check if all required keys are accessible before appending anything\n", - " required_keys = ['stl_areas', 'stl_coordinates', 'stl_centers']\n", + " required_keys = [\"stl_areas\", \"stl_coordinates\", \"stl_centers\"]\n", " all_keys_valid = True\n", - " \n", + "\n", " for key in required_keys:\n", " if key not in store:\n", " print(f\"Missing key '{key}' in store: {store_name}\")\n", " all_keys_valid = False\n", " break\n", - " \n", + "\n", " # Only append data if all keys are valid\n", " if all_keys_valid:\n", " for key in required_keys:\n", " data[key].append(store[key])\n", - " data['valid_stores'].append(store_name)\n", + " data[\"valid_stores\"].append(store_name)\n", " else:\n", " print(f\"Skipping store {store_name} due to missing keys\")\n", - " \n", + "\n", " except Exception as e:\n", " print(\"Error in store_name: \", store_name)\n", " print(e)\n", @@ -397,31 +443,36 @@ " print(f\"Successfully loaded {len(data['valid_stores'])} stores\")\n", " return data\n", "\n", - "drivaerml_zarr_path = '/mnt/datasets/drivaerml_stl_zarr'\n", - "ahmedml_zarr_path = '/mnt/datasets/ahmedml_stl_zarr'\n", "\n", - "drivaerml_extracted_data = extract_key_data(drivaerml_zarr_path, ['stl_areas', 'stl_coordinates', 'stl_centers'])\n", - "ahmedml_extracted_data = extract_key_data(ahmedml_zarr_path, ['stl_areas', 'stl_coordinates', 'stl_centers'])\n", + "drivaerml_zarr_path = \"/mnt/datasets/drivaerml_stl_zarr\"\n", + "ahmedml_zarr_path = \"/mnt/datasets/ahmedml_stl_zarr\"\n", + "\n", + "drivaerml_extracted_data = extract_key_data(\n", + " drivaerml_zarr_path, [\"stl_areas\", \"stl_coordinates\", \"stl_centers\"]\n", + ")\n", + "ahmedml_extracted_data = extract_key_data(\n", + " ahmedml_zarr_path, [\"stl_areas\", \"stl_coordinates\", \"stl_centers\"]\n", + ")\n", "\n", "drivaerml_descriptors = []\n", "ahmedml_descriptors = []\n", "\n", - "for i in range(len(drivaerml_extracted_data['valid_stores'])):\n", + "for i in range(len(drivaerml_extracted_data[\"valid_stores\"])):\n", " descriptor = compute_feature_descriptor(\n", - " drivaerml_extracted_data['stl_areas'][i],\n", - " drivaerml_extracted_data['stl_coordinates'][i],\n", - " drivaerml_extracted_data['stl_centers'][i]\n", + " drivaerml_extracted_data[\"stl_areas\"][i],\n", + " drivaerml_extracted_data[\"stl_coordinates\"][i],\n", + " drivaerml_extracted_data[\"stl_centers\"][i],\n", " )\n", " drivaerml_descriptors.append(descriptor)\n", "\n", "drivaerml_descriptors = cp.asarray(drivaerml_descriptors)\n", "print(\"Computed descriptors for drivaerml\")\n", "\n", - "for i in range(len(ahmedml_extracted_data['valid_stores'])):\n", + "for i in range(len(ahmedml_extracted_data[\"valid_stores\"])):\n", " descriptor = compute_feature_descriptor(\n", - " ahmedml_extracted_data['stl_areas'][i],\n", - " ahmedml_extracted_data['stl_coordinates'][i],\n", - " ahmedml_extracted_data['stl_centers'][i]\n", + " ahmedml_extracted_data[\"stl_areas\"][i],\n", + " ahmedml_extracted_data[\"stl_coordinates\"][i],\n", + " ahmedml_extracted_data[\"stl_centers\"][i],\n", " )\n", " ahmedml_descriptors.append(descriptor)\n", "\n", @@ -446,7 +497,9 @@ "num_ahmedml_points_to_add = 10\n", "\n", "# Stack both datasets together for UMAP\n", - "combined_descriptors = np.vstack([drivaerml_descriptors, ahmedml_descriptors[:num_ahmedml_points_to_add]])\n", + "combined_descriptors = np.vstack(\n", + " [drivaerml_descriptors, ahmedml_descriptors[:num_ahmedml_points_to_add]]\n", + ")\n", "combined_embedding = run_umap_gpu(combined_descriptors)" ] }, @@ -480,8 +533,10 @@ "clusterer.fit(cp.asarray(combined_embedding))\n", "\n", "labels_all = cp.asnumpy(clusterer.labels_)\n", - "probs_all = cp.asnumpy(clusterer.probabilities_)\n", - "print(\"Label count per cluster: \", pd.Series(cp.asnumpy(clusterer.labels_)).value_counts())\n", + "probs_all = cp.asnumpy(clusterer.probabilities_)\n", + "print(\n", + " \"Label count per cluster: \", pd.Series(cp.asnumpy(clusterer.labels_)).value_counts()\n", + ")\n", "\n", "# Split the combined embedding back into separate arrays for plotting\n", "drivaerml_embedding_np = combined_embedding[:drivaerml_count]\n", @@ -489,8 +544,8 @@ "\n", "# Usage:\n", "embeddings = {\n", - " 'DrivAerML': drivaerml_embedding_np,\n", - " 'AhmedML': ahmedml_embedding_np,\n", + " \"DrivAerML\": drivaerml_embedding_np,\n", + " \"AhmedML\": ahmedml_embedding_np,\n", "}\n", "\n", "plot_umap_embeddings_with_probabilities(\n", @@ -503,7 +558,7 @@ " train_dataset_probabilities=probs_all[:drivaerml_count],\n", " test_dataset_probabilities=probs_all[drivaerml_count:],\n", " title=\"UMAP Projection with Outlier Detection\",\n", - " outlier_threshold=0.5\n", + " outlier_threshold=0.5,\n", ")" ] } diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 2ee2c55..f921827 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -23,7 +23,7 @@ import pyvista as pv import vtk import zarr -from constants import DatasetKind, ModelType +from constants import DatasetKind, ModelType, get_physics_constants from paths import get_path_getter from schemas import ( ExternalAerodynamicsExtractedDataInMemory, @@ -124,6 +124,7 @@ def read_file(self, dirname: str) -> ExternalAerodynamicsExtractedDataInMemory: metadata = ExternalAerodynamicsMetadata( filename=dirname, dataset_type=self.model_type, # surface, volume, combined + physics_constants=get_physics_constants(self.kind), ) return ExternalAerodynamicsExtractedDataInMemory( @@ -234,12 +235,8 @@ def _write_zarr( zarr_store = zarr.DirectoryStore(output_path) root = zarr.group(store=zarr_store) - # Remove 'global_params_values' and `global_params_reference` that are written - # as part of the data itself - metadata_dict = asdict(data.metadata) - metadata_dict.pop("global_params_values", None) - metadata_dict.pop("global_params_reference", None) - root.attrs.update(metadata_dict) + # Write metadata as root attributes + root.attrs.update(asdict(data.metadata)) # Write required arrays for field in ["stl_coordinates", "stl_centers", "stl_faces", "stl_areas"]: diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 0e01660..caa98ac 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -70,8 +70,6 @@ def transform( # Create minimal metadata numpy_metadata = ExternalAerodynamicsNumpyMetadata( filename=data.metadata.filename, - global_params_values=data.metadata.global_params_values, - global_params_reference=data.metadata.global_params_reference, ) return ExternalAerodynamicsNumpyDataInMemory( @@ -86,6 +84,8 @@ def transform( surface_fields=to_float32(data.surface_fields), volume_mesh_centers=to_float32(data.volume_mesh_centers), volume_fields=to_float32(data.volume_fields), + global_params_values=to_float32(data.global_params_values), + global_params_reference=to_float32(data.global_params_reference), ) @@ -414,10 +414,10 @@ def transform( metadata=data.metadata, # `global_params_values` and `global_params_reference` are saved without compression global_params_values=self._prepare_array_no_compression( - data.metadata.global_params_values + data.global_params_values ), global_params_reference=self._prepare_array_no_compression( - data.metadata.global_params_reference + data.global_params_reference ), surface_mesh_centers=self._prepare_array(data.surface_mesh_centers), surface_normals=self._prepare_array(data.surface_normals), diff --git a/examples/external_aerodynamics/external_aero_geometry_data_processors.py b/examples/external_aerodynamics/external_aero_geometry_data_processors.py index e5242ec..27c9f5e 100644 --- a/examples/external_aerodynamics/external_aero_geometry_data_processors.py +++ b/examples/external_aerodynamics/external_aero_geometry_data_processors.py @@ -114,7 +114,7 @@ def filter_geometry_invalid_faces( logger.info( f"Filtered {n_filtered_faces} invalid geometry faces " - f"({n_filtered_faces/n_total_faces*100:.2f}% of {n_total_faces} total faces):" + f"({n_filtered_faces / n_total_faces * 100:.2f}% of {n_total_faces} total faces):" ) logger.info(f" - {n_filtered_faces} faces with area <= {tolerance}") diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index 616173b..67d8987 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -17,10 +17,7 @@ import logging import numpy as np - -from examples.external_aerodynamics.schemas import ( - ExternalAerodynamicsExtractedDataInMemory, -) +from schemas import ExternalAerodynamicsExtractedDataInMemory logging.basicConfig( format="%(asctime)s - Process %(process)d - %(levelname)s - %(message)s", @@ -50,7 +47,7 @@ def default_global_params_processing_for_external_aerodynamics( } Returns: - Updated data with global_params_reference set in metadata + Updated data with global_params_reference set """ # Build dictionaries for types and reference values global_params_types = { @@ -74,8 +71,8 @@ def default_global_params_processing_for_external_aerodynamics( f"Must be 'vector' or 'scalar'." ) - # Convert to numpy array and store in metadata - data.metadata.global_params_reference = np.array( + # Convert to numpy array and store in data container + data.global_params_reference = np.array( global_params_reference_list, dtype=np.float32 ) @@ -99,16 +96,16 @@ def process_global_params( global_parameters: Dict from config with parameter definitions Returns: - Updated data with global_params_values set in metadata + Updated data with global_params_values set """ - if data.metadata.global_params_reference is None: + if data.global_params_reference is None: logger.warning( "global_params_reference not set. Skipping global_params_values processing." ) raise ValueError("global_params_reference are absent in the configuration") # Default behavior: assume simulation values match reference - data.metadata.global_params_values = data.metadata.global_params_reference.copy() + data.global_params_values = data.global_params_reference.copy() return data @@ -137,7 +134,7 @@ def process_global_params_hlpw( Returns: Updated data with global_params_values extracted from simulation """ - if data.metadata.global_params_reference is None: + if data.global_params_reference is None: logger.warning( "global_params_reference not set. Skipping global_params_values processing." ) @@ -178,8 +175,6 @@ def process_global_params_hlpw( f"Must be 'vector' or 'scalar'." ) - data.metadata.global_params_values = np.array( - global_params_values_list, dtype=np.float32 - ) + data.global_params_values = np.array(global_params_values_list, dtype=np.float32) return data diff --git a/examples/external_aerodynamics/paths.py b/examples/external_aerodynamics/paths.py index 54af5a1..82f10c0 100644 --- a/examples/external_aerodynamics/paths.py +++ b/examples/external_aerodynamics/paths.py @@ -168,6 +168,7 @@ def volume_path(car_dir: Path) -> Path: dirname = car_dir.name return car_dir / f"volume_{dirname}.vtu" + def get_path_getter(kind: DatasetKind): """Returns path getter for a given dataset type.""" @@ -179,4 +180,4 @@ def get_path_getter(kind: DatasetKind): case DatasetKind.DRIVESIM: return DriveSimPaths case DatasetKind.HLPW: - return HLPWPaths \ No newline at end of file + return HLPWPaths diff --git a/examples/external_aerodynamics/schemas.py b/examples/external_aerodynamics/schemas.py index 76f4b44..44972f4 100644 --- a/examples/external_aerodynamics/schemas.py +++ b/examples/external_aerodynamics/schemas.py @@ -30,16 +30,18 @@ class ExternalAerodynamicsMetadata: Version history: - 1.0: Initial version with expected metadata fields. - - 1.1: Added AoA (Angle of Attack) field. + - 1.1: Added physics_constants dict for pipeline-specific constants. """ # Simulation identifiers filename: str dataset_type: ModelType - # Physical parameters - global_params_values: Optional[np.ndarray] = None - global_params_reference: Optional[np.ndarray] = None + # Physics constants - populated based on dataset kind from config + # Keys/values vary by pipeline, e.g.: + # CarAerodynamics: {"air_density": 1.205, "stream_velocity": 30.0} + # HLPW: {"pref": 176.352, "uref": 2679.505, "tref": 518.67} + physics_constants: Optional[dict[str, float]] = None # Geometry bounds x_bound: Optional[tuple[float, float]] = None # xmin, xmax @@ -87,6 +89,10 @@ class ExternalAerodynamicsExtractedDataInMemory: volume_mesh_centers: Optional[np.ndarray] = None volume_fields: Optional[np.ndarray] = None + # Global parameters (physical conditions for training) + global_params_values: Optional[np.ndarray] = None + global_params_reference: Optional[np.ndarray] = None + @dataclass(frozen=True) class PreparedZarrArrayInfo: @@ -142,8 +148,6 @@ class ExternalAerodynamicsNumpyMetadata: """ filename: str - global_params_values: np.ndarray - global_params_reference: np.ndarray @dataclass(frozen=True) @@ -173,3 +177,7 @@ class ExternalAerodynamicsNumpyDataInMemory: # Volume data volume_mesh_centers: Optional[np.ndarray] = None volume_fields: Optional[np.ndarray] = None + + # Global parameters + global_params_values: Optional[np.ndarray] = None + global_params_reference: Optional[np.ndarray] = None From 2f5746b6196af76d10892ec5046a9958f5bacd96 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 18:49:59 -0800 Subject: [PATCH 17/23] (chore) improve global_params extraction and handling --- examples/external_aerodynamics/constants.py | 2 +- .../external_aerodynamics/data_sources.py | 22 ++++++---------- .../data_transformations.py | 25 ++++++++----------- ...rnal_aero_global_params_data_processors.py | 20 +++++---------- 4 files changed, 26 insertions(+), 43 deletions(-) diff --git a/examples/external_aerodynamics/constants.py b/examples/external_aerodynamics/constants.py index 7f905eb..a28f3c4 100644 --- a/examples/external_aerodynamics/constants.py +++ b/examples/external_aerodynamics/constants.py @@ -67,7 +67,7 @@ class DefaultVariables: def get_physics_constants(kind: DatasetKind) -> dict[str, float]: """Get physics constants dict based on dataset kind. Add a branch to the if-elif pipeline below to populate metadata with values - used for non-dimensionalization + used for non-dimensionalization. Args: kind: The dataset kind (from config etl.common.kind) diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index f921827..f94b8f7 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -192,19 +192,23 @@ def _write_numpy( """ # Convert to dict for numpy storage save_dict = { - # Arrays + # Required arrays "stl_coordinates": data.stl_coordinates, "stl_centers": data.stl_centers, "stl_faces": data.stl_faces, "stl_areas": data.stl_areas, # Basic metadata "filename": data.metadata.filename, - "stream_velocity": data.metadata.stream_velocity, - "air_density": data.metadata.air_density, } - # Add optional arrays if present + # Add physics constants if present (pipeline-specific keys) + if data.metadata.physics_constants: + save_dict.update(data.metadata.physics_constants) + + # Add optional arrays if present (same fields as Zarr I/O) for field in [ + "global_params_values", + "global_params_reference", "surface_mesh_centers", "surface_normals", "surface_areas", @@ -241,13 +245,6 @@ def _write_zarr( # Write required arrays for field in ["stl_coordinates", "stl_centers", "stl_faces", "stl_areas"]: array_info = getattr(data, field) - self.logger.info( - f"Writing required field '{field}': array_info={array_info is not None}, type={type(array_info)}" - ) - if array_info is None: - raise ValueError( - f"Required field '{field}' is None - cannot write zarr dataset" - ) root.create_dataset( field, data=array_info.data, @@ -274,9 +271,6 @@ def _write_zarr( chunks=array_info.chunks, compressor=array_info.compressor, ) - self.logger.info( - f"Successfully wrote field '{field}' with shape {array_info.data.shape}" - ) else: self.logger.error(f"{field} is absent in the dataset") diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index caa98ac..42f4178 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -285,11 +285,10 @@ def __init__( super().__init__(cfg) self.global_parameters = global_parameters self.global_params_processors = global_params_processors - self.logger = logging.getLogger(__name__) if global_parameters is None: - self.logger.error( - "No global_parameters provided. Please provide global parameters" + raise ValueError( + "global_parameters is required but was not provided in config" ) def transform( @@ -299,18 +298,16 @@ def transform( Processes global parameter references from config and extracts values from simulation data. """ + # Apply default processing to set up reference arrays from config + data = default_global_params_processing_for_external_aerodynamics( + data, self.global_parameters + ) - if self.global_parameters is not None: - # Apply default processing to set up reference arrays from config - data = default_global_params_processing_for_external_aerodynamics( - data, self.global_parameters - ) - - # Apply any custom processors (e.g., extract values from simulation files) - # Pass global_parameters so processors know the types (vector vs scalar) - if self.global_params_processors is not None: - for processor in self.global_params_processors: - data = processor(data, self.global_parameters) + # Apply any custom processors (e.g., extract values from simulation files) + # Pass global_parameters so processors know the types (vector vs scalar) + if self.global_params_processors is not None: + for processor in self.global_params_processors: + data = processor(data, self.global_parameters) return data diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index 67d8987..efa8ee2 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -98,12 +98,6 @@ def process_global_params( Returns: Updated data with global_params_values set """ - if data.global_params_reference is None: - logger.warning( - "global_params_reference not set. Skipping global_params_values processing." - ) - raise ValueError("global_params_reference are absent in the configuration") - # Default behavior: assume simulation values match reference data.global_params_values = data.global_params_reference.copy() @@ -124,7 +118,7 @@ def process_global_params_hlpw( ) -> ExternalAerodynamicsExtractedDataInMemory: """Extract global parameters from HLPW simulation data. - For HLPW, typically: + For HLPW, : - AoA (Angle of Attack) varies per simulation and can be extracted from filename Args: @@ -134,11 +128,6 @@ def process_global_params_hlpw( Returns: Updated data with global_params_values extracted from simulation """ - if data.global_params_reference is None: - logger.warning( - "global_params_reference not set. Skipping global_params_values processing." - ) - raise ValueError("global_params_reference are absent in the configuration") # Build a dict of extracted values keyed by parameter name extracted_values = {} @@ -156,14 +145,17 @@ def process_global_params_hlpw( extracted_values["AoA"] = aoa logger.info(f"Extracted AoA={aoa} from filename: {filename}") else: - # Fallback to reference if not in filename raise ValueError(f"AoA pattern not found in filename '{filename}'.") # Build the flattened array using the same logic as reference processing global_params_values_list = [] for name, params in global_parameters.items(): param_type = params["type"] - value = extracted_values.get(name, params["reference"]) + if name not in extracted_values: + raise ValueError( + f"Global parameter '{name}' was not extracted from simulation data." + ) + value = extracted_values[name] if param_type == "vector": global_params_values_list.extend(value) From 0c8cd9ce52e79b86e8e45afe0f0fe9d97457d441 Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 26 Nov 2025 18:51:59 -0800 Subject: [PATCH 18/23] (chore) reformat comments and logs --- .../external_aero_global_params_data_processors.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/external_aerodynamics/external_aero_global_params_data_processors.py b/examples/external_aerodynamics/external_aero_global_params_data_processors.py index efa8ee2..d618243 100644 --- a/examples/external_aerodynamics/external_aero_global_params_data_processors.py +++ b/examples/external_aerodynamics/external_aero_global_params_data_processors.py @@ -47,8 +47,9 @@ def default_global_params_processing_for_external_aerodynamics( } Returns: - Updated data with global_params_reference set + Updated `data` with global_params_reference set """ + # Build dictionaries for types and reference values global_params_types = { name: params["type"] for name, params in global_parameters.items() @@ -96,7 +97,7 @@ def process_global_params( global_parameters: Dict from config with parameter definitions Returns: - Updated data with global_params_values set + Updated `data` with global_params_values set """ # Default behavior: assume simulation values match reference data.global_params_values = data.global_params_reference.copy() @@ -126,7 +127,7 @@ def process_global_params_hlpw( global_parameters: Dict from config with parameter definitions Returns: - Updated data with global_params_values extracted from simulation + Updated `data` with global_params_values extracted from simulation """ # Build a dict of extracted values keyed by parameter name From d403a48a8017b550521f73760b33afd86ffa07f7 Mon Sep 17 00:00:00 2001 From: snidhan Date: Mon, 1 Dec 2025 15:14:10 -0800 Subject: [PATCH 19/23] (chore) fix some more tests --- .../config/variables/global/ahmedml.yaml | 3 +- .../config/variables/global/drivaerml.yaml | 3 +- .../external_aerodynamics/data_sources.py | 2 +- .../test_constants.py | 40 +++- .../test_data_sources.py | 26 +++ .../test_data_transformations.py | 25 +++ .../test_global_params_processors.py | 208 ++++++++++++++++++ 7 files changed, 298 insertions(+), 9 deletions(-) create mode 100644 tests/test_examples/test_external_aerodynamics/test_global_params_processors.py diff --git a/examples/external_aerodynamics/config/variables/global/ahmedml.yaml b/examples/external_aerodynamics/config/variables/global/ahmedml.yaml index 69bf1f9..9ae6a99 100644 --- a/examples/external_aerodynamics/config/variables/global/ahmedml.yaml +++ b/examples/external_aerodynamics/config/variables/global/ahmedml.yaml @@ -18,11 +18,10 @@ global_parameters: inlet_velocity: type: vector - reference: [30.0] # Inlet velocity vector in m/s + reference: [30.0] # Inlet velocity vector in m/s, if a 2D vector, give [30, 30], if a 3D vector give [30, 30, 30] air_density: type: scalar reference: 1.205 # Air density in kg/m³ pressure: type: scalar reference: 101325.0 # Reference pressure in Pa - diff --git a/examples/external_aerodynamics/config/variables/global/drivaerml.yaml b/examples/external_aerodynamics/config/variables/global/drivaerml.yaml index 69bf1f9..9ae6a99 100644 --- a/examples/external_aerodynamics/config/variables/global/drivaerml.yaml +++ b/examples/external_aerodynamics/config/variables/global/drivaerml.yaml @@ -18,11 +18,10 @@ global_parameters: inlet_velocity: type: vector - reference: [30.0] # Inlet velocity vector in m/s + reference: [30.0] # Inlet velocity vector in m/s, if a 2D vector, give [30, 30], if a 3D vector give [30, 30, 30] air_density: type: scalar reference: 1.205 # Air density in kg/m³ pressure: type: scalar reference: 101325.0 # Reference pressure in Pa - diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index f94b8f7..701f395 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -272,7 +272,7 @@ def _write_zarr( compressor=array_info.compressor, ) else: - self.logger.error(f"{field} is absent in the dataset") + self.logger.warning(f"{field} is absent in the dataset") def should_skip(self, filename: str) -> bool: """Checks whether the file should be skipped. diff --git a/tests/test_examples/test_external_aerodynamics/test_constants.py b/tests/test_examples/test_external_aerodynamics/test_constants.py index d401271..509552e 100644 --- a/tests/test_examples/test_external_aerodynamics/test_constants.py +++ b/tests/test_examples/test_external_aerodynamics/test_constants.py @@ -17,13 +17,45 @@ import dataclasses import pytest -from constants import DatasetKind, ModelType, PhysicsConstants +from constants import ( + DatasetKind, + ModelType, + PhysicsConstantsCarAerodynamics, + PhysicsConstantsHLPW, + get_physics_constants, +) -def test_physics_constants_immutability(): - """Test that PhysicsConstants cannot be modified.""" +def test_physics_constants_car_aerodynamics_immutability(): + """Test that PhysicsConstantsCarAerodynamics cannot be modified.""" with pytest.raises(dataclasses.FrozenInstanceError): - PhysicsConstants().AIR_DENSITY = 2.0 + PhysicsConstantsCarAerodynamics().AIR_DENSITY = 2.0 + + +def test_physics_constants_hlpw_immutability(): + """Test that PhysicsConstantsHLPW cannot be modified.""" + with pytest.raises(dataclasses.FrozenInstanceError): + PhysicsConstantsHLPW().PREF = 200.0 + + +def test_get_physics_constants_drivaerml(): + """Test get_physics_constants returns correct dict for car aero datasets.""" + result = get_physics_constants(DatasetKind.DRIVAERML) + assert "air_density" in result + assert "stream_velocity" in result + assert result["air_density"] == PhysicsConstantsCarAerodynamics.AIR_DENSITY + assert result["stream_velocity"] == PhysicsConstantsCarAerodynamics.STREAM_VELOCITY + + +def test_get_physics_constants_hlpw(): + """Test get_physics_constants returns correct dict for HLPW dataset.""" + result = get_physics_constants(DatasetKind.HLPW) + assert "pref" in result + assert "uref" in result + assert "tref" in result + assert result["pref"] == PhysicsConstantsHLPW.PREF + assert result["uref"] == PhysicsConstantsHLPW.UREF + assert result["tref"] == PhysicsConstantsHLPW.TREF def test_model_type_validation(): diff --git a/tests/test_examples/test_external_aerodynamics/test_data_sources.py b/tests/test_examples/test_external_aerodynamics/test_data_sources.py index 0ad9fb3..c44dfd3 100644 --- a/tests/test_examples/test_external_aerodynamics/test_data_sources.py +++ b/tests/test_examples/test_external_aerodynamics/test_data_sources.py @@ -185,6 +185,8 @@ def test_drivaerml_write_numpy(self, temp_dir): stl_centers=np.array([[0.5, 0.5, 0.5]]), stl_faces=np.array([[0, 1, 2]]), stl_areas=np.array([1.0]), + global_params_values=np.array([30.0, 1.225], dtype=np.float32), + global_params_reference=np.array([30.0, 1.225], dtype=np.float32), ) source.write(test_data, "test_case") assert (temp_dir / "test_case.npz").exists() @@ -235,6 +237,17 @@ def test_drivaerml_write_zarr(self, temp_dir): decimation_reduction=0.5, decimation_algo="decimate_pro", ), + # Global parameters (no compression for small 1xN arrays) + global_params_values=PreparedZarrArrayInfo( + data=np.array([30.0, 1.225], dtype=np.float32), + chunks=(2,), + compressor=None, + ), + global_params_reference=PreparedZarrArrayInfo( + data=np.array([30.0, 1.225], dtype=np.float32), + chunks=(2,), + compressor=None, + ), # Surface data surface_mesh_centers=PreparedZarrArrayInfo( data=np.array([[0.5, 0.5, 0.5]]), @@ -427,6 +440,8 @@ def test_write_numpy_uses_temp_then_rename(self, temp_dir): stl_centers=np.array([[0.5, 0.5, 0.5]]), stl_faces=np.array([[0, 1, 2]]), stl_areas=np.array([1.0]), + global_params_values=np.array([30.0, 1.225], dtype=np.float32), + global_params_reference=np.array([30.0, 1.225], dtype=np.float32), ) # Write should create final file, not temp file @@ -478,6 +493,17 @@ def test_write_zarr_uses_temp_then_rename(self, temp_dir): filename="test_case", dataset_type=ModelType.COMBINED, ), + # Global parameters (no compression for small 1xN arrays) + global_params_values=PreparedZarrArrayInfo( + data=np.array([30.0, 1.225], dtype=np.float32), + chunks=(2,), + compressor=None, + ), + global_params_reference=PreparedZarrArrayInfo( + data=np.array([30.0, 1.225], dtype=np.float32), + chunks=(2,), + compressor=None, + ), ) # Write should create final file, not temp file diff --git a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py index 38b77ba..5cc30e4 100644 --- a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py +++ b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py @@ -185,6 +185,8 @@ def sample_data_processed(): surface_fields=np.array([[1.0, 0.0, 0.0]], dtype=np.float64), volume_mesh_centers=np.array([[0, 0, 0]], dtype=np.float64), volume_fields=np.array([[1.0, 0.0, 0.0]], dtype=np.float64), + global_params_values=np.array([30.0, 1.205], dtype=np.float32), + global_params_reference=np.array([30.0, 1.205], dtype=np.float32), ) @@ -227,6 +229,15 @@ def test_transform(self, sample_data_processed): result.volume_fields, sample_data_processed.volume_fields ) + # Check global params fields + np.testing.assert_array_equal( + result.global_params_values, sample_data_processed.global_params_values + ) + np.testing.assert_array_equal( + result.global_params_reference, + sample_data_processed.global_params_reference, + ) + class TestExternalAerodynamicsZarrTransformation: """Test the ExternalAerodynamicsZarrTransformation class.""" @@ -278,6 +289,20 @@ def test_transform(self, sample_data_processed): assert result.volume_mesh_centers.chunks == (1, 3) assert result.volume_mesh_centers.compressor == transform.compressor + # Check global params fields (no compression for small 1xN arrays) + assert isinstance(result.global_params_values, PreparedZarrArrayInfo) + np.testing.assert_array_equal( + result.global_params_values.data, sample_data_processed.global_params_values + ) + assert result.global_params_values.compressor is None + + assert isinstance(result.global_params_reference, PreparedZarrArrayInfo) + np.testing.assert_array_equal( + result.global_params_reference.data, + sample_data_processed.global_params_reference, + ) + assert result.global_params_reference.compressor is None + def test_prepare_array(self): """Test array preparation for Zarr storage.""" config = ProcessingConfig(num_processes=1) diff --git a/tests/test_examples/test_external_aerodynamics/test_global_params_processors.py b/tests/test_examples/test_external_aerodynamics/test_global_params_processors.py new file mode 100644 index 0000000..0de16af --- /dev/null +++ b/tests/test_examples/test_external_aerodynamics/test_global_params_processors.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest +from constants import ModelType +from external_aero_global_params_data_processors import ( + default_global_params_processing_for_external_aerodynamics, + process_global_params, + process_global_params_hlpw, +) +from schemas import ( + ExternalAerodynamicsExtractedDataInMemory, + ExternalAerodynamicsMetadata, +) + + +@pytest.fixture +def sample_metadata(): + """Create sample metadata for testing.""" + return ExternalAerodynamicsMetadata( + filename="test_sample", + dataset_type=ModelType.COMBINED, + ) + + +@pytest.fixture +def sample_data(sample_metadata): + """Create sample extracted data container for testing.""" + return ExternalAerodynamicsExtractedDataInMemory( + metadata=sample_metadata, + ) + + +class TestDefaultGlobalParamsProcessing: + """Test the default_global_params_processing_for_external_aerodynamics function.""" + + def test_scalar_parameters(self, sample_data): + """Test processing with scalar parameters only.""" + global_parameters = { + "air_density": {"type": "scalar", "reference": 1.205}, + "pressure": {"type": "scalar", "reference": 101325.0}, + } + + result = default_global_params_processing_for_external_aerodynamics( + sample_data, global_parameters + ) + + # Check that global_params_reference is set + assert result.global_params_reference is not None + + # Check dtype is float32 + assert result.global_params_reference.dtype == np.float32 + + # Check values are flattened correctly: [1.205, 101325.0] + expected = np.array([1.205, 101325.0], dtype=np.float32) + np.testing.assert_array_almost_equal(result.global_params_reference, expected) + + def test_vector_parameters(self, sample_data): + """Test processing with vector parameters only. + + Vectors can have variable length: + - [30.0] represents velocity in x-direction only + - [30.0, 30.0] represents 2D velocity + - [30.0, 30.0, 30.0] represents full 3D velocity + """ + global_parameters = { + "inlet_velocity_1d": {"type": "vector", "reference": [30.0]}, + "inlet_velocity_2d": {"type": "vector", "reference": [25.0, 10.0]}, + } + + result = default_global_params_processing_for_external_aerodynamics( + sample_data, global_parameters + ) + + # Check dtype is float32 + assert result.global_params_reference.dtype == np.float32 + + # Vectors are flattened in order: [30.0] + [25.0, 10.0] = [30.0, 25.0, 10.0] + expected = np.array([30.0, 25.0, 10.0], dtype=np.float32) + np.testing.assert_array_almost_equal(result.global_params_reference, expected) + + def test_mixed_parameters(self, sample_data): + """Test processing with mixed scalar and vector parameters. + + Mirrors drivaerml.yaml structure: + - inlet_velocity: vector + - air_density: scalar + - pressure: scalar + """ + global_parameters = { + "inlet_velocity": {"type": "vector", "reference": [30.0]}, + "air_density": {"type": "scalar", "reference": 1.205}, + "pressure": {"type": "scalar", "reference": 101325.0}, + } + + result = default_global_params_processing_for_external_aerodynamics( + sample_data, global_parameters + ) + + # Check dtype is float32 + assert result.global_params_reference.dtype == np.float32 + + # Flattened in order: [30.0] + 1.205 + 101325.0 = [30.0, 1.205, 101325.0] + expected = np.array([30.0, 1.205, 101325.0], dtype=np.float32) + np.testing.assert_array_almost_equal(result.global_params_reference, expected) + + def test_invalid_type_raises_error(self, sample_data): + """Test that unsupported parameter type raises ValueError.""" + global_parameters = { + "temperature": {"type": "tensor", "reference": [[1, 2], [3, 4]]}, + } + + with pytest.raises(ValueError, match="unsupported type"): + default_global_params_processing_for_external_aerodynamics( + sample_data, global_parameters + ) + + +class TestProcessGlobalParams: + """Test the process_global_params function.""" + + def test_copies_reference_to_values(self, sample_data): + """Test that process_global_params copies reference to values. + + This is the default behavior when simulation conditions match reference. + """ + global_parameters = { + "inlet_velocity": {"type": "vector", "reference": [30.0]}, + "air_density": {"type": "scalar", "reference": 1.205}, + } + + # First, set up global_params_reference + data = default_global_params_processing_for_external_aerodynamics( + sample_data, global_parameters + ) + + # Then apply process_global_params + result = process_global_params(data, global_parameters) + + # Check that global_params_values is set + assert result.global_params_values is not None + + # Check that values equal reference + np.testing.assert_array_equal( + result.global_params_values, result.global_params_reference + ) + + # Verify it's a copy, not the same object + assert result.global_params_values is not result.global_params_reference + + +class TestProcessGlobalParamsHLPW: + """Test the process_global_params_hlpw function.""" + + @pytest.mark.parametrize( + "filename, expected_aoa", + [ + ("geo_LHC001_AoA_16", 16.0), # Two-digit AoA + ("geo_LHC002_AoA_4", 4.0), # Single-digit AoA + ], + ) + def test_extracts_aoa_from_filename(self, filename, expected_aoa): + """Test that AoA is correctly extracted from HLPW filename patterns.""" + metadata = ExternalAerodynamicsMetadata( + filename=filename, + dataset_type=ModelType.COMBINED, + ) + data = ExternalAerodynamicsExtractedDataInMemory(metadata=metadata) + + global_parameters = { + "AoA": {"type": "scalar", "reference": 22.0}, + } + + result = process_global_params_hlpw(data, global_parameters) + + # Check extracted value matches expected (not the reference 22.0) + assert result.global_params_values is not None + expected = np.array([expected_aoa], dtype=np.float32) + np.testing.assert_array_equal(result.global_params_values, expected) + + def test_missing_aoa_pattern_raises_error(self): + """Test that missing AoA pattern in filename raises ValueError.""" + metadata = ExternalAerodynamicsMetadata( + filename="geo_LHC001_no_angle_info", + dataset_type=ModelType.COMBINED, + ) + data = ExternalAerodynamicsExtractedDataInMemory(metadata=metadata) + + global_parameters = { + "AoA": {"type": "scalar", "reference": 22.0}, + } + + with pytest.raises(ValueError, match="AoA pattern not found"): + process_global_params_hlpw(data, global_parameters) From f8b517b229bd90cf47f148b3ac67bbae144c494e Mon Sep 17 00:00:00 2001 From: snidhan Date: Wed, 3 Dec 2025 14:22:59 -0800 Subject: [PATCH 20/23] (chore) black formatting --- .../external_aerodynamics/data_sources.py | 6 ++- .../test_data_sources.py | 10 +---- .../test_data_transformations.py | 40 ++++++++----------- 3 files changed, 22 insertions(+), 34 deletions(-) diff --git a/examples/external_aerodynamics/data_sources.py b/examples/external_aerodynamics/data_sources.py index 701f395..356e307 100644 --- a/examples/external_aerodynamics/data_sources.py +++ b/examples/external_aerodynamics/data_sources.py @@ -202,8 +202,10 @@ def _write_numpy( } # Add physics constants if present (pipeline-specific keys) - if data.metadata.physics_constants: - save_dict.update(data.metadata.physics_constants) + # Use getattr since ExternalAerodynamicsNumpyMetadata doesn't have physics_constants + physics_constants = getattr(data.metadata, "physics_constants", None) + if physics_constants: + save_dict.update(physics_constants) # Add optional arrays if present (same fields as Zarr I/O) for field in [ diff --git a/tests/test_examples/test_external_aerodynamics/test_data_sources.py b/tests/test_examples/test_external_aerodynamics/test_data_sources.py index c44dfd3..b5cfc73 100644 --- a/tests/test_examples/test_external_aerodynamics/test_data_sources.py +++ b/tests/test_examples/test_external_aerodynamics/test_data_sources.py @@ -178,8 +178,6 @@ def test_drivaerml_write_numpy(self, temp_dir): test_data = ExternalAerodynamicsNumpyDataInMemory( metadata=ExternalAerodynamicsNumpyMetadata( filename="test_case", - stream_velocity=[30.0, 0.0, 0.0], - air_density=1.225, ), stl_coordinates=np.array([[0.0, 0.0, 0.0], [1.0, 1.0, 1.0]]), stl_centers=np.array([[0.5, 0.5, 0.5]]), @@ -225,10 +223,9 @@ def test_drivaerml_write_zarr(self, temp_dir): compressor=compressor_for_test, ), metadata=ExternalAerodynamicsMetadata( - stream_velocity=[30.0, 0.0, 0.0], - air_density=1.225, filename="test_case", dataset_type=ModelType.COMBINED, + physics_constants={"stream_velocity": 30.0, "air_density": 1.225}, x_bound=(0.0, 1.0), y_bound=(0.0, 1.0), z_bound=(0.0, 1.0), @@ -433,8 +430,6 @@ def test_write_numpy_uses_temp_then_rename(self, temp_dir): test_data = ExternalAerodynamicsNumpyDataInMemory( metadata=ExternalAerodynamicsNumpyMetadata( filename="test_case", - stream_velocity=[30.0, 0.0, 0.0], - air_density=1.225, ), stl_coordinates=np.array([[0.0, 0.0, 0.0], [1.0, 1.0, 1.0]]), stl_centers=np.array([[0.5, 0.5, 0.5]]), @@ -488,10 +483,9 @@ def test_write_zarr_uses_temp_then_rename(self, temp_dir): compressor=compressor_for_test, ), metadata=ExternalAerodynamicsMetadata( - stream_velocity=[30.0, 0.0, 0.0], - air_density=1.225, filename="test_case", dataset_type=ModelType.COMBINED, + physics_constants={"stream_velocity": 30.0, "air_density": 1.225}, ), # Global parameters (no compression for small 1xN arrays) global_params_values=PreparedZarrArrayInfo( diff --git a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py index 5cc30e4..f9e2813 100644 --- a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py +++ b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py @@ -138,8 +138,7 @@ def sample_data_raw(temp_dir): metadata=ExternalAerodynamicsMetadata( filename="test_sample", dataset_type=ModelType.COMBINED, - stream_velocity=30.0, - air_density=1.205, + physics_constants={"stream_velocity": 30.0, "air_density": 1.205}, ), stl_polydata=stl_polydata, surface_polydata=surface_polydata, @@ -165,8 +164,7 @@ def sample_data_processed(): metadata=ExternalAerodynamicsMetadata( filename="run_1234", dataset_type=ModelType.COMBINED, - stream_velocity=30.0, - air_density=1.205, + physics_constants={"stream_velocity": 30.0, "air_density": 1.205}, x_bound=(0.0, 1.0), y_bound=(0.0, 1.0), z_bound=(0.0, 1.0), @@ -721,14 +719,10 @@ def test_transform_with_default_processor_and_non_dimensionalization( # Check that the fields were non-dimensionalized assert result.surface_fields.shape == sample_data_raw.surface_fields.shape # Verify non-dimensionalization: result = original / (rho * V^2) - dynamic_pressure_factor = 1.00 * 10.00**2 # air_density # stream_velocity + dynamic_pressure_factor = 1.00 * 10.00**2 # air_density * stream_velocity^2 expected = np.array([[1.0, 0.5, 0.2, 101325.0]]) / dynamic_pressure_factor np.testing.assert_allclose(result.surface_fields, expected, rtol=1e-5) - # Verify that the metadata was updated - assert result.metadata.air_density == 1.00 - assert result.metadata.stream_velocity == 10.00 - def test_transform_with_default_processor_and_filter_invalid_surface_cells( self, sample_data_raw ): @@ -820,8 +814,12 @@ def test_validate_surface_sample_quality_with_valid_data(self, sample_data_raw): surface_processors=( partial( non_dimensionalize_surface_fields, - air_density=sample_data_raw.metadata.air_density, - stream_velocity=sample_data_raw.metadata.stream_velocity, + air_density=sample_data_raw.metadata.physics_constants[ + "air_density" + ], + stream_velocity=sample_data_raw.metadata.physics_constants[ + "stream_velocity" + ], ), ), ) @@ -846,8 +844,6 @@ def test_validate_surface_sample_quality_with_extreme_values(self): metadata = ExternalAerodynamicsMetadata( filename="test_extreme", dataset_type=ModelType.SURFACE, - stream_velocity=30.0, - air_density=1.205, ) data = ExternalAerodynamicsExtractedDataInMemory(metadata=metadata) @@ -1009,15 +1005,11 @@ def test_transform_with_default_processor_and_non_dimensionalization( / 10.00 # stream_velocity ) expected_pressure = np.array([[101325], [101300], [101320], [101310]]) / ( - 1.00 * 10.00**2 # air_density # stream_velocity + 1.00 * 10.00**2 # air_density * stream_velocity^2 ) expected = np.concatenate([expected_velocity, expected_pressure], axis=-1) np.testing.assert_allclose(result.volume_fields, expected, rtol=1e-5) - # Verify that the metadata was updated - assert result.metadata.air_density == 1.00 - assert result.metadata.stream_velocity == 10.00 - def test_transform_with_default_processor_and_shuffle_data(self, sample_data_raw): """Test volume transformation with shuffle on top of the default processor.""" config = ProcessingConfig(num_processes=1) @@ -1147,8 +1139,12 @@ def test_transform_with_default_processor_and_validate_volume_sample_quality_wit volume_processors=( partial( non_dimensionalize_volume_fields, - air_density=sample_data_raw.metadata.air_density, - stream_velocity=sample_data_raw.metadata.stream_velocity, + air_density=sample_data_raw.metadata.physics_constants[ + "air_density" + ], + stream_velocity=sample_data_raw.metadata.physics_constants[ + "stream_velocity" + ], ), ), ) @@ -1179,8 +1175,6 @@ def test_transform_with_default_processor_and_validate_volume_sample_quality_wit metadata = ExternalAerodynamicsMetadata( filename="test_outliers_velocity", dataset_type=ModelType.VOLUME, - stream_velocity=30.0, - air_density=1.205, ) data = ExternalAerodynamicsExtractedDataInMemory(metadata=metadata) @@ -1213,8 +1207,6 @@ def test_validate_volume_sample_quality_with_extreme_pressure(self): metadata = ExternalAerodynamicsMetadata( filename="test_extreme", dataset_type=ModelType.VOLUME, - stream_velocity=30.0, - air_density=1.205, ) data = ExternalAerodynamicsExtractedDataInMemory(metadata=metadata) From 23a0cc573b85fec19abe30b317c34dee099a1a2f Mon Sep 17 00:00:00 2001 From: snidhan Date: Thu, 11 Dec 2025 11:33:56 -0800 Subject: [PATCH 21/23] (chore) address PR comments --- .gitignore | 2 +- .../config/external_aero_etl_drivaerml.yaml | 2 +- .../config/external_aero_etl_hlpw.yaml | 16 ++++++++-------- .../config/variables/surface/hlpw.yaml | 2 +- .../data_transformations.py | 16 +++++++++++++--- .../external_aero_surface_data_processors.py | 13 +++++++------ examples/external_aerodynamics/schemas.py | 2 +- 7 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 8e35bb0..d6b9716 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ .coverage* .pytest_cache* .ruff_cache* -.vscode/ \ No newline at end of file +.vscode/ diff --git a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml index 304cab8..b5da9a3 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_drivaerml.yaml @@ -98,7 +98,7 @@ etl: _partial_: true - _target_: external_aero_volume_data_processors.shuffle_volume_data _partial_: true - + global_params_preprocessing: _target_: data_transformations.ExternalAerodynamicsGlobalParamsTransformation _convert_: all diff --git a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml index cbbc5c6..8601f16 100644 --- a/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml +++ b/examples/external_aerodynamics/config/external_aero_etl_hlpw.yaml @@ -24,10 +24,10 @@ defaults: etl: common: kind: hlpw - model_type: combined # produce data for which model? surface, volume, combined - + model_type: surface # produce data for which model? surface, volume, combined + processing: - num_processes: 1 + num_processes: 12 transformations: surface_preprocessing: @@ -35,16 +35,16 @@ etl: _target_: data_transformations.ExternalAerodynamicsSurfaceTransformationHLPW _convert_: all nbf_field_name: "N_BF" - + surface_processors: - _target_: external_aero_surface_data_processors.decimate_mesh _partial_: true algo: decimate_pro reduction: 0.0 preserve_topology: false - + # Note: normalize_surface_normals not needed - already done via N_BF normalization - + # HLPW-specific non-dimensionalization - _target_: external_aero_surface_data_processors.non_dimensionalize_surface_fields_hlpw _partial_: true @@ -66,11 +66,11 @@ etl: _partial_: true - _target_: external_aero_volume_data_processors.shuffle_volume_data _partial_: true - + global_params_preprocessing: _target_: data_transformations.ExternalAerodynamicsGlobalParamsTransformation _convert_: all global_parameters: ${etl.global_parameters} global_params_processors: - _target_: external_aero_global_params_data_processors.process_global_params_hlpw - _partial_: true \ No newline at end of file + _partial_: true diff --git a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml index c4cc4aa..29972b6 100644 --- a/examples/external_aerodynamics/config/variables/surface/hlpw.yaml +++ b/examples/external_aerodynamics/config/variables/surface/hlpw.yaml @@ -20,4 +20,4 @@ surface_variables: PROJ(AVG(P)): scalar AVG(TAU_WALL(0)): scalar AVG(TAU_WALL(1)): scalar - AVG(TAU_WALL(2)): scalar \ No newline at end of file + AVG(TAU_WALL(2)): scalar diff --git a/examples/external_aerodynamics/data_transformations.py b/examples/external_aerodynamics/data_transformations.py index 68bc96b..cfea1be 100644 --- a/examples/external_aerodynamics/data_transformations.py +++ b/examples/external_aerodynamics/data_transformations.py @@ -19,8 +19,8 @@ from typing import Callable, Optional import numpy as np -from constants import PhysicsConstantsCarAerodynamics import zarr +from constants import PhysicsConstantsCarAerodynamics from external_aero_geometry_data_processors import ( default_geometry_processing_for_external_aerodynamics, ) @@ -183,6 +183,15 @@ def __init__( surface_processors: Optional[tuple[Callable, ...]] = None, nbf_field_name: str = "N_BF", ): + """Initialize the HLPW surface transformation. + + Args: + cfg: Processing configuration object. + surface_variables: Mapping of variable names for surface data. + surface_processors: Optional tuple of callable processors to apply. + nbf_field_name: Name of the field containing the area vector of cells + in the surface mesh. Defaults to "N_BF". + """ super().__init__(cfg) self.logger = logging.getLogger(__name__) @@ -207,10 +216,11 @@ def __init__( def transform( self, data: ExternalAerodynamicsExtractedDataInMemory ) -> ExternalAerodynamicsExtractedDataInMemory: - """Transform surface data for HLPW using N_BF field.""" + """Transform surface data for HLPW using N_BF field. + The meaning of `N_BF` field is described above. + """ if data.surface_polydata is not None: - # Use HLPW-specific default processing (with N_BF field) data = default_surface_processing_for_external_aerodynamics_hlpw( data, self.surface_variables, self.nbf_field_name ) diff --git a/examples/external_aerodynamics/external_aero_surface_data_processors.py b/examples/external_aerodynamics/external_aero_surface_data_processors.py index 4dd62ef..f2d2a37 100644 --- a/examples/external_aerodynamics/external_aero_surface_data_processors.py +++ b/examples/external_aerodynamics/external_aero_surface_data_processors.py @@ -66,7 +66,7 @@ def default_surface_processing_for_external_aerodynamics_hlpw( """ Default surface processing for HLPW dataset. - Uses the N_BF (Normal Boundary Faces) field for computing normals and areas, + Uses the N_BF flag field for computing normals and areas, which is faster than computing them separately with PyVista for HLPW dataset. Important: Converts point data to cell data before processing, as HLPW @@ -105,7 +105,8 @@ def default_surface_processing_for_external_aerodynamics_hlpw( f"HLPW processing requires N_BF field for accurate normal and area computation." ) - # Use N_BF (area-weighted normal vectors) - HLPW-specific + # Use N_BF - HLPW-specific + # data.surface_polydata.cell_data['N_BF'] contains the area vector of each cell surface_normals_area = np.array( data.surface_polydata.cell_data[nbf_field_name] ).astype(np.float32) @@ -215,8 +216,8 @@ def normalize_surface_normals( def non_dimensionalize_surface_fields( data: ExternalAerodynamicsExtractedDataInMemory, - air_density: PhysicsConstantsCarAerodynamics.AIR_DENSITY, - stream_velocity: PhysicsConstantsCarAerodynamics.STREAM_VELOCITY, + air_density: float = PhysicsConstantsCarAerodynamics.AIR_DENSITY, + stream_velocity: float = PhysicsConstantsCarAerodynamics.STREAM_VELOCITY, ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using PhysicsConstantsCarAerodynamics. @@ -240,8 +241,8 @@ def non_dimensionalize_surface_fields( def non_dimensionalize_surface_fields_hlpw( data: ExternalAerodynamicsExtractedDataInMemory, - pref: PhysicsConstantsHLPW.PREF, - tref: PhysicsConstantsHLPW.TREF, + pref: float = PhysicsConstantsHLPW.PREF, + tref: float = PhysicsConstantsHLPW.TREF, ) -> ExternalAerodynamicsExtractedDataInMemory: """ Non-dimensionalize surface fields using PhysicsConstantsHLPW. diff --git a/examples/external_aerodynamics/schemas.py b/examples/external_aerodynamics/schemas.py index d322d67..e9752bf 100644 --- a/examples/external_aerodynamics/schemas.py +++ b/examples/external_aerodynamics/schemas.py @@ -92,7 +92,7 @@ class ExternalAerodynamicsExtractedDataInMemory: volume_mesh_centers: Optional[np.ndarray] = None volume_fields: Optional[np.ndarray] = None - # Global parameters (physical conditions for training) + # Global parameters (global simulation parameters used for training) global_params_values: Optional[np.ndarray] = None global_params_reference: Optional[np.ndarray] = None From 6dec34ee76ab213323d565782d734971e1236ec0 Mon Sep 17 00:00:00 2001 From: snidhan Date: Thu, 11 Dec 2025 11:41:51 -0800 Subject: [PATCH 22/23] (chore) update PR --- examples/external_aerodynamics/schemas.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/external_aerodynamics/schemas.py b/examples/external_aerodynamics/schemas.py index e9752bf..f905aca 100644 --- a/examples/external_aerodynamics/schemas.py +++ b/examples/external_aerodynamics/schemas.py @@ -37,7 +37,7 @@ class ExternalAerodynamicsMetadata: filename: str dataset_type: ModelType - # Physics constants - populated based on dataset kind from config + # Physics constants - populated based on dataset kind from config. # Keys/values vary by pipeline, e.g.: # CarAerodynamics: {"air_density": 1.205, "stream_velocity": 30.0} # HLPW: {"pref": 176.352, "uref": 2679.505, "tref": 518.67} @@ -92,7 +92,12 @@ class ExternalAerodynamicsExtractedDataInMemory: volume_mesh_centers: Optional[np.ndarray] = None volume_fields: Optional[np.ndarray] = None - # Global parameters (global simulation parameters used for training) + # Global parameters - simulation-wide global quantities used as conditioning inputs + # for ML models. These capture operating global conditions that affect the entire flow field. + + # global_params_values: Actual values of global parameters for this simulation + # Example: [stream_velocity, air_density, ...]. + # global_params_reference: Reference/normalization values for `global_params_values`, global_params_values: Optional[np.ndarray] = None global_params_reference: Optional[np.ndarray] = None @@ -130,10 +135,6 @@ class ExternalAerodynamicsZarrDataInMemory: stl_faces: PreparedZarrArrayInfo stl_areas: PreparedZarrArrayInfo - # Global parameters - global_params_values: Optional[PreparedZarrArrayInfo] = None - global_params_reference: Optional[PreparedZarrArrayInfo] = None - # Surface data surface_mesh_centers: Optional[PreparedZarrArrayInfo] = None surface_normals: Optional[PreparedZarrArrayInfo] = None @@ -144,6 +145,12 @@ class ExternalAerodynamicsZarrDataInMemory: volume_mesh_centers: Optional[PreparedZarrArrayInfo] = None volume_fields: Optional[PreparedZarrArrayInfo] = None + # Global parameters + # Refer to the description provided in dataclass + # ExternalAerodynamicsExtractedDataInMemory above + global_params_values: Optional[PreparedZarrArrayInfo] = None + global_params_reference: Optional[PreparedZarrArrayInfo] = None + @dataclass(frozen=True) class ExternalAerodynamicsNumpyMetadata: From c79eb9b259b333c499a8fef03789dc593627757b Mon Sep 17 00:00:00 2001 From: snidhan Date: Mon, 15 Dec 2025 14:50:24 -0800 Subject: [PATCH 23/23] (chore) fix blossom error --- .../test_external_aerodynamics/test_data_transformations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py index b6b18c6..16fbcce 100644 --- a/tests/test_examples/test_external_aerodynamics/test_data_transformations.py +++ b/tests/test_examples/test_external_aerodynamics/test_data_transformations.py @@ -448,8 +448,7 @@ def test_sharding_shape_for_large_arrays(self): metadata = ExternalAerodynamicsMetadata( filename="test_large", dataset_type=ModelType.COMBINED, - stream_velocity=30.0, - air_density=1.205, + physics_constants={"stream_velocity": 30.0, "air_density": 1.205}, ) large_data = ExternalAerodynamicsExtractedDataInMemory(