From 5b4687d9b40e3670a226c615fc8ee6d249ff4f93 Mon Sep 17 00:00:00 2001 From: travis Date: Wed, 11 Oct 2023 09:39:55 -0700 Subject: [PATCH 01/14] Update tools for speed --- tools/abel/abel.py | 3 +- tools/abel/model/connection.py | 16 ++-- tools/abel/model/entity.py | 35 +++++---- tools/abel/model/entity_field.py | 38 +++++----- tools/abel/model/entity_operation.py | 2 +- tools/abel/model/export_helper.py | 22 +++--- tools/abel/model/from_building_config.py | 39 +++++----- tools/abel/model/from_spreadsheet.py | 26 +++---- tools/abel/model/guid_to_entity_map.py | 74 +++++++++++++------ tools/abel/model/import_helper.py | 3 +- tools/abel/model/model_builder.py | 5 +- tools/abel/model/site.py | 18 +++-- tools/abel/model/state.py | 10 +-- tools/abel/model/units.py | 2 +- .../instance_validator/validate/handler.py | 5 +- .../validate/instance_parser.py | 28 +++++-- .../yamlformat/validator/parse_config_lib.py | 1 + 17 files changed, 191 insertions(+), 136 deletions(-) diff --git a/tools/abel/abel.py b/tools/abel/abel.py index b88eddfd43..df5427b047 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -19,10 +19,9 @@ from model.arg_parser import ParseArgs from model.workflow import Workflow - def main(parsed_args: ParseArgs) -> None: print( - '\nHow would you like to use ABEL?\n' + '\nHow would you like to use ABEL?\n'"" + '1: Modify a spreadsheet/building config for an existing building\n' + '2: Create a spreadsheet for a new building\n' + '3: Split a building config\n' diff --git a/tools/abel/model/connection.py b/tools/abel/model/connection.py index c45d8a4644..9642d40c8c 100644 --- a/tools/abel/model/connection.py +++ b/tools/abel/model/connection.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module for concrete model connections.""" - +import uuid from typing import Dict # pylint: disable=g-importing-member @@ -44,8 +44,8 @@ class Connection(object): def __init__( self, - source_entity_guid: str, - target_entity_guid: str, + source_entity_guid: uuid.UUID, + target_entity_guid: uuid.UUID, connection_type: ConnectionType, ) -> None: """Init. @@ -91,8 +91,8 @@ def FromDict(cls, connection_dict: Dict[str, object]) -> ...: An instance of Connection class. """ return cls( - source_entity_guid=connection_dict[SOURCE_ENTITY_GUID], - target_entity_guid=connection_dict[TARGET_ENTITY_GUID], + source_entity_guid=uuid.UUID(connection_dict[SOURCE_ENTITY_GUID]), + target_entity_guid=uuid.UUID(connection_dict[TARGET_ENTITY_GUID]), connection_type=ConnectionType[connection_dict[CONNECTION_TYPE]], ) @@ -114,7 +114,7 @@ def connection_type(self, value: ConnectionType) -> None: def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap - ) -> Dict[str, str]: + ) -> Dict[str, any]: """Returns a dictionary of Connection attributes by spreadsheet headers.""" return { VALUES: [ @@ -125,7 +125,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.source_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.source_entity_guid)}}, { USER_ENTERED_VALUE: {STRING_VALUE: self.connection_type.name}, DATA_VALIDATION: { @@ -147,6 +147,6 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.target_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.target_entity_guid)}}, ] } diff --git a/tools/abel/model/entity.py b/tools/abel/model/entity.py index 42c3a169d3..d1166b4505 100644 --- a/tools/abel/model/entity.py +++ b/tools/abel/model/entity.py @@ -14,6 +14,7 @@ """Module for concrete model entities.""" import abc +import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -61,10 +62,10 @@ class Entity(object): def __init__( self, code: str, - namespace: str, + namespace: EntityNamespace, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[str] = None, + bc_guid: Optional[uuid.UUID] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -85,6 +86,8 @@ def __init__( self._connections = [] self.type_name = type_name self.metadata = metadata + if bc_guid is not None and not isinstance(bc_guid, uuid.UUID): + raise Exception("Entity created with a string GUID") def __hash__(self): return hash((self.code, self.etag, self.bc_guid)) @@ -152,10 +155,10 @@ class VirtualEntity(Entity): def __init__( self, code: str, - namespace: str, + namespace: EntityNamespace, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[str] = None, + bc_guid: Optional[uuid.UUID] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -191,12 +194,15 @@ def FromDict(cls, entity_dict: Dict[str, str]) -> ...: # TODO(b/228973208) Add support for key errors for keys not in entity_dict. virtual_entity_instance = cls( code=entity_dict[ENTITY_CODE], - bc_guid=entity_dict[BC_GUID], + bc_guid=uuid.UUID(entity_dict[BC_GUID]), namespace=EntityNamespace(entity_dict.get(NAMESPACE).upper()), type_name=entity_dict[TYPE_NAME], ) if ETAG in entity_dict.keys(): - virtual_entity_instance.etag = entity_dict[ETAG] + etag = entity_dict[ETAG] + if isinstance(etag, list) and len(etag) == 0: + etag = "" + virtual_entity_instance.etag = etag # Merge all metadata cells in a row into one dictionary return virtual_entity_instance @@ -223,12 +229,12 @@ def AddLink(self, new_link: FieldTranslation) -> None: self._links.append(new_link) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: """Returns map of virtual entity attributes by spreadsheet headers.""" row_map_object = { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: self.bc_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.bc_guid)}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, { USER_ENTERED_VALUE: {STRING_VALUE: IS_REPORTING_FALSE}, @@ -277,11 +283,11 @@ class ReportingEntity(Entity): def __init__( self, code: str, - namespace: str, + namespace: EntityNamespace, cloud_device_id: Optional[str] = None, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[str] = None, + bc_guid: Optional[uuid.UUID] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -322,12 +328,15 @@ def FromDict(cls, entity_dict: Dict[str, str]) -> ...: # TODO(b/228973208) Add support for key errors for keys not in entity_dict. reporting_entity_instance = cls( code=entity_dict[ENTITY_CODE], - bc_guid=entity_dict[BC_GUID], + bc_guid=uuid.UUID(entity_dict[BC_GUID]), namespace=EntityNamespace(entity_dict.get(NAMESPACE).upper()), type_name=entity_dict[TYPE_NAME], cloud_device_id=entity_dict[CLOUD_DEVICE_ID], ) if ETAG in entity_dict.keys(): + etag = entity_dict[ETAG] + if isinstance(etag, list) and len(etag) == 0: + etag = "" reporting_entity_instance.etag = entity_dict[ETAG] return reporting_entity_instance @@ -356,12 +365,12 @@ def AddTranslation(self, new_translation: FieldTranslation) -> None: self._translations.append(new_translation) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: """Returns map of reporting entity attributes by spreadsheet headers.""" row_map_object = { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: self.bc_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.bc_guid)}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, { USER_ENTERED_VALUE: {STRING_VALUE: IS_REPORTING_TRUE}, diff --git a/tools/abel/model/entity_field.py b/tools/abel/model/entity_field.py index 1611c2dc68..d1d3414461 100644 --- a/tools/abel/model/entity_field.py +++ b/tools/abel/model/entity_field.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module to hold EntityField class.""" - +import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -64,8 +64,8 @@ class MissingField(field_translation.UndefinedField): def __init__( self, std_field_name: str, - entity_guid: str, - reporting_entity_guid: Optional[str] = None, + entity_guid: uuid.UUID, + reporting_entity_guid: Optional[uuid.UUID] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -120,8 +120,8 @@ def FromDict( reporting_entity_field_name=missing_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=missing_field_dict[BC_GUID], - reporting_entity_guid=missing_field_dict[REPORTING_ENTITY_GUID], + entity_guid=uuid.UUID(missing_field_dict.get(BC_GUID)), + reporting_entity_guid=uuid.UUID(missing_field_dict.get(REPORTING_ENTITY_GUID)), ) missing_field_instance.metadata = { k[len(METADATA) + 1 :]: v @@ -145,7 +145,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -153,7 +153,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_TRUE}, DATA_VALIDATION: { @@ -187,8 +187,8 @@ def __init__( self, std_field_name: str, raw_field_name: str, - entity_guid: str, - reporting_entity_guid: Optional[str] = None, + entity_guid: uuid.UUID, + reporting_entity_guid: Optional[uuid.UUID] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -266,8 +266,8 @@ def FromDict( reporting_entity_field_name=multistate_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=multistate_field_dict[BC_GUID], - reporting_entity_guid=multistate_field_dict[REPORTING_ENTITY_GUID], + entity_guid=uuid.UUID(multistate_field_dict[BC_GUID]), + reporting_entity_guid=uuid.UUID(multistate_field_dict[REPORTING_ENTITY_GUID]), ) multi_state_value_field_instance.metadata = { k[len(METADATA) + 1 :]: v @@ -313,7 +313,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -321,7 +321,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_FALSE}, DATA_VALIDATION: { @@ -365,8 +365,8 @@ def __init__( self, std_field_name: str, raw_field_name: str, - entity_guid: str, - reporting_entity_guid: Optional[str] = None, + entity_guid: uuid.UUID, + reporting_entity_guid: Optional[uuid.UUID] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -438,8 +438,8 @@ def FromDict( reporting_entity_field_name=dimensional_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=dimensional_field_dict[BC_GUID], - reporting_entity_guid=dimensional_field_dict[REPORTING_ENTITY_GUID], + entity_guid=uuid.UUID(dimensional_field_dict[BC_GUID]), + reporting_entity_guid=uuid.UUID(dimensional_field_dict[REPORTING_ENTITY_GUID]), ) # Create a units instance from a spreadsheet and to a Dimensional Field # instance. @@ -495,7 +495,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -503,7 +503,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_FALSE}, DATA_VALIDATION: { diff --git a/tools/abel/model/entity_operation.py b/tools/abel/model/entity_operation.py index 3230c1593c..fafd9934fe 100644 --- a/tools/abel/model/entity_operation.py +++ b/tools/abel/model/entity_operation.py @@ -86,7 +86,7 @@ def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap ) -> Dict[str, str]: """Returns map of entity attributes wih operation by spreadsheet headers.""" - entity_row_map = self.entity.GetSpreadsheetRowMapping(guid_to_entity_map) + entity_row_map = self.entity.GetSpreadsheetRowMapping() operation_row_map = { VALUES: [ { diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index ce7053abfc..4d792bca21 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper module for exporting a valid Building Configuration or spreadsheet.""" - +import uuid from typing import Any, Dict, List, Optional # pylint: disable=g-importing-member @@ -132,7 +132,7 @@ def ExportUpdateBuildingConfiguration( if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { - entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( + str(entity.bc_guid): self._GetReportingEntityBuildingConfigBlock( entity, operation ) } @@ -140,7 +140,7 @@ def ExportUpdateBuildingConfiguration( elif isinstance(entity, VirtualEntity): entity_yaml_dict.update( { - entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( + str(entity.bc_guid): self._GetVirtualEntityBuildingConfigBlock( entity, operation ) } @@ -148,7 +148,7 @@ def ExportUpdateBuildingConfiguration( entity_yaml_dict.update( { - site.guid: { + str(site.guid): { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, CONFIG_ETAG: site.etag, @@ -184,7 +184,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { - entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( + str(entity.bc_guid): self._GetReportingEntityBuildingConfigBlock( entity=entity, operation=None, ) @@ -193,7 +193,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: elif isinstance(entity, VirtualEntity): entity_yaml_dict.update( { - entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( + str(entity.bc_guid): self._GetVirtualEntityBuildingConfigBlock( entity=entity, operation=None ) } @@ -201,7 +201,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: entity_yaml_dict.update( { - site.guid: { + str(site.guid): { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, } @@ -302,11 +302,11 @@ def _GetVirtualEntityBuildingConfigBlock( virtual_entity_yaml.update(self._AddOperationToBlock(operation)) return virtual_entity_yaml - def _GetConnections(self, entity: Entity) -> Dict[str, List[str]]: + def _GetConnections(self, entity: Entity) -> Dict[str, any]: if entity.connections: return { CONFIG_CONNECTIONS: { - c.source_entity_guid: [c.connection_type.name] + str(c.source_entity_guid): [c.connection_type.name] for c in entity.connections } } @@ -335,11 +335,11 @@ def _SortLinks(self, entity: VirtualEntity) -> Dict[str, object]: if not field_value: field_value = field.std_field_name if field.reporting_entity_guid not in link_map: - link_map[field.reporting_entity_guid] = { + link_map[str(field.reporting_entity_guid)] = { field.std_field_name: field_value } else: - link_map.get(field.reporting_entity_guid).update( + link_map.get(str(field.reporting_entity_guid)).update( {field.std_field_name: field_value} ) return link_map diff --git a/tools/abel/model/from_building_config.py b/tools/abel/model/from_building_config.py index 98895d1f42..b11a93fb8a 100644 --- a/tools/abel/model/from_building_config.py +++ b/tools/abel/model/from_building_config.py @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper module for model_builder class.""" - +import uuid from typing import List, Tuple # pylint: disable=g-importing-member from model.connection import Connection as ABELConnection -from model.constants import CONNECTION_TYPE -from model.constants import SOURCE_ENTITY_GUID -from model.constants import TARGET_ENTITY_GUID from model.entity import Entity from model.entity import ReportingEntity from model.entity import VirtualEntity @@ -60,14 +57,14 @@ def EntityInstanceToEntity( connections = [] entity_operation = None enumerated_namespace = EntityNamespace(entity_instance.namespace.upper()) - + entity_guid = uuid.UUID(entity_instance.guid) if not entity_instance.cloud_device_id: entity = VirtualEntity( code=entity_instance.code, namespace=enumerated_namespace, etag=entity_instance.etag, type_name=entity_instance.type_name, - bc_guid=entity_instance.guid, + bc_guid=entity_guid, ) else: entity = ReportingEntity( @@ -76,33 +73,33 @@ def EntityInstanceToEntity( cloud_device_id=entity_instance.cloud_device_id, etag=entity_instance.etag, type_name=entity_instance.type_name, - bc_guid=entity_instance.guid, + bc_guid=entity_guid, ) if entity_instance.translation: for field in entity_instance.translation.values(): if isinstance(field, DimensionalValue): fields.append( _DimensionalValueToDimensionalValueField( - reporting_entity_guid=entity_instance.guid, field=field + reporting_entity_guid=entity_guid, field=field ) ) elif isinstance(field, MultiStateValue): this_field, this_states = _MultistateValueToMultistateValueField( - reporting_entity_guid=entity_instance.guid, field=field + reporting_entity_guid=entity_guid, field=field ) fields.append(this_field) states.extend(this_states) elif isinstance(field, IVUndefinedField): fields.append( _UndefinedFieldToUndefinedField( - reporting_entity_guid=entity_instance.guid, field=field + reporting_entity_guid=entity_guid, field=field ) ) if entity_instance.connections: for connection in entity_instance.connections: connections.append( - _TranslateConnectionsToABEL(entity_instance.guid, connection) + _TranslateConnectionsToABEL(entity_guid, connection) ) if entity_instance.operation: operation = EntityOperationType(entity_instance.operation.value) @@ -117,7 +114,7 @@ def EntityInstanceToEntity( def _DimensionalValueToDimensionalValueField( - reporting_entity_guid: str, field: DimensionalValue + reporting_entity_guid: uuid.UUID, field: DimensionalValue ) -> DimensionalValueField: """Maps DimensionalValue attributes to ABEL DimensionalValueField instance. @@ -144,7 +141,7 @@ def _DimensionalValueToDimensionalValueField( def _MultistateValueToMultistateValueField( - reporting_entity_guid: str, field: MultiStateValue + reporting_entity_guid: uuid.UUID, field: MultiStateValue ) -> Tuple[MultistateValueField, List[State]]: """Maps MultiStateValue attributes to ABEL MultistateValueField instances. @@ -169,7 +166,7 @@ def _MultistateValueToMultistateValueField( def _TranslateStatesToABEL( - entity_guid: str, field: MultiStateValue + entity_guid: uuid.UUID, field: MultiStateValue ) -> List[State]: """Maps MultiStateValue state attributes to ABEL State instance. @@ -196,7 +193,7 @@ def _TranslateStatesToABEL( def _UndefinedFieldToUndefinedField( - reporting_entity_guid: str, field: IVUndefinedField + reporting_entity_guid: uuid.UUID, field: IVUndefinedField ) -> MissingField: """Maps IV UndefinedField attributes to ABEL UndefinedField instances. @@ -215,7 +212,7 @@ def _UndefinedFieldToUndefinedField( def _TranslateConnectionsToABEL( - entity_guid: str, connection: IVConnection + entity_guid: uuid.UUID, connection: IVConnection ) -> ABELConnection: """Maps Instance Validator Connection attributes to ABEL Connection object. @@ -226,11 +223,11 @@ def _TranslateConnectionsToABEL( Returns: ABELConnection instance """ - return ABELConnection.FromDict({ - SOURCE_ENTITY_GUID: connection.source, - CONNECTION_TYPE: connection.ctype, - TARGET_ENTITY_GUID: entity_guid, - }) + return ABELConnection( + source_entity_guid=connection.source, + target_entity_guid=entity_guid, + connection_type=connection.ctype + ) def AddReportingEntitiesFromEntityInstance( diff --git a/tools/abel/model/from_spreadsheet.py b/tools/abel/model/from_spreadsheet.py index 738c4cd1b6..fa42f2aa9a 100644 --- a/tools/abel/model/from_spreadsheet.py +++ b/tools/abel/model/from_spreadsheet.py @@ -45,7 +45,7 @@ def LoadEntitiesFromSpreadsheet( - entity_entries: List[Dict[str, str]], guid_to_entity_map: GuidToEntityMap + entity_entries: List[Dict[str, any]], guid_to_entity_map: GuidToEntityMap ) -> List[Entity]: """Loads a list of entity maps into Entity instances. @@ -64,7 +64,7 @@ def LoadEntitiesFromSpreadsheet( else: new_entity = VirtualEntity.FromDict(entity_entry) if not new_entity.bc_guid: - new_entity.bc_guid = str(uuid.uuid4()) + new_entity.bc_guid = uuid.uuid4() guid_to_entity_map.AddEntity(new_entity) parsed_entities.append(new_entity) @@ -92,14 +92,14 @@ def LoadFieldsFromSpreadsheet( """ fields = [] for entity_field_entry in entity_field_entries: - entity_field_entry[BC_GUID] = guid_to_entity_map.GetEntityGuidByCode( + entity_field_entry[BC_GUID] = str(guid_to_entity_map.GetEntityGuidByCode( entity_field_entry[ENTITY_CODE] - ) + )) if entity_field_entry[REPORTING_ENTITY_CODE]: entity_field_entry[REPORTING_ENTITY_GUID] = ( - guid_to_entity_map.GetEntityGuidByCode( + str(guid_to_entity_map.GetEntityGuidByCode( entity_field_entry[REPORTING_ENTITY_CODE] - ) + )) ) if entity_field_entry[MISSING].upper() == MISSING_TRUE: fields.append(MissingField.FromDict(entity_field_entry)) @@ -128,9 +128,9 @@ def LoadStatesFromSpreadsheet( states = [] for state_entry in state_entries: - state_entry[BC_GUID] = guid_to_entity_map.GetEntityGuidByCode( + state_entry[BC_GUID] = str(guid_to_entity_map.GetEntityGuidByCode( state_entry[REPORTING_ENTITY_CODE] - ) + )) states.append(State.FromDict(states_dict=state_entry)) return states @@ -155,14 +155,14 @@ def LoadConnectionsFromSpreadsheet( for connection_entry in connection_entries: connection_entry[SOURCE_ENTITY_GUID] = ( - guid_to_entity_map.GetEntityGuidByCode( + str(guid_to_entity_map.GetEntityGuidByCode( connection_entry[SOURCE_ENTITY_CODE] - ) + )) ) connection_entry[TARGET_ENTITY_GUID] = ( - guid_to_entity_map.GetEntityGuidByCode( + str(guid_to_entity_map.GetEntityGuidByCode( connection_entry[TARGET_ENTITY_CODE] - ) + )) ) connections.append(ABELConnection.FromDict(connection_entry)) @@ -170,7 +170,7 @@ def LoadConnectionsFromSpreadsheet( def LoadOperationsFromSpreadsheet( - entity_entries: Dict[str, str], guid_to_entity_map: GuidToEntityMap + entity_entries: Dict[str, any], guid_to_entity_map: GuidToEntityMap ) -> List[EntityOperation]: """loads a list of entity dicitionary mappings into EntityOperation instances. diff --git a/tools/abel/model/guid_to_entity_map.py b/tools/abel/model/guid_to_entity_map.py index c8010b40c2..d03065c2b4 100644 --- a/tools/abel/model/guid_to_entity_map.py +++ b/tools/abel/model/guid_to_entity_map.py @@ -12,20 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. """Mapping of guids to Entity instances.""" - +import uuid from typing import Dict +GUID_MAP_SHARDS = 64 class GuidToEntityMap(object): """Container for mapping of Entity instances by entity guids. - Attributes: guid_to_entity_map(class variable): Mapping of entity guids + Attributes: guid_to_entity_map: Mapping of entity guids to Entity instances. """ + def _submap(self, key: uuid.UUID) -> dict[uuid.UUID, object]: + shard = key.int % GUID_MAP_SHARDS + return self._shards[shard] + def __init__(self): """Init.""" - self._guid_to_entity_map = {} + self._shards = [{} for i in range(0, GUID_MAP_SHARDS)] + self._guid_code_map = None def AddSite(self, site: ...) -> None: """Adds a site by guid to the mapping. @@ -44,11 +50,14 @@ def AddSite(self, site: ...) -> None: """ if not site.guid: raise AttributeError(f'{site.code}: guid missing') - elif site.guid not in self._guid_to_entity_map: - self._guid_to_entity_map.update({site.guid: site}) + + shard = self._submap(site.guid) + if site.guid not in shard: + shard[site.guid] = site else: raise KeyError( - f'{site.guid} maps to {self._guid_to_entity_map[site.guid]}') + f'{site.guid} maps to {shard[site.guid]}') + self._guid_code_map = None def AddEntity(self, entity: ...) -> None: """Adds an entity by guid to the mapping. @@ -67,14 +76,16 @@ def AddEntity(self, entity: ...) -> None: raise ValueError('Cannot add None values to the guid to entity map.') if not entity.bc_guid: raise AttributeError(f'{entity.code}: guid missing') - if entity.bc_guid not in self._guid_to_entity_map: - self._guid_to_entity_map[entity.bc_guid] = entity + shard = self._submap(entity.bc_guid) + if entity.bc_guid not in shard: + shard[entity.bc_guid] = entity + self._guid_code_map = None else: raise KeyError( - f'{entity.bc_guid} maps to {self._guid_to_entity_map[entity.bc_guid]}' + f'{entity.bc_guid} maps to {shard[entity.bc_guid]}' ) - def GetEntityByGuid(self, guid: str) ->...: + def GetEntityByGuid(self, guid: uuid.UUID) ->...: """Gets an Entity instance mapped to the input guid. Args: @@ -86,12 +97,12 @@ def GetEntityByGuid(self, guid: str) ->...: Raises: KeyError: When guid is not a valid key in the map. """ - entity = self._guid_to_entity_map.get(guid) + entity = self._submap(guid).get(guid) if entity is None: raise KeyError(f'{guid} is not a valid guid in the guid to entity map') return entity - def GetEntityCodeByGuid(self, guid: str) -> str: + def GetEntityCodeByGuid(self, guid: uuid.UUID) -> str: """Gets an entity code mapped by guid. Args: @@ -102,7 +113,16 @@ def GetEntityCodeByGuid(self, guid: str) -> str: """ return self.GetEntityByGuid(guid).code - def GetEntityGuidByCode(self, code: str) -> str: + def _GuidCodeMap(self) -> dict[str, uuid.UUID]: + if self._guid_code_map is not None: + return self._guid_code_map + guid_by_code = {} + for shard in self._shards: + for guid, entity in shard.items(): + guid_by_code[entity.code] = guid + return guid_by_code + + def GetEntityGuidByCode(self, code: str) -> uuid.UUID: """Returns entity code mapped by guid in the guid to entity mapping. Args: @@ -115,16 +135,14 @@ def GetEntityGuidByCode(self, code: str) -> str: AttributeError: If code is not an entity code contained in self._guid_to_entity_map """ - guid_by_code = { - entity.code: guid for guid, entity in self._guid_to_entity_map.items() - } + guid_by_code = self._GuidCodeMap() guid = guid_by_code.get(code) if not guid: raise AttributeError(f'{code} is not a valid entity code.') else: return guid - def RemoveEntity(self, guid: str) -> None: + def RemoveEntity(self, guid: uuid.UUID) -> None: """Removes a guid and entity pair from guid to entity mapping. Args: @@ -134,9 +152,10 @@ def RemoveEntity(self, guid: str) -> None: The removed Entity instance. """ - return self._guid_to_entity_map.pop(guid) + self._submap(guid).pop(guid) + self._guid_code_map = None - def UpdateEntityMapping(self, guid: str, entity: ...) -> None: + def UpdateEntityMapping(self, guid: uuid.UUID, entity: ...) -> None: """Maps existing guid key to new Entity instance. Args: @@ -147,19 +166,26 @@ def UpdateEntityMapping(self, guid: str, entity: ...) -> None: KeyError: When guid is not a valid key in the guid to entity map. ValueError: When entity is not an Entity instance. """ - if not self._guid_to_entity_map.get(guid): + shard = self._submap(guid) + if not shard.get(guid): raise KeyError(f'{guid} is not a valid guid in the guid to entity map') elif not entity: raise ValueError(f'{guid} cannot map to object of type None') - self._guid_to_entity_map.update({guid: entity}) + shard[guid] = entity + self._guid_code_map = None - def GetGuidToEntityMap(self) -> Dict[str, object]: + def GetGuidToEntityMap(self) -> Dict[uuid.UUID, object]: """Returns mapping of guids to Entity instances.""" - return self._guid_to_entity_map + full_map = {} + for shard in self._shards: + full_map.update(shard) + return full_map def Clear(self) -> None: """Clears global guid mapping. Adding for testing purposes. """ - self._guid_to_entity_map.clear() + for shard in self._shards: + shard.clear() + self._guid_code_map = None diff --git a/tools/abel/model/import_helper.py b/tools/abel/model/import_helper.py index f2655f0f5e..e461c63b45 100644 --- a/tools/abel/model/import_helper.py +++ b/tools/abel/model/import_helper.py @@ -14,6 +14,7 @@ """Module to import google sheets or Building Configurations into ABEL.""" import os +import uuid from typing import Any, Dict, List # pylint: disable=g-importing-member @@ -128,5 +129,5 @@ def DeserializeBuildingConfiguration(filepath: str) -> Dict[str, Any]: if instance.type_name == SITE_TYPE_NAME: site = instance del deserialized_bc[site.guid] - abel_site = Site(code=site.code, guid=site.guid, etag=site.etag) + abel_site = Site(code=site.code, guid=uuid.UUID(site.guid), etag=site.etag) return (abel_site, deserialized_bc) diff --git a/tools/abel/model/model_builder.py b/tools/abel/model/model_builder.py index f229253339..1a2ea1e62f 100644 --- a/tools/abel/model/model_builder.py +++ b/tools/abel/model/model_builder.py @@ -14,6 +14,7 @@ """Helper module for concrete model construction.""" import datetime +import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -271,13 +272,13 @@ def connections(self) -> List[ABELConnection]: def guid_to_entity_map(self) -> GuidToEntityMap: return self._guid_to_entity_map - def GetEntity(self, entity_guid: str) -> Entity: + def GetEntity(self, entity_guid: uuid.UUID) -> Entity: """Helper function to get an Entity instance for a guid.""" return self.guid_to_entity_map.GetEntityByGuid(entity_guid) def GetStates( self, - entity_guid: str, + entity_guid: uuid.UUID, std_field_name: str, ) -> List[State]: """Helper function to get State instances for a field name and guid.""" diff --git a/tools/abel/model/site.py b/tools/abel/model/site.py index 213e138925..a7ae847fc6 100644 --- a/tools/abel/model/site.py +++ b/tools/abel/model/site.py @@ -25,6 +25,7 @@ from model.constants import USER_ENTERED_VALUE from model.constants import VALUES from model.entity import Entity +import uuid # TODO(b/247621096): Combine site namespace and type name into one attribute. @@ -41,11 +42,11 @@ class Site(object): namespace: A site's standardized DBO namespace. type_name: A site's standardized DBO entity type id. guid: A globally unique identifier(uuid4) for a site. - entities: A list of GUIDs for entities cointained in a site. + entities: A list of GUIDs for entities contained in a site. """ def __init__( - self, code: str, etag: Optional[str], guid: Optional[str] = None + self, code: str, etag: Optional[str], guid: Optional[uuid.UUID] = None ) -> None: """Init. @@ -71,15 +72,18 @@ def __eq__(self, other: ...) -> bool: @classmethod def FromDict(cls, site_dict: Dict[str, object]) -> ...: + etag = site_dict.get(ETAG) + if isinstance(etag, list) and len(etag) == 0: + etag = "" site_instance = cls( code=site_dict.get(BUILDING_CODE), - etag=site_dict.get(ETAG), - guid=site_dict.get(BC_GUID), + etag=etag, + guid=uuid.UUID(site_dict.get(BC_GUID)), ) return site_instance @property - def entities(self) -> List[str]: + def entities(self) -> List[uuid.UUID]: """Returns a list of entity guids contained in a site.""" return self._entities @@ -102,12 +106,12 @@ def AddEntity(self, entity: Entity) -> None: self._entities.append(entity.bc_guid) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: """Returns a dictionary of Site attributes by spreadsheet headers.""" return { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: self.guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.guid)}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, ] } diff --git a/tools/abel/model/state.py b/tools/abel/model/state.py index 0bb2cdfa93..8dbafcfe27 100644 --- a/tools/abel/model/state.py +++ b/tools/abel/model/state.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module for concrete model states.""" - +import uuid from typing import Dict # pylint: disable=g-importing-member @@ -39,7 +39,7 @@ class State(object): def __init__( self, - reporting_entity_guid: str, + reporting_entity_guid: uuid.UUID, std_field_name: str, standard_state: str, raw_state: str, @@ -76,7 +76,7 @@ def __eq__(self, other: ...) -> bool: @classmethod def FromDict(cls, states_dict: Dict[str, str]) -> ...: return cls( - reporting_entity_guid=states_dict[REPORTING_ENTITY_GUID], + reporting_entity_guid=uuid.UUID(states_dict[REPORTING_ENTITY_GUID]), std_field_name=states_dict[REPORTING_ENTITY_FIELD_NAME], standard_state=states_dict[STANDARD_STATE], raw_state=states_dict[RAW_STATE], @@ -84,7 +84,7 @@ def FromDict(cls, states_dict: Dict[str, str]) -> ...: def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap - ) -> Dict[str, str]: + ) -> Dict[str, any]: """Returns a dictionary of State attributes by spreadsheet headers.""" return { VALUES: [ @@ -95,7 +95,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, + {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.std_field_name}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.standard_state}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.raw_state}}, diff --git a/tools/abel/model/units.py b/tools/abel/model/units.py index c9f4e46e38..c97035647e 100644 --- a/tools/abel/model/units.py +++ b/tools/abel/model/units.py @@ -51,7 +51,7 @@ def __eq__(self, other): and self.standard_to_raw_unit_map == other.standard_to_raw_unit_map ) - def GetSpreadsheetRowMapping(self) -> Dict[str, str]: + def GetSpreadsheetRowMapping(self) -> Dict[str, any]: """Returns a dictionary of EntityField attributes by spreadsheet headers. Corresponds to a single row in a concrete model spreadsheet. diff --git a/tools/validators/instance_validator/validate/handler.py b/tools/validators/instance_validator/validate/handler.py index 310471276b..8f50d219d2 100644 --- a/tools/validators/instance_validator/validate/handler.py +++ b/tools/validators/instance_validator/validate/handler.py @@ -20,6 +20,7 @@ import json import os import sys +import uuid from typing import Dict, List, Tuple from validate import constants @@ -73,7 +74,7 @@ def GetDefaultOperation( def Deserialize( yaml_files: List[str], ) -> Tuple[ - Dict[str, entity_instance.EntityInstance], instance_parser.ConfigMode + Dict[uuid.UUID, entity_instance.EntityInstance], instance_parser.ConfigMode ]: """Parses a yaml configuration file and deserializes it. @@ -98,7 +99,7 @@ def Deserialize( for entity_key, entity_yaml in parser.GetEntities().items(): try: entity = entity_instance.EntityInstance.FromYaml( - entity_key, entity_yaml, default_entity_operation + str(entity_key), entity_yaml, default_entity_operation ) entities[entity.guid] = entity except ValueError as ex: diff --git a/tools/validators/instance_validator/validate/instance_parser.py b/tools/validators/instance_validator/validate/instance_parser.py index fd1af8c382..197ed34587 100644 --- a/tools/validators/instance_validator/validate/instance_parser.py +++ b/tools/validators/instance_validator/validate/instance_parser.py @@ -19,11 +19,13 @@ import enum import re import sys +import uuid from typing import Callable, Dict, List, Optional, Type, TypeVar import warnings import ruamel import strictyaml as syaml +import yaml #### Program constants #### # Size of entity block to send to the syntax validator @@ -264,7 +266,7 @@ class InstanceParser: def __init__(self): self._queued_entity_blocks = collections.deque() self._config_mode = None - self._validated_entities = {} + self._validated_entities = {} # type: dict[uuid.UUID, dict] self._is_final = False def Finalize(self) -> None: @@ -276,7 +278,7 @@ def Finalize(self) -> None: self._ProcessEntities() self._is_final = True - def GetEntities(self) -> syaml.YAML: + def GetEntities(self) -> dict[uuid.UUID]: """Returns the YAML object derived from parsing the input files. Raises: @@ -305,8 +307,18 @@ def AddFile(self, filename: str) -> None: entity_instance_block = '' found_entities = 0 in_config = False + total_lines = 0 with open(filename, encoding='utf-8') as file: + for _ in file: + total_lines += 1 + print(f"[Instance Parser] Parsing started...") + with open(filename, encoding='utf-8') as file: + line_count = 0 for line in file: + line_count += 1 + if (line_count % 5) == 0: + percentage = '{:.3%}'.format(line_count / total_lines) + print(f"[Instance Parser] ({line_count}/{total_lines}) {percentage}% parsed") if _IGNORE_PATTERN.match(line): continue @@ -340,6 +352,7 @@ def AddFile(self, filename: str) -> None: entity_instance_block = entity_instance_block + line + print(f"[Instance Parser] Parsed all lines") # handle the singleton case if in_config: # parse the config block @@ -355,6 +368,7 @@ def _ProcessEntities(self) -> None: if not self._config_mode: return + print(f"[Instance Parser] Processing entities...") # Validate all queued blocks while True: try: @@ -433,11 +447,13 @@ def _ValidateEntityBlock(self, block: syaml.YAML) -> None: Raises: ValueError: if block contains a key that has already been found. """ - for key in block.keys(): - if key in self._validated_entities: - raise ValueError('Duplicate key {key}') + data = block.data + for key in data: + guid = uuid.UUID(key) + # if guid in self._validated_entities: + # raise ValueError(f'Duplicate key {guid}') self._ValidateEntityContent(block.get(key)) - self._validated_entities.update(block.data) + self._validated_entities[guid] = data[key] def _ValidateBlock( self, unvalidated_block: str, validation_fn: Callable[[syaml.YAML], None] diff --git a/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py b/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py index b774658896..dd7664d075 100644 --- a/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py +++ b/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py @@ -126,6 +126,7 @@ def _CreateFolder(folderpath, global_namespace, create_folder_fn, file_tuples): """Creates a ConfigFolder for the given folderpath.""" folder = create_folder_fn(folderpath, global_namespace) for ft in file_tuples: + print(f"ontology open {os.path.join(ft.root, ft.relative_path)}") with open(os.path.join(ft.root, ft.relative_path), 'r', encoding='utf-8') as f: try: From 6b8c4f11f63f3ad5e2b708d24b7cc0ccf1530c99 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 16 Oct 2023 16:11:41 -0700 Subject: [PATCH 02/14] add schema.py --- .../instance_validator/validate/schema.py | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 tools/validators/instance_validator/validate/schema.py diff --git a/tools/validators/instance_validator/validate/schema.py b/tools/validators/instance_validator/validate/schema.py new file mode 100644 index 0000000000..d7a32e59b9 --- /dev/null +++ b/tools/validators/instance_validator/validate/schema.py @@ -0,0 +1,169 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from validate import enumerations + +#### Public Text parsing Constants #### +ENTITY_ID_KEY = 'id' # deprecated; kept for legacy reasons +ENTITY_GUID_KEY = 'guid' +ENTITY_CODE_KEY = 'code' +ENTITY_CLOUD_DEVICE_ID_KEY = 'cloud_device_id' +ENTITY_TYPE_KEY = 'type' +ENTITY_OPERATION_KEY = 'operation' + +LINKS_KEY = 'links' +TRANSLATION_KEY = 'translation' +CONNECTIONS_KEY = 'connections' +METADATA_KEY = 'metadata' +PRESENT_VALUE_KEY = 'present_value' +POINTS = 'points' +VALUE_RANGE_KEY = 'value_range' +UNITS_KEY = 'units' +UNIT_NAME_KEY = 'key' +UNIT_VALUES_KEY = 'values' +STATES_KEY = 'states' +UPDATE_MASK_KEY = 'update_mask' +ETAG_KEY = 'etag' + +# Minimum threshold for a valid entity name. Additional validation is required +# check adherence to more specific naming conventions +# Note: As-written this will capture the metadata key below, so logic should +# check for it first +_ENTITY_CODE_REGEX = r'^[a-zA-Z][a-zA-Z0-9/\-_ ]+:' +_ENTITY_GUID_REGEX = r'^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}:' +_ENTITY_CODE_PATTERN = re.compile(_ENTITY_CODE_REGEX) +_ENTITY_GUID_PATTERN = re.compile(_ENTITY_GUID_REGEX) + +# Exact key for the configuration metadata block +_CONFIG_METADATA_KEY = 'CONFIG_METADATA' +_CONFIG_METADATA_REGEX = f'^{_CONFIG_METADATA_KEY}:' +_CONFIG_METADATA_PATTERN = re.compile(_CONFIG_METADATA_REGEX) +# Key that marks the mode to parse file in. +_CONFIG_MODE_KEY = 'operation' + +_TRANSLATION_SCHEMA = { + '$id': '/schemas/translation', + 'type': 'object', + 'properties': { + PRESENT_VALUE_KEY: {'type': 'string'}, + STATES_KEY: { + 'type': 'array', + 'prefixItems': [ + # Not sure if this right? + {{'type': 'string'}: {'type': 'string'}} + ] + }, + UNITS_KEY: { + 'type': 'object', + 'properties': { + UNIT_NAME_KEY: {'type': 'string'}, + UNIT_VALUES_KEY: { + 'type': 'array' + } + } + } + }, + 'required': [PRESENT_VALUE_KEY] +} + +_METADATA_SCHEMA = { + '$id': '/schemas/config-metadata', + _CONFIG_MODE_KEY: { + 'oneOf': [ + {'const': enumerations.ConfigMode.UPDATE.value}, + {'const': enumerations.ConfigMode.INITIALIZE.value}, + {'const': enumerations.ConfigMode.EXPORT.value}, + ] + } +} + +_ENTITY_ATTR_SCHEMA = { + '$id': '/schemas/entity-attributes', + 'type': 'object', + 'properties': { + ENTITY_CODE_KEY: {'type': 'string'}, + CONNECTIONS_KEY: { + 'type': 'array' + }, + LINKS_KEY: {'type': 'array'}, + TRANSLATION_KEY: {'$ref': '/schemas/translation'} + } +} + +_ENTITY_BASE_SCHEMA = { + '$id': '/schemas/entity-base-schema', + 'type': 'object', + 'properties': { + ENTITY_CLOUD_DEVICE_ID_KEY: {'type': 'string'}, + ENTITY_CODE_KEY: {'type': 'string'}, + ENTITY_GUID_KEY: {'type': 'string'}, + 'allOf': [ + {'$ref': '/schemas/entity-attributes'} + ] + } + +} + +_ENTITY_UPDATE_SCHEMA = { + 'type': 'object', + 'properties': { + ETAG_KEY: {'type': 'string'}, + ENTITY_TYPE_KEY: {'type': 'string'}, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], + ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.UPDATE.value}, + UPDATE_MASK_KEY: { + 'type': 'array', + 'uniqueItems': True + }, + }, + 'required': [ETAG_KEY, ENTITY_TYPE_KEY], + 'dependentRequired': { + UPDATE_MASK_KEY: [ENTITY_OPERATION_KEY] + } +} + +_ENTITY_ADD_SCHEMA = { + 'type': 'object', + 'properties': { + ENTITY_TYPE_KEY: {'type': 'string'}, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], + ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.ADD.value}, + } +} + +_ENTITY_DELETE_SCHEMA = { + 'type': 'object', + 'properties': { + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], + ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.DELETE.value}, + } +} + +_ENTITY_EXPORT_SCHEMA = { + 'type': 'object', + 'properties': { + ETAG_KEY: {'type': 'string'}, + ENTITY_TYPE_KEY: {'type': 'string'}, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], + ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.EXPORT.value}, + } +} \ No newline at end of file From 6f94bbeee53f3878a9f1b5a8c05f84c32470e650 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 16 Oct 2023 16:12:16 -0700 Subject: [PATCH 03/14] refactor global enumerations --- .../validate/enumerations.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tools/validators/instance_validator/validate/enumerations.py diff --git a/tools/validators/instance_validator/validate/enumerations.py b/tools/validators/instance_validator/validate/enumerations.py new file mode 100644 index 0000000000..8cbb6b1f71 --- /dev/null +++ b/tools/validators/instance_validator/validate/enumerations.py @@ -0,0 +1,51 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import enum + +class ConfigMode(enum.Enum): + """Enumerated building config file processing modes.""" + + INITIALIZE = 'INITIALIZE' + UPDATE = 'UPDATE' + EXPORT = 'EXPORT' + + @classmethod + def FromString(cls, value: str): + """Returns a ConfigMode instance matching the provided string.""" + for _, member in cls.__members__.items(): + if member.value == value: + return member + raise LookupError + + @classmethod + def Default(cls): + """Returns the default ConfigMode if no config block is provided.""" + return cls.INITIALIZE + + +class EntityOperation(enum.Enum): + """Enumerated building config entity processing modes.""" + + UPDATE = 'UPDATE' + ADD = 'ADD' + DELETE = 'DELETE' + EXPORT = 'EXPORT' + + @classmethod + def FromString(cls, value: str): + """Returns a ConfigMode instance matching the provided string.""" + for _, member in cls.__members__.items(): + if member.value == value: + return member + raise LookupError \ No newline at end of file From cb99d459e94081778a55353599c71026f1dd81ad Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 23 Oct 2023 09:15:24 -0700 Subject: [PATCH 04/14] migrate strictyaml to pyyaml and json schema --- .../schemas/config-metadata.schema.json | 20 +++ .../schemas/entity-add-schema.schema.json | 16 ++ .../schemas/entity-attributes.schema.json | 48 +++++ .../schemas/entity-base-schema.schema.json | 26 +++ .../schemas/entity-block-schema.schema.json | 22 +++ .../schemas/entity-delete-schema.schema.json | 17 ++ .../schemas/entity-export-schema.schema.json | 22 +++ .../schemas/entity-noop-schema.schema.json | 10 ++ .../schemas/entity-update-schema.schema.json | 31 ++++ .../schemas/translation.schema.json | 38 ++++ .../fake_instances/BAD/duplicate_keys.yaml | 3 + .../fake_instances/BAD/entity_add_mask.yaml | 3 + .../fake_instances/BAD/missing_colon.yaml | 4 + ...onfigmode.yaml => no_config_metadata.yaml} | 3 - .../GOOD/building_connections.yaml | 8 +- .../tests/instance_parser_test.py | 4 + .../instance_validator/tests/parser_test.py | 91 ++++++++++ .../instance_validator/validate/handler.py | 72 ++------ .../validate/instance_parser.py | 3 +- .../instance_validator/validate/parser.py | 167 ++++++++++++++++++ .../instance_validator/validate/schema.py | 114 ++++++++---- 21 files changed, 622 insertions(+), 100 deletions(-) create mode 100644 tools/validators/instance_validator/schemas/config-metadata.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-add-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-attributes.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-base-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-block-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-delete-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-export-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-noop-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/entity-update-schema.schema.json create mode 100644 tools/validators/instance_validator/schemas/translation.schema.json rename tools/validators/instance_validator/tests/fake_instances/BAD/{configmode.yaml => no_config_metadata.yaml} (95%) create mode 100644 tools/validators/instance_validator/tests/parser_test.py create mode 100644 tools/validators/instance_validator/validate/parser.py diff --git a/tools/validators/instance_validator/schemas/config-metadata.schema.json b/tools/validators/instance_validator/schemas/config-metadata.schema.json new file mode 100644 index 0000000000..5bd948a803 --- /dev/null +++ b/tools/validators/instance_validator/schemas/config-metadata.schema.json @@ -0,0 +1,20 @@ +{ + "$id": "config-metadata.schema.json", + "type": "object", + "properties": { + "operation": { + "oneOf": [ + { + "const": "UPDATE" + }, + { + "const": "INITIALIZE" + }, + { + "const": "EXPORT" + } + ] + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-add-schema.schema.json b/tools/validators/instance_validator/schemas/entity-add-schema.schema.json new file mode 100644 index 0000000000..88b0f7ad51 --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-add-schema.schema.json @@ -0,0 +1,16 @@ +{ + "$id": "entity-add-schema.schema.json", + "type": "object", + "properties": { + "operation": { + "const": "ADD" + } + }, + "required": ["operation"], + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-attributes.schema.json b/tools/validators/instance_validator/schemas/entity-attributes.schema.json new file mode 100644 index 0000000000..59f1ef4af0 --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-attributes.schema.json @@ -0,0 +1,48 @@ +{ + "$id": "entity-attributes.schema.json", + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "connections": { + "type": "object", + "patternProperties": { + "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + "links": { + "type": "object", + "patternProperties": { + "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$": { + "patternProperties": { + ".*": {"type": "string"} + } + } + } + }, + "translation": { + "type": "object", + "patternProperties": { + ".*": {"$ref": "translation.schema.json"} + } + }, + "type": { + "type": "string" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-base-schema.schema.json b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json new file mode 100644 index 0000000000..16d7a7043b --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json @@ -0,0 +1,26 @@ +{ + "$id": "entity-base-schema.schema.json", + "type": "object", + "properties": { + "cloud_device_id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "guid": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "allOf": [ + { + "$ref": "entity-attributes.schema.json" + } + ], + "required": ["type"], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-block-schema.schema.json b/tools/validators/instance_validator/schemas/entity-block-schema.schema.json new file mode 100644 index 0000000000..aef2b0040e --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-block-schema.schema.json @@ -0,0 +1,22 @@ +{ + "$id": "entity-block-schema.schema.json", + "type": "object", + "oneOf": [ + { + "$ref": "entity-update-schema.schema.json" + }, + { + "$ref": "entity-add-schema.schema.json" + }, + { + "$ref": "entity-delete-schema.schema.json" + }, + { + "$ref": "entity-export-schema.schema.json" + }, + { + "$ref": "entity-noop-schema.schema.json" + } + ], + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json b/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json new file mode 100644 index 0000000000..889512668d --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json @@ -0,0 +1,17 @@ +{ + "$id": "entity-delete-schema.schema.json", + "type": "object", + "properties": { + "operation": { + "const": "DELETE" + } + }, + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], + "required": ["operation"], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} diff --git a/tools/validators/instance_validator/schemas/entity-export-schema.schema.json b/tools/validators/instance_validator/schemas/entity-export-schema.schema.json new file mode 100644 index 0000000000..d89c3c5189 --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-export-schema.schema.json @@ -0,0 +1,22 @@ +{ + "$id": "entity-export-schema.schema.json", + "type": "object", + "properties": { + "etag": { + "type": "string" + }, + "type": { + "type": "string" + }, + "operation": { + "const": "EXPORT" + } + }, + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], + "required": [ "operation"], + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json b/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json new file mode 100644 index 0000000000..42890cc98e --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json @@ -0,0 +1,10 @@ +{ + "$id": "entity-noop-schema.schema.json", + "type": "object", + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-update-schema.schema.json b/tools/validators/instance_validator/schemas/entity-update-schema.schema.json new file mode 100644 index 0000000000..7cc59363a0 --- /dev/null +++ b/tools/validators/instance_validator/schemas/entity-update-schema.schema.json @@ -0,0 +1,31 @@ +{ + "$id": "entity-update-schema.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], + "dependentRequired": { + "update_mask": [ + "operation" + ] + }, + "properties": { + "etag": { + "type": "string" + }, + "operation": { + "const": "UPDATE" + }, + "update_mask": { + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "etag", + "operation" + ], + "type": "object" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/translation.schema.json b/tools/validators/instance_validator/schemas/translation.schema.json new file mode 100644 index 0000000000..96e06cbe33 --- /dev/null +++ b/tools/validators/instance_validator/schemas/translation.schema.json @@ -0,0 +1,38 @@ +{ + "$id": "translation.schema.json", + "type": "object", + "properties": { + "present_value": { + "type": "string" + }, + "states": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "units": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "values": { + "type": "object", + "properties": { + ".*": { + "type": "string" + } + } + } + } + } + } + }, + "required": [ + "present_value" + ], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_keys.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_keys.yaml index f89ed71c63..79ef3798c7 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_keys.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_keys.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +CONFIG_METADATA: + operation: EXPORT + US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING type: FACILITIES/123456b diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_add_mask.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_add_mask.yaml index 503a918d2b..76e7b565dc 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_add_mask.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_add_mask.yaml @@ -13,6 +13,9 @@ # limitations under the License. # Cannot have update mask on an ADD operation +CONFIG_METADATA: + operation: UPDATE + SDC_EXT-17-GUID: type: HVAC/SDC_EXT code: SDC_EXT-17 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/missing_colon.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/missing_colon.yaml index d51c63ed32..b66d36462a 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/missing_colon.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/missing_colon.yaml @@ -13,6 +13,10 @@ # limitations under the License. # Building +CONFIG_MODE: + operation: EXPORT + US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING + etag: 123456 code US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/configmode.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/no_config_metadata.yaml similarity index 95% rename from tools/validators/instance_validator/tests/fake_instances/BAD/configmode.yaml rename to tools/validators/instance_validator/tests/fake_instances/BAD/no_config_metadata.yaml index 579080b6ab..55c3c39e33 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/configmode.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/no_config_metadata.yaml @@ -19,9 +19,6 @@ SDC_EXT-17-GUID: etag: a12345 code: SDC_EXT-17 -CONFIG_METADATA: - operation: EXPORT - US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING etag: b23456 diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml index e57a3974a8..2c01dc0a67 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml @@ -12,11 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +CONFIG_METADATA: + operation: INITIALIZE + # Building US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING + etag: 123456 code: US-SEA-BLDG1 connections: # Listed entities are sources on connections - ANOTHER-ENTITY-GUID: FEEDS - A-THIRD-ENTITY-GUID: CONTAINS + 62b81ce8-f8bf-47e6-86b8-20870f9712e3: FEEDS + 8dff2e92-b619-4259-a683-c946bd0ff612: CONTAINS diff --git a/tools/validators/instance_validator/tests/instance_parser_test.py b/tools/validators/instance_validator/tests/instance_parser_test.py index 80d7cb453f..fb7b30645d 100644 --- a/tools/validators/instance_validator/tests/instance_parser_test.py +++ b/tools/validators/instance_validator/tests/instance_parser_test.py @@ -57,11 +57,13 @@ def testInstanceValidator_DetectDuplicateKeys_Fails(self): [path.join(_TESTCASE_PATH, 'BAD', 'duplicate_keys.yaml')]) del parser + # Migrated def testInstanceValidator_DetectMissingColon_Fails(self): with self.assertRaises(SystemExit): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'missing_colon.yaml')]) del parser + # Migrated def testInstanceValidator_DetectImproperSpacing_Fails(self): with self.assertRaises(SystemExit): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'spacing.yaml')]) @@ -72,6 +74,8 @@ def testInstanceValidator_DetectImproperTabbing_Fails(self): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'tabbing.yaml')]) del parser + # Don't think the following test case should be handled by parser...Maybe? + # Come back to it def testInstanceValidator_ParseProperFormat_Success(self): parser = _Helper([path.join(_TESTCASE_PATH, 'GOOD', 'building_type.yaml')]) del parser diff --git a/tools/validators/instance_validator/tests/parser_test.py b/tools/validators/instance_validator/tests/parser_test.py new file mode 100644 index 0000000000..a1c1cfd0d3 --- /dev/null +++ b/tools/validators/instance_validator/tests/parser_test.py @@ -0,0 +1,91 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests tools.validators.instance_validator.instance_parser.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os.path +import yaml +from os import path + +from absl.testing import absltest + +from tests import test_constants +from validate import handler +from validate import parser as p +from validate import enumerations + + +# _TESTCASE_PATH = test_constants.TEST_INSTANCES + +class ParserTest(absltest.TestCase): + def setUp(self): + self.parser = p.Parser(schema_folder='../schemas') + def testGetDefaultEntityOperation(self): + pass + + def testDeserializeGoodBuildingConfig(self): + pass + + def testDeserialize_DuplicateKeys_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'duplicate_keys.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertEqual(config_mode, enumerations.ConfigMode.EXPORT) + self.assertLen(entities, 0) + + def testDeserialize_NoConfigModeRaisesKeyError(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'no_config_metadata.yaml')] + + with self.assertRaises(KeyError): + self.parser.Deserialize(yaml_files=bad_testcase_path) + + def testDeserialize_MissingColon_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'missing_colon.yaml')] + + with self.assertRaises(yaml.scanner.ScannerError): + self.parser.Deserialize(yaml_files=bad_testcase_path) + + def testDeserialize_BadSpacing_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'spacing.yaml')] + + with self.assertRaises(yaml.scanner.ScannerError): + self.parser.Deserialize(yaml_files=bad_testcase_path) + + def testDeserialize_BadSpacing_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'tabbing.yaml')] + + with self.assertRaises(yaml.parser.ParserError): + self.parser.Deserialize(yaml_files=bad_testcase_path) + + def testDeserialize_BuildingConnections_Success(self): + good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'building_connections.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + + self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) + self.assertLen(entities, 1) + self.assertIn('US-SEA-BLDG1-GUID', entities) + + + + + + + +if __name__ == '__main__': + absltest.main() \ No newline at end of file diff --git a/tools/validators/instance_validator/validate/handler.py b/tools/validators/instance_validator/validate/handler.py index 8f50d219d2..bfb2a9d7f1 100644 --- a/tools/validators/instance_validator/validate/handler.py +++ b/tools/validators/instance_validator/validate/handler.py @@ -21,16 +21,22 @@ import os import sys import uuid -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Any +import yaml from validate import constants from validate import entity_instance +from validate import enumerations from validate import generate_universe -from validate import instance_parser +from validate import parser as p from validate import subscriber from validate import telemetry_validation_report as tvr from validate import telemetry_validator from yamlformat.validator import presubmit_validate_types_lib as pvt +try: + from yaml import CLOader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper INSTANCE_VALIDATION_FILENAME = 'instance_validation_report.txt' @@ -56,27 +62,12 @@ def FileNameEnumerationHelper(filename: str) -> str: )) -def GetDefaultOperation( - config_mode: instance_parser.ConfigMode, -) -> instance_parser.EntityOperation: - """Returns the default EntityOperation for the ConfigMode.""" - if config_mode == instance_parser.ConfigMode.INITIALIZE: - return instance_parser.EntityOperation.ADD - # we default to export for a config update when no operation is specified - elif config_mode == instance_parser.ConfigMode.UPDATE: - return instance_parser.EntityOperation.EXPORT - elif config_mode == instance_parser.ConfigMode.EXPORT: - return instance_parser.EntityOperation.EXPORT - else: - raise LookupError - - def Deserialize( yaml_files: List[str], ) -> Tuple[ - Dict[uuid.UUID, entity_instance.EntityInstance], instance_parser.ConfigMode + Dict[uuid.UUID, entity_instance.EntityInstance], enumerations.ConfigMode ]: - """Parses a yaml configuration file and deserializes it. + """Wrapper function to parse a yaml configuration file and deserializes it. Args: yaml_files: list of building configuration files. @@ -86,39 +77,8 @@ def Deserialize( ConfigMode: INITIALIZE or UPDATE """ - print('[INFO]\tStarting syntax validation.') - parser = instance_parser.InstanceParser() - for yaml_file in yaml_files: - print(f'[INFO]\tOpening file: {yaml_file}.') - parser.AddFile(yaml_file) - parser.Finalize() - - default_entity_operation = GetDefaultOperation(parser.GetConfigMode()) - - entities = {} - for entity_key, entity_yaml in parser.GetEntities().items(): - try: - entity = entity_instance.EntityInstance.FromYaml( - str(entity_key), entity_yaml, default_entity_operation - ) - entities[entity.guid] = entity - except ValueError as ex: - print( - '[ERROR]\tInvalid Entity syntax found for this entity: ' - f'{entity_key} and this content: "{entity_yaml}" and with error' - f': "{ex}"' - ) - raise ex - except KeyError as ex: - print( - '[ERROR]\tInvalid Entity syntax found for this entity: ' - f'{entity_key} and this content: "{entity_yaml}" and with error' - f': "{ex}"' - ) - raise ex - - return entities, parser.GetConfigMode() - + parser = p.Parser() + return parser.Deserialize(yaml_files=yaml_files) def _ValidateConfig( filenames: List[str], universe: pvt.ConfigUniverse, is_udmi @@ -379,7 +339,7 @@ def _IsDuplicateCDMIds( def Validate( self, entities: Dict[str, entity_instance.EntityInstance], - config_mode: instance_parser.ConfigMode, + config_mode: enumerations.ConfigMode, is_udmi: bool = True, ) -> Dict[str, entity_instance.EntityInstance]: """Validates entity instances that are already deserialized. @@ -413,7 +373,7 @@ def Validate( 'than 2 operations; one being EXPORT.' ) if ( - current_entity.operation is not instance_parser.EntityOperation.DELETE + current_entity.operation is not enumerations.EntityOperation.DELETE and current_entity.type_name.lower() == 'building' ): building_found = True @@ -447,7 +407,7 @@ def GetValidationState(self) -> bool: return self.__validation_state def ValidateAndUpdateState( - self, entity_operation: instance_parser.EntityOperation + self, entity_operation: enumerations.EntityOperation ) -> bool: """Validates entity instance operation against v1 Alpha Milestones. @@ -461,7 +421,7 @@ def ValidateAndUpdateState( True if entities seen thus far conform to v1 Alpha constraint; False o.w. """ # first branch by EXPORT - if entity_operation == instance_parser.EntityOperation.EXPORT: + if entity_operation == enumerations.EntityOperation.EXPORT: return self.GetValidationState() # either: ADD, DELETE, UPDATE # an entity operation of the three has been detected diff --git a/tools/validators/instance_validator/validate/instance_parser.py b/tools/validators/instance_validator/validate/instance_parser.py index 197ed34587..63fc3e7b76 100644 --- a/tools/validators/instance_validator/validate/instance_parser.py +++ b/tools/validators/instance_validator/validate/instance_parser.py @@ -79,7 +79,7 @@ def _OrRegex(values: List[str]) -> syaml.Regex: E = TypeVar('E', bound=enum.Enum) - +# WHY? What's the purpose of the below function. def EnumToRegex( enum_type: Optional[Type[E]] = None, omit: Optional[List[E]] = None, @@ -465,6 +465,7 @@ def _ValidateBlock( validation_fn: a validation function that takes YAML as an argument """ try: + validated = syaml.load( unvalidated_block, syaml.MapPattern(syaml.Str(), syaml.Any()) ) diff --git a/tools/validators/instance_validator/validate/parser.py b/tools/validators/instance_validator/validate/parser.py new file mode 100644 index 0000000000..747611c56f --- /dev/null +++ b/tools/validators/instance_validator/validate/parser.py @@ -0,0 +1,167 @@ +import jsonschema +import json +import os +import re +from referencing import Registry, Resource +from referencing.jsonschema import DRAFT202012 +from typing import Dict, Tuple, Any, List +import uuid +import yaml + +from validate import enumerations +from validate import entity_instance +from validate import schema + +try: + from yaml import CLOader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +_ENTITY_CODE_REGEX = r'^[a-zA-Z][a-zA-Z0-9/\-_ ]+:' +_ENTITY_GUID_REGEX = r'^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}:' +_ENTITY_CODE_PATTERN = re.compile(_ENTITY_CODE_REGEX) +_ENTITY_GUID_PATTERN = re.compile(_ENTITY_GUID_REGEX) + +# Exact key for the configuration metadata block +_CONFIG_METADATA_KEY = 'CONFIG_METADATA' +_CONFIG_METADATA_REGEX = f'^{_CONFIG_METADATA_KEY}:' +_CONFIG_METADATA_PATTERN = re.compile(_CONFIG_METADATA_REGEX) +_SCHEMA_ID = '$id' + +class Parser(object): + """A simple parser for building config yaml files. + + Attributes: + config_mode: An instance of ConfigMode enumeration. + """ + + def __init__(self, schema_folder): + """Init. + Args: + schema_folder: Absolute path to a folder of json schema files. + """ + self.config_mode = enumerations.ConfigMode.EXPORT + self.schema_folder = schema_folder + + def GetDefaultEntityOperation(self) -> enumerations.EntityOperation: + """Returns the default EntityOperation for the ConfigMode.""" + if self.config_mode == enumerations.ConfigMode.EXPORT: + return enumerations.EntityOperation.EXPORT + elif self.config_mode == enumerations.ConfigMode.UPDATE: + return enumerations.EntityOperation.EXPORT + elif self.config_mode == enumerations.ConfigMode.INITIALIZE: + return enumerations.EntityOperation.ADD + + def _CreateSchemaRegistry(self) -> Registry: + resources = [] + for root, dir_names, filenames in os.walk(os.path.abspath(self.schema_folder)): + for file in filenames: + with open(os.path.abspath(os.path.join(root, file)), 'r') as f: + parsed_json = json.loads(f.read()) + new_resource = Resource.from_contents(parsed_json) + resources.append((file, new_resource)) + return Registry().with_resources(pairs=resources) + + + def _ValidateMetadataSchema(self, metadata_block: Dict[str, Any]) -> bool: + """Helper function to validate a building config's CONFIG METADATA block. + + Args: + metadata_block: Dictionary mapping for a building config CONFIG METADATA block. + + Returns: + a boolean representing if the metadata has been validated? is valid? + """ + with open(os.path.abspath(os.path.join(self.schema_folder, 'config-metadata.schema.json')), 'r') as f: + config_metadata_schema = json.loads(f.read()) + try: + jsonschema.validate(instance=metadata_block, schema=config_metadata_schema) + self.config_mode = enumerations.ConfigMode(metadata_block.get('operation')) + except jsonschema.ValidationError as ve: + print('CONFIG_METADATA is invalid') + print(ve) + return False + except ValueError: + print('CONFIG_METADATA operation invalid. Operation must be one of INITLIAZE, EXPORT, UPDATE') + return False + return True + + + def _ValidateEntityInstance(self, guid: str, config_dict: Dict[str, Any], validator: jsonschema.Draft202012Validator) -> bool: + """Helper function to validate a building config schema using jsonschema. + + Args: + config_dict: A parsed dictionary representation of a building config yaml file. + validator: + + Returns: + a boolean indicating whether the block is valid + """ + try: + validator.validate(instance=config_dict) + except jsonschema.ValidationError as ve: + print(ve) + return False + return True + + + def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_instance.EntityInstance], enumerations.ConfigMode]: + """New deserialize logic that uses pyyaml and jsonschema rather than strictyaml. + + Steps: + 1. Parse yaml file + 2. need to validate block by block + 3. separate out config metadata and validate + 4. For each block, ensure it matches one of the entity block schemas defined in schema. + + Args: + yaml_files: A list of absolute paths for a collection of building config files. + + Returns: + A tuple containing a dictionary mapping of uuid objects to entity instances + and the building config's config mode. + """ + + for yaml_file in yaml_files: + absolute_bc_path = os.path.expanduser(yaml_file) + with open(absolute_bc_path) as f: + yaml_dict = yaml.load(f.read(), Loader=Loader) + + # Fix metadata validation here + # Enforce the existence opf + metadata_block = yaml_dict.get(_CONFIG_METADATA_KEY) + if not metadata_block: + raise KeyError('[ERROR]\tBuilding config must contain CONFIG_METADATA') + else: + self._ValidateMetadataSchema(metadata_block) + del yaml_dict[_CONFIG_METADATA_KEY] + + with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-block-schema.schema.json')), 'r') as f: + entity_block_schema = json.loads(f.read()) + try: + jsonschema.Draft202012Validator.check_schema(schema=entity_block_schema) + except jsonschema.SchemaError as schema_error: + raise schema_error + validator = jsonschema.Draft202012Validator(schema=entity_block_schema, registry=self._CreateSchemaRegistry()) + + return_dict = {} + for guid, entity_yaml in yaml_dict.items(): + if guid == _CONFIG_METADATA_KEY: + print('Cannot have more than one config metadata block') + continue + default_entity_operation = self.GetDefaultEntityOperation() + is_valid = False + try: + valid_uuid = uuid.UUID(guid) + except ValueError: + # Great spot to append to a log here + print('Not a valid guid') + continue + is_valid = self._ValidateEntityInstance(guid, entity_yaml, validator) + if is_valid: + entity = entity_instance.EntityInstance.FromYaml( + str(valid_uuid), entity_yaml, default_entity_operation + ) + return_dict.update({valid_uuid: entity}) + return return_dict, self.config_mode + diff --git a/tools/validators/instance_validator/validate/schema.py b/tools/validators/instance_validator/validate/schema.py index d7a32e59b9..149a95d6d1 100644 --- a/tools/validators/instance_validator/validate/schema.py +++ b/tools/validators/instance_validator/validate/schema.py @@ -11,7 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import re +import os + +from referencing import Resource, Registry +from referencing.jsonschema import DRAFT202012 from validate import enumerations #### Public Text parsing Constants #### @@ -58,11 +63,7 @@ 'properties': { PRESENT_VALUE_KEY: {'type': 'string'}, STATES_KEY: { - 'type': 'array', - 'prefixItems': [ - # Not sure if this right? - {{'type': 'string'}: {'type': 'string'}} - ] + 'type': 'array' }, UNITS_KEY: { 'type': 'object', @@ -77,24 +78,22 @@ 'required': [PRESENT_VALUE_KEY] } -_METADATA_SCHEMA = { - '$id': '/schemas/config-metadata', - _CONFIG_MODE_KEY: { - 'oneOf': [ - {'const': enumerations.ConfigMode.UPDATE.value}, - {'const': enumerations.ConfigMode.INITIALIZE.value}, - {'const': enumerations.ConfigMode.EXPORT.value}, - ] - } -} - _ENTITY_ATTR_SCHEMA = { '$id': '/schemas/entity-attributes', 'type': 'object', 'properties': { ENTITY_CODE_KEY: {'type': 'string'}, CONNECTIONS_KEY: { - 'type': 'array' + 'type': 'object', + 'patternProperties': { + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$': { + 'anyOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'} + ] + } + }, + 'additionalProperties': False }, LINKS_KEY: {'type': 'array'}, TRANSLATION_KEY: {'$ref': '/schemas/translation'} @@ -108,21 +107,18 @@ ENTITY_CLOUD_DEVICE_ID_KEY: {'type': 'string'}, ENTITY_CODE_KEY: {'type': 'string'}, ENTITY_GUID_KEY: {'type': 'string'}, - 'allOf': [ - {'$ref': '/schemas/entity-attributes'} - ] - } - + }, + 'allOf': [ + {'$ref': '/schemas/entity-attributes'} + ] } _ENTITY_UPDATE_SCHEMA = { + '$id': '/schemas/entity-update-schema', 'type': 'object', 'properties': { ETAG_KEY: {'type': 'string'}, ENTITY_TYPE_KEY: {'type': 'string'}, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.UPDATE.value}, UPDATE_MASK_KEY: { 'type': 'array', @@ -132,38 +128,80 @@ 'required': [ETAG_KEY, ENTITY_TYPE_KEY], 'dependentRequired': { UPDATE_MASK_KEY: [ENTITY_OPERATION_KEY] - } + }, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], } _ENTITY_ADD_SCHEMA = { + '$id': '/schemas/entity-add-schema', 'type': 'object', 'properties': { ENTITY_TYPE_KEY: {'type': 'string'}, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.ADD.value}, - } + }, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], } _ENTITY_DELETE_SCHEMA = { + '$id': '/schemas/entity-delete-schema', 'type': 'object', 'properties': { - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.DELETE.value}, - } + }, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], } _ENTITY_EXPORT_SCHEMA = { + '$id': '/schemas/entity-export-schema', 'type': 'object', 'properties': { ETAG_KEY: {'type': 'string'}, ENTITY_TYPE_KEY: {'type': 'string'}, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.EXPORT.value}, + }, + 'allOf': [ + {'$ref': '/schemas/entity-base-schema'} + ], +} + +# Generalized entity schema. All entity blocks must adhere to the below schema. +ENTITY_BLOCK_SCHEMA = { + '$id': '/schemas/entity-block-schema', + 'type': 'object', + 'oneOf': [ + {'$ref': '/schemas/entity-update-schema'}, + {'$ref': '/schemas/entity-add-schema'}, + {'$ref': '/schemas/entity-delete-schema'}, + {'$ref': '/schemas/entity-export-schema'}, + ] +} + +METADATA_SCHEMA = { + '$id': '/schemas/config-metadata', + 'type': 'object', + 'properties': { + _CONFIG_MODE_KEY: { + 'oneOf': [ + {'const': enumerations.ConfigMode.UPDATE.value}, + {'const': enumerations.ConfigMode.INITIALIZE.value}, + {'const': enumerations.ConfigMode.EXPORT.value}, + ] } -} \ No newline at end of file + } +} +def ExportSchemaRegistry() -> Registry: + id_tag = '$id' + resources = [] + for root, dir_names, filenames in os.walk(os.path.abspath('../schemas')): + for file in filenames: + with open(os.path.abspath(os.path.join(root, file)), 'r') as f: + parsed_json = json.loads(f.read()) + new_resource = Resource.from_contents(json.loads(parsed_json)) + resources.append((file, new_resource)) + return Registry().with_resources(pairs=resources) \ No newline at end of file From a19e4e90ae15ab36fd4ac3706c541a3ff32f31db Mon Sep 17 00:00:00 2001 From: travis Date: Tue, 24 Oct 2023 13:48:30 -0700 Subject: [PATCH 05/14] migrate to jsonschema --- .../schemas/entity-attributes.schema.json | 3 +- .../schemas/entity-base-schema.schema.json | 8 ++- .../schemas/translation.schema.json | 29 ++++---- ...ice_id.yaml => additional_properties.yaml} | 20 +++--- .../fake_instances/BAD/entity_operation.yaml | 5 +- .../BAD/translation_compliant.yaml | 3 + .../fake_instances/BAD/translation_keys.yaml | 5 +- .../translation_missing_cloud_device_id.yaml | 7 +- .../BAD/translation_units_format.yaml | 5 +- .../GOOD/building_connection_list.yaml | 7 +- .../GOOD/building_connections.yaml | 3 +- .../fake_instances/GOOD/multi_instances.yaml | 21 +++--- .../tests/instance_parser_test.py | 11 +++ .../instance_validator/tests/parser_test.py | 72 ++++++++++++++++++- .../instance_validator/validate/parser.py | 4 +- 15 files changed, 152 insertions(+), 51 deletions(-) rename tools/validators/instance_validator/tests/fake_instances/BAD/{translation_no_cloud_device_id.yaml => additional_properties.yaml} (60%) diff --git a/tools/validators/instance_validator/schemas/entity-attributes.schema.json b/tools/validators/instance_validator/schemas/entity-attributes.schema.json index 59f1ef4af0..180cece260 100644 --- a/tools/validators/instance_validator/schemas/entity-attributes.schema.json +++ b/tools/validators/instance_validator/schemas/entity-attributes.schema.json @@ -21,8 +21,7 @@ } ] } - }, - "additionalProperties": false + } }, "links": { "type": "object", diff --git a/tools/validators/instance_validator/schemas/entity-base-schema.schema.json b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json index 16d7a7043b..3324d26b94 100644 --- a/tools/validators/instance_validator/schemas/entity-base-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json @@ -13,14 +13,18 @@ }, "type": { "type": "string" - } + }, + "connections": true, + "links": true, + "translation": true }, "allOf": [ { "$ref": "entity-attributes.schema.json" } ], - "required": ["type"], "additionalProperties": false, + "required": ["type"], + "dependentRequired": {"translation": ["cloud_device_id"]}, "$schema": "https://json-schema.org/draft/2020-12/schema" } \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/translation.schema.json b/tools/validators/instance_validator/schemas/translation.schema.json index 96e06cbe33..005a2f0e4b 100644 --- a/tools/validators/instance_validator/schemas/translation.schema.json +++ b/tools/validators/instance_validator/schemas/translation.schema.json @@ -11,23 +11,24 @@ ".*": { "type": "string" } - }, - "units": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "values": { - "type": "object", - "properties": { - ".*": { - "type": "string" - } + } + }, + "units": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "values": { + "type": "object", + "properties": { + ".*": { + "type": "string" } } } - } + }, + "required": ["key", "values"] } }, "required": [ diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_no_cloud_device_id.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/additional_properties.yaml similarity index 60% rename from tools/validators/instance_validator/tests/fake_instances/BAD/translation_no_cloud_device_id.yaml rename to tools/validators/instance_validator/tests/fake_instances/BAD/additional_properties.yaml index 7051ae5366..0c5c991f1d 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_no_cloud_device_id.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/additional_properties.yaml @@ -12,17 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This config file is invalid because it fails to pass the syntax expectations -# for translation. The devices listed in translation need to follow specific -# formats, and `invalid_key` is not a valid part of any of them. +CONFIG_METADATA: + operation: INITIALIZE -US-SEA-BLDG1-GUID: +# Building +3cf600dc-0e48-40ad-9807-ba98018e9946: type: FACILITIES/BUILDING code: US-SEA-BLDG1 - translation: - zone_air_temperature_sensor: - present_value: "temp_1" - units: - key: "units" - values: - degrees_celsius: "degC" + connections: + # Listed entities are sources on connections + 62b81ce8-f8bf-47e6-86b8-20870f9712e3: FEEDS + 8dff2e92-b619-4259-a683-c946bd0ff612: CONTAINS + blah: bad additional property \ No newline at end of file diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_operation.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_operation.yaml index ddba5c0d20..fd148f3ca5 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_operation.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_operation.yaml @@ -13,7 +13,10 @@ # limitations under the License. # Cannot have operation when config is in INITIALIZE mode -US-SEA-BLDG1-GUID: +CONFIG_MODE: + operation: INITIALIZE + +eb15ee68-795f-430a-bb1f-8e70eaf2e66a: type: FACILITIES/BUILDING operation: DELETE code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_compliant.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_compliant.yaml index 7ca7f06374..32d3030c8e 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_compliant.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_compliant.yaml @@ -15,6 +15,9 @@ # This is an invalid config file because the content of translation needs to # case-sensitive match the string `COMPLIANT` if the translation is compliant. +CONFIG_METADATA: + operation: INITIALIZE + US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING translation: COMPLIANT_INVALID diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_keys.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_keys.yaml index 2eaf74c8ff..567bd50974 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_keys.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_keys.yaml @@ -16,7 +16,10 @@ # for translation. The devices listed in translation need to follow specific # formats, and `invalid_key` is not a valid part of any of them. -US-SEA-BLDG1-GUID: +CONFIG_METADATA: + operation: INITIALIZE + +eb15ee68-795f-430a-bb1f-8e70eaf2e66a: type: FACILITIES/BUILDING code: US-SEA-BLDG1 cloud_device_id: "foobar" diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_missing_cloud_device_id.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_missing_cloud_device_id.yaml index 83c8fb5d77..0f7815172a 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_missing_cloud_device_id.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_missing_cloud_device_id.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -SDC_EXT-17-GUID: +CONFIG_METADATA: + operation: INITIALIZE + +eb15ee68-795f-430a-bb1f-8e70eaf2e66a: type: HVAC/SDC_EXT code: SDC_EXT-17 translation: @@ -23,6 +26,6 @@ SDC_EXT-17-GUID: values: percent: "%" -US-SEA-BLDG1-GUID: +135d08f4-8df0-46ae-86cb-16b953870aeb: type: FACILITIES/BUILDING code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_units_format.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_units_format.yaml index 0535c4a845..150959e605 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/translation_units_format.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/translation_units_format.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -US-SEA-BLDG1-GUID: +CONFIG_METADATA: + operation: INITIALIZE + +135d08f4-8df0-46ae-86cb-16b953870aeb: type: FACILITIES/BUILDING code: US-SEA-BLDG1 translation: diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connection_list.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connection_list.yaml index adb5b152a5..d706223f5f 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connection_list.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connection_list.yaml @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +CONFIG_METADATA: + operation: INITIALIZE + # Building -US-SEA-BLDG1-GUID: +3cf600dc-0e48-40ad-9807-ba98018e9946: type: FACILITIES/BUILDING code: US-SEA-BLDG1 connections: # Listed entities are sources on connections - ANOTHER-ENTITY-GUID: + d5568a44-f78f-4704-a3f8-d8c9ecf6111a: - FEEDS - CONTAINS diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml index 2c01dc0a67..334e3e4c2d 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/building_connections.yaml @@ -16,9 +16,8 @@ CONFIG_METADATA: operation: INITIALIZE # Building -US-SEA-BLDG1-GUID: +3cf600dc-0e48-40ad-9807-ba98018e9946: type: FACILITIES/BUILDING - etag: 123456 code: US-SEA-BLDG1 connections: # Listed entities are sources on connections diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml index 5b0f6019b3..0fa950344e 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml @@ -14,17 +14,18 @@ # Multiple instances aiming to test the parsing of several entities in a file. -AHU-11-GUID: - connections: - FARSR-1: FEEDS +CONFIG_METADATA: + operation: INITIALIZE + +37bc3537-9c19-42f9-968e-893d2ce1c6b6: code: AHU-11 type: FACILITIES/BUILDING -FCU-1-GUID: +9fe360c5-3ee2-4ca5-a395-fd818f2d9fe: connections: - VLV-1: CONTROLS - VLV-2: CONTROLS - ZONE-G00: FEEDS + 135d08f4-8df0-46ae-86cb-16b953870aeb: CONTROLS + 04e831e8-7e08-4c68-a920-b23065e41e5e: CONTROLS + 709e5ed6-d1a9-49c3-b6b1-b9bd6099b028: FEEDS code: FCU-1 cloud_device_id: "1234567890123456" translation: @@ -142,10 +143,10 @@ FCU-1-GUID: degrees_celsius: degC type: HVAC/FCU_DFSS_DFVSC_ZTC_ZHC_CHWDC_HWDC_FDPM_RMM -FCU-10-GUID: +2778562f-8600-4c55-bb36-0802cdf63956: connections: - VLV-17: CONTROLS - ZONE-G03: FEEDS + 71965f14-9690-4218-9c9e-cb9550b8a07f: CONTROLS + 1f3f9c94-0e3b-4389-b1ad-bdbd5ea1a927: FEEDS code: FCU-10 cloud_device_id: "1234567890123457" translation: diff --git a/tools/validators/instance_validator/tests/instance_parser_test.py b/tools/validators/instance_validator/tests/instance_parser_test.py index fb7b30645d..b1ef05d987 100644 --- a/tools/validators/instance_validator/tests/instance_parser_test.py +++ b/tools/validators/instance_validator/tests/instance_parser_test.py @@ -69,6 +69,7 @@ def testInstanceValidator_DetectImproperSpacing_Fails(self): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'spacing.yaml')]) del parser + # Migrated def testInstanceValidator_DetectImproperTabbing_Fails(self): with self.assertRaises(SystemExit): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'tabbing.yaml')]) @@ -80,11 +81,13 @@ def testInstanceValidator_ParseProperFormat_Success(self): parser = _Helper([path.join(_TESTCASE_PATH, 'GOOD', 'building_type.yaml')]) del parser + # Migrated def testInstanceValidator_ParseProperConnections_Success(self): parser = _Helper( [path.join(_TESTCASE_PATH, 'GOOD', 'building_connections.yaml')]) del parser + # Migrated def testInstanceValidator_ParseProperConnectionList_Success(self): parser = _Helper( [path.join(_TESTCASE_PATH, 'GOOD', 'building_connection_list.yaml')]) @@ -99,12 +102,14 @@ def testInstanceValidator_ParseMultipleEntities_Success(self): self.assertIn('FCU-1-GUID', parser.keys()) self.assertIn('FCU-10-GUID', parser.keys()) + # Migrated def testInstanceValidator_DetectImproperTranslationCompliance(self): with self.assertRaises(SystemExit): parser = _Helper( [path.join(_TESTCASE_PATH, 'BAD', 'translation_compliant.yaml')]) del parser + # Migrated def testInstanceValidator_DetectImproperTranslationKeys(self): with self.assertRaises(SystemExit): parser = _Helper( @@ -117,6 +122,7 @@ def testInstanceValidator_DetectImproperUnitsKeys(self): [path.join(_TESTCASE_PATH, 'BAD', 'translation_units_format.yaml')]) del parser + # Migrated def testInstanceValidator_CloudDeviceIdNotSetWithTranslation(self): with self.assertRaises(KeyError): parser = _Helper([ @@ -125,23 +131,28 @@ def testInstanceValidator_CloudDeviceIdNotSetWithTranslation(self): ]) del parser + #Don't think parser should test for below... def testInstanceValidator_DetectDuplicateEntityKeys(self): with self.assertRaises(SystemExit): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'duplicate_key.yaml')]) del parser + # Don't know if I need this either def testInstanceValidator_DetectDuplicateMetadata(self): with self.assertRaises(SystemExit): parser = _Helper( [path.join(_TESTCASE_PATH, 'BAD', 'duplicate_metadata.yaml')]) del parser + # Not yet + # Need to add validation for having entity operations under INITIALIZE config mode def testInstanceValidator_RejectsOperationOnInitialize(self): with self.assertRaises(SystemExit): parser = _Helper( [path.join(_TESTCASE_PATH, 'BAD', 'entity_operation.yaml')]) del parser + # Add validation for this def testInstanceValidator_RejectsMaskOnInitialize(self): with self.assertRaises(SystemExit): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'entity_mask.yaml')]) diff --git a/tools/validators/instance_validator/tests/parser_test.py b/tools/validators/instance_validator/tests/parser_test.py index a1c1cfd0d3..f60ba69ea9 100644 --- a/tools/validators/instance_validator/tests/parser_test.py +++ b/tools/validators/instance_validator/tests/parser_test.py @@ -27,6 +27,7 @@ from validate import handler from validate import parser as p from validate import enumerations +import uuid # _TESTCASE_PATH = test_constants.TEST_INSTANCES @@ -72,6 +73,7 @@ def testDeserialize_BadSpacing_Fails(self): with self.assertRaises(yaml.parser.ParserError): self.parser.Deserialize(yaml_files=bad_testcase_path) + # NOT WORKING def testDeserialize_BuildingConnections_Success(self): good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'building_connections.yaml')] @@ -79,7 +81,75 @@ def testDeserialize_BuildingConnections_Success(self): self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) self.assertLen(entities, 1) - self.assertIn('US-SEA-BLDG1-GUID', entities) + self.assertIn(uuid.UUID('3cf600dc-0e48-40ad-9807-ba98018e9946'), entities) + + def testDeserialize_ParseConnectionList_Succeeds(self): + good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'building_connection_list.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + + self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) + self.assertLen(entities, 1) + self.assertIn(uuid.UUID('3cf600dc-0e48-40ad-9807-ba98018e9946'), entities) + + def testDeserialize_AdditionalProperties_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'additional_properties.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) + self.assertLen(entities, 0) + + def testDeserialize_ParseMultipleEntities_Success(self): + good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'multi_instances.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + + self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) + self.assertLen(entities, 3) + self.assertIn(uuid.UUID('37bc3537-9c19-42f9-968e-893d2ce1c6b6'), entities) + self.assertIn(uuid.UUID('9fe360c5-3ee2-4ca5-a395-fd818f2d9fe'), entities) + self.assertIn(uuid.UUID('2778562f-8600-4c55-bb36-0802cdf63956'), entities) + + def testDeserialize_BadTranslation_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_compliant.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertLen(entities, 0) + + def testDeserialize_BadTranslationKeys_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_keys.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertLen(entities, 0) + + def testDeserialize_BadTranslationUnits_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_units_format.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertLen(entities, 0) + + def testDeserialize_BadTranslationNoCDID_Fails(self): + bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_missing_cloud_device_id.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + self.assertLen(entities, 1) + self.assertIn(uuid.UUID('135d08f4-8df0-46ae-86cb-16b953870aeb'), entities) + self.assertNotIn(uuid.UUID('eb15ee68-795f-430a-bb1f-8e70eaf2e66a'), entities) + + def testDeserialize_DuplicateMetadata_Fails(self): + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'duplicate_metadata.yaml')] + + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + + + + diff --git a/tools/validators/instance_validator/validate/parser.py b/tools/validators/instance_validator/validate/parser.py index 747611c56f..35cd04a8d6 100644 --- a/tools/validators/instance_validator/validate/parser.py +++ b/tools/validators/instance_validator/validate/parser.py @@ -82,7 +82,7 @@ def _ValidateMetadataSchema(self, metadata_block: Dict[str, Any]) -> bool: print(ve) return False except ValueError: - print('CONFIG_METADATA operation invalid. Operation must be one of INITLIAZE, EXPORT, UPDATE') + print('CONFIG_METADATA operation invalid. Operation must be one of INITIALIZE, EXPORT, UPDATE') return False return True @@ -136,7 +136,7 @@ def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_ins self._ValidateMetadataSchema(metadata_block) del yaml_dict[_CONFIG_METADATA_KEY] - with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-block-schema.schema.json')), 'r') as f: + with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-noop-schema.schema.json')), 'r') as f: entity_block_schema = json.loads(f.read()) try: jsonschema.Draft202012Validator.check_schema(schema=entity_block_schema) From aed5296c600dd851ce658df888c11b33983be15b Mon Sep 17 00:00:00 2001 From: travis Date: Thu, 26 Oct 2023 12:42:11 -0700 Subject: [PATCH 06/14] migrate instance parsing to pyyaml and jsonschema --- .../schemas/entity-add-schema.schema.json | 9 +- .../schemas/entity-base-schema.schema.json | 4 +- .../schemas/entity-delete-schema.schema.json | 3 +- .../schemas/entity-export-schema.schema.json | 10 +- .../schemas/entity-noop-schema.schema.json | 10 + .../schemas/entity-update-schema.schema.json | 17 +- .../fake_instances/BAD/building_type.yaml | 5 +- .../BAD/duplicate_metadata.yaml | 28 --- .../tests/fake_instances/BAD/entity_etag.yaml | 2 +- .../BAD/entity_export_operation.yaml | 4 +- .../BAD/update_mask_operation.yaml | 2 +- .../fake_instances/BAD/update_mask_value.yaml | 4 +- .../fake_instances/GOOD/code_with_spaces.yaml | 24 --- .../GOOD/entity_export_operation.yaml | 2 +- .../fake_instances/GOOD/multi_instances.yaml | 196 +----------------- .../GOOD/new_format_singleton.yaml | 31 --- .../GOOD/states_case_insensitive.yaml | 3 + .../fake_instances/GOOD/update_mask.yaml | 4 +- .../tests/instance_parser_test.py | 12 ++ .../instance_validator/tests/parser_test.py | 39 +++- .../validate/entity_instance.py | 4 +- .../instance_validator/validate/parser.py | 8 +- 22 files changed, 102 insertions(+), 319 deletions(-) delete mode 100644 tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_metadata.yaml delete mode 100644 tools/validators/instance_validator/tests/fake_instances/GOOD/code_with_spaces.yaml delete mode 100644 tools/validators/instance_validator/tests/fake_instances/GOOD/new_format_singleton.yaml diff --git a/tools/validators/instance_validator/schemas/entity-add-schema.schema.json b/tools/validators/instance_validator/schemas/entity-add-schema.schema.json index 88b0f7ad51..b87f962b0d 100644 --- a/tools/validators/instance_validator/schemas/entity-add-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-add-schema.schema.json @@ -4,7 +4,13 @@ "properties": { "operation": { "const": "ADD" - } + }, + "code": true, + "connections": true, + "type": true, + "links": true, + "translation": true, + "cloud_device_id": true }, "required": ["operation"], "allOf": [ @@ -12,5 +18,6 @@ "$ref": "entity-base-schema.schema.json" } ], + "additionalProperties": false, "$schema": "https://json-schema.org/draft/2020-12/schema" } \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-base-schema.schema.json b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json index 3324d26b94..137c9a4aaa 100644 --- a/tools/validators/instance_validator/schemas/entity-base-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-base-schema.schema.json @@ -12,7 +12,8 @@ "type": "string" }, "type": { - "type": "string" + "type": "string", + "pattern": "^(.*\/.*)" }, "connections": true, "links": true, @@ -23,7 +24,6 @@ "$ref": "entity-attributes.schema.json" } ], - "additionalProperties": false, "required": ["type"], "dependentRequired": {"translation": ["cloud_device_id"]}, "$schema": "https://json-schema.org/draft/2020-12/schema" diff --git a/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json b/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json index 889512668d..0dcc239fec 100644 --- a/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-delete-schema.schema.json @@ -4,7 +4,8 @@ "properties": { "operation": { "const": "DELETE" - } + }, + "code": true }, "allOf": [ { diff --git a/tools/validators/instance_validator/schemas/entity-export-schema.schema.json b/tools/validators/instance_validator/schemas/entity-export-schema.schema.json index d89c3c5189..1e45efba8f 100644 --- a/tools/validators/instance_validator/schemas/entity-export-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-export-schema.schema.json @@ -10,13 +10,19 @@ }, "operation": { "const": "EXPORT" - } + }, + "code": true, + "connections": true, + "links": true, + "translation": true, + "cloud_device_id": true }, "allOf": [ { "$ref": "entity-base-schema.schema.json" } ], - "required": [ "operation"], + "required": ["operation", "etag"], + "additionProperties": false, "$schema": "https://json-schema.org/draft/2020-12/schema" } \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json b/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json index 42890cc98e..f439907b81 100644 --- a/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-noop-schema.schema.json @@ -6,5 +6,15 @@ "$ref": "entity-base-schema.schema.json" } ], + "properties": { + "code": true, + "connections": true, + "type": true, + "links": true, + "translation": true, + "etag": true, + "cloud_device_id": true + }, + "additionalProperties": false, "$schema": "https://json-schema.org/draft/2020-12/schema" } \ No newline at end of file diff --git a/tools/validators/instance_validator/schemas/entity-update-schema.schema.json b/tools/validators/instance_validator/schemas/entity-update-schema.schema.json index 7cc59363a0..0eca19ada3 100644 --- a/tools/validators/instance_validator/schemas/entity-update-schema.schema.json +++ b/tools/validators/instance_validator/schemas/entity-update-schema.schema.json @@ -1,11 +1,6 @@ { "$id": "entity-update-schema.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "allOf": [ - { - "$ref": "entity-base-schema.schema.json" - } - ], "dependentRequired": { "update_mask": [ "operation" @@ -21,8 +16,18 @@ "update_mask": { "type": "array", "uniqueItems": true - } + }, + "code": true, + "connections": true, + "links": true, + "translation": true, + "cloud_device_id": true }, + "allOf": [ + { + "$ref": "entity-base-schema.schema.json" + } + ], "required": [ "etag", "operation" diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/building_type.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/building_type.yaml index bcd59771fb..b07255dde9 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/building_type.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/building_type.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +CONFIG_METADATA: + operation: INITIALIZE + # Building -US-SEA-BLDG1-GUID: +135d08f4-8df0-46ae-86cb-16b953870aeb: type: FACILITIES/BUILDING/ERRORORO code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_metadata.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_metadata.yaml deleted file mode 100644 index 5dda06ea3a..0000000000 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/duplicate_metadata.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Two metadata blocks (bad) - -SDC_EXT-18-GUID: - code: SDC_EXT-18 - -CONFIG_METADATA: - operation: UPDATE - -CONFIG_METADATA: - operation: INITIALIZE - -US-SEA-BLDG1-GUID: - type: FACILITIES/BUILDING - code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_etag.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_etag.yaml index a7ecc7117f..c4c3029d73 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_etag.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_etag.yaml @@ -17,7 +17,7 @@ CONFIG_METADATA: operation: UPDATE -US-SEA-BLDG1-GUID: +04e831e8-7e08-4c68-a920-b23065e41e5e: operation: UPDATE type: FACILITIES/BUILDING code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_export_operation.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_export_operation.yaml index 590fb6c89c..c920611c2f 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/entity_export_operation.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/entity_export_operation.yaml @@ -15,9 +15,9 @@ # Bad entity export operation missing etag CONFIG_METADATA: - operation: UPDATE + operation: EXPORT -SDC_EXT-19-GUID: +04e831e8-7e08-4c68-a920-b23065e41e5e: type: HVAC/SDC_EXT code: SDC_EXT-19 cloud_device_id: "baz" diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_operation.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_operation.yaml index 4bcc1fdeb8..581fbf5c44 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_operation.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_operation.yaml @@ -17,7 +17,7 @@ CONFIG_METADATA: operation: UPDATE -SDC_EXT-19-GUID: +eb15ee68-795f-430a-bb1f-8e70eaf2e66a: type: HVAC/SDC_EXT code: SDC_EXT-19 cloud_device_id: "baz" diff --git a/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_value.yaml b/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_value.yaml index 9e742a6a09..2a87ecc66f 100644 --- a/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_value.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/BAD/update_mask_value.yaml @@ -18,13 +18,13 @@ CONFIG_METADATA: operation: UPDATE -SDC_EXT-19-GUID: +709e5ed6-d1a9-49c3-b6b1-b9bd6099b028: type: HVAC/SDC_EXT code: SDC_EXT-19 cloud_device_id: "baz" etag: a56790 update_mask: - translation + - translation - connection translation: shade_extent_percentage_command: diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/code_with_spaces.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/code_with_spaces.yaml deleted file mode 100644 index 3840f5354a..0000000000 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/code_with_spaces.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SDC_EXT 2-1 / Rm 2D2-GUID: - type: HVAC/SDC_EXT - code: SDC_EXT-2-1 - cloud_device_id: "1234567890123456" - translation: - shade_extent_percentage_command: - present_value: "points.shade_extent_percentage_command.present_value" - units: - key: "points.shade_extent_percentage_command.units" - values: - percent: "%" diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/entity_export_operation.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/entity_export_operation.yaml index 8e32f831a0..008f00f264 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/entity_export_operation.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/entity_export_operation.yaml @@ -17,7 +17,7 @@ CONFIG_METADATA: operation: UPDATE -SDC_EXT-19-GUID: +71965f14-9690-4218-9c9e-cb9550b8a07f: type: HVAC/SDC_EXT code: SDC_EXT-19 cloud_device_id: "1234567890123456" diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml index 0fa950344e..abb1ebf02b 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/multi_instances.yaml @@ -21,7 +21,7 @@ CONFIG_METADATA: code: AHU-11 type: FACILITIES/BUILDING -9fe360c5-3ee2-4ca5-a395-fd818f2d9fe: +71965f14-9690-4218-9c9e-cb9550b8a07f: connections: 135d08f4-8df0-46ae-86cb-16b953870aeb: CONTROLS 04e831e8-7e08-4c68-a920-b23065e41e5e: CONTROLS @@ -29,118 +29,12 @@ CONFIG_METADATA: code: FCU-1 cloud_device_id: "1234567890123456" translation: - chilled_water_flowrate_sensor: - present_value: points.chilled_water_flowrate_sensor.present_value - units: - key: pointset.points.chilled_water_flowrate_sensor.units - values: - liters_per_second: L/s - chilled_water_valve_percentage_command: - present_value: points.cooling_valve_percentage_command.present_value - units: - key: pointset.points.cooling_valve_percentage_command.units - values: - percent: '%' - chilled_water_valve_percentage_sensor: - present_value: points.cooling_valve_percentage_sensor.present_value - units: - key: pointset.points.cooling_valve_percentage_sensor.units - values: - percent: '%' - discharge_air_static_pressure_sensor: - present_value: points.supply_air_static_pressure_sensor.present_value - units: - key: pointset.points.supply_air_static_pressure_sensor.units - values: - pascals: Pa - discharge_air_temperature_sensor: - present_value: points.supply_air_temperature_sensor.present_value - units: - key: pointset.points.supply_air_temperature_sensor.units - values: - degrees_celsius: degC - discharge_air_temperature_setpoint: - present_value: points.supply_air_temperature_setpoint_humidity.present_value - units: - key: pointset.points.supply_air_temperature_setpoint_humidity.units - values: - degrees_celsius: degC - discharge_fan_run_command: - present_value: points.supply_fan_run_command.present_value - states: - OFF: 'false' - ON: 'true' - discharge_fan_run_status: - present_value: points.supply_fan_run_status.present_value - states: - OFF: 'false' - ON: 'true' - discharge_fan_speed_percentage_command: - present_value: points.supply_fan_speed_percentage_command.present_value - units: - key: pointset.points.supply_fan_speed_percentage_command.units - values: - percent: '%' - filter_differential_pressure_sensor: - present_value: points.filter_differential_pressure_sensor.present_value - units: - key: pointset.points.filter_differential_pressure_sensor.units - values: - pascals: Pa - heating_water_flowrate_sensor: - present_value: points.heating_water_flowrate_sensor.present_value - units: - key: pointset.points.heating_water_flowrate_sensor.units - values: - liters_per_second: L/s heating_water_valve_percentage_command: present_value: points.heating_valve_percentage_command.present_value units: key: pointset.points.heating_valve_percentage_command.units values: percent: '%' - heating_water_valve_percentage_sensor: - present_value: points.heating_valve_percentage_sensor.present_value - units: - key: pointset.points.heating_valve_percentage_sensor.units - values: - percent: '%' - return_air_temperature_sensor: - present_value: points.return_air_temperature_sensor.present_value - units: - key: pointset.points.return_air_temperature_sensor.units - values: - degrees_celsius: degC - run_mode: - present_value: points.device_mode.present_value - states: - OFF: '0.0' - AUTO: '1.0' - MANUAL: '2.0' - zone_air_relative_humidity_sensor: - present_value: points.space_air_relative_humidity_sensor.present_value - units: - key: pointset.points.space_air_relative_humidity_sensor.units - values: - percent_relative_humidity: '%RH' - zone_air_relative_humidity_setpoint: - present_value: points.space_air_relative_humidity_setpoint.present_value - units: - key: pointset.points.space_air_relative_humidity_setpoint.units - values: - percent_relative_humidity: '%RH' - zone_air_temperature_sensor: - present_value: points.space_air_temperature_sensor.present_value - units: - key: pointset.points.space_air_temperature_sensor.units - values: - degrees_celsius: degC - zone_air_temperature_setpoint: - present_value: points.space_air_temperature_setpoint.present_value - units: - key: pointset.points.space_air_temperature_setpoint.units - values: - degrees_celsius: degC type: HVAC/FCU_DFSS_DFVSC_ZTC_ZHC_CHWDC_HWDC_FDPM_RMM 2778562f-8600-4c55-bb36-0802cdf63956: @@ -156,92 +50,4 @@ CONFIG_METADATA: key: pointset.points.chilled_water_flowrate_sensor.units values: liters_per_second: L/s - chilled_water_valve_percentage_command: - present_value: points.cooling_valve_percentage_command.present_value - units: - key: pointset.points.cooling_valve_percentage_command.units - values: - percent: '%' - chilled_water_valve_percentage_sensor: - present_value: points.cooling_valve_percentage_sensor.present_value - units: - key: pointset.points.cooling_valve_percentage_sensor.units - values: - percent: '%' - discharge_air_static_pressure_sensor: - present_value: points.supply_air_static_pressure_sensor.present_value - units: - key: pointset.points.supply_air_static_pressure_sensor.units - values: - pascals: Pa - discharge_air_temperature_sensor: - present_value: points.supply_air_temperature_sensor.present_value - units: - key: pointset.points.supply_air_temperature_sensor.units - values: - degrees_celsius: degC - discharge_air_temperature_setpoint: - present_value: points.supply_air_temperature_setpoint_humidity.present_value - units: - key: pointset.points.supply_air_temperature_setpoint_humidity.units - values: - degrees_celsius: degC - discharge_fan_run_command: - present_value: points.supply_fan_run_command.present_value - states: - OFF: 'false' - ON: 'true' - discharge_fan_run_status: - present_value: points.supply_fan_run_status.present_value - states: - OFF: 'false' - ON: 'true' - discharge_fan_speed_percentage_command: - present_value: points.supply_fan_speed_percentage_command.present_value - units: - key: pointset.points.supply_fan_speed_percentage_command.units - values: - percent: '%' - filter_differential_pressure_sensor: - present_value: points.filter_differential_pressure_sensor.present_value - units: - key: pointset.points.filter_differential_pressure_sensor.units - values: - pascals: Pa - return_air_temperature_sensor: - present_value: points.return_air_temperature_sensor.present_value - units: - key: pointset.points.return_air_temperature_sensor.units - values: - degrees_celsius: degC - run_mode: - present_value: points.device_mode.present_value - states: - OFF: '0.0' - AUTO: '1.0' - MANUAL: '2.0' - zone_air_relative_humidity_sensor: - present_value: points.space_air_relative_humidity_sensor.present_value - units: - key: pointset.points.space_air_relative_humidity_sensor.units - values: - percent_relative_humidity: '%RH' - zone_air_relative_humidity_setpoint: - present_value: points.space_air_relative_humidity_setpoint.present_value - units: - key: pointset.points.space_air_relative_humidity_setpoint.units - values: - percent_relative_humidity: '%RH' - zone_air_temperature_sensor: - present_value: points.space_air_temperature_sensor.present_value - units: - key: pointset.points.space_air_temperature_sensor.units - values: - degrees_celsius: degC - zone_air_temperature_setpoint: - present_value: points.space_air_temperature_setpoint.present_value - units: - key: pointset.points.space_air_temperature_setpoint.units - values: - degrees_celsius: degC type: HVAC/FCU_DFSS_DFVSC_ZTC_ZHC_CHWDC_FDPM_RMM diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/new_format_singleton.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/new_format_singleton.yaml deleted file mode 100644 index 0c3fc3dfaf..0000000000 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/new_format_singleton.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -9a86a19b-b687-4db1-888e-2cf34d04b74c: - type: HVAC/CHWS_WDT - code: CHWS-1 - cloud_device_id: "1234567890123456" - translation: - supply_water_temperature_sensor: - present_value: points.supply_water_temperature_sensor.present_value - units: - key: points.supply_water_temperature_sensor.units - values: - degrees_celsius: degC - return_water_temperature_sensor: - present_value: points.return_water_temperature_sensor.present_value - units: - key: points.return_water_temperature_sensor.units - values: - degrees_celsius: degC diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/states_case_insensitive.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/states_case_insensitive.yaml index 3dfcab154c..cc40cc2cb1 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/states_case_insensitive.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/states_case_insensitive.yaml @@ -17,6 +17,9 @@ # instance validator: passes # entity instance: fails +CONFIG_METADATA: + operation: INITIALIZE + US-SEA-BLDG1-GUID: type: FACILITIES/BUILDING code: US-SEA-BLDG1 diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/update_mask.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/update_mask.yaml index 8770c8164d..6e86cf0079 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/update_mask.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/update_mask.yaml @@ -17,11 +17,12 @@ CONFIG_METADATA: operation: UPDATE -SDC_EXT-17-GUID: +71965f14-9690-4218-9c9e-cb9550b8a07f: type: HVAC/SDC_EXT code: SDC_EXT-17 cloud_device_id: "1234567890123456" etag: a56789 + operation: UPDATE update_mask: - translation - connection @@ -32,4 +33,3 @@ SDC_EXT-17-GUID: key: "points.shade_extent_percentage_command.units" values: percent: "%" - type: HVAC/SDC_EXT diff --git a/tools/validators/instance_validator/tests/instance_parser_test.py b/tools/validators/instance_validator/tests/instance_parser_test.py index b1ef05d987..326935a73a 100644 --- a/tools/validators/instance_validator/tests/instance_parser_test.py +++ b/tools/validators/instance_validator/tests/instance_parser_test.py @@ -169,18 +169,21 @@ def testInstanceValidator_RejectsUpdateWithoutEtag(self): parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'entity_etag.yaml')]) del parser + # Don't need this def testInstanceValidator_ReadsMetadata_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'with_metadata.yaml')]) self.assertLen(parser.GetEntities().keys(), 2) self.assertEqual(parser.GetConfigMode(), instance_parser.ConfigMode.UPDATE) + # Don't need this def testInstanceValidator_ReadsMetadataAtEnd_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'with_metadata_at_end.yaml')]) self.assertLen(parser.GetEntities().keys(), 2) self.assertEqual(parser.GetConfigMode(), instance_parser.ConfigMode.UPDATE) + # won't migrate def testInstanceValidator_HandlesUpdateMode_Success(self): parser = _ParserHelper([ path.join(_TESTCASE_PATH, 'GOOD', @@ -202,12 +205,14 @@ def testInstanceValidator_UsesDefaultMode_Success(self): self.assertEqual(parser.GetConfigMode(), instance_parser.ConfigMode.Default()) + # File configmode.yaml not found def testInstanceValidator_InvalidConfigModeExport_RaisesKeyError(self): with self.assertWarns(Warning): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'BAD', 'configmode.yaml')]) del parser + # Won't Migrate def testEntityBlock_NewFormatSingleton_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'new_format_singleton.yaml')]) @@ -215,23 +220,27 @@ def testEntityBlock_NewFormatSingleton_Success(self): list(parser.GetEntities().keys()).pop(), '9a86a19b-b687-4db1-888e-2cf34d04b74c') + # This test doesn't make sense def testEntityBlock_CodeWithSpace_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'code_with_spaces.yaml')]) self.assertEqual( list(parser.GetEntities().keys()).pop(), 'SDC_EXT 2-1 / Rm 2D2-GUID') + # Migrated def testEntityBlock_ValidUpdateMaskValueTypes_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'update_mask.yaml')]) self.assertLen(parser.GetEntities().keys(), 1) + #Migrated def testEntityBlock_InvalidUpdateMaskInconsistentTypes_Fails(self): with self.assertRaises(SystemExit): parser = _Helper( [path.join(_TESTCASE_PATH, 'BAD', 'update_mask_value.yaml')]) del parser + #Migrated def testEntityBlock_ValidUpdateMaskValue_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'update_mask.yaml')]) @@ -261,6 +270,7 @@ def testEntityBlock_ValidUpdateMaskValue_Success(self): self.assertLen(parser.GetEntities().keys(), 1) self.assertEqual(parser.GetEntities(), expected) + # Migrated def testGoodEntity_DefaultExportOperationParses_Success(self): parser = _ParserHelper( [path.join(_TESTCASE_PATH, 'GOOD', 'entity_export_operation.yaml')]) @@ -274,12 +284,14 @@ def testGoodEntity_DefaultExportOperationParses_Success(self): self.assertLen(parser.GetEntities().keys(), 1) self.assertEqual(default_operation, instance_parser.EntityOperation.EXPORT) + # Migrated def testEntityBlock_InvalidExportOperation_Fails(self): with self.assertRaises(SystemExit): parser = _Helper( [path.join(_TESTCASE_PATH, 'BAD', 'entity_export_operation.yaml')]) del parser + # Migrated def testEntityBlock_InvalidUpdateMaskAndOperation_Fails(self): with self.assertRaises(SystemExit): parser = _Helper( diff --git a/tools/validators/instance_validator/tests/parser_test.py b/tools/validators/instance_validator/tests/parser_test.py index f60ba69ea9..da033be0d1 100644 --- a/tools/validators/instance_validator/tests/parser_test.py +++ b/tools/validators/instance_validator/tests/parser_test.py @@ -18,6 +18,8 @@ from __future__ import print_function import os.path + +import jsonschema import yaml from os import path @@ -92,14 +94,6 @@ def testDeserialize_ParseConnectionList_Succeeds(self): self.assertLen(entities, 1) self.assertIn(uuid.UUID('3cf600dc-0e48-40ad-9807-ba98018e9946'), entities) - def testDeserialize_AdditionalProperties_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'additional_properties.yaml')] - - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) - - self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) - self.assertLen(entities, 0) - def testDeserialize_ParseMultipleEntities_Success(self): good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'multi_instances.yaml')] @@ -108,7 +102,7 @@ def testDeserialize_ParseMultipleEntities_Success(self): self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) self.assertLen(entities, 3) self.assertIn(uuid.UUID('37bc3537-9c19-42f9-968e-893d2ce1c6b6'), entities) - self.assertIn(uuid.UUID('9fe360c5-3ee2-4ca5-a395-fd818f2d9fe'), entities) + self.assertIn(uuid.UUID('71965f14-9690-4218-9c9e-cb9550b8a07f'), entities) self.assertIn(uuid.UUID('2778562f-8600-4c55-bb36-0802cdf63956'), entities) def testDeserialize_BadTranslation_Fails(self): @@ -141,20 +135,43 @@ def testDeserialize_BadTranslationNoCDID_Fails(self): self.assertIn(uuid.UUID('135d08f4-8df0-46ae-86cb-16b953870aeb'), entities) self.assertNotIn(uuid.UUID('eb15ee68-795f-430a-bb1f-8e70eaf2e66a'), entities) - def testDeserialize_DuplicateMetadata_Fails(self): + def testDeserializer_UpdateMaskWithoutUpdateOperation_Fails(self): bad_testcase_path = [ - os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'duplicate_metadata.yaml')] + os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'update_mask_value.yaml')] entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + self.assertLen(entities, 0) + def testDeserializer_UpdateMaskDependency_Success(self): + good_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'update_mask.yaml')] + entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + self.assertEqual(config_mode, enumerations.ConfigMode.UPDATE) + self.assertLen(entities, 1) + self.assertIn(uuid.UUID('71965f14-9690-4218-9c9e-cb9550b8a07f'), entities) + + def testDeserialize_DefaultExportOperationForUpdateMode_Success(self): + good_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'entity_export_operation.yaml')] + entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + self.assertEqual(config_mode, enumerations.ConfigMode.UPDATE) + self.assertEqual(self.parser.GetDefaultEntityOperation(), enumerations.EntityOperation.EXPORT) + self.assertLen(entities, 1) + self.assertIn(uuid.UUID('71965f14-9690-4218-9c9e-cb9550b8a07f'), entities) + def testDeserialize_ExportEntityMissingEtag_Fails(self): + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'entity_export_operation.yaml')] + entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + self.assertEqual(config_mode, enumerations.ConfigMode.EXPORT) + self.assertLen(entities, 0) if __name__ == '__main__': diff --git a/tools/validators/instance_validator/validate/entity_instance.py b/tools/validators/instance_validator/validate/entity_instance.py index 6a70e363f6..70d1ad08ff 100644 --- a/tools/validators/instance_validator/validate/entity_instance.py +++ b/tools/validators/instance_validator/validate/entity_instance.py @@ -1316,9 +1316,7 @@ def _ParseOperationAndUpdateMask( ) != parse.EntityOperation.UPDATE ): - raise ValueError( - 'Only specify UPDATE operation when "update_mask" is present.' - ) + print(f'[ERROR]\t{entity_yaml.get("code")} Only specify UPDATE operation when "update_mask" is present.') update_mask = entity_yaml[parse.UPDATE_MASK_KEY] operation = parse.EntityOperation.UPDATE # case 2: update_mask implies update operation diff --git a/tools/validators/instance_validator/validate/parser.py b/tools/validators/instance_validator/validate/parser.py index 35cd04a8d6..9c462a0a95 100644 --- a/tools/validators/instance_validator/validate/parser.py +++ b/tools/validators/instance_validator/validate/parser.py @@ -136,7 +136,7 @@ def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_ins self._ValidateMetadataSchema(metadata_block) del yaml_dict[_CONFIG_METADATA_KEY] - with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-noop-schema.schema.json')), 'r') as f: + with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-block-schema.schema.json')), 'r') as f: entity_block_schema = json.loads(f.read()) try: jsonschema.Draft202012Validator.check_schema(schema=entity_block_schema) @@ -147,8 +147,7 @@ def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_ins return_dict = {} for guid, entity_yaml in yaml_dict.items(): if guid == _CONFIG_METADATA_KEY: - print('Cannot have more than one config metadata block') - continue + raise jsonschema.ValidationError('Cannot have more than one config metadata block!') default_entity_operation = self.GetDefaultEntityOperation() is_valid = False try: @@ -163,5 +162,4 @@ def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_ins str(valid_uuid), entity_yaml, default_entity_operation ) return_dict.update({valid_uuid: entity}) - return return_dict, self.config_mode - + return return_dict, self.config_mode \ No newline at end of file From e606713cbf64a81f6b01f90b1b2d4b8a3e53347f Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 11:10:34 -0700 Subject: [PATCH 07/14] move abel uuid changes to new branch --- tools/abel/abel.py | 3 +- tools/abel/model/connection.py | 16 ++--- tools/abel/model/entity.py | 35 +++++------ tools/abel/model/entity_field.py | 43 ++++++++------ tools/abel/model/entity_operation.py | 2 +- tools/abel/model/export_helper.py | 22 +++---- tools/abel/model/from_building_config.py | 39 +++++++------ tools/abel/model/from_spreadsheet.py | 26 ++++----- tools/abel/model/guid_to_entity_map.py | 74 ++++++++---------------- tools/abel/model/import_helper.py | 3 +- tools/abel/model/model_builder.py | 5 +- tools/abel/model/site.py | 18 +++--- tools/abel/model/state.py | 10 ++-- tools/abel/model/units.py | 2 +- tools/abel/tests/entity_field_test.py | 3 + 15 files changed, 136 insertions(+), 165 deletions(-) diff --git a/tools/abel/abel.py b/tools/abel/abel.py index df5427b047..b88eddfd43 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -19,9 +19,10 @@ from model.arg_parser import ParseArgs from model.workflow import Workflow + def main(parsed_args: ParseArgs) -> None: print( - '\nHow would you like to use ABEL?\n'"" + '\nHow would you like to use ABEL?\n' + '1: Modify a spreadsheet/building config for an existing building\n' + '2: Create a spreadsheet for a new building\n' + '3: Split a building config\n' diff --git a/tools/abel/model/connection.py b/tools/abel/model/connection.py index 9642d40c8c..c45d8a4644 100644 --- a/tools/abel/model/connection.py +++ b/tools/abel/model/connection.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module for concrete model connections.""" -import uuid + from typing import Dict # pylint: disable=g-importing-member @@ -44,8 +44,8 @@ class Connection(object): def __init__( self, - source_entity_guid: uuid.UUID, - target_entity_guid: uuid.UUID, + source_entity_guid: str, + target_entity_guid: str, connection_type: ConnectionType, ) -> None: """Init. @@ -91,8 +91,8 @@ def FromDict(cls, connection_dict: Dict[str, object]) -> ...: An instance of Connection class. """ return cls( - source_entity_guid=uuid.UUID(connection_dict[SOURCE_ENTITY_GUID]), - target_entity_guid=uuid.UUID(connection_dict[TARGET_ENTITY_GUID]), + source_entity_guid=connection_dict[SOURCE_ENTITY_GUID], + target_entity_guid=connection_dict[TARGET_ENTITY_GUID], connection_type=ConnectionType[connection_dict[CONNECTION_TYPE]], ) @@ -114,7 +114,7 @@ def connection_type(self, value: ConnectionType) -> None: def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap - ) -> Dict[str, any]: + ) -> Dict[str, str]: """Returns a dictionary of Connection attributes by spreadsheet headers.""" return { VALUES: [ @@ -125,7 +125,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.source_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.source_entity_guid}}, { USER_ENTERED_VALUE: {STRING_VALUE: self.connection_type.name}, DATA_VALIDATION: { @@ -147,6 +147,6 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.target_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.target_entity_guid}}, ] } diff --git a/tools/abel/model/entity.py b/tools/abel/model/entity.py index d1166b4505..42c3a169d3 100644 --- a/tools/abel/model/entity.py +++ b/tools/abel/model/entity.py @@ -14,7 +14,6 @@ """Module for concrete model entities.""" import abc -import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -62,10 +61,10 @@ class Entity(object): def __init__( self, code: str, - namespace: EntityNamespace, + namespace: str, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[uuid.UUID] = None, + bc_guid: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -86,8 +85,6 @@ def __init__( self._connections = [] self.type_name = type_name self.metadata = metadata - if bc_guid is not None and not isinstance(bc_guid, uuid.UUID): - raise Exception("Entity created with a string GUID") def __hash__(self): return hash((self.code, self.etag, self.bc_guid)) @@ -155,10 +152,10 @@ class VirtualEntity(Entity): def __init__( self, code: str, - namespace: EntityNamespace, + namespace: str, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[uuid.UUID] = None, + bc_guid: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -194,15 +191,12 @@ def FromDict(cls, entity_dict: Dict[str, str]) -> ...: # TODO(b/228973208) Add support for key errors for keys not in entity_dict. virtual_entity_instance = cls( code=entity_dict[ENTITY_CODE], - bc_guid=uuid.UUID(entity_dict[BC_GUID]), + bc_guid=entity_dict[BC_GUID], namespace=EntityNamespace(entity_dict.get(NAMESPACE).upper()), type_name=entity_dict[TYPE_NAME], ) if ETAG in entity_dict.keys(): - etag = entity_dict[ETAG] - if isinstance(etag, list) and len(etag) == 0: - etag = "" - virtual_entity_instance.etag = etag + virtual_entity_instance.etag = entity_dict[ETAG] # Merge all metadata cells in a row into one dictionary return virtual_entity_instance @@ -229,12 +223,12 @@ def AddLink(self, new_link: FieldTranslation) -> None: self._links.append(new_link) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: """Returns map of virtual entity attributes by spreadsheet headers.""" row_map_object = { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.bc_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.bc_guid}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, { USER_ENTERED_VALUE: {STRING_VALUE: IS_REPORTING_FALSE}, @@ -283,11 +277,11 @@ class ReportingEntity(Entity): def __init__( self, code: str, - namespace: EntityNamespace, + namespace: str, cloud_device_id: Optional[str] = None, etag: Optional[str] = None, type_name: Optional[str] = None, - bc_guid: Optional[uuid.UUID] = None, + bc_guid: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): """Init. @@ -328,15 +322,12 @@ def FromDict(cls, entity_dict: Dict[str, str]) -> ...: # TODO(b/228973208) Add support for key errors for keys not in entity_dict. reporting_entity_instance = cls( code=entity_dict[ENTITY_CODE], - bc_guid=uuid.UUID(entity_dict[BC_GUID]), + bc_guid=entity_dict[BC_GUID], namespace=EntityNamespace(entity_dict.get(NAMESPACE).upper()), type_name=entity_dict[TYPE_NAME], cloud_device_id=entity_dict[CLOUD_DEVICE_ID], ) if ETAG in entity_dict.keys(): - etag = entity_dict[ETAG] - if isinstance(etag, list) and len(etag) == 0: - etag = "" reporting_entity_instance.etag = entity_dict[ETAG] return reporting_entity_instance @@ -365,12 +356,12 @@ def AddTranslation(self, new_translation: FieldTranslation) -> None: self._translations.append(new_translation) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: """Returns map of reporting entity attributes by spreadsheet headers.""" row_map_object = { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.bc_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.bc_guid}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, { USER_ENTERED_VALUE: {STRING_VALUE: IS_REPORTING_TRUE}, diff --git a/tools/abel/model/entity_field.py b/tools/abel/model/entity_field.py index d1d3414461..7a3d406504 100644 --- a/tools/abel/model/entity_field.py +++ b/tools/abel/model/entity_field.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module to hold EntityField class.""" -import uuid + from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -64,8 +64,8 @@ class MissingField(field_translation.UndefinedField): def __init__( self, std_field_name: str, - entity_guid: uuid.UUID, - reporting_entity_guid: Optional[uuid.UUID] = None, + entity_guid: str, + reporting_entity_guid: Optional[str] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -120,8 +120,8 @@ def FromDict( reporting_entity_field_name=missing_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=uuid.UUID(missing_field_dict.get(BC_GUID)), - reporting_entity_guid=uuid.UUID(missing_field_dict.get(REPORTING_ENTITY_GUID)), + entity_guid=missing_field_dict[BC_GUID], + reporting_entity_guid=missing_field_dict[REPORTING_ENTITY_GUID], ) missing_field_instance.metadata = { k[len(METADATA) + 1 :]: v @@ -138,6 +138,11 @@ def GetSpreadsheetRowMapping( VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.std_field_name}}, {USER_ENTERED_VALUE: {STRING_VALUE: ''}}, + { + USER_ENTERED_VALUE: { + STRING_VALUE: self.reporting_entity_field_name + } + }, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -145,7 +150,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -153,7 +158,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_TRUE}, DATA_VALIDATION: { @@ -187,8 +192,8 @@ def __init__( self, std_field_name: str, raw_field_name: str, - entity_guid: uuid.UUID, - reporting_entity_guid: Optional[uuid.UUID] = None, + entity_guid: str, + reporting_entity_guid: Optional[str] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -266,8 +271,8 @@ def FromDict( reporting_entity_field_name=multistate_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=uuid.UUID(multistate_field_dict[BC_GUID]), - reporting_entity_guid=uuid.UUID(multistate_field_dict[REPORTING_ENTITY_GUID]), + entity_guid=multistate_field_dict[BC_GUID], + reporting_entity_guid=multistate_field_dict[REPORTING_ENTITY_GUID], ) multi_state_value_field_instance.metadata = { k[len(METADATA) + 1 :]: v @@ -313,7 +318,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -321,7 +326,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_FALSE}, DATA_VALIDATION: { @@ -365,8 +370,8 @@ def __init__( self, std_field_name: str, raw_field_name: str, - entity_guid: uuid.UUID, - reporting_entity_guid: Optional[uuid.UUID] = None, + entity_guid: str, + reporting_entity_guid: Optional[str] = None, reporting_entity_field_name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ): @@ -438,8 +443,8 @@ def FromDict( reporting_entity_field_name=dimensional_field_dict[ REPORTING_ENTITY_FIELD_NAME ], - entity_guid=uuid.UUID(dimensional_field_dict[BC_GUID]), - reporting_entity_guid=uuid.UUID(dimensional_field_dict[REPORTING_ENTITY_GUID]), + entity_guid=dimensional_field_dict[BC_GUID], + reporting_entity_guid=dimensional_field_dict[REPORTING_ENTITY_GUID], ) # Create a units instance from a spreadsheet and to a Dimensional Field # instance. @@ -495,7 +500,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.entity_guid}}, { USER_ENTERED_VALUE: { STRING_VALUE: guid_to_entity_map.GetEntityCodeByGuid( @@ -503,7 +508,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, { USER_ENTERED_VALUE: {STRING_VALUE: MISSING_FALSE}, DATA_VALIDATION: { diff --git a/tools/abel/model/entity_operation.py b/tools/abel/model/entity_operation.py index fafd9934fe..3230c1593c 100644 --- a/tools/abel/model/entity_operation.py +++ b/tools/abel/model/entity_operation.py @@ -86,7 +86,7 @@ def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap ) -> Dict[str, str]: """Returns map of entity attributes wih operation by spreadsheet headers.""" - entity_row_map = self.entity.GetSpreadsheetRowMapping() + entity_row_map = self.entity.GetSpreadsheetRowMapping(guid_to_entity_map) operation_row_map = { VALUES: [ { diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index 4d792bca21..ce7053abfc 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper module for exporting a valid Building Configuration or spreadsheet.""" -import uuid + from typing import Any, Dict, List, Optional # pylint: disable=g-importing-member @@ -132,7 +132,7 @@ def ExportUpdateBuildingConfiguration( if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { - str(entity.bc_guid): self._GetReportingEntityBuildingConfigBlock( + entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( entity, operation ) } @@ -140,7 +140,7 @@ def ExportUpdateBuildingConfiguration( elif isinstance(entity, VirtualEntity): entity_yaml_dict.update( { - str(entity.bc_guid): self._GetVirtualEntityBuildingConfigBlock( + entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( entity, operation ) } @@ -148,7 +148,7 @@ def ExportUpdateBuildingConfiguration( entity_yaml_dict.update( { - str(site.guid): { + site.guid: { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, CONFIG_ETAG: site.etag, @@ -184,7 +184,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { - str(entity.bc_guid): self._GetReportingEntityBuildingConfigBlock( + entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( entity=entity, operation=None, ) @@ -193,7 +193,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: elif isinstance(entity, VirtualEntity): entity_yaml_dict.update( { - str(entity.bc_guid): self._GetVirtualEntityBuildingConfigBlock( + entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( entity=entity, operation=None ) } @@ -201,7 +201,7 @@ def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: entity_yaml_dict.update( { - str(site.guid): { + site.guid: { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, } @@ -302,11 +302,11 @@ def _GetVirtualEntityBuildingConfigBlock( virtual_entity_yaml.update(self._AddOperationToBlock(operation)) return virtual_entity_yaml - def _GetConnections(self, entity: Entity) -> Dict[str, any]: + def _GetConnections(self, entity: Entity) -> Dict[str, List[str]]: if entity.connections: return { CONFIG_CONNECTIONS: { - str(c.source_entity_guid): [c.connection_type.name] + c.source_entity_guid: [c.connection_type.name] for c in entity.connections } } @@ -335,11 +335,11 @@ def _SortLinks(self, entity: VirtualEntity) -> Dict[str, object]: if not field_value: field_value = field.std_field_name if field.reporting_entity_guid not in link_map: - link_map[str(field.reporting_entity_guid)] = { + link_map[field.reporting_entity_guid] = { field.std_field_name: field_value } else: - link_map.get(str(field.reporting_entity_guid)).update( + link_map.get(field.reporting_entity_guid).update( {field.std_field_name: field_value} ) return link_map diff --git a/tools/abel/model/from_building_config.py b/tools/abel/model/from_building_config.py index b11a93fb8a..98895d1f42 100644 --- a/tools/abel/model/from_building_config.py +++ b/tools/abel/model/from_building_config.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Helper module for model_builder class.""" -import uuid + from typing import List, Tuple # pylint: disable=g-importing-member from model.connection import Connection as ABELConnection +from model.constants import CONNECTION_TYPE +from model.constants import SOURCE_ENTITY_GUID +from model.constants import TARGET_ENTITY_GUID from model.entity import Entity from model.entity import ReportingEntity from model.entity import VirtualEntity @@ -57,14 +60,14 @@ def EntityInstanceToEntity( connections = [] entity_operation = None enumerated_namespace = EntityNamespace(entity_instance.namespace.upper()) - entity_guid = uuid.UUID(entity_instance.guid) + if not entity_instance.cloud_device_id: entity = VirtualEntity( code=entity_instance.code, namespace=enumerated_namespace, etag=entity_instance.etag, type_name=entity_instance.type_name, - bc_guid=entity_guid, + bc_guid=entity_instance.guid, ) else: entity = ReportingEntity( @@ -73,33 +76,33 @@ def EntityInstanceToEntity( cloud_device_id=entity_instance.cloud_device_id, etag=entity_instance.etag, type_name=entity_instance.type_name, - bc_guid=entity_guid, + bc_guid=entity_instance.guid, ) if entity_instance.translation: for field in entity_instance.translation.values(): if isinstance(field, DimensionalValue): fields.append( _DimensionalValueToDimensionalValueField( - reporting_entity_guid=entity_guid, field=field + reporting_entity_guid=entity_instance.guid, field=field ) ) elif isinstance(field, MultiStateValue): this_field, this_states = _MultistateValueToMultistateValueField( - reporting_entity_guid=entity_guid, field=field + reporting_entity_guid=entity_instance.guid, field=field ) fields.append(this_field) states.extend(this_states) elif isinstance(field, IVUndefinedField): fields.append( _UndefinedFieldToUndefinedField( - reporting_entity_guid=entity_guid, field=field + reporting_entity_guid=entity_instance.guid, field=field ) ) if entity_instance.connections: for connection in entity_instance.connections: connections.append( - _TranslateConnectionsToABEL(entity_guid, connection) + _TranslateConnectionsToABEL(entity_instance.guid, connection) ) if entity_instance.operation: operation = EntityOperationType(entity_instance.operation.value) @@ -114,7 +117,7 @@ def EntityInstanceToEntity( def _DimensionalValueToDimensionalValueField( - reporting_entity_guid: uuid.UUID, field: DimensionalValue + reporting_entity_guid: str, field: DimensionalValue ) -> DimensionalValueField: """Maps DimensionalValue attributes to ABEL DimensionalValueField instance. @@ -141,7 +144,7 @@ def _DimensionalValueToDimensionalValueField( def _MultistateValueToMultistateValueField( - reporting_entity_guid: uuid.UUID, field: MultiStateValue + reporting_entity_guid: str, field: MultiStateValue ) -> Tuple[MultistateValueField, List[State]]: """Maps MultiStateValue attributes to ABEL MultistateValueField instances. @@ -166,7 +169,7 @@ def _MultistateValueToMultistateValueField( def _TranslateStatesToABEL( - entity_guid: uuid.UUID, field: MultiStateValue + entity_guid: str, field: MultiStateValue ) -> List[State]: """Maps MultiStateValue state attributes to ABEL State instance. @@ -193,7 +196,7 @@ def _TranslateStatesToABEL( def _UndefinedFieldToUndefinedField( - reporting_entity_guid: uuid.UUID, field: IVUndefinedField + reporting_entity_guid: str, field: IVUndefinedField ) -> MissingField: """Maps IV UndefinedField attributes to ABEL UndefinedField instances. @@ -212,7 +215,7 @@ def _UndefinedFieldToUndefinedField( def _TranslateConnectionsToABEL( - entity_guid: uuid.UUID, connection: IVConnection + entity_guid: str, connection: IVConnection ) -> ABELConnection: """Maps Instance Validator Connection attributes to ABEL Connection object. @@ -223,11 +226,11 @@ def _TranslateConnectionsToABEL( Returns: ABELConnection instance """ - return ABELConnection( - source_entity_guid=connection.source, - target_entity_guid=entity_guid, - connection_type=connection.ctype - ) + return ABELConnection.FromDict({ + SOURCE_ENTITY_GUID: connection.source, + CONNECTION_TYPE: connection.ctype, + TARGET_ENTITY_GUID: entity_guid, + }) def AddReportingEntitiesFromEntityInstance( diff --git a/tools/abel/model/from_spreadsheet.py b/tools/abel/model/from_spreadsheet.py index fa42f2aa9a..de25018bb3 100644 --- a/tools/abel/model/from_spreadsheet.py +++ b/tools/abel/model/from_spreadsheet.py @@ -45,7 +45,7 @@ def LoadEntitiesFromSpreadsheet( - entity_entries: List[Dict[str, any]], guid_to_entity_map: GuidToEntityMap + entity_entries: List[Dict[str, str]], guid_to_entity_map: GuidToEntityMap ) -> List[Entity]: """Loads a list of entity maps into Entity instances. @@ -64,7 +64,7 @@ def LoadEntitiesFromSpreadsheet( else: new_entity = VirtualEntity.FromDict(entity_entry) if not new_entity.bc_guid: - new_entity.bc_guid = uuid.uuid4() + new_entity.bc_guid = str(uuid.uuid4()) guid_to_entity_map.AddEntity(new_entity) parsed_entities.append(new_entity) @@ -92,14 +92,14 @@ def LoadFieldsFromSpreadsheet( """ fields = [] for entity_field_entry in entity_field_entries: - entity_field_entry[BC_GUID] = str(guid_to_entity_map.GetEntityGuidByCode( + entity_field_entry[BC_GUID] = guid_to_entity_map.GetEntityGuidByCode( entity_field_entry[ENTITY_CODE] - )) + ) if entity_field_entry[REPORTING_ENTITY_CODE]: entity_field_entry[REPORTING_ENTITY_GUID] = ( - str(guid_to_entity_map.GetEntityGuidByCode( + guid_to_entity_map.GetEntityGuidByCode( entity_field_entry[REPORTING_ENTITY_CODE] - )) + ) ) if entity_field_entry[MISSING].upper() == MISSING_TRUE: fields.append(MissingField.FromDict(entity_field_entry)) @@ -128,9 +128,9 @@ def LoadStatesFromSpreadsheet( states = [] for state_entry in state_entries: - state_entry[BC_GUID] = str(guid_to_entity_map.GetEntityGuidByCode( + state_entry[REPORTING_ENTITY_GUID] = guid_to_entity_map.GetEntityGuidByCode( state_entry[REPORTING_ENTITY_CODE] - )) + ) states.append(State.FromDict(states_dict=state_entry)) return states @@ -155,14 +155,14 @@ def LoadConnectionsFromSpreadsheet( for connection_entry in connection_entries: connection_entry[SOURCE_ENTITY_GUID] = ( - str(guid_to_entity_map.GetEntityGuidByCode( + guid_to_entity_map.GetEntityGuidByCode( connection_entry[SOURCE_ENTITY_CODE] - )) + ) ) connection_entry[TARGET_ENTITY_GUID] = ( - str(guid_to_entity_map.GetEntityGuidByCode( + guid_to_entity_map.GetEntityGuidByCode( connection_entry[TARGET_ENTITY_CODE] - )) + ) ) connections.append(ABELConnection.FromDict(connection_entry)) @@ -170,7 +170,7 @@ def LoadConnectionsFromSpreadsheet( def LoadOperationsFromSpreadsheet( - entity_entries: Dict[str, any], guid_to_entity_map: GuidToEntityMap + entity_entries: Dict[str, str], guid_to_entity_map: GuidToEntityMap ) -> List[EntityOperation]: """loads a list of entity dicitionary mappings into EntityOperation instances. diff --git a/tools/abel/model/guid_to_entity_map.py b/tools/abel/model/guid_to_entity_map.py index d03065c2b4..c8010b40c2 100644 --- a/tools/abel/model/guid_to_entity_map.py +++ b/tools/abel/model/guid_to_entity_map.py @@ -12,26 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. """Mapping of guids to Entity instances.""" -import uuid + from typing import Dict -GUID_MAP_SHARDS = 64 class GuidToEntityMap(object): """Container for mapping of Entity instances by entity guids. - Attributes: guid_to_entity_map: Mapping of entity guids + Attributes: guid_to_entity_map(class variable): Mapping of entity guids to Entity instances. """ - def _submap(self, key: uuid.UUID) -> dict[uuid.UUID, object]: - shard = key.int % GUID_MAP_SHARDS - return self._shards[shard] - def __init__(self): """Init.""" - self._shards = [{} for i in range(0, GUID_MAP_SHARDS)] - self._guid_code_map = None + self._guid_to_entity_map = {} def AddSite(self, site: ...) -> None: """Adds a site by guid to the mapping. @@ -50,14 +44,11 @@ def AddSite(self, site: ...) -> None: """ if not site.guid: raise AttributeError(f'{site.code}: guid missing') - - shard = self._submap(site.guid) - if site.guid not in shard: - shard[site.guid] = site + elif site.guid not in self._guid_to_entity_map: + self._guid_to_entity_map.update({site.guid: site}) else: raise KeyError( - f'{site.guid} maps to {shard[site.guid]}') - self._guid_code_map = None + f'{site.guid} maps to {self._guid_to_entity_map[site.guid]}') def AddEntity(self, entity: ...) -> None: """Adds an entity by guid to the mapping. @@ -76,16 +67,14 @@ def AddEntity(self, entity: ...) -> None: raise ValueError('Cannot add None values to the guid to entity map.') if not entity.bc_guid: raise AttributeError(f'{entity.code}: guid missing') - shard = self._submap(entity.bc_guid) - if entity.bc_guid not in shard: - shard[entity.bc_guid] = entity - self._guid_code_map = None + if entity.bc_guid not in self._guid_to_entity_map: + self._guid_to_entity_map[entity.bc_guid] = entity else: raise KeyError( - f'{entity.bc_guid} maps to {shard[entity.bc_guid]}' + f'{entity.bc_guid} maps to {self._guid_to_entity_map[entity.bc_guid]}' ) - def GetEntityByGuid(self, guid: uuid.UUID) ->...: + def GetEntityByGuid(self, guid: str) ->...: """Gets an Entity instance mapped to the input guid. Args: @@ -97,12 +86,12 @@ def GetEntityByGuid(self, guid: uuid.UUID) ->...: Raises: KeyError: When guid is not a valid key in the map. """ - entity = self._submap(guid).get(guid) + entity = self._guid_to_entity_map.get(guid) if entity is None: raise KeyError(f'{guid} is not a valid guid in the guid to entity map') return entity - def GetEntityCodeByGuid(self, guid: uuid.UUID) -> str: + def GetEntityCodeByGuid(self, guid: str) -> str: """Gets an entity code mapped by guid. Args: @@ -113,16 +102,7 @@ def GetEntityCodeByGuid(self, guid: uuid.UUID) -> str: """ return self.GetEntityByGuid(guid).code - def _GuidCodeMap(self) -> dict[str, uuid.UUID]: - if self._guid_code_map is not None: - return self._guid_code_map - guid_by_code = {} - for shard in self._shards: - for guid, entity in shard.items(): - guid_by_code[entity.code] = guid - return guid_by_code - - def GetEntityGuidByCode(self, code: str) -> uuid.UUID: + def GetEntityGuidByCode(self, code: str) -> str: """Returns entity code mapped by guid in the guid to entity mapping. Args: @@ -135,14 +115,16 @@ def GetEntityGuidByCode(self, code: str) -> uuid.UUID: AttributeError: If code is not an entity code contained in self._guid_to_entity_map """ - guid_by_code = self._GuidCodeMap() + guid_by_code = { + entity.code: guid for guid, entity in self._guid_to_entity_map.items() + } guid = guid_by_code.get(code) if not guid: raise AttributeError(f'{code} is not a valid entity code.') else: return guid - def RemoveEntity(self, guid: uuid.UUID) -> None: + def RemoveEntity(self, guid: str) -> None: """Removes a guid and entity pair from guid to entity mapping. Args: @@ -152,10 +134,9 @@ def RemoveEntity(self, guid: uuid.UUID) -> None: The removed Entity instance. """ - self._submap(guid).pop(guid) - self._guid_code_map = None + return self._guid_to_entity_map.pop(guid) - def UpdateEntityMapping(self, guid: uuid.UUID, entity: ...) -> None: + def UpdateEntityMapping(self, guid: str, entity: ...) -> None: """Maps existing guid key to new Entity instance. Args: @@ -166,26 +147,19 @@ def UpdateEntityMapping(self, guid: uuid.UUID, entity: ...) -> None: KeyError: When guid is not a valid key in the guid to entity map. ValueError: When entity is not an Entity instance. """ - shard = self._submap(guid) - if not shard.get(guid): + if not self._guid_to_entity_map.get(guid): raise KeyError(f'{guid} is not a valid guid in the guid to entity map') elif not entity: raise ValueError(f'{guid} cannot map to object of type None') - shard[guid] = entity - self._guid_code_map = None + self._guid_to_entity_map.update({guid: entity}) - def GetGuidToEntityMap(self) -> Dict[uuid.UUID, object]: + def GetGuidToEntityMap(self) -> Dict[str, object]: """Returns mapping of guids to Entity instances.""" - full_map = {} - for shard in self._shards: - full_map.update(shard) - return full_map + return self._guid_to_entity_map def Clear(self) -> None: """Clears global guid mapping. Adding for testing purposes. """ - for shard in self._shards: - shard.clear() - self._guid_code_map = None + self._guid_to_entity_map.clear() diff --git a/tools/abel/model/import_helper.py b/tools/abel/model/import_helper.py index e461c63b45..f2655f0f5e 100644 --- a/tools/abel/model/import_helper.py +++ b/tools/abel/model/import_helper.py @@ -14,7 +14,6 @@ """Module to import google sheets or Building Configurations into ABEL.""" import os -import uuid from typing import Any, Dict, List # pylint: disable=g-importing-member @@ -129,5 +128,5 @@ def DeserializeBuildingConfiguration(filepath: str) -> Dict[str, Any]: if instance.type_name == SITE_TYPE_NAME: site = instance del deserialized_bc[site.guid] - abel_site = Site(code=site.code, guid=uuid.UUID(site.guid), etag=site.etag) + abel_site = Site(code=site.code, guid=site.guid, etag=site.etag) return (abel_site, deserialized_bc) diff --git a/tools/abel/model/model_builder.py b/tools/abel/model/model_builder.py index 1a2ea1e62f..f229253339 100644 --- a/tools/abel/model/model_builder.py +++ b/tools/abel/model/model_builder.py @@ -14,7 +14,6 @@ """Helper module for concrete model construction.""" import datetime -import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -272,13 +271,13 @@ def connections(self) -> List[ABELConnection]: def guid_to_entity_map(self) -> GuidToEntityMap: return self._guid_to_entity_map - def GetEntity(self, entity_guid: uuid.UUID) -> Entity: + def GetEntity(self, entity_guid: str) -> Entity: """Helper function to get an Entity instance for a guid.""" return self.guid_to_entity_map.GetEntityByGuid(entity_guid) def GetStates( self, - entity_guid: uuid.UUID, + entity_guid: str, std_field_name: str, ) -> List[State]: """Helper function to get State instances for a field name and guid.""" diff --git a/tools/abel/model/site.py b/tools/abel/model/site.py index a7ae847fc6..213e138925 100644 --- a/tools/abel/model/site.py +++ b/tools/abel/model/site.py @@ -25,7 +25,6 @@ from model.constants import USER_ENTERED_VALUE from model.constants import VALUES from model.entity import Entity -import uuid # TODO(b/247621096): Combine site namespace and type name into one attribute. @@ -42,11 +41,11 @@ class Site(object): namespace: A site's standardized DBO namespace. type_name: A site's standardized DBO entity type id. guid: A globally unique identifier(uuid4) for a site. - entities: A list of GUIDs for entities contained in a site. + entities: A list of GUIDs for entities cointained in a site. """ def __init__( - self, code: str, etag: Optional[str], guid: Optional[uuid.UUID] = None + self, code: str, etag: Optional[str], guid: Optional[str] = None ) -> None: """Init. @@ -72,18 +71,15 @@ def __eq__(self, other: ...) -> bool: @classmethod def FromDict(cls, site_dict: Dict[str, object]) -> ...: - etag = site_dict.get(ETAG) - if isinstance(etag, list) and len(etag) == 0: - etag = "" site_instance = cls( code=site_dict.get(BUILDING_CODE), - etag=etag, - guid=uuid.UUID(site_dict.get(BC_GUID)), + etag=site_dict.get(ETAG), + guid=site_dict.get(BC_GUID), ) return site_instance @property - def entities(self) -> List[uuid.UUID]: + def entities(self) -> List[str]: """Returns a list of entity guids contained in a site.""" return self._entities @@ -106,12 +102,12 @@ def AddEntity(self, entity: Entity) -> None: self._entities.append(entity.bc_guid) # pylint: disable=unused-argument - def GetSpreadsheetRowMapping(self, *args) -> Dict[str, any]: + def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: """Returns a dictionary of Site attributes by spreadsheet headers.""" return { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.guid}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, ] } diff --git a/tools/abel/model/state.py b/tools/abel/model/state.py index 8dbafcfe27..0bb2cdfa93 100644 --- a/tools/abel/model/state.py +++ b/tools/abel/model/state.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module for concrete model states.""" -import uuid + from typing import Dict # pylint: disable=g-importing-member @@ -39,7 +39,7 @@ class State(object): def __init__( self, - reporting_entity_guid: uuid.UUID, + reporting_entity_guid: str, std_field_name: str, standard_state: str, raw_state: str, @@ -76,7 +76,7 @@ def __eq__(self, other: ...) -> bool: @classmethod def FromDict(cls, states_dict: Dict[str, str]) -> ...: return cls( - reporting_entity_guid=uuid.UUID(states_dict[REPORTING_ENTITY_GUID]), + reporting_entity_guid=states_dict[REPORTING_ENTITY_GUID], std_field_name=states_dict[REPORTING_ENTITY_FIELD_NAME], standard_state=states_dict[STANDARD_STATE], raw_state=states_dict[RAW_STATE], @@ -84,7 +84,7 @@ def FromDict(cls, states_dict: Dict[str, str]) -> ...: def GetSpreadsheetRowMapping( self, guid_to_entity_map: GuidToEntityMap - ) -> Dict[str, any]: + ) -> Dict[str, str]: """Returns a dictionary of State attributes by spreadsheet headers.""" return { VALUES: [ @@ -95,7 +95,7 @@ def GetSpreadsheetRowMapping( ) } }, - {USER_ENTERED_VALUE: {STRING_VALUE: str(self.reporting_entity_guid)}}, + {USER_ENTERED_VALUE: {STRING_VALUE: self.reporting_entity_guid}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.std_field_name}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.standard_state}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.raw_state}}, diff --git a/tools/abel/model/units.py b/tools/abel/model/units.py index c97035647e..c9f4e46e38 100644 --- a/tools/abel/model/units.py +++ b/tools/abel/model/units.py @@ -51,7 +51,7 @@ def __eq__(self, other): and self.standard_to_raw_unit_map == other.standard_to_raw_unit_map ) - def GetSpreadsheetRowMapping(self) -> Dict[str, any]: + def GetSpreadsheetRowMapping(self) -> Dict[str, str]: """Returns a dictionary of EntityField attributes by spreadsheet headers. Corresponds to a single row in a concrete model spreadsheet. diff --git a/tools/abel/tests/entity_field_test.py b/tools/abel/tests/entity_field_test.py index 0c0819a93b..52ad026423 100644 --- a/tools/abel/tests/entity_field_test.py +++ b/tools/abel/tests/entity_field_test.py @@ -110,6 +110,9 @@ def testMissingFieldGetSpreadsheetRowMapping(self, test_get_code): } }, {USER_ENTERED_VALUE: {STRING_VALUE: ''}}, + {USER_ENTERED_VALUE: { + STRING_VALUE: TEST_MISSING_STANDARD_FIELD_NAME} + }, {USER_ENTERED_VALUE: {STRING_VALUE: TEST_REPORTING_ENTITY_CODE}}, {USER_ENTERED_VALUE: {STRING_VALUE: TEST_REPORTING_GUID}}, {USER_ENTERED_VALUE: {STRING_VALUE: TEST_REPORTING_ENTITY_CODE}}, From e428d92477bcdbd37b327c92b8b81d72f4840f63 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 11:18:44 -0700 Subject: [PATCH 08/14] revert ontology validator changes --- .../ontology_validator/yamlformat/validator/parse_config_lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py b/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py index dd7664d075..b774658896 100644 --- a/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py +++ b/tools/validators/ontology_validator/yamlformat/validator/parse_config_lib.py @@ -126,7 +126,6 @@ def _CreateFolder(folderpath, global_namespace, create_folder_fn, file_tuples): """Creates a ConfigFolder for the given folderpath.""" folder = create_folder_fn(folderpath, global_namespace) for ft in file_tuples: - print(f"ontology open {os.path.join(ft.root, ft.relative_path)}") with open(os.path.join(ft.root, ft.relative_path), 'r', encoding='utf-8') as f: try: From 16a3c9be9f57c4e93c4b8e65e59ade06fcdbf640 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 14:13:18 -0700 Subject: [PATCH 09/14] fix pylint errors --- .../instance_validator/tests/parser_test.py | 122 +++++++---- .../validate/enumerations.py | 3 +- .../instance_validator/validate/handler.py | 28 +-- .../instance_validator/validate/parser.py | 102 +++++---- .../instance_validator/validate/schema.py | 207 ------------------ 5 files changed, 150 insertions(+), 312 deletions(-) delete mode 100644 tools/validators/instance_validator/validate/schema.py diff --git a/tools/validators/instance_validator/tests/parser_test.py b/tools/validators/instance_validator/tests/parser_test.py index da033be0d1..d77cc9f32b 100644 --- a/tools/validators/instance_validator/tests/parser_test.py +++ b/tools/validators/instance_validator/tests/parser_test.py @@ -11,93 +11,97 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests tools.validators.instance_validator.instance_parser.""" +"""Tests tools.validators.instance_validator.validate.parser.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import os.path - -import jsonschema import yaml -from os import path - from absl.testing import absltest - -from tests import test_constants -from validate import handler from validate import parser as p from validate import enumerations import uuid - -# _TESTCASE_PATH = test_constants.TEST_INSTANCES - +# pylint: disable=unused-variable class ParserTest(absltest.TestCase): def setUp(self): self.parser = p.Parser(schema_folder='../schemas') - def testGetDefaultEntityOperation(self): - pass - - def testDeserializeGoodBuildingConfig(self): - pass def testDeserialize_DuplicateKeys_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'duplicate_keys.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'duplicate_keys.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.EXPORT) self.assertLen(entities, 0) def testDeserialize_NoConfigModeRaisesKeyError(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'no_config_metadata.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'no_config_metadata.yaml')] with self.assertRaises(KeyError): self.parser.Deserialize(yaml_files=bad_testcase_path) def testDeserialize_MissingColon_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'missing_colon.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'missing_colon.yaml')] with self.assertRaises(yaml.scanner.ScannerError): self.parser.Deserialize(yaml_files=bad_testcase_path) def testDeserialize_BadSpacing_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'spacing.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'spacing.yaml')] with self.assertRaises(yaml.scanner.ScannerError): self.parser.Deserialize(yaml_files=bad_testcase_path) - def testDeserialize_BadSpacing_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'tabbing.yaml')] + def testDeserialize_BadTabbing_Fails(self): + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'tabbing.yaml')] with self.assertRaises(yaml.parser.ParserError): self.parser.Deserialize(yaml_files=bad_testcase_path) # NOT WORKING def testDeserialize_BuildingConnections_Success(self): - good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'building_connections.yaml')] + good_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', + 'building_connections.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=good_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) self.assertLen(entities, 1) self.assertIn(uuid.UUID('3cf600dc-0e48-40ad-9807-ba98018e9946'), entities) def testDeserialize_ParseConnectionList_Succeeds(self): - good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'building_connection_list.yaml')] + good_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', + 'building_connection_list.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=good_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) self.assertLen(entities, 1) self.assertIn(uuid.UUID('3cf600dc-0e48-40ad-9807-ba98018e9946'), entities) def testDeserialize_ParseMultipleEntities_Success(self): - good_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'multi_instances.yaml')] + good_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', + 'multi_instances.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=good_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.INITIALIZE) self.assertLen(entities, 3) @@ -106,48 +110,65 @@ def testDeserialize_ParseMultipleEntities_Success(self): self.assertIn(uuid.UUID('2778562f-8600-4c55-bb36-0802cdf63956'), entities) def testDeserialize_BadTranslation_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_compliant.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'translation_compliant.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertLen(entities, 0) def testDeserialize_BadTranslationKeys_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_keys.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'translation_keys.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertLen(entities, 0) def testDeserialize_BadTranslationUnits_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_units_format.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'translation_units_format.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertLen(entities, 0) def testDeserialize_BadTranslationNoCDID_Fails(self): - bad_testcase_path = [os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'translation_missing_cloud_device_id.yaml')] + bad_testcase_path = [ + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'translation_missing_cloud_device_id.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertLen(entities, 1) self.assertIn(uuid.UUID('135d08f4-8df0-46ae-86cb-16b953870aeb'), entities) - self.assertNotIn(uuid.UUID('eb15ee68-795f-430a-bb1f-8e70eaf2e66a'), entities) + self.assertNotIn(uuid.UUID('eb15ee68-795f-430a-bb1f-8e70eaf2e66a'), + entities) def testDeserializer_UpdateMaskWithoutUpdateOperation_Fails(self): bad_testcase_path = [ - os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'update_mask_value.yaml')] + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'update_mask_value.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertLen(entities, 0) def testDeserializer_UpdateMaskDependency_Success(self): good_testcase_path = [ - os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'update_mask.yaml')] + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', + 'update_mask.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=good_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.UPDATE) self.assertLen(entities, 1) @@ -155,24 +176,29 @@ def testDeserializer_UpdateMaskDependency_Success(self): def testDeserialize_DefaultExportOperationForUpdateMode_Success(self): good_testcase_path = [ - os.path.join(os.path.abspath('./fake_instances'), 'GOOD', 'entity_export_operation.yaml')] + os.path.join(os.path.abspath('./fake_instances'), 'GOOD', + 'entity_export_operation.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=good_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=good_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.UPDATE) - self.assertEqual(self.parser.GetDefaultEntityOperation(), enumerations.EntityOperation.EXPORT) + self.assertEqual(self.parser.GetDefaultEntityOperation(), + enumerations.EntityOperation.EXPORT) self.assertLen(entities, 1) self.assertIn(uuid.UUID('71965f14-9690-4218-9c9e-cb9550b8a07f'), entities) def testDeserialize_ExportEntityMissingEtag_Fails(self): bad_testcase_path = [ - os.path.join(os.path.abspath('./fake_instances'), 'BAD', 'entity_export_operation.yaml')] + os.path.join(os.path.abspath('./fake_instances'), 'BAD', + 'entity_export_operation.yaml')] - entities, config_mode = self.parser.Deserialize(yaml_files=bad_testcase_path) + entities, config_mode = self.parser.Deserialize( + yaml_files=bad_testcase_path) self.assertEqual(config_mode, enumerations.ConfigMode.EXPORT) self.assertLen(entities, 0) if __name__ == '__main__': - absltest.main() \ No newline at end of file + absltest.main() diff --git a/tools/validators/instance_validator/validate/enumerations.py b/tools/validators/instance_validator/validate/enumerations.py index 8cbb6b1f71..03ae123d00 100644 --- a/tools/validators/instance_validator/validate/enumerations.py +++ b/tools/validators/instance_validator/validate/enumerations.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Module to store instance validator enumerations.""" import enum class ConfigMode(enum.Enum): @@ -48,4 +49,4 @@ def FromString(cls, value: str): for _, member in cls.__members__.items(): if member.value == value: return member - raise LookupError \ No newline at end of file + raise LookupError diff --git a/tools/validators/instance_validator/validate/handler.py b/tools/validators/instance_validator/validate/handler.py index bfb2a9d7f1..7425be7294 100644 --- a/tools/validators/instance_validator/validate/handler.py +++ b/tools/validators/instance_validator/validate/handler.py @@ -21,9 +21,8 @@ import os import sys import uuid -from typing import Dict, List, Tuple, Any +from typing import Dict, List, Tuple -import yaml from validate import constants from validate import entity_instance from validate import enumerations @@ -33,18 +32,15 @@ from validate import telemetry_validation_report as tvr from validate import telemetry_validator from yamlformat.validator import presubmit_validate_types_lib as pvt -try: - from yaml import CLOader as Loader, CDumper as Dumper -except ImportError: - from yaml import Loader, Dumper INSTANCE_VALIDATION_FILENAME = 'instance_validation_report.txt' TELEMETRY_VALIDATION_FILENAME = 'telemetry_validation_report.json' +SCHEMA_FOLDER = 'schemas' def FileNameEnumerationHelper(filename: str) -> str: - """Adds an UTC timestamp enurmation prefix to the filename. + """Adds a UTC timestamp enumeration prefix to the filename. Args: filename: string representing the filename to be enumerated with a local @@ -53,7 +49,7 @@ def FileNameEnumerationHelper(filename: str) -> str: Returns: the filename enumerated as _. example: 2020_10_15T17_21_59Z_instance_validation_report.txt where the timestamp - is given as year_month_dayThour_min_secondZ and the filename as + is given as year_month_day_hour_min_secondZ and the filename as instance_validation_report.txt """ return '_'.join(( @@ -77,12 +73,12 @@ def Deserialize( ConfigMode: INITIALIZE or UPDATE """ - parser = p.Parser() + parser = p.Parser(schema_folder=SCHEMA_FOLDER) return parser.Deserialize(yaml_files=yaml_files) def _ValidateConfig( filenames: List[str], universe: pvt.ConfigUniverse, is_udmi -) -> List[entity_instance.EntityInstance]: +) -> Dict[uuid.UUID, entity_instance.EntityInstance]: """Runs all config validation checks.""" print(f'[INFO]\tLoading config files: {filenames}') entities, config_mode = Deserialize(filenames) @@ -116,7 +112,7 @@ def RunValidation( report_directory: str = None, timeout: int = constants.DEFAULT_TIMEOUT, is_udmi: bool = True, -) -> None: +) -> str: """Top level runner for all validations. Args: @@ -136,7 +132,7 @@ def RunValidation( Report file name or None if no report file is generated. """ saved_stdout = sys.stdout - report_file = None + report_file = '' print('[INFO]\tStarting validation process.') if report_directory: @@ -190,9 +186,8 @@ def RunValidation( report_file.close() print(f'[INFO]\tInstance validation report generated: {report_file.name}') print('[INFO]\tInstance validation completed.') - if report_file: - return report_file.name - return None + return report_file.name + class TelemetryHelper(object): @@ -200,7 +195,6 @@ class TelemetryHelper(object): Attributes: subscription: resource string referencing the subscription to check - service_account_file: path to file with service account information report_directory: fully qualified path to report output directory """ @@ -341,7 +335,7 @@ def Validate( entities: Dict[str, entity_instance.EntityInstance], config_mode: enumerations.ConfigMode, is_udmi: bool = True, - ) -> Dict[str, entity_instance.EntityInstance]: + ) -> Tuple[Dict[str, entity_instance.EntityInstance], bool]: """Validates entity instances that are already deserialized. Args: diff --git a/tools/validators/instance_validator/validate/parser.py b/tools/validators/instance_validator/validate/parser.py index 9c462a0a95..0d2d8e75a3 100644 --- a/tools/validators/instance_validator/validate/parser.py +++ b/tools/validators/instance_validator/validate/parser.py @@ -1,21 +1,34 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Instance Validator parser module.""" + import jsonschema import json import os import re from referencing import Registry, Resource -from referencing.jsonschema import DRAFT202012 from typing import Dict, Tuple, Any, List import uuid import yaml from validate import enumerations from validate import entity_instance -from validate import schema try: - from yaml import CLOader as Loader, CDumper as Dumper + from yaml import CLOader as Loader except ImportError: - from yaml import Loader, Dumper + from yaml import Loader _ENTITY_CODE_REGEX = r'^[a-zA-Z][a-zA-Z0-9/\-_ ]+:' _ENTITY_GUID_REGEX = r'^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}:' @@ -28,6 +41,7 @@ _CONFIG_METADATA_PATTERN = re.compile(_CONFIG_METADATA_REGEX) _SCHEMA_ID = '$id' + class Parser(object): """A simple parser for building config yaml files. @@ -54,45 +68,55 @@ def GetDefaultEntityOperation(self) -> enumerations.EntityOperation: def _CreateSchemaRegistry(self) -> Registry: resources = [] - for root, dir_names, filenames in os.walk(os.path.abspath(self.schema_folder)): - for file in filenames: - with open(os.path.abspath(os.path.join(root, file)), 'r') as f: - parsed_json = json.loads(f.read()) - new_resource = Resource.from_contents(parsed_json) - resources.append((file, new_resource)) + # pylint: disable=unused-variable + for root, dir_names, filenames in os.walk( + os.path.abspath(self.schema_folder)): + for file in filenames: + with open(os.path.abspath(os.path.join(root, file)), 'r', + encoding='utf-8') as f: + parsed_json = json.loads(f.read()) + new_resource = Resource.from_contents(parsed_json) + resources.append((file, new_resource)) return Registry().with_resources(pairs=resources) - def _ValidateMetadataSchema(self, metadata_block: Dict[str, Any]) -> bool: """Helper function to validate a building config's CONFIG METADATA block. Args: - metadata_block: Dictionary mapping for a building config CONFIG METADATA block. + metadata_block: Dictionary for a building config CONFIG METADATA block. Returns: a boolean representing if the metadata has been validated? is valid? """ - with open(os.path.abspath(os.path.join(self.schema_folder, 'config-metadata.schema.json')), 'r') as f: + with open(os.path.abspath( + os.path.join(self.schema_folder, 'config-metadata.schema.json')), + 'r', encoding='utf-8') as f: config_metadata_schema = json.loads(f.read()) try: - jsonschema.validate(instance=metadata_block, schema=config_metadata_schema) - self.config_mode = enumerations.ConfigMode(metadata_block.get('operation')) + jsonschema.validate(instance=metadata_block, + schema=config_metadata_schema) + self.config_mode = enumerations.ConfigMode( + metadata_block.get('operation')) except jsonschema.ValidationError as ve: print('CONFIG_METADATA is invalid') print(ve) return False except ValueError: - print('CONFIG_METADATA operation invalid. Operation must be one of INITIALIZE, EXPORT, UPDATE') + print( + 'CONFIG_METADATA operation must be one of INITIALIZE, EXPORT, UPDATE') return False return True - - def _ValidateEntityInstance(self, guid: str, config_dict: Dict[str, Any], validator: jsonschema.Draft202012Validator) -> bool: + def _ValidateEntityInstance( + self, + config_dict: Dict[str, Any], + validator: jsonschema.Draft202012Validator + ) -> bool: """Helper function to validate a building config schema using jsonschema. Args: - config_dict: A parsed dictionary representation of a building config yaml file. - validator: + config_dict: A dictionary representation of a building config yaml file. + validator: Validation engine used to validate instance. Returns: a boolean indicating whether the block is valid @@ -104,27 +128,22 @@ def _ValidateEntityInstance(self, guid: str, config_dict: Dict[str, Any], valida return False return True - - def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_instance.EntityInstance], enumerations.ConfigMode]: - """New deserialize logic that uses pyyaml and jsonschema rather than strictyaml. - - Steps: - 1. Parse yaml file - 2. need to validate block by block - 3. separate out config metadata and validate - 4. For each block, ensure it matches one of the entity block schemas defined in schema. + def Deserialize(self, yaml_files: List[str]) -> Tuple[ + Dict[uuid.UUID, entity_instance.EntityInstance], enumerations.ConfigMode]: + """pyyaml and jsonschema to parse building config file. Args: - yaml_files: A list of absolute paths for a collection of building config files. + yaml_files: A list of absolute paths for a collection of building + config files. Returns: - A tuple containing a dictionary mapping of uuid objects to entity instances - and the building config's config mode. + A tuple containing a dictionary mapping of uuid objects to entity + instances and the building config's config mode. """ for yaml_file in yaml_files: absolute_bc_path = os.path.expanduser(yaml_file) - with open(absolute_bc_path) as f: + with open(absolute_bc_path, encoding='utf-8') as f: yaml_dict = yaml.load(f.read(), Loader=Loader) # Fix metadata validation here @@ -136,30 +155,35 @@ def Deserialize(self, yaml_files: List[str]) -> Tuple[Dict[uuid.UUID, entity_ins self._ValidateMetadataSchema(metadata_block) del yaml_dict[_CONFIG_METADATA_KEY] - with open(os.path.abspath(os.path.join(self.schema_folder, 'entity-block-schema.schema.json')), 'r') as f: + with open(os.path.abspath( + os.path.join(self.schema_folder, 'entity-block-schema.schema.json')), + 'r', encoding='utf-8') as f: entity_block_schema = json.loads(f.read()) try: jsonschema.Draft202012Validator.check_schema(schema=entity_block_schema) except jsonschema.SchemaError as schema_error: raise schema_error - validator = jsonschema.Draft202012Validator(schema=entity_block_schema, registry=self._CreateSchemaRegistry()) + validator = jsonschema.Draft202012Validator( + schema=entity_block_schema, + registry=self._CreateSchemaRegistry() + ) return_dict = {} for guid, entity_yaml in yaml_dict.items(): if guid == _CONFIG_METADATA_KEY: - raise jsonschema.ValidationError('Cannot have more than one config metadata block!') + raise jsonschema.ValidationError( + 'Cannot have more than one config metadata block!') default_entity_operation = self.GetDefaultEntityOperation() - is_valid = False try: valid_uuid = uuid.UUID(guid) except ValueError: # Great spot to append to a log here print('Not a valid guid') continue - is_valid = self._ValidateEntityInstance(guid, entity_yaml, validator) + is_valid = self._ValidateEntityInstance(entity_yaml, validator) if is_valid: entity = entity_instance.EntityInstance.FromYaml( str(valid_uuid), entity_yaml, default_entity_operation ) return_dict.update({valid_uuid: entity}) - return return_dict, self.config_mode \ No newline at end of file + return return_dict, self.config_mode diff --git a/tools/validators/instance_validator/validate/schema.py b/tools/validators/instance_validator/validate/schema.py deleted file mode 100644 index 149a95d6d1..0000000000 --- a/tools/validators/instance_validator/validate/schema.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import re -import os - -from referencing import Resource, Registry -from referencing.jsonschema import DRAFT202012 -from validate import enumerations - -#### Public Text parsing Constants #### -ENTITY_ID_KEY = 'id' # deprecated; kept for legacy reasons -ENTITY_GUID_KEY = 'guid' -ENTITY_CODE_KEY = 'code' -ENTITY_CLOUD_DEVICE_ID_KEY = 'cloud_device_id' -ENTITY_TYPE_KEY = 'type' -ENTITY_OPERATION_KEY = 'operation' - -LINKS_KEY = 'links' -TRANSLATION_KEY = 'translation' -CONNECTIONS_KEY = 'connections' -METADATA_KEY = 'metadata' -PRESENT_VALUE_KEY = 'present_value' -POINTS = 'points' -VALUE_RANGE_KEY = 'value_range' -UNITS_KEY = 'units' -UNIT_NAME_KEY = 'key' -UNIT_VALUES_KEY = 'values' -STATES_KEY = 'states' -UPDATE_MASK_KEY = 'update_mask' -ETAG_KEY = 'etag' - -# Minimum threshold for a valid entity name. Additional validation is required -# check adherence to more specific naming conventions -# Note: As-written this will capture the metadata key below, so logic should -# check for it first -_ENTITY_CODE_REGEX = r'^[a-zA-Z][a-zA-Z0-9/\-_ ]+:' -_ENTITY_GUID_REGEX = r'^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}:' -_ENTITY_CODE_PATTERN = re.compile(_ENTITY_CODE_REGEX) -_ENTITY_GUID_PATTERN = re.compile(_ENTITY_GUID_REGEX) - -# Exact key for the configuration metadata block -_CONFIG_METADATA_KEY = 'CONFIG_METADATA' -_CONFIG_METADATA_REGEX = f'^{_CONFIG_METADATA_KEY}:' -_CONFIG_METADATA_PATTERN = re.compile(_CONFIG_METADATA_REGEX) -# Key that marks the mode to parse file in. -_CONFIG_MODE_KEY = 'operation' - -_TRANSLATION_SCHEMA = { - '$id': '/schemas/translation', - 'type': 'object', - 'properties': { - PRESENT_VALUE_KEY: {'type': 'string'}, - STATES_KEY: { - 'type': 'array' - }, - UNITS_KEY: { - 'type': 'object', - 'properties': { - UNIT_NAME_KEY: {'type': 'string'}, - UNIT_VALUES_KEY: { - 'type': 'array' - } - } - } - }, - 'required': [PRESENT_VALUE_KEY] -} - -_ENTITY_ATTR_SCHEMA = { - '$id': '/schemas/entity-attributes', - 'type': 'object', - 'properties': { - ENTITY_CODE_KEY: {'type': 'string'}, - CONNECTIONS_KEY: { - 'type': 'object', - 'patternProperties': { - '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$': { - 'anyOf': [ - {'type': 'array', 'items': {'type': 'string'}}, - {'type': 'string'} - ] - } - }, - 'additionalProperties': False - }, - LINKS_KEY: {'type': 'array'}, - TRANSLATION_KEY: {'$ref': '/schemas/translation'} - } -} - -_ENTITY_BASE_SCHEMA = { - '$id': '/schemas/entity-base-schema', - 'type': 'object', - 'properties': { - ENTITY_CLOUD_DEVICE_ID_KEY: {'type': 'string'}, - ENTITY_CODE_KEY: {'type': 'string'}, - ENTITY_GUID_KEY: {'type': 'string'}, - }, - 'allOf': [ - {'$ref': '/schemas/entity-attributes'} - ] -} - -_ENTITY_UPDATE_SCHEMA = { - '$id': '/schemas/entity-update-schema', - 'type': 'object', - 'properties': { - ETAG_KEY: {'type': 'string'}, - ENTITY_TYPE_KEY: {'type': 'string'}, - ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.UPDATE.value}, - UPDATE_MASK_KEY: { - 'type': 'array', - 'uniqueItems': True - }, - }, - 'required': [ETAG_KEY, ENTITY_TYPE_KEY], - 'dependentRequired': { - UPDATE_MASK_KEY: [ENTITY_OPERATION_KEY] - }, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], -} - -_ENTITY_ADD_SCHEMA = { - '$id': '/schemas/entity-add-schema', - 'type': 'object', - 'properties': { - ENTITY_TYPE_KEY: {'type': 'string'}, - ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.ADD.value}, - }, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], -} - -_ENTITY_DELETE_SCHEMA = { - '$id': '/schemas/entity-delete-schema', - 'type': 'object', - 'properties': { - ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.DELETE.value}, - }, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], -} - -_ENTITY_EXPORT_SCHEMA = { - '$id': '/schemas/entity-export-schema', - 'type': 'object', - 'properties': { - ETAG_KEY: {'type': 'string'}, - ENTITY_TYPE_KEY: {'type': 'string'}, - ENTITY_OPERATION_KEY: {'const': enumerations.EntityOperation.EXPORT.value}, - }, - 'allOf': [ - {'$ref': '/schemas/entity-base-schema'} - ], -} - -# Generalized entity schema. All entity blocks must adhere to the below schema. -ENTITY_BLOCK_SCHEMA = { - '$id': '/schemas/entity-block-schema', - 'type': 'object', - 'oneOf': [ - {'$ref': '/schemas/entity-update-schema'}, - {'$ref': '/schemas/entity-add-schema'}, - {'$ref': '/schemas/entity-delete-schema'}, - {'$ref': '/schemas/entity-export-schema'}, - ] -} - -METADATA_SCHEMA = { - '$id': '/schemas/config-metadata', - 'type': 'object', - 'properties': { - _CONFIG_MODE_KEY: { - 'oneOf': [ - {'const': enumerations.ConfigMode.UPDATE.value}, - {'const': enumerations.ConfigMode.INITIALIZE.value}, - {'const': enumerations.ConfigMode.EXPORT.value}, - ] - } - } -} -def ExportSchemaRegistry() -> Registry: - id_tag = '$id' - resources = [] - for root, dir_names, filenames in os.walk(os.path.abspath('../schemas')): - for file in filenames: - with open(os.path.abspath(os.path.join(root, file)), 'r') as f: - parsed_json = json.loads(f.read()) - new_resource = Resource.from_contents(json.loads(parsed_json)) - resources.append((file, new_resource)) - return Registry().with_resources(pairs=resources) \ No newline at end of file From ffe476cee2331ce00d20e705abfb0ee3907a0e11 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 14:41:15 -0700 Subject: [PATCH 10/14] fix pylint errors --- .../instance_validator/tests/instance_parser_test.py | 2 -- .../instance_validator/validate/instance_parser.py | 11 ++++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/validators/instance_validator/tests/instance_parser_test.py b/tools/validators/instance_validator/tests/instance_parser_test.py index 326935a73a..c5c7eb4fc9 100644 --- a/tools/validators/instance_validator/tests/instance_parser_test.py +++ b/tools/validators/instance_validator/tests/instance_parser_test.py @@ -144,8 +144,6 @@ def testInstanceValidator_DetectDuplicateMetadata(self): [path.join(_TESTCASE_PATH, 'BAD', 'duplicate_metadata.yaml')]) del parser - # Not yet - # Need to add validation for having entity operations under INITIALIZE config mode def testInstanceValidator_RejectsOperationOnInitialize(self): with self.assertRaises(SystemExit): parser = _Helper( diff --git a/tools/validators/instance_validator/validate/instance_parser.py b/tools/validators/instance_validator/validate/instance_parser.py index 63fc3e7b76..4395e3f72c 100644 --- a/tools/validators/instance_validator/validate/instance_parser.py +++ b/tools/validators/instance_validator/validate/instance_parser.py @@ -25,7 +25,6 @@ import ruamel import strictyaml as syaml -import yaml #### Program constants #### # Size of entity block to send to the syntax validator @@ -311,14 +310,16 @@ def AddFile(self, filename: str) -> None: with open(filename, encoding='utf-8') as file: for _ in file: total_lines += 1 - print(f"[Instance Parser] Parsing started...") + print('[Instance Parser] Parsing started...') with open(filename, encoding='utf-8') as file: line_count = 0 for line in file: line_count += 1 + # pylint: disable=consider-using-f-string if (line_count % 5) == 0: percentage = '{:.3%}'.format(line_count / total_lines) - print(f"[Instance Parser] ({line_count}/{total_lines}) {percentage}% parsed") + # pylint: disable=line-too-long + print(f'[Instance Parser] ({line_count}/{total_lines}) {percentage}% parsed') if _IGNORE_PATTERN.match(line): continue @@ -352,7 +353,7 @@ def AddFile(self, filename: str) -> None: entity_instance_block = entity_instance_block + line - print(f"[Instance Parser] Parsed all lines") + print('[Instance Parser] Parsed all lines') # handle the singleton case if in_config: # parse the config block @@ -368,7 +369,7 @@ def _ProcessEntities(self) -> None: if not self._config_mode: return - print(f"[Instance Parser] Processing entities...") + print('[Instance Parser] Processing entities...') # Validate all queued blocks while True: try: From cae34d157a0fb77fcda217de4a097e1ce0b69915 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 14:46:12 -0700 Subject: [PATCH 11/14] update instance validator dependencies --- tools/validators/instance_validator/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/validators/instance_validator/setup.py b/tools/validators/instance_validator/setup.py index fa9d66dff2..158ea61946 100644 --- a/tools/validators/instance_validator/setup.py +++ b/tools/validators/instance_validator/setup.py @@ -29,9 +29,8 @@ 'Nigel Kilmer', packages=find_packages(), install_requires=[ - 'ruamel.yaml==0.17.4', 'strictyaml==1.4.2', 'google-cloud-pubsub', 'googleapis-common-protos', 'google-auth', 'google-auth-oauthlib', - 'protobuf', 'proto-plus'], + 'protobuf', 'proto-plus', 'jsonschema'], python_requires='>=3.9', ) From fc05a995a23a2276554f63a803fb2a8bf99e0cb7 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 14:50:07 -0700 Subject: [PATCH 12/14] update instance validator dependencies --- tools/validators/instance_validator/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/validators/instance_validator/setup.py b/tools/validators/instance_validator/setup.py index 158ea61946..246ed33b29 100644 --- a/tools/validators/instance_validator/setup.py +++ b/tools/validators/instance_validator/setup.py @@ -29,6 +29,7 @@ 'Nigel Kilmer', packages=find_packages(), install_requires=[ + 'ruamel.yaml==0.17.4', 'strictyaml==1.4.2' 'google-cloud-pubsub', 'googleapis-common-protos', 'google-auth', 'google-auth-oauthlib', 'protobuf', 'proto-plus', 'jsonschema'], From 51b4c87f820e943be02a35fc4c9140349cdccf45 Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 14:55:18 -0700 Subject: [PATCH 13/14] update instance validator dependencies --- tools/validators/instance_validator/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/validators/instance_validator/setup.py b/tools/validators/instance_validator/setup.py index 246ed33b29..241a296b54 100644 --- a/tools/validators/instance_validator/setup.py +++ b/tools/validators/instance_validator/setup.py @@ -29,7 +29,7 @@ 'Nigel Kilmer', packages=find_packages(), install_requires=[ - 'ruamel.yaml==0.17.4', 'strictyaml==1.4.2' + 'ruamel.yaml==0.17.4', 'strictyaml==1.4.2', 'google-cloud-pubsub', 'googleapis-common-protos', 'google-auth', 'google-auth-oauthlib', 'protobuf', 'proto-plus', 'jsonschema'], From aae11d4a03e0f96d20c8b8301fb657f304ebddca Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 30 Oct 2023 15:11:21 -0700 Subject: [PATCH 14/14] deprecate instance parser test --- .../tests/instance_parser_test.py | 320 ------------------ 1 file changed, 320 deletions(-) delete mode 100644 tools/validators/instance_validator/tests/instance_parser_test.py diff --git a/tools/validators/instance_validator/tests/instance_parser_test.py b/tools/validators/instance_validator/tests/instance_parser_test.py deleted file mode 100644 index c5c7eb4fc9..0000000000 --- a/tools/validators/instance_validator/tests/instance_parser_test.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests tools.validators.instance_validator.instance_parser.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from os import path - -from absl.testing import absltest -import strictyaml as syaml - -from tests import test_constants -from validate import handler -from validate import instance_parser - - -_TESTCASE_PATH = test_constants.TEST_INSTANCES - - -def _ParserHelper(testpaths): - parser = instance_parser.InstanceParser() - for filepath in testpaths: - parser.AddFile(filepath) - parser.Finalize() - return parser - - -def _Helper(testpaths): - return _ParserHelper(testpaths).GetEntities() - - -class ParserTest(absltest.TestCase): - - def testFunction_EnumToRegex_Success(self): - expected = syaml.Regex('^(ADD) | (UPDATE)$') - actual = instance_parser.EnumToRegex( - instance_parser.EntityOperation, - [instance_parser.EntityOperation.DELETE]) - self.assertEqual(str(expected), str(actual)) - - def testInstanceValidator_DetectDuplicateKeys_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'duplicate_keys.yaml')]) - del parser - - # Migrated - def testInstanceValidator_DetectMissingColon_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'missing_colon.yaml')]) - del parser - - # Migrated - def testInstanceValidator_DetectImproperSpacing_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'spacing.yaml')]) - del parser - - # Migrated - def testInstanceValidator_DetectImproperTabbing_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'tabbing.yaml')]) - del parser - - # Don't think the following test case should be handled by parser...Maybe? - # Come back to it - def testInstanceValidator_ParseProperFormat_Success(self): - parser = _Helper([path.join(_TESTCASE_PATH, 'GOOD', 'building_type.yaml')]) - del parser - - # Migrated - def testInstanceValidator_ParseProperConnections_Success(self): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'GOOD', 'building_connections.yaml')]) - del parser - - # Migrated - def testInstanceValidator_ParseProperConnectionList_Success(self): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'GOOD', 'building_connection_list.yaml')]) - del parser - - def testInstanceValidator_ParseMultipleEntities_Success(self): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'GOOD', 'multi_instances.yaml')]) - - self.assertLen(parser.keys(), 3) - self.assertIn('AHU-11-GUID', parser.keys()) - self.assertIn('FCU-1-GUID', parser.keys()) - self.assertIn('FCU-10-GUID', parser.keys()) - - # Migrated - def testInstanceValidator_DetectImproperTranslationCompliance(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'translation_compliant.yaml')]) - del parser - - # Migrated - def testInstanceValidator_DetectImproperTranslationKeys(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'translation_keys.yaml')]) - del parser - - def testInstanceValidator_DetectImproperUnitsKeys(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'translation_units_format.yaml')]) - del parser - - # Migrated - def testInstanceValidator_CloudDeviceIdNotSetWithTranslation(self): - with self.assertRaises(KeyError): - parser = _Helper([ - path.join(_TESTCASE_PATH, 'BAD', - 'translation_no_cloud_device_id.yaml') - ]) - del parser - - #Don't think parser should test for below... - def testInstanceValidator_DetectDuplicateEntityKeys(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'duplicate_key.yaml')]) - del parser - - # Don't know if I need this either - def testInstanceValidator_DetectDuplicateMetadata(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'duplicate_metadata.yaml')]) - del parser - - def testInstanceValidator_RejectsOperationOnInitialize(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'entity_operation.yaml')]) - del parser - - # Add validation for this - def testInstanceValidator_RejectsMaskOnInitialize(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'entity_mask.yaml')]) - del parser - - def testInstanceValidator_RejectsMaskOnAdd(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'entity_add_mask.yaml')]) - del parser - - def testInstanceValidator_RejectsUpdateWithoutEtag(self): - with self.assertRaises(SystemExit): - parser = _Helper([path.join(_TESTCASE_PATH, 'BAD', 'entity_etag.yaml')]) - del parser - - # Don't need this - def testInstanceValidator_ReadsMetadata_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'with_metadata.yaml')]) - self.assertLen(parser.GetEntities().keys(), 2) - self.assertEqual(parser.GetConfigMode(), instance_parser.ConfigMode.UPDATE) - - # Don't need this - def testInstanceValidator_ReadsMetadataAtEnd_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'with_metadata_at_end.yaml')]) - self.assertLen(parser.GetEntities().keys(), 2) - self.assertEqual(parser.GetConfigMode(), instance_parser.ConfigMode.UPDATE) - - # won't migrate - def testInstanceValidator_HandlesUpdateMode_Success(self): - parser = _ParserHelper([ - path.join(_TESTCASE_PATH, 'GOOD', - 'update_change_subset_of_entities.yaml') - ]) - self.assertLen(parser.GetEntities().keys(), 4) - - def testInstanceValidator_RejectsUpdateMaskWithoutUpdateMode_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper([ - path.join(_TESTCASE_PATH, 'BAD', - 'update_with_incorrect_metadata.yaml') - ]) - del parser - - def testInstanceValidator_UsesDefaultMode_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'building_type.yaml')]) - self.assertEqual(parser.GetConfigMode(), - instance_parser.ConfigMode.Default()) - - # File configmode.yaml not found - def testInstanceValidator_InvalidConfigModeExport_RaisesKeyError(self): - with self.assertWarns(Warning): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'BAD', 'configmode.yaml')]) - del parser - - # Won't Migrate - def testEntityBlock_NewFormatSingleton_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'new_format_singleton.yaml')]) - self.assertEqual( - list(parser.GetEntities().keys()).pop(), - '9a86a19b-b687-4db1-888e-2cf34d04b74c') - - # This test doesn't make sense - def testEntityBlock_CodeWithSpace_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'code_with_spaces.yaml')]) - self.assertEqual( - list(parser.GetEntities().keys()).pop(), 'SDC_EXT 2-1 / Rm 2D2-GUID') - - # Migrated - def testEntityBlock_ValidUpdateMaskValueTypes_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'update_mask.yaml')]) - self.assertLen(parser.GetEntities().keys(), 1) - - #Migrated - def testEntityBlock_InvalidUpdateMaskInconsistentTypes_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'update_mask_value.yaml')]) - del parser - - #Migrated - def testEntityBlock_ValidUpdateMaskValue_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'update_mask.yaml')]) - expected = { - 'SDC_EXT-17-GUID': { - 'type': 'HVAC/SDC_EXT', - 'code': 'SDC_EXT-17', - 'cloud_device_id': '1234567890123456', - 'etag': 'a56789', - 'update_mask': ['translation', 'connection'], - 'translation': { - 'shade_extent_percentage_command': { - 'present_value': - 'points.shade_extent_percentage_command.present_value', - 'units': { - 'key': 'points.shade_extent_percentage_command.units', - 'values': { - 'percent': '%', - 'type': 'HVAC/SDC_EXT' - } - } - } - } - } - } - - self.assertLen(parser.GetEntities().keys(), 1) - self.assertEqual(parser.GetEntities(), expected) - - # Migrated - def testGoodEntity_DefaultExportOperationParses_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'entity_export_operation.yaml')]) - - parsed = parser.GetEntities() - _, entity = next(iter(parsed.items())) - entity_operation = entity.get(instance_parser.ENTITY_OPERATION_KEY, None) - default_operation = handler.GetDefaultOperation(parser.GetConfigMode()) - - self.assertIsNone(entity_operation) - self.assertLen(parser.GetEntities().keys(), 1) - self.assertEqual(default_operation, instance_parser.EntityOperation.EXPORT) - - # Migrated - def testEntityBlock_InvalidExportOperation_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'entity_export_operation.yaml')]) - del parser - - # Migrated - def testEntityBlock_InvalidUpdateMaskAndOperation_Fails(self): - with self.assertRaises(SystemExit): - parser = _Helper( - [path.join(_TESTCASE_PATH, 'BAD', 'update_mask_operation.yaml')]) - del parser - - def testInstanceValidator_translationFieldStatesCaseInsensitive_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'states_case_insensitive.yaml')]) - self.assertLen(parser.GetEntities().keys(), 2) - - def testEntityBlock_EntityWithId_Success(self): - parser = _ParserHelper( - [path.join(_TESTCASE_PATH, 'GOOD', 'bc_entity_with_id.yaml')]) - - parsed = parser.GetEntities() - iterator = iter(parsed.items()) - _, entity_1 = next(iterator) - _, entity_2 = next(iterator) - - self.assertLen(parser.GetEntities().keys(), 2) - self.assertIsNone(entity_1.get(instance_parser.ENTITY_ID_KEY)) - self.assertEqual( - entity_2.get(instance_parser.ENTITY_ID_KEY), - "deprecated-but-doesn't-break") - -if __name__ == '__main__': - absltest.main()