diff --git a/seed/static/seed/js/controllers/column_settings_controller.js b/seed/static/seed/js/controllers/column_settings_controller.js index 0f20fd9822..f8e1c7c9ed 100644 --- a/seed/static/seed/js/controllers/column_settings_controller.js +++ b/seed/static/seed/js/controllers/column_settings_controller.js @@ -15,6 +15,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting 'auth_payload', 'columns_service', 'modified_service', + 'organization_service', + 'uploader_service', 'spinner_utility', 'urls', 'naturalSort', @@ -33,6 +35,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting auth_payload, columns_service, modified_service, + organization_service, + uploader_service, spinner_utility, urls, naturalSort, @@ -124,7 +128,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting $scope.matching_status = (column) => { if (column.is_extra_data) return 'ineligible'; - if ($scope.org.inventory_count && initial_matching_ids.includes(column.id)) return 'locked'; + if ($scope.org.access_level_names.length > 1 && initial_matching_ids.includes(column.id)) return 'locked'; return 'eligible'; }; @@ -247,6 +251,32 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting $scope.isModified = () => modified_service.isModified(); + $scope.complete_column_update = function () { + var matching_criteria_changed = _.find(_.values(diff), function (delta) { + return _.has(delta, 'is_matching_criteria'); + }); + + if (matching_criteria_changed) { + // reset the spinner and run whole org match merge link + spinner_utility.show(undefined, $('.display')[0]); + + organization_service.match_merge_link($scope.org.id, $scope.inventory_type).then(function (response) { + uploader_service.check_progress_loop( + response.progress_key, + 0, + 1, + (response) => { + organization_service.get_match_merge_link_result($scope.org.id, response.unique_id).then(() => column_update_complete()); + }, + () => {}, + { progress: 0 }, + ); + }) + } else { + column_update_complete(); + } + }; + // Table Sorting const default_sort_toggle = () => { $scope.column_sort = 'default'; @@ -350,6 +380,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting } modified_service.resetModified(); + $scope.modal_instance.dismiss(); $state.reload(); }; @@ -368,14 +399,7 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting return; } - const modal_instance = $scope.open_confirm_column_settings_modal(); - modal_instance.result - .then((data) => { - column_update_complete(data); - }) - .catch(() => { - // User cancelled - }); + $scope.modal_instance = $scope.open_confirm_column_settings_modal(); }; $scope.open_create_column_modal = () => $uibModal.open({ @@ -402,7 +426,8 @@ angular.module('SEED.controller.column_settings', []).controller('column_setting columns_service: () => columns_service, spinner_utility: () => spinner_utility, table_name: () => ($scope.inventory_type === 'properties' ? 'PropertyState' : 'TaxLotState'), - $q: () => $q + $q: () => $q, + complete_column_update: () => $scope.complete_column_update, } }); diff --git a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js index 3369abd25e..adc067fc8d 100644 --- a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js +++ b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js @@ -15,7 +15,9 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' 'proposed_changes', 'columns_service', 'spinner_utility', + 'uiGridGroupingConstants', '$q', + 'complete_column_update', '$interval', 'uploader_service', 'table_name', @@ -35,7 +37,9 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' proposed_changes, columns_service, spinner_utility, + uiGridGroupingConstants, $q, + complete_column_update, $interval, uploader_service, table_name, @@ -194,7 +198,7 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' organization_service.matching_criteria_columns($scope.org_id); } - $scope.confirm_changes_and_rehash = () => { + $scope.confirm_changes_and_rehash = () => { const api_ready_proposed_changes = Object.keys(proposed_changes).reduce((acc, col_id) => { const col = proposed_changes[col_id]; col.display_name = col.displayName; // Add display_name for backend @@ -220,6 +224,7 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' 0, 1, (response) => { + complete_column_update(); $scope.result = `${response.message} in ${$scope.elapsed}`; $scope.state = 'done'; $interval.cancel($scope.interval); @@ -260,6 +265,217 @@ angular.module('SEED.controller.confirm_column_settings_modal', []).controller(' } }; + // Preview + // Agg function returning last value of matching criteria field (all should be the same if they match) + $scope.matching_field_value = function (aggregation, fieldValue) { + aggregation.value = fieldValue; + }; + + var prioritize_sort = function (grid, sortColumns) { + // To maintain grouping while giving users the ability to have some sorting, + // matching columns are given top priority followed by the hidden linking ID column. + // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. + if (sortColumns.length > 1) { + var matching_cols = _.filter(sortColumns, function (col) { + return col.colDef.is_matching_criteria; + }); + var linking_id_col = _.find(sortColumns, ['name', 'id']); + var remaining_cols = _.filter(sortColumns, function (col) { + return !col.colDef.is_matching_criteria && !(col.name === 'id'); + }); + sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); + _.forEach(sortColumns, function (col, index) { + col.sort.priority = index; + }); + } + }; + + // Takes raw cycle-partitioned records and returns array of cycle-aware records + var format_preview_records = function (raw_inventory) { + return _.reduce(raw_inventory, function (all_records, records, cycle_id) { + var cycle = _.find($scope.cycles, { id: parseInt(cycle_id) }); + _.forEach(records, function (record) { + record.cycle_name = cycle.name; + record.cycle_start = cycle.start; + all_records.push(record); + }); + return all_records; + }, []); + }; + + // Builds preview columns using non-extra_data columns + var build_preview_columns = function () { + // create copy in order to not change original column objects. + var preview_column_defs = _.reject(_.cloneDeep($scope.columns), 'is_extra_data'); + var default_min_width = 50; + var autopin_width = 100; + var column_def_defaults = { + headerCellFilter: 'translate', + minWidth: default_min_width, + width: 125, + groupingShowAggregationMenu: false + }; + + _.map(preview_column_defs, function (col) { + var options = {}; + if (col.data_type === 'datetime') { + options.cellFilter = 'date:\'yyyy-MM-dd h:mm a\''; + options.filter = inventory_service.dateFilter(); + } else { + options.filter = inventory_service.combinedFilter(); + } + + // For matching criteria values, always pin left and show values in aggregate rows. + if ($scope.proposed_matching_criteria_columns.includes(col.column_name)) { + col.pinnedLeft = true; + + // Help indicate matching columns are given preferred sort priority + col.displayName = col.displayName + '*'; + options.headerCellClass = 'matching-column-header'; + + options.customTreeAggregationFn = $scope.matching_field_value; + options.width = autopin_width; + } + return _.defaults(col, options, column_def_defaults); + }); + + // Grouping Settings + preview_column_defs.unshift( + { + displayName: 'Linking ID', + grouping: { groupPriority: 0 }, + name: 'id', + sort: { priority: 0, direction: 'desc' }, + pinnedLeft: true, + visible: false, + suppressRemoveSort: true, // since grouping relies on sorting + minWidth: default_min_width, + width: autopin_width + }, + { + name: 'cycle_name', + displayName: 'Cycle', + pinnedLeft: true, + treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, + customTreeAggregationFinalizerFn: function (aggregation) { + aggregation.rendered = 'total cycles: ' + aggregation.value; + }, + minWidth: default_min_width, + width: autopin_width, + groupingShowAggregationMenu: false + }, + { + name: 'cycle_start', + displayName: 'Cycle Start', + cellFilter: 'date:\'yyyy-MM-dd\'', + filter: inventory_service.dateFilter(), + type: 'date', + sort: { priority: 1, direction: 'asc' }, + pinnedLeft: true, + minWidth: default_min_width, + width: autopin_width, + groupingShowAggregationMenu: false + } + ); + + return preview_column_defs; + }; + + // Initialize preview table as empty for now. + $scope.match_merge_link_preview = { + data: 'data', + enableColumnResizing: true, + enableFiltering: true, + onRegisterApi: function (gridApi) { + $scope.gridApi = gridApi; + + // used to allow filtering for child branches of grouping tree + $scope.gridApi.table_category = 'year-over-year'; + + $scope.gridApi.core.on.filterChanged($scope, function () { + // This is a workaround for losing the state of expanded rows during filtering. + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); + + // Prioritized to maintain grouping. + $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); + } + }; + + // Preview Loading Helpers + var build_proposed_matching_columns = function (result) { + // Summarize proposed matching_criteria_columns for pinning and to create preview + var criteria_additions = _.filter($scope.change_summary_data, function (change) { + return change.is_matching_criteria; + }); + var criteria_removals = _.filter($scope.change_summary_data, function (change) { + return change.is_matching_criteria === false; + }); + + $scope.criteria_changes = { + add: _.map(criteria_additions, 'column_name'), + remove: _.map(criteria_removals, 'column_name') + }; + + var base_and_add; + if ($scope.inventory_type == 'properties') { + base_and_add = _.union(result.PropertyState, $scope.criteria_changes.add); + } else { + base_and_add = _.union(result.TaxLotState, $scope.criteria_changes.add); + } + $scope.proposed_matching_criteria_columns = _.difference(base_and_add, $scope.criteria_changes.remove); + }; + + var build_preview = function (summary) { + $scope.data = format_preview_records(summary); + $scope.preview_columns = build_preview_columns(); + $scope.match_merge_link_preview.columnDefs = $scope.preview_columns; + }; + + var preview_loading_complete = function () { + $scope.preview_loading = false; + spinner_utility.hide(); + }; + + var get_preview = function () { + // Use new proposed matching_criteria_columns to request a preview then render this preview. + var spinner_options = { + scale: 0.40, + position: 'relative', + left: '100%' + }; + spinner_utility.show(spinner_options, $('#spinner_placeholder')[0]); + + organization_service.match_merge_link_preview($scope.org_id, $scope.inventory_type, $scope.criteria_changes) + .then(function (response) { + uploader_service.check_progress_loop( + response.progress_key, + 0, + 1, + (completion_notice) => { + organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) + .then(build_preview) + .then(preview_loading_complete); + }, + () => resultHandler('error', 'Unexpected Error'), + { progress: 0 } + ); + }); + }; + + // Get and Show Preview (If matching criteria changes exist.) + $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), function (delta) { + return _.has(delta, 'is_matching_criteria'); + }); + + if ($scope.matching_criteria_exists) { + $scope.preview_loading = true; + + organization_service.matching_criteria_columns($scope.org_id) + .then(build_proposed_matching_columns) + .then(get_preview); + } + $scope.setRunningState = () => { $scope.eta = $scope.etaFn(); if ($scope.eta) { diff --git a/seed/static/seed/js/services/organization_service.js b/seed/static/seed/js/services/organization_service.js index ebac53836a..7801fc8107 100644 --- a/seed/static/seed/js/services/organization_service.js +++ b/seed/static/seed/js/services/organization_service.js @@ -163,6 +163,42 @@ angular.module('SEED.service.organization', []).factory('organization_service', organization_factory.geocoding_columns = (org_id) => $http.get(`/api/v3/organizations/${org_id}/geocoding_columns/`).then((response) => response.data); organization_factory.reset_all_passwords = (org_id) => $http.post(`/api/v3/organizations/${org_id}/reset_all_passwords/`).then((response) => response.data); + + organization_factory.match_merge_link = function (org_id, inventory_type) { + return $http.post('/api/v3/organizations/' + org_id + '/match_merge_link/', { + inventory_type: inventory_type + }).then(function (response) { + return response.data; + }); + }; + + organization_factory.geocoding_columns = function (org_id) { + return $http.get('/api/v3/organizations/' + org_id + '/geocoding_columns/').then(function (response) { + return response.data; + }); + }; + + organization_factory.match_merge_link_preview = function (org_id, inventory_type, criteria_change_columns) { + return $http.post('/api/v3/organizations/' + org_id + '/match_merge_link_preview/', { + inventory_type: inventory_type, + add: criteria_change_columns.add, + remove: criteria_change_columns.remove + }).then(function (response) { + return response.data; + }); + }; + + organization_factory.get_match_merge_link_result = function (org_id, match_merge_link_id) { + return $http.get('/api/v3/organizations/' + org_id + '/match_merge_link_result/' + '?match_merge_link_id=' + match_merge_link_id).then(function (response) { + return response.data; + }); + }; + + organization_factory.reset_all_passwords = function (org_id) { + return $http.post('/api/v3/organizations/' + org_id + '/reset_all_passwords/').then(function (response) { + return response.data; + }); + }; organization_factory.insert_sample_data = (org_id) => $http.get(`/api/v3/organizations/${org_id}/insert_sample_data/`).then((response) => response.data); diff --git a/seed/static/seed/partials/confirm_column_settings_modal.html b/seed/static/seed/partials/confirm_column_settings_modal.html index 641ff047fd..58df13edb4 100644 --- a/seed/static/seed/partials/confirm_column_settings_modal.html +++ b/seed/static/seed/partials/confirm_column_settings_modal.html @@ -1,75 +1,51 @@ -
-

Review Changes

+
+

Review Changes

+

Matching Criteria Preview

-
-
-
-
- - +
+
+
+
+ + +
-
-
-
-
+
+
+
+
-
-
-
- ORG_MATCH_MERGE_LINK_WARNING +
+
ORG_MATCH_MERGE_LINK_WARNING
+ +
+ + +
-
- - +
+
+
+
-
-
-
-
-
-
-

REHASH_TEXT

-

LONG_OP_CONFIRM_TEXT

+
+
+
+ +
+
+
@@ -109,40 +85,13 @@

Review Changes

{$ result $}
+
+
+ +
-
- - - +
+
-
diff --git a/seed/tests/test_organization_views.py b/seed/tests/test_organization_views.py index 55c967b02c..3e0ea2d449 100644 --- a/seed/tests/test_organization_views.py +++ b/seed/tests/test_organization_views.py @@ -22,6 +22,17 @@ FakePropertyViewFactory, ) from seed.tests.util import AccessLevelBaseTestCase, DataMappingBaseTestCase +from seed.data_importer.tasks import geocode_and_match_buildings_task +from seed.landing.models import SEEDUser as User +from seed.lib.progress_data.progress_data import ProgressData +from seed.models import ASSESSED_RAW, DATA_STATE_MAPPING +from seed.test_helpers.fake import ( + FakeCycleFactory, + FakePropertyStateFactory, + FakeTaxLotStateFactory +) +from seed.tests.util import DataMappingBaseTestCase +from seed.utils.cache import get_cache_raw from seed.utils.organizations import create_organization @@ -284,3 +295,238 @@ def test_filtered_report(self): url += f"&filter_group_id={self.office_filter_group.id}" resp = self.client.get(url, content_type="application/json") assert resp.json()["data"]["property_counts"][0]["num_properties"] == 1 + + def test_whole_org_match_merge_link_endpoint_properties(self): + url = reverse('api:v3:organizations-match-merge-link', args=[self.org.id]) + post_params = json.dumps({"inventory_type": "properties"}) + raw_result = self.client.post(url, post_params, content_type='application/json') + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id'] + result_key = "org_match_merge_link_result__%s" % identifier + summary = get_cache_raw(result_key) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(['PropertyState', 'TaxLotState'], summary_keys) + + # try to get result using results endpoint + get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier) + + get_result_raw_response = self.client.get(get_result_url) + summary = json.loads(get_result_raw_response.content) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(['PropertyState', 'TaxLotState'], summary_keys) + + def test_whole_org_match_merge_link_endpoint_taxlots(self): + url = reverse('api:v3:organizations-match-merge-link', args=[self.org.id]) + post_params = json.dumps({"inventory_type": "taxlots"}) + raw_result = self.client.post(url, post_params, content_type='application/json') + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id'] + result_key = "org_match_merge_link_result__%s" % identifier + summary = get_cache_raw(result_key) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(['PropertyState', 'TaxLotState'], summary_keys) + + # try to get result using results endpoint + get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier) + + get_result_raw_response = self.client.get(get_result_url) + summary = json.loads(get_result_raw_response.content) + + summary_keys = list(summary.keys()) + + self.assertCountEqual(['PropertyState', 'TaxLotState'], summary_keys) + + +class TestOrganizationPreviewViews(DataMappingBaseTestCase): + def setUp(self): + selfvars = self.set_up(ASSESSED_RAW) + self.user, self.org, self.import_file_1, self.import_record_1, self.cycle_1 = selfvars + + cycle_factory = FakeCycleFactory(organization=self.org, user=self.user) + self.cycle_2 = cycle_factory.get_cycle(name="Cycle 2") + self.import_record_2, self.import_file_2 = self.create_import_file( + self.user, self.org, self.cycle_2 + ) + + user_details = { + 'username': 'test_user@demo.com', + 'password': 'test_pass', + 'email': 'test_user@demo.com' + } + + self.client.login(**user_details) + + self.property_state_factory = FakePropertyStateFactory(organization=self.org) + self.taxlot_state_factory = FakeTaxLotStateFactory(organization=self.org) + + def test_whole_org_match_merge_link_preview_endpoint_invalid_columns(self): + url = reverse('api:v3:organizations-match-merge-link-preview', args=[self.org.id]) + post_params = json.dumps({ + "inventory_type": "properties", + "add": ['DNE col 1'], + "remove": ['DNE col 2'] + }) + raw_result = self.client.post(url, post_params, content_type='application/json') + self.assertEqual(404, raw_result.status_code) + + def test_whole_org_match_merge_link_preview_endpoint_properties(self): + # Cycle 1 / ImportFile 1 - Create 1 property + base_property_details = { + 'pm_property_id': '1st Non-Match Set', + 'city': 'City 1', + 'property_name': 'Match Set', + 'import_file_id': self.import_file_1.id, + 'data_state': DATA_STATE_MAPPING, + 'no_default_data': False, + "raw_access_level_instance_id": self.org.root.id, + } + print(PropertyView.objects.count()) + print() + + ps_1 = self.property_state_factory.get_property_state(**base_property_details) + + self.import_file_1.mapping_done = True + self.import_file_1.save() + geocode_and_match_buildings_task(self.import_file_1.id) + + # Cycle 2 / ImportFile 2 - Create 1 unlinked property + base_property_details['pm_property_id'] = '2nd Non-Match Set' + base_property_details['property_name'] = 'Match Set' + base_property_details['import_file_id'] = self.import_file_2.id + ps_2 = self.property_state_factory.get_property_state(**base_property_details) + + self.import_file_2.mapping_done = True + self.import_file_2.save() + geocode_and_match_buildings_task(self.import_file_2.id) + + # Check there doesn't exist links + print(PropertyView.objects.count()) + print(ps_1.propertyview_set.values("id")) + print(ps_2.propertyview_set.values("id")) + self.assertNotEqual(ps_1.propertyview_set.first().property_id, ps_2.propertyview_set.first().property_id) + + url = reverse('api:v3:organizations-match-merge-link-preview', args=[self.org.id]) + post_params = json.dumps({ + "inventory_type": "properties", + "add": ['property_name'], + "remove": ['pm_property_id'] + }) + raw_result = self.client.post(url, post_params, content_type='application/json') + + # Check there *still* doesn't exist links + self.assertNotEqual(ps_1.propertyview_set.first().property_id, ps_2.propertyview_set.first().property_id) + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id'] + result_key = "org_match_merge_link_result__%s" % identifier + raw_summary = get_cache_raw(result_key) + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id']) + + # try to get result using results endpoint + get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier) + + get_result_raw_response = self.client.get(get_result_url) + raw_summary = json.loads(get_result_raw_response.content) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id']) + + def test_whole_org_match_merge_link_preview_endpoint_taxlots(self): + # Cycle 1 / ImportFile 1 - Create 1 taxlot + base_taxlot_details = { + 'jurisdiction_tax_lot_id': '1st Non-Match Set', + 'city': 'City 1', + 'district': 'Match Set', + 'import_file_id': self.import_file_1.id, + 'data_state': DATA_STATE_MAPPING, + 'no_default_data': False, + "raw_access_level_instance_id": self.org.root.id, + } + + tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) + + self.import_file_1.mapping_done = True + self.import_file_1.save() + geocode_and_match_buildings_task(self.import_file_1.id) + + # Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot + base_taxlot_details['jurisdiction_tax_lot_id'] = '2nd Non-Match Set' + base_taxlot_details['district'] = 'Match Set' + base_taxlot_details['import_file_id'] = self.import_file_2.id + tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) + + self.import_file_2.mapping_done = True + self.import_file_2.save() + geocode_and_match_buildings_task(self.import_file_2.id) + + # Check there doesn't exist links + self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) + + url = reverse('api:v3:organizations-match-merge-link-preview', args=[self.org.id]) + post_params = json.dumps({ + "inventory_type": "taxlots", + "add": ['district'], + "remove": ['jurisdiction_tax_lot_id'] + }) + raw_result = self.client.post(url, post_params, content_type='application/json') + + # Check there *still* doesn't exist links + self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) + + self.assertEqual(200, raw_result.status_code) + + raw_content = json.loads(raw_result.content) + + identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id'] + result_key = "org_match_merge_link_result__%s" % identifier + raw_summary = get_cache_raw(result_key) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id']) + + # try to get result using results endpoint + get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier) + + get_result_raw_response = self.client.get(get_result_url) + raw_summary = json.loads(get_result_raw_response.content) + + summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles + + # Check format of summary + self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) + + # Check that preview shows links would be created + self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id']) diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 5eaeeb8ab1..65161d40ba 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -12,8 +12,11 @@ from io import BytesIO from numbers import Number from pathlib import Path +from random import randint import numpy as np +import dateutil +from celery import shared_task from django.conf import settings from django.contrib.auth.forms import PasswordResetForm from django.contrib.postgres.aggregates.general import ArrayAgg @@ -36,6 +39,7 @@ from seed.decorators import ajax_request from seed.landing.models import SEEDUser as User from seed.lib.superperms.orgs.decorators import has_hierarchy_access, has_perm +from seed.lib.progress_data.progress_data import ProgressData from seed.lib.superperms.orgs.models import ( ROLE_MEMBER, ROLE_OWNER, @@ -71,8 +75,14 @@ from seed.utils.api import api_endpoint from seed.utils.api_schema import AutoSchemaHelper from seed.utils.encrypt import decrypt, encrypt +from seed.utils.cache import get_cache_raw, set_cache_raw +from seed.utils.generic import median, round_down_hundred_thousand from seed.utils.geocode import geocode_buildings -from seed.utils.match import match_merge_link +from seed.utils.match import ( + match_merge_link, + get_matching_criteria_column_names, + whole_org_match_merge_link +) from seed.utils.merge import merge_properties from seed.utils.organizations import create_organization, create_suborganization, set_default_2fa_method from seed.utils.properties import pair_unpair_property_taxlot @@ -225,6 +235,19 @@ def _dict_org_brief(request, organizations): return orgs +def _get_match_merge_link_key(identifier): + return "org_match_merge_link_result__%s" % identifier + + +@shared_task(serializer='pickle', ignore_result=True) +def cache_match_merge_link_result(summary, identifier, progress_key): + result_key = _get_match_merge_link_key(identifier) + set_cache_raw(result_key, summary) + + progress_data = ProgressData.from_key(progress_key) + progress_data.finish_with_success() + + class OrganizationViewSet(viewsets.ViewSet): # allow using `pk` in url path for authorization (i.e., for has_perm) authz_org_id_kwarg = "pk" @@ -323,6 +346,21 @@ def column_mappings(self, request, pk=None): else: return JsonResponse({"status": "success"}) + def _start_whole_org_match_merge_link(self, org_id, state_class_name, proposed_columns=[]): + identifier = randint(100, 100000) + result_key = _get_match_merge_link_key(identifier) + set_cache_raw(result_key, {}) + + progress_data = ProgressData(func_name='org_match_merge_link', unique_id=identifier) + progress_data.delete() + + whole_org_match_merge_link.apply_async( + args=(org_id, state_class_name, proposed_columns), + link=cache_match_merge_link_result.s(identifier, progress_data.key) + ) + + return progress_data.key + @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.query_boolean_field( @@ -911,6 +949,139 @@ def matching_criteria_columns(self, request, pk=None): return JsonResponse(matching_criteria_column_names) + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory( + { + 'inventory_type': 'string' + }, + required=['inventory_type'], + description='Properties:\n' + '- inventory_type: either "properties" or "taxlots"' + ) + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=['POST']) + def match_merge_link(self, request, pk=None): + """ + Run match_merge_link for an org. + """ + inventory_type = request.data.get('inventory_type', None) + if inventory_type not in ['properties', 'taxlots']: + return JsonResponse({'status': 'error', + 'message': 'Provided inventory type should either be "properties" or "taxlots".'}, + status=status.HTTP_404_NOT_FOUND) + + try: + org = Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse({'status': 'error', + 'message': 'Could not retrieve organization at pk = ' + str(pk)}, + status=status.HTTP_404_NOT_FOUND) + + state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState' + + progress_key = self._start_whole_org_match_merge_link(org.id, state_class_name) + + return JsonResponse({'progress_key': progress_key}) + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory( + { + 'inventory_type': 'string', + 'add': ['string'], + 'remove': ['string'], + }, + required=['inventory_type'], + description='Properties:\n' + '- inventory_type: either "properties" or "taxlots"\n' + '- add: list of column names\n' + '- remove: list of column names' + ) + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=['POST']) + def match_merge_link_preview(self, request, pk=None): + """ + Run match_merge_link preview for an org and record type. + """ + inventory_type = request.data.get('inventory_type', None) + if inventory_type not in ['properties', 'taxlots']: + return JsonResponse({'status': 'error', + 'message': 'Provided inventory type should either be "properties" or "taxlots".'}, + status=status.HTTP_404_NOT_FOUND) + + try: + org = Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse({'status': 'error', + 'message': 'Could not retrieve organization at pk = ' + str(pk)}, + status=status.HTTP_404_NOT_FOUND) + + state_class_name = 'PropertyState' if inventory_type == 'properties' else 'TaxLotState' + + current_columns = get_matching_criteria_column_names(org.id, state_class_name) + + add = set(request.data.get('add', [])) + remove = set(request.data.get('remove', [])) + + provided_columns = Column.objects.filter( + column_name__in=add.union(remove), + organization_id=org.id, + table_name=state_class_name + ) + if provided_columns.count() != (len(add) + len(remove)): + return JsonResponse({'status': 'error', + 'message': 'Invalid column names provided.'}, + status=status.HTTP_404_NOT_FOUND) + + proposed_columns = current_columns.union(add).difference(remove) + + progress_key = self._start_whole_org_match_merge_link(org.id, state_class_name, list(proposed_columns)) + + return JsonResponse({'progress_key': progress_key}) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_integer_field( + 'match_merge_link_id', + required=True, + description='ID of match merge link' + )] + ) + @method_decorator( + [ + api_endpoint, + ajax_request, + has_perm("requires_member"), + ] + ) + @action(detail=True, methods=['GET']) + def match_merge_link_result(self, request, pk=None): + try: + Organization.objects.get(pk=pk) + except ObjectDoesNotExist: + return JsonResponse({'status': 'error', + 'message': 'Could not retrieve organization at pk = ' + str(pk)}, + status=status.HTTP_404_NOT_FOUND) + + identifier = request.query_params['match_merge_link_id'] + result_key = _get_match_merge_link_key(identifier) + + # using unsafe serialization b/c the result might not be a dict + return JsonResponse(get_cache_raw(result_key), safe=False) + + @method_decorator( [ api_endpoint,