From 1741011c1d0dc5cc2583bd9e1e2bc52a17605fe9 Mon Sep 17 00:00:00 2001 From: Mirthala Lopez Date: Mon, 2 May 2022 12:18:38 -0700 Subject: [PATCH 1/4] Changes in index_everything (es.py) and index_search_box (textsearch.py) --- es.py | 25 +++++++++++++++++++++++-- textsearch.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/es.py b/es.py index 70a4560..7de1acf 100644 --- a/es.py +++ b/es.py @@ -9,6 +9,8 @@ import csv from typing import Dict, List, Tuple from collections import OrderedDict +from dfply import * +import pandas as pd #when you load this pacakge these global variables are defined #es = Elasticsearch('http://localhost:9200') # es = Elasticsearch( @@ -89,7 +91,7 @@ def add_to_index(filepath:str) -> None: filepath (str): a filepath to a txt file general plan """ - i = get_max_index() + i = get_max_index() try: filename = os.path.basename(filepath) @@ -118,6 +120,7 @@ def index_everything(): """Adds all of the txt files in the data directory to the elasticsearch index """ global es + global index_to_info_map wd = os.getcwd() data_dir = os.path.join(wd, 'static', 'data', 'places') filepaths = glob.glob(data_dir+'/*.txt') @@ -138,9 +141,26 @@ def index_everything(): hash_to_prop_mapping[keyhash] = parsed_filename es.index(index='test_4', id=keyhash, body={'text': txt, 'filename': filename}, ) i += 1 - with open('key_hash_mapping.json', 'w') as fp: json.dump(hash_to_prop_mapping, fp) + index_to_info_map = None + +# def check_lists(): +# plan_df = pd.read_json('key_hash_mapping.json', orient='index') +# plan_df = plan_df.sort_values(by='plan_date', ascending=False) +# plan_df = plan_df.sort_values(by='place_name') + +# city_df = plan_df[plan_df.is_city == 'true'] +# county_df = plan_df[plan_df.is_city == 'false'] + +# print(city_df) +# print(county_df) + + + +# @app.route('/recentplans/', methods=['GET']) +# def get_recentyear() -> + def elastic_search(query) -> Tuple[List[int], List[float]]: """Puts a query into elasticsearch and returns the ids and score @@ -211,6 +231,7 @@ def map_index_to_vals(search_result_indices, key_to_hash_path='key_hash_mapping. else: my_dict = index_to_info_map + # print(index_to_info_map) return list(map(lambda x:my_dict[str(x)], search_result_indices)) def elastic_search_highlight(query): diff --git a/textsearch.py b/textsearch.py index 9ec52d6..dc42ede 100644 --- a/textsearch.py +++ b/textsearch.py @@ -343,7 +343,7 @@ def index_search_box(): resultsDiv = Div(text = """ {} out of {} cities mention '{}'

{} out of {} counties mention '{}' - """.format(uniqueCities, numCities, wordinput, uniqueCounties, numCounties, wordinput), + """.format(uniqueCities, numCities, twitQuery, uniqueCounties, numCounties, twitQuery), margin = (40, 0, 20, 30), css_classes=["results-div"]) From c70bb8328cc0f97dfd516f28ce72a60ebc30b461 Mon Sep 17 00:00:00 2001 From: Mirthala Lopez Date: Tue, 3 May 2022 11:16:35 -0700 Subject: [PATCH 2/4] Fixed quotations for search term and indexing issue. --- es.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/es.py b/es.py index 7de1acf..02b1f48 100644 --- a/es.py +++ b/es.py @@ -145,23 +145,6 @@ def index_everything(): json.dump(hash_to_prop_mapping, fp) index_to_info_map = None -# def check_lists(): -# plan_df = pd.read_json('key_hash_mapping.json', orient='index') -# plan_df = plan_df.sort_values(by='plan_date', ascending=False) -# plan_df = plan_df.sort_values(by='place_name') - -# city_df = plan_df[plan_df.is_city == 'true'] -# county_df = plan_df[plan_df.is_city == 'false'] - -# print(city_df) -# print(county_df) - - - -# @app.route('/recentplans/', methods=['GET']) -# def get_recentyear() -> - - def elastic_search(query) -> Tuple[List[int], List[float]]: """Puts a query into elasticsearch and returns the ids and score Args: From 5df2d1c0fd9114a0bad611479d0aa40c74991b82 Mon Sep 17 00:00:00 2001 From: Mirthala Lopez Date: Wed, 4 May 2022 15:53:37 -0700 Subject: [PATCH 3/4] added button that triggers download of csv file containing city/county names and associated years. --- es.py | 14 +++++++++++++- templates/upload_index.html | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/es.py b/es.py index 02b1f48..73818e3 100644 --- a/es.py +++ b/es.py @@ -9,7 +9,6 @@ import csv from typing import Dict, List, Tuple from collections import OrderedDict -from dfply import * import pandas as pd #when you load this pacakge these global variables are defined #es = Elasticsearch('http://localhost:9200') @@ -143,8 +142,21 @@ def index_everything(): i += 1 with open('key_hash_mapping.json', 'w') as fp: json.dump(hash_to_prop_mapping, fp) + create_csv() index_to_info_map = None +def create_csv(): + with open('key_hash_mapping.json', 'r') as key_hash: + json_df = pd.read_json(key_hash, orient='index') + + json_df = json_df.drop(json_df.columns[[0, 1, 5]], axis=1) + json_df.rename(columns={"is_city": "city/county"}, inplace=True) + json_df = json_df.replace({'city/county':{True:'city', False:'county'}}) + + path_for_csv = 'static/data/names-and-years-in-database.csv' + json_df.to_csv(path_for_csv) + print(".csv file saved!") + def elastic_search(query) -> Tuple[List[int], List[float]]: """Puts a query into elasticsearch and returns the ids and score Args: diff --git a/templates/upload_index.html b/templates/upload_index.html index c6ecc43..e5dd8ba 100644 --- a/templates/upload_index.html +++ b/templates/upload_index.html @@ -143,6 +143,10 @@

Upload file

+
+ Download a .csv file with all city/county names and associated years listed in our database. + Download +
From 61c5c953d75c883cd441dcdb6da1b6e8bbe3c3af Mon Sep 17 00:00:00 2001 From: Mirthala Lopez Date: Thu, 26 May 2022 12:49:56 -0700 Subject: [PATCH 4/4] changes to colors for text --- templates/upload_index.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/upload_index.html b/templates/upload_index.html index e5dd8ba..9c45c47 100644 --- a/templates/upload_index.html +++ b/templates/upload_index.html @@ -143,11 +143,11 @@

Upload file

-
- Download a .csv file with all city/county names and associated years listed in our database. - Download -
- + +
+ Download a .csv file with all city/county names and associated years listed in our database.
+ Download +