delete code blocks containing TextFileReader

cdm_reader_mapper/cdm_mapper/mapper.py

@@ -3,23 +3,19 @@
 
 Created on Thu Apr 11 13:45:38 2019
 
-Maps data contained in a pandas DataFrame (or pd.io.parsers.TextFileReader) to
-the C3S Climate Data Store Common Data Model (CDM) header and observational
-tables using the mapping information available in the tool's mapping library
+Maps data contained in a pandas DataFrame to the C3S Climate Data Store Common Data Model (CDM)
+header and observational tables using the mapping information available in the tool's mapping library
 for the input data model.
 
 @author: iregon
 """
 
 from __future__ import annotations
 
-from copy import deepcopy
-from io import StringIO
-
 import numpy as np
 import pandas as pd
 
-from cdm_reader_mapper.common import logging_hdlr, pandas_TextParser_hdlr
+from cdm_reader_mapper.common import logging_hdlr
 
 from . import properties
 from .codes.codes import get_code_table
@@ -223,11 +219,11 @@ def _map_and_convert(
     null_label,
     imodel_functions,
     codes_subset,
-    cdm_tables,
     cdm_complete,
+    cdm_atts,
     logger,
 ):
-    atts = deepcopy(cdm_tables[table]["atts"])
+    atts = cdm_atts.get(table)
     columns = (
         [x for x in atts.keys() if x in idata.columns]
         if not cdm_complete
@@ -258,10 +254,7 @@ def _map_and_convert(
 
     table_df_i.columns = pd.MultiIndex.from_product([[table], columns])
     table_df_i = drop_duplicates(table_df_i)
-    table_df_i = table_df_i.fillna(null_label)
-    table_df_i.to_csv(cdm_tables[table]["buffer"], header=False, index=False, mode="a")
-    cdm_tables[table]["columns"] = table_df_i.columns
-    return cdm_tables
+    return table_df_i.fillna(null_label)
 
 
 def map_and_convert(
@@ -284,11 +277,6 @@ def map_and_convert(
 
     imodel_functions = mapping_functions("_".join([data_model] + list(sub_models)))
 
-    # Initialize dictionary to store temporal tables (buffer) and table attributes
-    cdm_tables = {
-        k: {"buffer": StringIO(), "atts": cdm_atts.get(k)} for k in imodel_maps.keys()
-    }
-
     date_columns = {}
     for table, values in imodel_maps.items():
         date_columns[table] = [
@@ -297,39 +285,22 @@ def map_and_convert(
             if "timestamp" in cdm_atts.get(table, {}).get(x, {}).get("data_type")
         ]
 
-    for idata in data:
-        cols = [x for x in idata]
-        for table, mapping in imodel_maps.items():
-            cdm_tables = _map_and_convert(
-                idata,
-                mapping,
-                table,
-                cols,
-                null_label,
-                imodel_functions,
-                codes_subset,
-                cdm_tables,
-                cdm_complete,
-                logger,
-            )
-
     table_list = []
-    for table in cdm_tables.keys():
-        # Convert dtime to object to be parsed by the reader
-        logger.debug(
-            f"\tParse datetime by reader; Table: {table}; Columns: {date_columns[table]}"
-        )
-        cdm_tables[table]["buffer"].seek(0)
-        data = pd.read_csv(
-            cdm_tables[table]["buffer"],
-            names=cdm_tables[table]["columns"],
-            na_values=[],
-            dtype="object",
-            keep_default_na=False,
+    for table in cdm_subset:
+        mapping = imodel_maps[table]
+        table_df = _map_and_convert(
+            data,
+            mapping,
+            table,
+            data.columns,
+            null_label,
+            imodel_functions,
+            codes_subset,
+            cdm_complete,
+            cdm_atts,
+            logger,
         )
-        cdm_tables[table]["buffer"].close()
-        cdm_tables[table].pop("buffer")
-        table_list.append(data)
+        table_list.append(table_df)
 
     merged = pd.concat(table_list, axis=1, join="outer")
     return merged.reset_index(drop=True)
@@ -377,25 +348,14 @@ def map_model(
         return
 
     # Check input data type and content (empty?)
-    # Make sure data is an iterable: this is to homogenize how we handle
-    # dataframes and textreaders
-    if isinstance(data, pd.DataFrame):
-        logger.debug("Input data is a pd.DataFrame")
-        if len(data) == 0:
-            logger.error("Input data is empty")
-            return
-        else:
-            data = [data]
-    elif isinstance(data, pd.io.parsers.TextFileReader):
-        logger.debug("Input is a pd.TextFileReader")
-        not_empty = pandas_TextParser_hdlr.is_not_empty(data)
-        if not not_empty:
-            logger.error("Input data is empty")
-            return
-    else:
+    if not isinstance(data, pd.DataFrame):
         logger.error("Input data type " f"{type(data)}" " not supported")
         return
 
+    if data.empty:
+        logger.error("Input data is empty")
+        return
+
     return map_and_convert(
         imodel[0],
         *imodel[1:],

cdm_reader_mapper/cdm_mapper/writer.py

@@ -4,8 +4,7 @@
 Created on Thu Apr 11 13:45:38 2019
 
 Exports tables written in the C3S Climate Data Store Common Data Model (CDM) format to ascii files,
-The tables format is contained in a python dictionary, stored as an attribute in a pandas.DataFrame
-(or pd.io.parsers.TextFileReader).
+The tables format is contained in a python dictionary, stored as an attribute in a pandas.DataFrame.
 
 This module uses a set of printer functions to "print" element values to a
 string object before exporting them to a final ascii file.

cdm_reader_mapper/common/inspect.py

@@ -9,9 +9,6 @@
 from __future__ import annotations
 
 import numpy as np
-import pandas as pd
-
-from cdm_reader_mapper.common import pandas_TextParser_hdlr
 
 
 def count_by_cat_i(series):
@@ -34,10 +31,7 @@ def get_length(data):
     int
         Total row count
     """
-    if not isinstance(data, pd.io.parsers.TextFileReader):
-        return len(data)
-    else:
-        return pandas_TextParser_hdlr.get_length(data)
+    return len(data)
 
 
 def count_by_cat(data, columns=None):
@@ -60,23 +54,6 @@ def count_by_cat(data, columns=None):
     if not isinstance(columns, list):
         columns = [columns]
     counts = {}
-    if not isinstance(data, pd.io.parsers.TextFileReader):
-        for column in columns:
-            counts[column] = count_by_cat_i(data[column])
-        return counts
-    else:
-        for column in columns:
-            data_cp = pandas_TextParser_hdlr.make_copy(data)
-            count_dicts = []
-            for df in data_cp:
-                count_dicts.append(count_by_cat_i(df[column]))
-
-            data_cp.close()
-            cats = [list(x.keys()) for x in count_dicts]
-            cats = list({x for y in cats for x in y})
-            cats.sort
-            count_dict = {}
-            for cat in cats:
-                count_dict[cat] = sum([x.get(cat) for x in count_dicts if x.get(cat)])
-            counts[column] = count_dict
-        return counts
+    for column in columns:
+        counts[column] = count_by_cat_i(data[column])
+    return counts