From 71cf00fa6b9a3cca7a090ed9a7c9584b11486aae Mon Sep 17 00:00:00 2001 From: Harish <102517942+HarishC727@users.noreply.github.com> Date: Mon, 23 Jun 2025 13:20:41 +0530 Subject: [PATCH 1/5] changed cron schedule (#1432) --- statvar_imports/world_bank/worldbank_ids/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/world_bank/worldbank_ids/manifest.json b/statvar_imports/world_bank/worldbank_ids/manifest.json index 18d22d202b..29070f6f00 100644 --- a/statvar_imports/world_bank/worldbank_ids/manifest.json +++ b/statvar_imports/world_bank/worldbank_ids/manifest.json @@ -35,7 +35,7 @@ "cleaned_csv": "output/disbursed/disbursed_output.csv" } ], - "cron_schedule": "00 11 1,15 * *" + "cron_schedule": "0 4 7,23 * *" } ] } \ No newline at end of file From 52f76eae7141422615631cd849ee71ea881a4491 Mon Sep 17 00:00:00 2001 From: shamimansari1988 <136044697+shamimansari1988@users.noreply.github.com> Date: Mon, 23 Jun 2025 08:34:24 +0000 Subject: [PATCH 2/5] EPAAirQualityIndex (#1226) * Auxilio Brazil Test Data and Readme file added * reverted the changes * EPAAirqualityIndex automation related changes * Manifest file change to add input_files * fixed PR comments * test case written for process method * fixing lint issue * fixed core team comments * PR comments fixed * method process and download_url made it as parameterized * Lint issue fix --- scripts/us_epa/airqualityindex/EPA_AQI.tmcf | 10 + scripts/us_epa/airqualityindex/README.md | 48 ++++ .../airqualityindex/air_quality_aggregate.py | 252 ++++++++++++++++++ .../air_quality_aggregate_test.py | 149 +++++++++++ scripts/us_epa/airqualityindex/manifest.json | 19 ++ scripts/us_epa/airqualityindex/openssl.cnf | 10 + .../test_data/test_aggregate_import_cbsa.csv | 11 + .../test_aggregate_import_county.csv | 11 + .../test_aggregate_import_data_cbsa.csv | 11 + .../test_aggregate_import_data_county.csv | 11 + 10 files changed, 532 insertions(+) create mode 100644 scripts/us_epa/airqualityindex/EPA_AQI.tmcf create mode 100644 scripts/us_epa/airqualityindex/README.md create mode 100644 scripts/us_epa/airqualityindex/air_quality_aggregate.py create mode 100644 scripts/us_epa/airqualityindex/air_quality_aggregate_test.py create mode 100644 scripts/us_epa/airqualityindex/manifest.json create mode 100644 scripts/us_epa/airqualityindex/openssl.cnf create mode 100644 scripts/us_epa/airqualityindex/test_data/test_aggregate_import_cbsa.csv create mode 100644 scripts/us_epa/airqualityindex/test_data/test_aggregate_import_county.csv create mode 100644 scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_cbsa.csv create mode 100644 scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_county.csv diff --git a/scripts/us_epa/airqualityindex/EPA_AQI.tmcf b/scripts/us_epa/airqualityindex/EPA_AQI.tmcf new file mode 100644 index 0000000000..1c05205f5e --- /dev/null +++ b/scripts/us_epa/airqualityindex/EPA_AQI.tmcf @@ -0,0 +1,10 @@ + +Node: E:EPA_AQI->E0 +typeOf: dcs:StatVarObservation +variableMeasured: dcs:AirQualityIndex_AirPollutant +observationDate: C:EPA_AQI->Date +observationAbout: C:EPA_AQI->Place +observationPeriod: "P1D" +value: C:EPA_AQI->AQI +definingPollutant: C:EPA_AQI->Pollutant +definingAirQualitySite: C:EPA_AQI->Site diff --git a/scripts/us_epa/airqualityindex/README.md b/scripts/us_epa/airqualityindex/README.md new file mode 100644 index 0000000000..537be5b335 --- /dev/null +++ b/scripts/us_epa/airqualityindex/README.md @@ -0,0 +1,48 @@ +# Importing EPA AirData +This directory imports [Outdoor Air Quality Data](https://aqs.epa.gov/aqsweb/airdata/download_files.html) into Data Commons. This includes mean/max concentration and AQI for Ozone, SO2, CO, NO2, PM2.5, and PM10 measured at various monitors throughout the US, Puerto Rico and US Virgin Islands. Source CSVs are downloaded directly from the pre-generated data files. + +The scripts generate: +- `EPA_AQI.csv` +- `EPA_AQI.tmcf` + +and relies on the following StatisticalVariables: +- Mean_Concentration_AirPollutant_Ozone +- Max_Concentration_AirPollutant_Ozone +- AirQualityIndex_AirPollutant_Ozone +- Mean_Concentration_AirPollutant_SO2 +- Max_Concentration_AirPollutant_SO2 +- AirQualityIndex_AirPollutant_SO2 +- Mean_Concentration_AirPollutant_CO +- Max_Concentration_AirPollutant_CO +- AirQualityIndex_AirPollutant_CO +- Mean_Concentration_AirPollutant_NO2 +- Max_Concentration_AirPollutant_NO2 +- AirQualityIndex_AirPollutant_NO2 +- Mean_Concentration_AirPollutant_PM2.5 +- Max_Concentration_AirPollutant_PM2.5 +- AirQualityIndex_AirPollutant_PM2.5 +- Mean_Concentration_AirPollutant_PM10 +- Max_Concentration_AirPollutant_PM10 +- AirQualityIndex_AirPollutant_PM10 +- AirQualityIndex_AirPollutant (only for county/CSBA) + +## Notes on the Data +Pollutant-specific metrics are provided on the site monitor level. For simplicity in the place mapping, we select a single monitor per site and pollutant and provide the monitor id (POC) as airQualitySiteMonitor in the observation. So, a given observation is distinguished by observationDate, observationAbout (AirQualitySite), and measurementMethod (pollutant standard). + +County/CBSA level AQI metrics include the defining site and pollutant, which are included in the StatVarObservation. + +## Generating Artifacts + +## To generate `EPA_AQI.csv` and `EPA_AQI.tmcf`, run: +## To process + python3 scripts/us_epa/airqualityindex/air_quality_aggregate.py --mode=process +## To download + python3 scripts/us_epa/airqualityindex/air_quality_aggregate.py --mode=download + + +### Running Tests +To run unit tests: +``` +OPENSSL_CONF=openssl.cnf python3 air_quality_aggregate_test.py + +``` \ No newline at end of file diff --git a/scripts/us_epa/airqualityindex/air_quality_aggregate.py b/scripts/us_epa/airqualityindex/air_quality_aggregate.py new file mode 100644 index 0000000000..e28ebcc99c --- /dev/null +++ b/scripts/us_epa/airqualityindex/air_quality_aggregate.py @@ -0,0 +1,252 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +Generates cleaned CSV and template MCF files for the EPA AirData. +This includes aggregate API measurements at the county/CSBA level. + +Usage: python3 air_quality_aggregate.py +''' +import csv, os, sys, requests, io, zipfile + +from absl import app +from absl import flags +from absl import logging +from datetime import datetime +from retry import retry +import frozendict +import os +from pathlib import Path +import time + +_FLAGS = flags.FLAGS + +flags.DEFINE_integer('aggregate_start_year', os.getenv('START_YEAR', '1980'), + 'Process data starting from this year.') +flags.DEFINE_integer('aggregate_end_year', + os.getenv('END_YEAR', + datetime.now().year), + 'Process data upto this year.') +flags.DEFINE_string('input_file_path', 'input_files', 'Input files path') +flags.DEFINE_string('output_file_path', 'output', 'Output files path') +flags.DEFINE_string('mode', '', 'Options: download or process') +MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +@retry(tries=3, delay=5, backoff=5) +def retry_method(url): + return requests.get(url) + + +POLLUTANTS = { + 'Ozone': 'Ozone', + 'SO2': 'SulfurDioxide', + 'CO': 'CarbonMonoxide', + 'NO2': 'NitrogenDioxide', + 'PM2.5': 'PM2.5', + 'PM10': 'PM10', +} + +CSV_COLUMNS = ['Date', 'Place', 'AQI', 'Pollutant', 'Site'] + + +def get_place(observation): + if 'State Code' and 'County Code' in observation: + return 'dcid:geoId/' + observation['State Code'] + observation[ + 'County Code'] + elif 'CBSA Code' in observation: + return 'dcid:geoId/C' + observation['CBSA Code'] + else: + return None + + +def create_csv(csv_file_path): + logging.info(f'Inside create csv value of {csv_file_path}') + output_file_path = os.path.join(MODULE_DIR, _FLAGS.output_file_path) + file_path = os.path.join(output_file_path, csv_file_path) + logging.info(f'file_path {file_path}') + with open(file_path, 'w', newline='') as f_out: + writer = csv.DictWriter(f_out, + fieldnames=CSV_COLUMNS, + lineterminator='\n') + writer.writeheader() + + +def write_csv(csv_file_path, reader): + with open(csv_file_path, 'a', newline='') as f_out: + writer = csv.DictWriter(f_out, + fieldnames=CSV_COLUMNS, + lineterminator='\n') + for observation in reader: + place = get_place(observation) + if not place: + continue + new_row = { + 'Date': + observation['Date'], + 'Place': + place, + 'AQI': + observation['AQI'], + 'Pollutant': + 'dcs:' + POLLUTANTS[observation['Defining Parameter']], + 'Site': + 'dcid:epa/' + observation['Defining Site'].replace('-', '') + } + writer.writerow(new_row) + + +def download_url(_INPUT_FILE_PATH, start_year, end_year, CSV_URLS): + """ + Downloads daily Air Quality Index (AQI) data zip files from the EPA AirData + website for a specified range of years and saves them locally. + + It iterates through years from `start_year` (inclusive) to `end_year` (exclusive). + For each year, it constructs file names for both county-level and CBSA-level + AQI data (e.g., 'daily_aqi_by_county_YYYY.zip', 'daily_aqi_by_cbsa_YYYY.zip'). + Args: + _INPUT_FILE_PATH (str): The local directory path where the downloaded + zip files will be saved. + start_year (int): The starting year (inclusive) from which to download data. + end_year (int): The ending year (exclusive) up to which to download data. + Data for `end_year - 1` will be the last year downloaded. + CSV_URLS (frozendict.frozendict): A frozendict where keys are series names + (e.g., "daily_aqi") and values are the + base URLs for the EPA AirData files. + This dictionary is expected to provide + the root URL to which file names are appended. + """ + + for year in range(start_year, int(end_year)): + logging.info(f'year: {year}') + file_names = [ + f'daily_aqi_by_county_{year}', f'daily_aqi_by_cbsa_{year}' + ] + # Iterate over the list of file names (county and cbsa) + for file_name in file_names: + logging.info(f'Processing file: {file_name}') + + for series_name, url in CSV_URLS.items(): + #Retry method is calling + url = f"{url}{file_name}.zip" + logging.info(f'============url {url}') + response = retry_method(url) + logging.info( + f'Downloading files from url {url} and save to path {_INPUT_FILE_PATH}' + ) + logging.info(f'raise_for_status {response.status_code}') + response.raise_for_status() + + if response.status_code != 200: + logging.fatal( + f"No data available for URL: {url}. Aborting download.") + elif (response.status_code == 404): + logging.fatal( + f"Not Found for url: {url}. Aborting download.") + elif response.status_code == 200: + if not response.content: + logging.fatal( + f"No data available for URL: {url}. Aborting download." + ) + input_file_name = f"{file_name}.zip" + logging.info( + f'filename: {input_file_name} _INPUT_FILE_PATH : {_INPUT_FILE_PATH}' + ) + file_path = os.path.join(_INPUT_FILE_PATH, input_file_name) + #download zip file locally + with open(file_path, 'wb') as f: + f.write(response.content) + + +def process(_INPUT_FILE_PATH, start_year, end_year): + """ + Processes downloaded EPA AirData zip files, extracts, transforms, and + consolidates the data into a single output CSV file. + + This method iterates through a specified range of years, from `start_year` + up to (but not including) `end_year`. For each year, it expects to find + two zipped files in the `_INPUT_FILE_PATH` directory: + 'daily_aqi_by_county_{year}.zip' and 'daily_aqi_by_cbsa_{year}.zip'. + + Args: + _INPUT_FILE_PATH (str): The local directory path where the downloaded + zip files (e.g., 'daily_aqi_by_county_2020.zip') + are stored. + start_year (int): The starting year (inclusive) from which to begin + processing data. + end_year (int): The ending year (exclusive) for which to process data. + Processing will stop at `end_year - 1`. + """ + output_file_name = f'EPA_AQI.csv' + #Creating the output csv file beofre + create_csv(output_file_name) + for year in range(start_year, int(end_year)): + logging.info(f'year: {year}') + file_names = [ + f'daily_aqi_by_county_{year}', f'daily_aqi_by_cbsa_{year}' + ] + # Iterate over the list of file names (county and cbsa) + for file_name in file_names: + logging.info(f'Processing file: {file_name}') + + try: + file_path = os.path.join(_INPUT_FILE_PATH, file_name) + output_file_path = os.path.join(MODULE_DIR, + _FLAGS.output_file_path, + output_file_name) + logging.info( + f'csv_file_path, output_file_path from request_and_write_csv {output_file_name} {output_file_path}' + ) + file_path_with_filename = f"{file_path}.zip" + with open(file_path_with_filename, 'rb') as zip_file: + with zipfile.ZipFile(zip_file) as zf: + try: + with zf.open(f'{file_name}.csv', 'r') as infile: + reader = csv.DictReader( + io.TextIOWrapper(infile, 'utf-8')) + write_csv(output_file_path, reader) + except FileNotFoundError: + raise FileNotFoundError( + f"CSV file '{file_name}.csv' not found in the ZIP archive" + ) + + except Exception as e: + logging.fatal( + f"Error while processing input files {e} {output_file_name}{output_file_name}" + ) + + +def main(_): + mode = _FLAGS.mode + input_file_path = os.path.join(MODULE_DIR, _FLAGS.input_file_path) + output_file_path = os.path.join(MODULE_DIR, _FLAGS.output_file_path) + Path(output_file_path).mkdir(parents=True, exist_ok=True) + Path(input_file_path).mkdir(parents=True, exist_ok=True) + start_year = _FLAGS.aggregate_start_year + end_year = _FLAGS.aggregate_end_year + CSV_URLS = frozendict.frozendict( + {"daily_aqi": "https://aqs.epa.gov/aqsweb/airdata/"}) + + if mode == "" or mode == "download": + logging.info(f'inside mode {mode}') + #downloading zip file for 'daily_aqi_by_county' + logging.info(f'downloading zipped files') + download_url(input_file_path, start_year, end_year, CSV_URLS) + + if mode == "" or mode == "process": + logging.info(f'inside mode {mode}') + process(input_file_path, start_year, end_year) + + +if __name__ == '__main__': + app.run(main) diff --git a/scripts/us_epa/airqualityindex/air_quality_aggregate_test.py b/scripts/us_epa/airqualityindex/air_quality_aggregate_test.py new file mode 100644 index 0000000000..0562e6ec61 --- /dev/null +++ b/scripts/us_epa/airqualityindex/air_quality_aggregate_test.py @@ -0,0 +1,149 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +Unit tests for air_quality_aggregate.py + +Usage: python3 -m unittest discover -v -s ../ -p "air_quality_aggregate_test.py" +''' +import unittest, csv, os, sys, tempfile, zipfile, io +from absl import app +from absl import flags +from absl import logging + +_FLAGS = flags.FLAGS + +module_dir_ = os.path.dirname(__file__) + +sys.path.append(module_dir_) + +from air_quality_aggregate import create_csv, write_csv, process + +MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestCriteriaGasesTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from absl import flags + flags.FLAGS( + ["air_quality_aggregate_test.py", "--output_file_path", "output"]) + + def test_write_csv_county(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with open( + os.path.join( + module_dir_, + 'test_data/test_aggregate_import_data_county.csv'), + 'r') as f: + test_csv = os.path.join(tmp_dir, 'test_csv.csv') + create_csv(test_csv) + + reader = csv.DictReader(f) + write_csv(test_csv, reader) + + expected_csv = os.path.join( + module_dir_, 'test_data/test_aggregate_import_county.csv') + with open(test_csv, 'r') as test: + test_str: str = test.read() + with open(expected_csv, 'r') as expected: + expected_str: str = expected.read() + self.assertEqual(test_str, expected_str) + os.remove(test_csv) + + def test_write_csv_csba(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with open( + os.path.join( + module_dir_, + 'test_data/test_aggregate_import_data_cbsa.csv'), + 'r') as f: + test_csv = os.path.join(tmp_dir, 'test_csv.csv') + create_csv(test_csv) + + reader = csv.DictReader(f) + write_csv(test_csv, reader) + expected_csv = os.path.join( + module_dir_, 'test_data/test_aggregate_import_cbsa.csv') + with open(test_csv, 'r') as test: + test_str: str = test.read() + with open(expected_csv, 'r') as expected: + expected_str: str = expected.read() + self.assertEqual(test_str, expected_str) + os.remove(test_csv) + + @classmethod + def setUpClass(cls): + #from absl import flags + if not flags.FLAGS.is_parsed(): + flags.FLAGS(["air_quality_aggregate_test.py"]) + + def test_process_generates_correct_output(self): + with tempfile.TemporaryDirectory( + ) as tmp_input_dir, tempfile.TemporaryDirectory() as tmp_output_dir: + # Update FLAGS + _FLAGS.input_file_path = tmp_input_dir + _FLAGS.output_file_path = tmp_output_dir + _FLAGS.aggregate_start_year = 2020 + _FLAGS.aggregate_end_year = 2021 # only 2020 will be processed + + # Sample CSV row + row = { + 'Date': '2020-01-01', + 'State Code': '06', + 'County Code': '075', + 'AQI': '45', + 'Defining Parameter': 'Ozone', + 'Defining Site': '06-075-0001' + } + + # Generate CSV content + csv_content = io.StringIO() + writer = csv.DictWriter(csv_content, fieldnames=row.keys()) + writer.writeheader() + writer.writerow(row) + csv_bytes = csv_content.getvalue().encode('utf-8') + + # Create ZIP files for county and cbsa + for prefix in [ + 'daily_aqi_by_county_2020', 'daily_aqi_by_cbsa_2020' + ]: + zip_path = os.path.join(tmp_input_dir, f"{prefix}.zip") + with zipfile.ZipFile(zip_path, 'w') as zipf: + zipf.writestr(f"{prefix}.csv", csv_bytes) + + # Run the process method + process(tmp_input_dir, 2020, 2021) + + # Check if the output files were created + expected_csv_path = os.path.join(MODULE_DIR, tmp_output_dir, + 'EPA_AQI.csv') + expected_tmcf_path = os.path.join(MODULE_DIR, tmp_output_dir, + 'EPA_AQI.tmcf') + + self.assertTrue(os.path.isfile(expected_csv_path), + "CSV output file not created") + + # Validate CSV content + with open(expected_csv_path, 'r') as f: + lines = f.readlines() + self.assertGreaterEqual(len(lines), + 2) # At least header + 1 data row + self.assertIn("dcid:geoId/06075", lines[1]) + self.assertIn("dcs:Ozone", lines[1]) + self.assertIn("dcid:epa/060750001", lines[1]) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/us_epa/airqualityindex/manifest.json b/scripts/us_epa/airqualityindex/manifest.json new file mode 100644 index 0000000000..13f5d7074d --- /dev/null +++ b/scripts/us_epa/airqualityindex/manifest.json @@ -0,0 +1,19 @@ +{ + "import_specifications": [ + { + "import_name": "EPAAirQualityIndex", + "curator_emails": ["support@datacommons.org"], + "provenance_url": "https://aqs.epa.gov/aqsweb/airdata/download_files.html", + "provenance_description": "Air quality data collected from outdoor monitors on the county, CBSA, and site monitor level.", + "scripts": ["air_quality_aggregate.py"], + "source_files": ["input_files/*"], + "import_inputs": [ + { + "template_mcf": "EPA_AQI.tmcf", + "cleaned_csv": "output/EPA_AQI.csv" + } + ], + "cron_schedule": "15 22 10 1,7 *" + } + ] +} diff --git a/scripts/us_epa/airqualityindex/openssl.cnf b/scripts/us_epa/airqualityindex/openssl.cnf new file mode 100644 index 0000000000..48eb2a3152 --- /dev/null +++ b/scripts/us_epa/airqualityindex/openssl.cnf @@ -0,0 +1,10 @@ +openssl_conf = openssl_init + +[openssl_init] +ssl_conf = ssl_sect + +[ssl_sect] +system_default = system_default_sect + +[system_default_sect] +Options = UnsafeLegacyRenegotiation \ No newline at end of file diff --git a/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_cbsa.csv b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_cbsa.csv new file mode 100644 index 0000000000..958d9a1bbd --- /dev/null +++ b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_cbsa.csv @@ -0,0 +1,11 @@ +Date,Place,AQI,Pollutant,Site +1989-01-01,dcid:geoId/C10140,43,dcs:SulfurDioxide,dcid:epa/530270006 +1989-01-02,dcid:geoId/C10140,43,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-03,dcid:geoId/C10140,69,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-04,dcid:geoId/C10140,29,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-05,dcid:geoId/C10140,81,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-06,dcid:geoId/C10140,14,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-07,dcid:geoId/C10140,43,dcs:SulfurDioxide,dcid:epa/530270006 +1989-01-08,dcid:geoId/C10140,94,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-09,dcid:geoId/C10140,121,dcs:SulfurDioxide,dcid:epa/530270003 +1989-01-10,dcid:geoId/C10140,29,dcs:SulfurDioxide,dcid:epa/530270003 diff --git a/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_county.csv b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_county.csv new file mode 100644 index 0000000000..564ed600a9 --- /dev/null +++ b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_county.csv @@ -0,0 +1,11 @@ +Date,Place,AQI,Pollutant,Site +2021-01-01,dcid:geoId/01049,30,dcs:Ozone,dcid:epa/010499991 +2021-01-02,dcid:geoId/01049,27,dcs:Ozone,dcid:epa/010499991 +2021-01-03,dcid:geoId/01049,34,dcs:Ozone,dcid:epa/010499991 +2021-01-04,dcid:geoId/01049,36,dcs:Ozone,dcid:epa/010499991 +2021-01-05,dcid:geoId/01049,31,dcs:Ozone,dcid:epa/010499991 +2021-01-06,dcid:geoId/01049,27,dcs:Ozone,dcid:epa/010499991 +2021-01-07,dcid:geoId/01049,29,dcs:Ozone,dcid:epa/010499991 +2021-01-08,dcid:geoId/01049,22,dcs:Ozone,dcid:epa/010499991 +2021-01-09,dcid:geoId/01049,26,dcs:Ozone,dcid:epa/010499991 +2021-01-10,dcid:geoId/01049,25,dcs:Ozone,dcid:epa/010499991 diff --git a/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_cbsa.csv b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_cbsa.csv new file mode 100644 index 0000000000..12d9060835 --- /dev/null +++ b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_cbsa.csv @@ -0,0 +1,11 @@ +"CBSA","CBSA Code","Date","AQI","Category","Defining Parameter","Defining Site","Number of Sites Reporting" +"Aberdeen, WA","10140","1989-01-01",43,"Good","SO2","53-027-0006",2 +"Aberdeen, WA","10140","1989-01-02",43,"Good","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-03",69,"Moderate","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-04",29,"Good","SO2","53-027-0003",1 +"Aberdeen, WA","10140","1989-01-05",81,"Moderate","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-06",14,"Good","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-07",43,"Good","SO2","53-027-0006",2 +"Aberdeen, WA","10140","1989-01-08",94,"Moderate","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-09",121,"Unhealthy for Sensitive Groups","SO2","53-027-0003",2 +"Aberdeen, WA","10140","1989-01-10",29,"Good","SO2","53-027-0003",2 diff --git a/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_county.csv b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_county.csv new file mode 100644 index 0000000000..d7029ee1f2 --- /dev/null +++ b/scripts/us_epa/airqualityindex/test_data/test_aggregate_import_data_county.csv @@ -0,0 +1,11 @@ +"State Name","county Name","State Code","County Code","Date","AQI","Category","Defining Parameter","Defining Site","Number of Sites Reporting" +"Alabama","DeKalb","01","049","2021-01-01",30,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-02",27,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-03",34,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-04",36,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-05",31,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-06",27,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-07",29,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-08",22,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-09",26,"Good","Ozone","01-049-9991",1 +"Alabama","DeKalb","01","049","2021-01-10",25,"Good","Ozone","01-049-9991",1 From c83290e4aa305faffbb4592b75d6ccc0c782ef3f Mon Sep 17 00:00:00 2001 From: SudhishaK Date: Tue, 24 Jun 2025 09:57:57 +0000 Subject: [PATCH 3/5] Commerce_NTIA Statvar update configurations (#1340) * Commerce_NTIA Statvat update configurations * Commerce_NTIA modified the preprocess script * Commerce_NTIA modified the preprocess script and manifest file * Commerce_NTIA Modified the download and age pv map files * Commerce_NTIA renamed the folder * Commerce_NTIA modified download file * commerce_ntia modified manifest file * commerce_ntia modified pvmap * commerce_ntia modified pvmap 2 * Commerce_NTIA usage of download util script. * Commerce_NTIA addressed pr comments. * Commerce_NTIA addressed pr comments.. * Commerce_NTIA removing conflicts. --- .../commerce_ntia/README.md | 52 +++ .../commerce_ntia/config.py | 1 + .../commerce_ntia/manifest.json | 29 ++ .../commerce_ntia/ntia_age_pvmap.csv | 211 +++++++++ .../commerce_ntia/ntia_metadata.csv | 13 + .../commerce_ntia/ntia_pvmap.csv | 211 +++++++++ .../commerce_ntia/preprocess.py | 94 ++++ .../testdata/ntia-data-age-only.csv | 6 + .../commerce_ntia/testdata/ntia-data.csv | 8 + .../testdata/ntia_age_output.csv | 22 + .../testdata/ntia_age_output.tmcf | 6 + .../testdata/ntia_age_output_stat_vars.mcf | 89 ++++ .../ntia_age_output_stat_vars_schema.mcf | 10 + .../commerce_ntia/testdata/ntia_output.csv | 441 ++++++++++++++++++ .../commerce_ntia/testdata/ntia_output.tmcf | 6 + .../testdata/ntia_output_stat_vars.mcf | 223 +++++++++ .../testdata/ntia_output_stat_vars_schema.mcf | 5 + util/statvar_dcid_generator.py | 3 + 18 files changed, 1430 insertions(+) create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/config.py create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data-age-only.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.tmcf create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars.mcf create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars_schema.mcf create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.csv create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.tmcf create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars.mcf create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars_schema.mcf diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md new file mode 100644 index 0000000000..eb6e1e8ea1 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -0,0 +1,52 @@ +# Commerce_NTIA + +- source: `https://www.ntia.gov/sites/default/files/data_central_downloads/datasets/ntia-analyze-table.csv` + +- NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users. + +- how to download data: + To download and process the data, you'll need to run the provided preprocess script, `preprocess.py`. This script will automatically create an "input_files" folder where you should place the file to be processed. + By using this script, we are creating two more columns in the input files such as 'universeAgeResol', 'variableAgeResol'. This columns are created based on the universe and variable columns in the existing data. + +- type of place: Demographics. + +- statvars: Demographics + +### How to run: + +``` +python3 stat_var_processor.py +--input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' +--pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +``` + +#### Download the data: + +For download and preprocess the source data, run: +`python3 preprocess.py` + +Notes: +Files will be downloaded inside "input_files" folder (input_files/ntia-analyze-table.csv). +This preprocess script will split the downloaded input file into two files based on the age. (input_files/ntia-data.csv, input_files/ntia-data-age-only.csv) + +#### Process the data: + +Execute the script inside the folder `/data/tools/statvar_importer/` + +``` +python3 stat_var_processor.py +--input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output +``` + +``` +python3 stat_var_processor.py +--input_data=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/ntia-data-age-only.csv +--pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv +--config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf +--output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output +``` + diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/config.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/config.py new file mode 100644 index 0000000000..7b4f49bda0 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/config.py @@ -0,0 +1 @@ +Commerce_NTIA_URL = "https://www.ntia.gov/sites/default/files/data_central_downloads/datasets/ntia-analyze-table.csv" \ No newline at end of file diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json new file mode 100644 index 0000000000..f123cd64e9 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -0,0 +1,29 @@ +{ + "import_specifications": [ + { + "import_name": "Commerce_NTIA", + "curator_emails": ["support@datacommons.org"], + "provenance_url": "https://www.ntia.gov/sites/default/files/data_central_downloads/datasets/ntia-analyze-table.csv", + "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.", + "scripts": [ + "preprocess.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output" + ], + "source_files": [ + "input_files/ntia-analyze-table.csv" + ], + "import_inputs": [ + { + "template_mcf": "output_files/ntia_output.tmcf", + "cleaned_csv": "output_files/ntia_output.csv" + }, + { + "template_mcf": "output_files/ntia_age_output.tmcf", + "cleaned_csv": "output_files/ntia_age_output.csv" + } + ], + "cron_schedule": "0 06 * * 5" + } + ] +} diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv new file mode 100644 index 0000000000..f9286c33c4 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv @@ -0,0 +1,211 @@ +key,prop,value,,,,,,,,,,,,,, +isHouseholder,populationType,Household,,,institutionalization,USC_NonInstitutionalized,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +isPerson,populationType,Person,armedForcesStatus,Civilian,,,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},, +isAdult,populationType,Person,,,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, +Adult,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, +CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, +internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +,,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +Nov 1994,observationDate,1994-11,,,,,,,,,,,,,, +Oct 1997,observationDate,1997-10,,,,,,,,,,,,,, +Dec 1998,observationDate,1998-12,,,,,,,,,,,,,, +Aug 2000,observationDate,2000-08,,,,,,,,,,,,,, +Sep 2001,observationDate,2001-09,,,,,,,,,,,,,, +Oct 2003,observationDate,2003-10,,,,,,,,,,,,,, +Oct 2007,observationDate,2007-10,,,,,,,,,,,,,, +Oct 2009,observationDate,2009-10,,,,,,,,,,,,,, +Oct 2010,observationDate,2010-10,,,,,,,,,,,,,, +Jul 2011,observationDate,2011-07,,,,,,,,,,,,,, +Oct 2012,observationDate,2012-10,,,,,,,,,,,,,, +Jul 2013,observationDate,2013-07,,,,,,,,,,,,,, +Jul 2015,observationDate,2015-07,,,,,,,,,,,,,, +Nov 2017,observationDate,2017-11,,,,,,,,,,,,,, +Nov 2019,observationDate,2019-11,,,,,,,,,,,,,, +Nov 2021,observationDate,2021-11,,,,,,,,,,,,,, +Nov 2023,observationDate,2023-11,,,,,,,,,,,,,, +Nov 2024,observationDate,2024-11,,,,,,,,,,,,,, +Nov 2025,observationDate,2025-11,,,,,,,,,,,,,, +Nov 2026,observationDate,2026-11,,,,,,,,,,,,,, +Nov 2027,observationDate,2027-11,,,,,,,,,,,,,, +Nov 2028,observationDate,2028-11,,,,,,,,,,,,,, +Nov 2029,observationDate,2029-11,,,,,,,,,,,,,, +Nov 2030,observationDate,2030-11,,,,,,,,,,,,,, +Nov 2031,observationDate,2031-11,,,,,,,,,,,,,, +Nov 2032,observationDate,2032-11,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +Household Reference Person in Universe: Non-Institutional,DescriptionVar,Household Reference Person in Universe: Non-Institutional,,,institutionalization,USC_NonInstitutionalized,,,,,,,,,, +Person in Universe: Ages 3+ Not Active-Duty Military,DescriptionVar,"Person Aged 3 and above,Not Active-Duty Military",,,armedForcesStatus,Civilian,,,,,,,,,, +"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",DescriptionVar,"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",,,hasComputer,True,computerUsageLocation,Home,,,,,,,, +Person Ages 15+,#ignore,ignore,,,,,,,,,,,,,, +Anyone in Household Uses Internet From Any Location,#ignore,ignore,,,,,,,,,,,,,, +Anyone in Household Uses Internet at Home,#ignore,ignore,,,,,,,,,,,,,, +Uses the Internet (Any Location),DescriptionVar,Uses the Internet (Any Location),isInternetUser,True,internetUsageLocation,AnyLocation,,,,,,,,,, +Uses the Internet at Home,DescriptionVar,Uses the Internet at Home,isInternetUser,True,internetUsageLocation,Home,,,,,,,,,, +Uses the Internet at School,DescriptionVar,Uses the Internet at School,isInternetUser,True,internetUsageLocation,School,,,,,,,,,, +Uses the Internet at Work,DescriptionVar,Uses the Internet at Work,isInternetUser,True,internetUsageLocation,Work,,,,,,,,,, +"Uses the Internet at a Public Place Like the Library, Community Center, Park",DescriptionVar,"Uses the Internet at a Public Place Like the Library, Community Center, Park",isInternetUser,True,internetUsageLocation,PublicPlace,,,,,,,,,, +Household Used to Go Online from Home,DescriptionVar,Household Used to Go Online from Home,isInternetUser,True,internetUsageLocation,Home,,,,,,,,,, +No One in Household Uses Internet at Home,DescriptionVar,No One in Household Uses Internet at Home,residenceType,Household,isInternetUser,False,internetUsageLocation,Home,,,,,,,, +Dial-up Internet Service Used at Home,#ignore,ignore,,,,,,,,,,,,,, +Wired High-Speed Internet Service Used at Home,#ignore,ignore,,,,,,,,,,,,,, +Searches for a Job Online,DescriptionVar,Searches for a Job Online,isInternetUser,True,internetUsagePurpose,JobSearchUsage,,,,,,,,,, +"Shops, Makes Travel Reservations, or Uses Other Consumer Services Online",#ignore,Ignore,,,,,,,,,,,,,, +Takes Class or Participates in Job Training Online,DescriptionVar,Takes Class or Participates in Job Training Online,isInternetUser,True,internetUsagePurpose,OnlineClassUsage__JobTrainningUsage,,,,,,,,,, +"Uses Online Financial Services Like Banking, Investing, Paying Bills",DescriptionVar,"Uses Online Financial Services Like Banking, Investing, Paying Bills",isInternetUser,True,internetUsagePurpose,FinancialServicesUsage,,,,,,,,,, +Main Reason for Household Not Online at Home: Can Use Elsewhere,DescriptionVar,Main Reason for Household Not Online at Home: Can Use Elsewhere,isInternetUser,True,reasonNotUsingInternetAtHome,InternetUseElsewhere,,,,,,,,,, +Main Reason for Household Not Online at Home: Don't Need It or Not Interested,DescriptionVar,Main Reason for Household Not Online at Home: Don't Need It or Not Interested,isInternetUser,True,reasonNotUsingInternetAtHome,InternetNotNeeded__InternetNotInterested,,,,,,,,,, +Main Reason for Household Not Online at Home: No/Inadequate Computer,DescriptionVar,Main Reason for Household Not Online at Home: No/Inadequate Computer,isInternetUser,True,reasonNotUsingInternetAtHome,InternetInadequateComputer,,,,,,,,,, +Main Reason for Household Not Online at Home: Too Expensive,DescriptionVar,Main Reason for Household Not Online at Home: Too Expensive,isInternetUser,True,reasonNotUsingInternetAtHome,InternetExpensive,,,,,,,,,, +Satellite Internet Service Used at Home,DescriptionVar,Satellite Internet Service Used at Home,isInternetUser,True,internetSubscriptionType,SatelliteInternetService,,,,,,,,,, +Main Reason for Household Not Online at Home: Privacy or Security Concerns,DescriptionVar,Main Reason for Household Not Online at Home: Privacy or Security Concerns,isInternetUser,True,reasonNotUsingInternetAtHome,InternetPrivacyConcerns__InternetSecurityConcerns,,,,,,,,,, +Main Reason for Household Not Online at Home: Not Available in Area,DescriptionVar,Main Reason for Household Not Online at Home: Not Available in Area,isInternetUser,True,reasonNotUsingInternetAtHome,InternetUnavailableInArea,,,,,,,,,, +Mobile Internet Service Used at Home,DescriptionVar,Mobile Internet Service Used at Home,isInternetUser,True,internetSubscriptionType,BroadbandOfAnyType,,,,,,,,,, +Affordability is Most Important Factor for Home Internet Service,DescriptionVar,Affordability is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetAffordability,,,,,,,, +Customer Service is Most Important Factor for Home Internet Service,DescriptionVar,Customer Service is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetCustomerService,,,,,,,, +Home Internet is Part of a Bundle,DescriptionVar,Home Internet is Part of a Bundle,internetUsageLocation,Home,internetSubscriptionType,InternetBundle,,,,,,,,,, +Mobility is Most Important Factor for Home Internet Service,DescriptionVar,Mobility is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetMobility,,,,,,,, +Reliability is Most Important Factor for Home Internet Service,DescriptionVar,Reliability is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetReliability,,,,,,,, +Service Speed is Most Important Factor for Home Internet Service,DescriptionVar,Service Speed is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetServiceSpeed,,,,,,,, +Uses a Desktop Computer,DescriptionVar,Uses a Desktop Computer,electronicGadgetUsed,DesktopUser,,,,,,,,,,,, +"Uses a Desktop, Laptop, or Tablet Computer",DescriptionVar,"Uses a Desktop, Laptop, or Tablet Computer",electronicGadgetUsed,DesktopUser__LaptopUser__TabletUser,,,,,,,,,,,, +Uses a Laptop Computer,DescriptionVar,Uses a Laptop Computer,electronicGadgetUsed,LaptopUser,,,,,,,,,,,, +Uses a Mobile Phone,DescriptionVar,Uses a Mobile Phone,electronicGadgetUsed,MobileUser,,,,,,,,,,,, +Uses a Smart TV or Connected Device,DescriptionVar,Uses a Smart TV or Connected Device,electronicGadgetUsed,SmartTelevisionUser,,,,,,,,,,,, +Uses a Tablet or e-Book Reader,DescriptionVar,Uses a Tablet or e-Book Reader,electronicGadgetUsed,TabletUser__EbookReaderUser,,,,,,,,,,,, +Home Phone Service Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +TV Channels Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +"Access Electronic Health Records or Insurance Records, or Talks w Doctor Online",#ignore,Ignore,,,,,,,,,,,,,, +Researches Health Information Online,DescriptionVar,Researches Health Information Online,isInternetUser,True,internetUsagePurpose,HealthDataResearchUsage,,,,,,,,,, +Telecommutes Using the Internet,DescriptionVar,Telecommutes Using the Internet,isInternetUser,True,internetUsagePurpose,TelecommutesUsage,,,,,,,,,, +Uses Email,DescriptionVar,Uses Email,isInternetUser,True,internetUsagePurpose,EmailUsage,,,,,,,,,, +Uses Online Location-Based (On-the-Go) Services,#ignore,Ignore,,,,,,,,,,,,,, +Uses Text Messaging or Instant Messaging,DescriptionVar,Uses Text Messaging or Instant Messaging,isInternetUser,True,internetUsagePurpose,TextMessagingUsage,,,,,,,,,, +Watches Videos Online,DescriptionVar,Watches Videos Online,isInternetUser,True,internetUsagePurpose,WatchingVideosUsage,,,,,,,,,, +Data Cap is Most Important Factor for Home Internet Service,DescriptionVar,Data Cap is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetDataLimit,,,,,,,, +Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, +Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, +Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, +Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, +Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, +Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, +Mobile Internet Service Used Outside the Home,DescriptionVar,Mobile Internet Service Used Outside the Home,isInternetUser,True,internetSubscriptionType,CellularDataPlan,internetUsageLocation,OutsideHome,,,,,,,, +Internet Service Provided for Entire Building/Campus/Community at Home,#ignore,Ignore,,,,,,,,,,,,,, +Publicly-Available Free Internet Service at Home,DescriptionVar,Publicly-Available Free Internet Service at Home,isInternetUser,True,internetSubscriptionType,FreeInternet,internetUsageLocation,Home,,,,,,,, +Subscription to Private ISP Service at Home,#ignore,Ignore,,,,,,,,,,,,,, +"Subscription to Public, Nonprofit, or Coop ISP Service at Home",#ignore,Ignore,,,,,,,,,,,,,, +Uses a Wearable Device,DescriptionVar,Uses a Wearable Device,electronicGadgetUsed,WearableDeviceUser,,,,,,,,,,,, +Uses the Internet While Traveling Between Places,DescriptionVar,Uses the Internet While Traveling Between Places,isInternetUser,True,internetUsageLocation,Traveling,,,,,,,,,, +Uses the Internet at a Coffee Shop or Other Business,DescriptionVar,Uses the Internet at a Coffee Shop or Other Business,isInternetUser,True,internetUsageLocation,Cafeteria,,,,,,,,,, +Home Security or Monitoring Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +Mobile Phone Service Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +Offers Services for Sale via the Internet,DescriptionVar,Offers Services for Sale via the Internet,isInternetUser,True,internetUsagePurpose,ServiceForSaleUsage,,,,,,,,,, +"Posts or Uploads Blog Posts, Videos, or Other Original Content",DescriptionVar,"Posts or Uploads Blog Posts, Videos, or Other Original Content",isInternetUser,True,internetUsagePurpose,UploadMediaUsage,,,,,,,,,, +Requests Services Provided by Other People via the Internet,#ignore,Ignore,,,,,,,,,,,,,, +Uses the Internet to Sell Goods,DescriptionVar,Uses the Internet to Sell Goods,isInternetUser,True,internetUsagePurpose,GoodsSellingUsage,,,,,,,,,, +Household Subscribes to Cable or Satellite TV Service,DescriptionVar,Household Subscribes to Cable or Satellite TV Service,internetSubscriptionType,BroadbandInternetSuchAsCableFiberOpticOrDsl__SatelliteInternetService,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +usCount,observationAbout,country/USA,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +age314Count,observationAbout,country/USA,measuredProperty,count,Age,[3 14 Years],value,{Number},name,"""{DescriptionVar}, age between 3 and 14 years""",,,,,, +age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Years],value,{Number},name,"""{DescriptionVar}, age between 15 and 24 years""",,,,,, +age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, +age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, +age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, +income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, +income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, +income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, +income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, +sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, +raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, +raceBlackCount,observationAbout,country/USA,measuredProperty,count,Race,BlackOrAfricanAmericanAlone,value,{Number},name,"""{DescriptionVar}, African American""",,,,,, +raceHispanicCount,observationAbout,country/USA,measuredProperty,count,ethnicity,HispanicOrLatino,value,{Number},name,"""{DescriptionVar}, Hispanic""",,,,,, +raceAsianCount,observationAbout,country/USA,measuredProperty,count,Race,Asian,value,{Number},name,"""{DescriptionVar}, Asian American""",,,,,, +raceAmIndianCount,observationAbout,country/USA,measuredProperty,count,Race,AmericanIndianOrAlaskaNative,value,{Number},name,"""{DescriptionVar}, American Indian or Alaska Native""",,,,,, +raceOtherCount,observationAbout,country/USA,measuredProperty,count,Race,NTIA_OtherRace,ethnicity,NotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, Other non-Hispanic""",,,, +disabilityNoCount,observationAbout,country/USA,measuredProperty,count,disabilityStatus,NoDisability,value,{Number},name,"""{DescriptionVar}, without disability""",,,,,, +disabilityYesCount,observationAbout,country/USA,measuredProperty,count,disabilityStatus,WithDisability,value,{Number},name,"""{DescriptionVar}, with disability""",,,,,, +metroNoCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,Rural,value,{Number},name,"""{DescriptionVar}, Rural""",,,,,, +metroYesCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,Urban,value,{Number},name,"""{DescriptionVar}, Urban""",,,,,, +metroUnknownCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,UnknownPlaceOfResidence,value,{Number},name,"""{DescriptionVar}, Unknown place of residence""",,,,,, +scChldHomeNoCount,observationAbout,country/USA,measuredProperty,count,schoolEnrollment,NotEnrolledInSchool,value,{Number},name,"""{DescriptionVar}, No School-Aged Child at Home""",,,,,, +scChldHomeYesCount,observationAbout,country/USA,measuredProperty,count,schoolEnrollment,EnrolledInSchool,value,{Number},name,"""{DescriptionVar}, School-Aged Child at Home""",,,,,, +veteranNoCount,observationAbout,country/USA,measuredProperty,count,veteranStatus,Nonveteran,value,{Number},name,"""{DescriptionVar}, Not Veteran""",,,,,, +veteranYesCount,observationAbout,country/USA,measuredProperty,count,veteranStatus,Veteran,value,{Number},name,"""{DescriptionVar}, Veteran""",,,,,, +ALCount,observationAbout,geoId/01,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +AKCount,observationAbout,geoId/02,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +AZCount,observationAbout,geoId/04,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ARCount,observationAbout,geoId/05,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +CACount,observationAbout,geoId/06,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +COCount,observationAbout,geoId/08,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +CTCount,observationAbout,geoId/09,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +DECount,observationAbout,geoId/10,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +DCCount,observationAbout,geoId/11,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +FLCount,observationAbout,geoId/12,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +GACount,observationAbout,geoId/13,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +HICount,observationAbout,geoId/15,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +IDCount,observationAbout,geoId/16,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ILCount,observationAbout,geoId/17,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +INCount,observationAbout,geoId/18,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +IACount,observationAbout,geoId/19,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +KSCount,observationAbout,geoId/20,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +KYCount,observationAbout,geoId/21,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +LACount,observationAbout,geoId/22,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MECount,observationAbout,geoId/23,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MDCount,observationAbout,geoId/24,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MACount,observationAbout,geoId/25,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MICount,observationAbout,geoId/26,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MNCount,observationAbout,geoId/27,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MSCount,observationAbout,geoId/28,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MOCount,observationAbout,geoId/29,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MTCount,observationAbout,geoId/30,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NECount,observationAbout,geoId/31,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NVCount,observationAbout,geoId/32,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NHCount,observationAbout,geoId/33,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NJCount,observationAbout,geoId/34,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NMCount,observationAbout,geoId/35,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NYCount,observationAbout,geoId/36,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NCCount,observationAbout,geoId/37,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NDCount,observationAbout,geoId/38,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +OHCount,observationAbout,geoId/39,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +OKCount,observationAbout,geoId/40,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ORCount,observationAbout,geoId/41,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +PACount,observationAbout,geoId/42,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +RICount,observationAbout,geoId/44,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +SCCount,observationAbout,geoId/45,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +SDCount,observationAbout,geoId/46,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +TNCount,observationAbout,geoId/47,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +TXCount,observationAbout,geoId/48,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +UTCount,observationAbout,geoId/49,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +VTCount,observationAbout,geoId/50,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +VACount,observationAbout,geoId/51,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WACount,observationAbout,geoId/53,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WVCount,observationAbout,geoId/54,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WICount,observationAbout,geoId/55,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WYCount,observationAbout,geoId/56,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +,,,,,,,,,,,,,,,, +#age314Count,Age,[3 14 Years],,,,,,,,,,,,,, +#age1524Count,Age,[15 24 Years],,,,,,,,,,,,,, +#age2544Count,Age,[25 44 Years],,,,,,,,,,,,,, +#age4564Count,Age,[45 64 Years],,,,,,,,,,,,,, +#age65pCount,Age,[65 - Years],,,,,,,,,,,,,, +#workEmployedCount,EmploymentStatus,Employed,,,,,,,,,,,,,, +#workUnemployedCount,EmploymentStatus,Unemployed,,,,,,,,,,,,,, +#workNILFCount,EmploymentStatus,NotInLaborForce,,,,,,,,,,,,,, +#isPerson,age,{Age},race,{Race},gender,{Gender},employmentStatus,{EmploymentStatus},,,,,,,, +#isHouseholder,housederAge,{Age},householderRace,{Race},householderGender,{Gender},householderEmploymentStatus,{EmploymentStatus},,,,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv new file mode 100644 index 0000000000..595050f0a2 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_metadata.csv @@ -0,0 +1,13 @@ +parameter,value +url,https://www.ntia.gov/other-publication/2022/digital-nation-data-explorer#sel=homeEverOnline&demo=scChldHome&pc=count&disp=both +description,National Telecommunication and Information administration Internet Use Survey +#place_type,"Country,AdministrativeArea1" +start_date,1994 +end_date,2023 +release_frequency,1years +process, +comments,"In the data sheet, moved universe column to the Left of description,places resolved manually in pvmap." +output_columns,"observationAbout, observationDate, value, variableMeasured" +#input_rows,25 +#input_columns,15 +header_rows,1 diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv new file mode 100644 index 0000000000..8797927dc2 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv @@ -0,0 +1,211 @@ +key,prop,value,,,,,,,,,,,,,, +isHouseholder,populationType,Household,,,institutionalization,USC_NonInstitutionalized,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +isPerson,populationType,Person,armedForcesStatus,Civilian,Age,[3 - Years],race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},, +isAdult,populationType,Person,Age,[15 - Years],race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, +Adult,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, +CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, +internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +,,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +Nov 1994,observationDate,1994-11,,,,,,,,,,,,,, +Oct 1997,observationDate,1997-10,,,,,,,,,,,,,, +Dec 1998,observationDate,1998-12,,,,,,,,,,,,,, +Aug 2000,observationDate,2000-08,,,,,,,,,,,,,, +Sep 2001,observationDate,2001-09,,,,,,,,,,,,,, +Oct 2003,observationDate,2003-10,,,,,,,,,,,,,, +Oct 2007,observationDate,2007-10,,,,,,,,,,,,,, +Oct 2009,observationDate,2009-10,,,,,,,,,,,,,, +Oct 2010,observationDate,2010-10,,,,,,,,,,,,,, +Jul 2011,observationDate,2011-07,,,,,,,,,,,,,, +Oct 2012,observationDate,2012-10,,,,,,,,,,,,,, +Jul 2013,observationDate,2013-07,,,,,,,,,,,,,, +Jul 2015,observationDate,2015-07,,,,,,,,,,,,,, +Nov 2017,observationDate,2017-11,,,,,,,,,,,,,, +Nov 2019,observationDate,2019-11,,,,,,,,,,,,,, +Nov 2021,observationDate,2021-11,,,,,,,,,,,,,, +Nov 2023,observationDate,2023-11,,,,,,,,,,,,,, +Nov 2024,observationDate,2024-11,,,,,,,,,,,,,, +Nov 2025,observationDate,2025-11,,,,,,,,,,,,,, +Nov 2026,observationDate,2026-11,,,,,,,,,,,,,, +Nov 2027,observationDate,2027-11,,,,,,,,,,,,,, +Nov 2028,observationDate,2028-11,,,,,,,,,,,,,, +Nov 2029,observationDate,2029-11,,,,,,,,,,,,,, +Nov 2030,observationDate,2030-11,,,,,,,,,,,,,, +Nov 2031,observationDate,2031-11,,,,,,,,,,,,,, +Nov 2032,observationDate,2032-11,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +Household Reference Person in Universe: Non-Institutional,DescriptionVar,Household Reference Person in Universe: Non-Institutional,,,institutionalization,USC_NonInstitutionalized,,,,,,,,,, +Person in Universe: Ages 3+ Not Active-Duty Military,DescriptionVar,"Person Aged 3 and above,Not Active-Duty Military",Age,[3 - Years],armedForcesStatus,Civilian,,,,,,,,,, +"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",DescriptionVar,"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",,,hasComputer,True,computerUsageLocation,Home,,,,,,,, +Person Ages 15+,DescriptionVar,Person Aged 15 and above,Age,[15 - Years],,,,,,,,,,,, +Anyone in Household Uses Internet From Any Location,#ignore,ignore,,,,,,,,,,,,,, +Anyone in Household Uses Internet at Home,#ignore,ignore,,,,,,,,,,,,,, +Uses the Internet (Any Location),DescriptionVar,Uses the Internet (Any Location),isInternetUser,True,internetUsageLocation,AnyLocation,,,,,,,,,, +Uses the Internet at Home,DescriptionVar,Uses the Internet at Home,isInternetUser,True,internetUsageLocation,Home,,,,,,,,,, +Uses the Internet at School,DescriptionVar,Uses the Internet at School,isInternetUser,True,internetUsageLocation,School,,,,,,,,,, +Uses the Internet at Work,DescriptionVar,Uses the Internet at Work,isInternetUser,True,internetUsageLocation,Work,,,,,,,,,, +"Uses the Internet at a Public Place Like the Library, Community Center, Park",DescriptionVar,"Uses the Internet at a Public Place Like the Library, Community Center, Park",isInternetUser,True,internetUsageLocation,PublicPlace,,,,,,,,,, +Household Used to Go Online from Home,DescriptionVar,Household Used to Go Online from Home,isInternetUser,True,internetUsageLocation,Home,,,,,,,,,, +No One in Household Uses Internet at Home,DescriptionVar,No One in Household Uses Internet at Home,residenceType,Household,isInternetUser,False,internetUsageLocation,Home,,,,,,,, +Dial-up Internet Service Used at Home,#ignore,ignore,,,,,,,,,,,,,, +Wired High-Speed Internet Service Used at Home,#ignore,ignore,,,,,,,,,,,,,, +Searches for a Job Online,DescriptionVar,Searches for a Job Online,isInternetUser,True,internetUsagePurpose,JobSearchUsage,,,,,,,,,, +"Shops, Makes Travel Reservations, or Uses Other Consumer Services Online",#ignore,Ignore,,,,,,,,,,,,,, +Takes Class or Participates in Job Training Online,DescriptionVar,Takes Class or Participates in Job Training Online,isInternetUser,True,internetUsagePurpose,OnlineClassUsage__JobTrainningUsage,,,,,,,,,, +"Uses Online Financial Services Like Banking, Investing, Paying Bills",DescriptionVar,"Uses Online Financial Services Like Banking, Investing, Paying Bills",isInternetUser,True,internetUsagePurpose,FinancialServicesUsage,,,,,,,,,, +Main Reason for Household Not Online at Home: Can Use Elsewhere,DescriptionVar,Main Reason for Household Not Online at Home: Can Use Elsewhere,isInternetUser,True,reasonNotUsingInternetAtHome,InternetUseElsewhere,,,,,,,,,, +Main Reason for Household Not Online at Home: Don't Need It or Not Interested,DescriptionVar,Main Reason for Household Not Online at Home: Don't Need It or Not Interested,isInternetUser,True,reasonNotUsingInternetAtHome,InternetNotNeeded__InternetNotInterested,,,,,,,,,, +Main Reason for Household Not Online at Home: No/Inadequate Computer,DescriptionVar,Main Reason for Household Not Online at Home: No/Inadequate Computer,isInternetUser,True,reasonNotUsingInternetAtHome,InternetInadequateComputer,,,,,,,,,, +Main Reason for Household Not Online at Home: Too Expensive,DescriptionVar,Main Reason for Household Not Online at Home: Too Expensive,isInternetUser,True,reasonNotUsingInternetAtHome,InternetExpensive,,,,,,,,,, +Satellite Internet Service Used at Home,DescriptionVar,Satellite Internet Service Used at Home,isInternetUser,True,internetSubscriptionType,SatelliteInternetService,,,,,,,,,, +Main Reason for Household Not Online at Home: Privacy or Security Concerns,DescriptionVar,Main Reason for Household Not Online at Home: Privacy or Security Concerns,isInternetUser,True,reasonNotUsingInternetAtHome,InternetPrivacyConcerns__InternetSecurityConcerns,,,,,,,,,, +Main Reason for Household Not Online at Home: Not Available in Area,DescriptionVar,Main Reason for Household Not Online at Home: Not Available in Area,isInternetUser,True,reasonNotUsingInternetAtHome,InternetUnavailableInArea,,,,,,,,,, +Mobile Internet Service Used at Home,DescriptionVar,Mobile Internet Service Used at Home,isInternetUser,True,internetSubscriptionType,BroadbandOfAnyType,,,,,,,,,, +Affordability is Most Important Factor for Home Internet Service,DescriptionVar,Affordability is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetAffordability,,,,,,,, +Customer Service is Most Important Factor for Home Internet Service,DescriptionVar,Customer Service is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetCustomerService,,,,,,,, +Home Internet is Part of a Bundle,DescriptionVar,Home Internet is Part of a Bundle,internetUsageLocation,Home,internetSubscriptionType,InternetBundle,,,,,,,,,, +Mobility is Most Important Factor for Home Internet Service,DescriptionVar,Mobility is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetMobility,,,,,,,, +Reliability is Most Important Factor for Home Internet Service,DescriptionVar,Reliability is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetReliability,,,,,,,, +Service Speed is Most Important Factor for Home Internet Service,DescriptionVar,Service Speed is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetServiceSpeed,,,,,,,, +Uses a Desktop Computer,DescriptionVar,Uses a Desktop Computer,electronicGadgetUsed,DesktopUser,,,,,,,,,,,, +"Uses a Desktop, Laptop, or Tablet Computer",DescriptionVar,"Uses a Desktop, Laptop, or Tablet Computer",electronicGadgetUsed,DesktopUser__LaptopUser__TabletUser,,,,,,,,,,,, +Uses a Laptop Computer,DescriptionVar,Uses a Laptop Computer,electronicGadgetUsed,LaptopUser,,,,,,,,,,,, +Uses a Mobile Phone,DescriptionVar,Uses a Mobile Phone,electronicGadgetUsed,MobileUser,,,,,,,,,,,, +Uses a Smart TV or Connected Device,DescriptionVar,Uses a Smart TV or Connected Device,electronicGadgetUsed,SmartTelevisionUser,,,,,,,,,,,, +Uses a Tablet or e-Book Reader,DescriptionVar,Uses a Tablet or e-Book Reader,electronicGadgetUsed,TabletUser__EbookReaderUser,,,,,,,,,,,, +Home Phone Service Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +TV Channels Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +"Access Electronic Health Records or Insurance Records, or Talks w Doctor Online",#ignore,Ignore,,,,,,,,,,,,,, +Researches Health Information Online,DescriptionVar,Researches Health Information Online,isInternetUser,True,internetUsagePurpose,HealthDataResearchUsage,,,,,,,,,, +Telecommutes Using the Internet,DescriptionVar,Telecommutes Using the Internet,isInternetUser,True,internetUsagePurpose,TelecommutesUsage,,,,,,,,,, +Uses Email,DescriptionVar,Uses Email,isInternetUser,True,internetUsagePurpose,EmailUsage,,,,,,,,,, +Uses Online Location-Based (On-the-Go) Services,#ignore,Ignore,,,,,,,,,,,,,, +Uses Text Messaging or Instant Messaging,DescriptionVar,Uses Text Messaging or Instant Messaging,isInternetUser,True,internetUsagePurpose,TextMessagingUsage,,,,,,,,,, +Watches Videos Online,DescriptionVar,Watches Videos Online,isInternetUser,True,internetUsagePurpose,WatchingVideosUsage,,,,,,,,,, +Data Cap is Most Important Factor for Home Internet Service,DescriptionVar,Data Cap is Most Important Factor for Home Internet Service,isInternetUser,True,internetUsageLocation,Home,factorForInternetService,FactorForHomeInternetDataLimit,,,,,,,, +Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, +Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, +Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, +Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, +Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, +Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, +Mobile Internet Service Used Outside the Home,DescriptionVar,Mobile Internet Service Used Outside the Home,isInternetUser,True,internetSubscriptionType,CellularDataPlan,internetUsageLocation,OutsideHome,,,,,,,, +Internet Service Provided for Entire Building/Campus/Community at Home,#ignore,Ignore,,,,,,,,,,,,,, +Publicly-Available Free Internet Service at Home,DescriptionVar,Publicly-Available Free Internet Service at Home,isInternetUser,True,internetSubscriptionType,FreeInternet,internetUsageLocation,Home,,,,,,,, +Subscription to Private ISP Service at Home,#ignore,Ignore,,,,,,,,,,,,,, +"Subscription to Public, Nonprofit, or Coop ISP Service at Home",#ignore,Ignore,,,,,,,,,,,,,, +Uses a Wearable Device,DescriptionVar,Uses a Wearable Device,electronicGadgetUsed,WearableDeviceUser,,,,,,,,,,,, +Uses the Internet While Traveling Between Places,DescriptionVar,Uses the Internet While Traveling Between Places,isInternetUser,True,internetUsageLocation,Traveling,,,,,,,,,, +Uses the Internet at a Coffee Shop or Other Business,DescriptionVar,Uses the Internet at a Coffee Shop or Other Business,isInternetUser,True,internetUsageLocation,Cafeteria,,,,,,,,,, +Home Security or Monitoring Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +Mobile Phone Service Bundled with Internet at Home,#ignore,Ignore,,,,,,,,,,,,,, +Offers Services for Sale via the Internet,DescriptionVar,Offers Services for Sale via the Internet,isInternetUser,True,internetUsagePurpose,ServiceForSaleUsage,,,,,,,,,, +"Posts or Uploads Blog Posts, Videos, or Other Original Content",DescriptionVar,"Posts or Uploads Blog Posts, Videos, or Other Original Content",isInternetUser,True,internetUsagePurpose,UploadMediaUsage,,,,,,,,,, +Requests Services Provided by Other People via the Internet,#ignore,Ignore,,,,,,,,,,,,,, +Uses the Internet to Sell Goods,DescriptionVar,Uses the Internet to Sell Goods,isInternetUser,True,internetUsagePurpose,GoodsSellingUsage,,,,,,,,,, +Household Subscribes to Cable or Satellite TV Service,DescriptionVar,Household Subscribes to Cable or Satellite TV Service,internetSubscriptionType,BroadbandInternetSuchAsCableFiberOpticOrDsl__SatelliteInternetService,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +,,,,,,,,,,,,,,,, +usCount,observationAbout,country/USA,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +age314Count,observationAbout,country/USA,measuredProperty,count,Age,[3 14 Years],value,{Number},name,"""{DescriptionVar}, age between 3 and 14 years""",,,,,, +age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Years],value,{Number},name,"""{DescriptionVar}, age between 15 and 24 years""",,,,,, +age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, +age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, +age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, +income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, +income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, +income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, +income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, +sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, +raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, +raceBlackCount,observationAbout,country/USA,measuredProperty,count,Race,BlackOrAfricanAmericanAlone,value,{Number},name,"""{DescriptionVar}, African American""",,,,,, +raceHispanicCount,observationAbout,country/USA,measuredProperty,count,ethnicity,HispanicOrLatino,value,{Number},name,"""{DescriptionVar}, Hispanic""",,,,,, +raceAsianCount,observationAbout,country/USA,measuredProperty,count,Race,Asian,value,{Number},name,"""{DescriptionVar}, Asian American""",,,,,, +raceAmIndianCount,observationAbout,country/USA,measuredProperty,count,Race,AmericanIndianOrAlaskaNative,value,{Number},name,"""{DescriptionVar}, American Indian or Alaska Native""",,,,,, +raceOtherCount,observationAbout,country/USA,measuredProperty,count,Race,NTIA_OtherRace,ethnicity,NotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, Other non-Hispanic""",,,, +disabilityNoCount,observationAbout,country/USA,measuredProperty,count,disabilityStatus,NoDisability,value,{Number},name,"""{DescriptionVar}, without disability""",,,,,, +disabilityYesCount,observationAbout,country/USA,measuredProperty,count,disabilityStatus,WithDisability,value,{Number},name,"""{DescriptionVar}, with disability""",,,,,, +metroNoCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,Rural,value,{Number},name,"""{DescriptionVar}, Rural""",,,,,, +metroYesCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,Urban,value,{Number},name,"""{DescriptionVar}, Urban""",,,,,, +metroUnknownCount,observationAbout,country/USA,measuredProperty,count,placeOfResidenceClassification,UnknownPlaceOfResidence,value,{Number},name,"""{DescriptionVar}, Unknown place of residence""",,,,,, +scChldHomeNoCount,observationAbout,country/USA,measuredProperty,count,schoolEnrollment,NotEnrolledInSchool,value,{Number},name,"""{DescriptionVar}, No School-Aged Child at Home""",,,,,, +scChldHomeYesCount,observationAbout,country/USA,measuredProperty,count,schoolEnrollment,EnrolledInSchool,value,{Number},name,"""{DescriptionVar}, School-Aged Child at Home""",,,,,, +veteranNoCount,observationAbout,country/USA,measuredProperty,count,veteranStatus,Nonveteran,value,{Number},name,"""{DescriptionVar}, Not Veteran""",,,,,, +veteranYesCount,observationAbout,country/USA,measuredProperty,count,veteranStatus,Veteran,value,{Number},name,"""{DescriptionVar}, Veteran""",,,,,, +ALCount,observationAbout,geoId/01,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +AKCount,observationAbout,geoId/02,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +AZCount,observationAbout,geoId/04,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ARCount,observationAbout,geoId/05,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +CACount,observationAbout,geoId/06,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +COCount,observationAbout,geoId/08,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +CTCount,observationAbout,geoId/09,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +DECount,observationAbout,geoId/10,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +DCCount,observationAbout,geoId/11,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +FLCount,observationAbout,geoId/12,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +GACount,observationAbout,geoId/13,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +HICount,observationAbout,geoId/15,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +IDCount,observationAbout,geoId/16,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ILCount,observationAbout,geoId/17,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +INCount,observationAbout,geoId/18,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +IACount,observationAbout,geoId/19,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +KSCount,observationAbout,geoId/20,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +KYCount,observationAbout,geoId/21,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +LACount,observationAbout,geoId/22,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MECount,observationAbout,geoId/23,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MDCount,observationAbout,geoId/24,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MACount,observationAbout,geoId/25,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MICount,observationAbout,geoId/26,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MNCount,observationAbout,geoId/27,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MSCount,observationAbout,geoId/28,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MOCount,observationAbout,geoId/29,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +MTCount,observationAbout,geoId/30,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NECount,observationAbout,geoId/31,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NVCount,observationAbout,geoId/32,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NHCount,observationAbout,geoId/33,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NJCount,observationAbout,geoId/34,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NMCount,observationAbout,geoId/35,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NYCount,observationAbout,geoId/36,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NCCount,observationAbout,geoId/37,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +NDCount,observationAbout,geoId/38,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +OHCount,observationAbout,geoId/39,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +OKCount,observationAbout,geoId/40,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +ORCount,observationAbout,geoId/41,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +PACount,observationAbout,geoId/42,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +RICount,observationAbout,geoId/44,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +SCCount,observationAbout,geoId/45,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +SDCount,observationAbout,geoId/46,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +TNCount,observationAbout,geoId/47,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +TXCount,observationAbout,geoId/48,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +UTCount,observationAbout,geoId/49,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +VTCount,observationAbout,geoId/50,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +VACount,observationAbout,geoId/51,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WACount,observationAbout,geoId/53,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WVCount,observationAbout,geoId/54,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WICount,observationAbout,geoId/55,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +WYCount,observationAbout,geoId/56,measuredProperty,count,value,{Number},name,"""{DescriptionVar}""",,,,,,,, +,,,,,,,,,,,,,,,, +#age314Count,Age,[3 14 Years],,,,,,,,,,,,,, +#age1524Count,Age,[15 24 Years],,,,,,,,,,,,,, +#age2544Count,Age,[25 44 Years],,,,,,,,,,,,,, +#age4564Count,Age,[45 64 Years],,,,,,,,,,,,,, +#age65pCount,Age,[65 - Years],,,,,,,,,,,,,, +#workEmployedCount,EmploymentStatus,Employed,,,,,,,,,,,,,, +#workUnemployedCount,EmploymentStatus,Unemployed,,,,,,,,,,,,,, +#workNILFCount,EmploymentStatus,NotInLaborForce,,,,,,,,,,,,,, +#isPerson,age,{Age},race,{Race},gender,{Gender},employmentStatus,{EmploymentStatus},,,,,,,, +#isHouseholder,housederAge,{Age},householderRace,{Race},householderGender,{Gender},householderEmploymentStatus,{EmploymentStatus},,,,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py new file mode 100644 index 0000000000..9bcf786a14 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -0,0 +1,94 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, sys +import pandas as pd +from absl import app, logging +from pathlib import Path +import config + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +sys.path.append(os.path.join(script_dir, '../../../util')) + +from download_util_script import download_file + +Commerce_NTIA_URL = config.Commerce_NTIA_URL + +INPUT_DIR = os.path.join(script_dir, "input_files") +Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) + + +COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] +AGE_COLUMNS = ["age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount"] +INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") +INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") +INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") + + +def move_column_left(df, column_to_move, target_column): + """Moves the universe column to the left of variable column.""" + cols = df.columns.tolist() + if column_to_move in cols and target_column in cols: + cols.remove(column_to_move) + target_index = cols.index(target_column) + new_cols = cols[:target_index] + [column_to_move] + cols[target_index:] + return df[new_cols] + return df + + +def preprocess_data(): + try: + org_df = pd.read_csv(INPUT_FILE) + + df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() + df1['universeAgeResol'] = df1['universe'].apply( + lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) + ) + df1['variableAgeResol'] = df1['variable'].apply( + lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) + ) + df1_moved = move_column_left(df1, 'universe', 'variable') + df1_moved.to_csv(INPUT_FILE_1, index=False) + + df2_cols_to_keep = [col for col in org_df.columns if not col.startswith('age')] + df2 = org_df[df2_cols_to_keep].copy() + df2['universeAgeResol'] = df2['universe'].apply( + lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) + ) + df2['variableAgeResol'] = df2['variable'].apply( + lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) + ) + df2_moved = move_column_left(df2, 'universe', 'variable') + df2_moved.to_csv(INPUT_FILE_2, index=False) + + except Exception as e: + logging.fatal(f"An error occurred while preprocessing the input data: {e}") + return None + +def main(argv): + try: + download_file(url=Commerce_NTIA_URL, + output_folder=INPUT_DIR, + unzip=False, + headers= None, + tries= 3, + delay= 5, + backoff= 2) + except Exception as e: + logging.fatal(f"Failed to download Commerce_NTIA file: {e}") + preprocess_data() + +if __name__ == "__main__": + app.run(main) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data-age-only.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data-age-only.csv new file mode 100644 index 0000000000..7448eab8cb --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data-age-only.csv @@ -0,0 +1,6 @@ +dataset,variable,description,universe,age314Count,age1524Count,age2544Count,age4564Count,age65pCount,universeAgeResol,variableAgeResol +Nov 1994,isHouseholder,Household Reference Person in Universe: Non-Institutional,,,5573616,43123860,29830455,21180087,, +Nov 1994,computerAtHome,"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",isHouseholder,,1021876,13039306,8525086,1622202,, +Oct 1997,computerAtHome,"Anyone in Household Uses a Desktop, Laptop, or Tablet at Home",isHouseholder,,1594328,19680204,13716027,2789026,, +Dec 1998,isPerson,Person in Universe: Ages 3+ Not Active-Duty Military,,48287009,37477648,82833709,57582133,32272407,,CivilPerson +Dec 1998,workInternetUser,Uses the Internet at Work,isAdult,,1798778,14741701,7969110,271065,Adult, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data.csv new file mode 100644 index 0000000000..e75281e5eb --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia-data.csv @@ -0,0 +1,8 @@ +dataset,variable,description,universe,usProp,usPropSE,usCount,usCountSE,workEmployedProp,workEmployedPropSE,workEmployedCount,workEmployedCountSE,workUnemployedProp,workUnemployedPropSE,workUnemployedCount,workUnemployedCountSE,workNILFProp,workNILFPropSE,workNILFCount,workNILFCountSE,incomeU25Prop,incomeU25PropSE,incomeU25Count,incomeU25CountSE,income2549Prop,income2549PropSE,income2549Count,income2549CountSE,income5074Prop,income5074PropSE,income5074Count,income5074CountSE,income7599Prop,income7599PropSE,income7599Count,income7599CountSE,income100pProp,income100pPropSE,income100pCount,income100pCountSE,edNoDiplomaProp,edNoDiplomaPropSE,edNoDiplomaCount,edNoDiplomaCountSE,edHSGradProp,edHSGradPropSE,edHSGradCount,edHSGradCountSE,edSomeCollegeProp,edSomeCollegePropSE,edSomeCollegeCount,edSomeCollegeCountSE,edCollegeGradProp,edCollegeGradPropSE,edCollegeGradCount,edCollegeGradCountSE,sexMaleProp,sexMalePropSE,sexMaleCount,sexMaleCountSE,sexFemaleProp,sexFemalePropSE,sexFemaleCount,sexFemaleCountSE,raceWhiteProp,raceWhitePropSE,raceWhiteCount,raceWhiteCountSE,raceBlackProp,raceBlackPropSE,raceBlackCount,raceBlackCountSE,raceHispanicProp,raceHispanicPropSE,raceHispanicCount,raceHispanicCountSE,raceAsianProp,raceAsianPropSE,raceAsianCount,raceAsianCountSE,raceAmIndianProp,raceAmIndianPropSE,raceAmIndianCount,raceAmIndianCountSE,raceOtherProp,raceOtherPropSE,raceOtherCount,raceOtherCountSE,disabilityNoProp,disabilityNoPropSE,disabilityNoCount,disabilityNoCountSE,disabilityYesProp,disabilityYesPropSE,disabilityYesCount,disabilityYesCountSE,metroNoProp,metroNoPropSE,metroNoCount,metroNoCountSE,metroYesProp,metroYesPropSE,metroYesCount,metroYesCountSE,metroUnknownProp,metroUnknownPropSE,metroUnknownCount,metroUnknownCountSE,scChldHomeNoProp,scChldHomeNoPropSE,scChldHomeNoCount,scChldHomeNoCountSE,scChldHomeYesProp,scChldHomeYesPropSE,scChldHomeYesCount,scChldHomeYesCountSE,veteranNoProp,veteranNoPropSE,veteranNoCount,veteranNoCountSE,veteranYesProp,veteranYesPropSE,veteranYesCount,veteranYesCountSE,ALProp,ALPropSE,ALCount,ALCountSE,AKProp,AKPropSE,AKCount,AKCountSE,AZProp,AZPropSE,AZCount,AZCountSE,ARProp,ARPropSE,ARCount,ARCountSE,CAProp,CAPropSE,CACount,CACountSE,COProp,COPropSE,COCount,COCountSE,CTProp,CTPropSE,CTCount,CTCountSE,DEProp,DEPropSE,DECount,DECountSE,DCProp,DCPropSE,DCCount,DCCountSE,FLProp,FLPropSE,FLCount,FLCountSE,GAProp,GAPropSE,GACount,GACountSE,HIProp,HIPropSE,HICount,HICountSE,IDProp,IDPropSE,IDCount,IDCountSE,ILProp,ILPropSE,ILCount,ILCountSE,INProp,INPropSE,INCount,INCountSE,IAProp,IAPropSE,IACount,IACountSE,KSProp,KSPropSE,KSCount,KSCountSE,KYProp,KYPropSE,KYCount,KYCountSE,LAProp,LAPropSE,LACount,LACountSE,MEProp,MEPropSE,MECount,MECountSE,MDProp,MDPropSE,MDCount,MDCountSE,MAProp,MAPropSE,MACount,MACountSE,MIProp,MIPropSE,MICount,MICountSE,MNProp,MNPropSE,MNCount,MNCountSE,MSProp,MSPropSE,MSCount,MSCountSE,MOProp,MOPropSE,MOCount,MOCountSE,MTProp,MTPropSE,MTCount,MTCountSE,NEProp,NEPropSE,NECount,NECountSE,NVProp,NVPropSE,NVCount,NVCountSE,NHProp,NHPropSE,NHCount,NHCountSE,NJProp,NJPropSE,NJCount,NJCountSE,NMProp,NMPropSE,NMCount,NMCountSE,NYProp,NYPropSE,NYCount,NYCountSE,NCProp,NCPropSE,NCCount,NCCountSE,NDProp,NDPropSE,NDCount,NDCountSE,OHProp,OHPropSE,OHCount,OHCountSE,OKProp,OKPropSE,OKCount,OKCountSE,ORProp,ORPropSE,ORCount,ORCountSE,PAProp,PAPropSE,PACount,PACountSE,RIProp,RIPropSE,RICount,RICountSE,SCProp,SCPropSE,SCCount,SCCountSE,SDProp,SDPropSE,SDCount,SDCountSE,TNProp,TNPropSE,TNCount,TNCountSE,TXProp,TXPropSE,TXCount,TXCountSE,UTProp,UTPropSE,UTCount,UTCountSE,VTProp,VTPropSE,VTCount,VTCountSE,VAProp,VAPropSE,VACount,VACountSE,WAProp,WAPropSE,WACount,WACountSE,WVProp,WVPropSE,WVCount,WVCountSE,WIProp,WIPropSE,WICount,WICountSE,WYProp,WYPropSE,WYCount,WYCountSE,universeAgeResol,variableAgeResol +Nov 1994,isHouseholder,Household Reference Person in Universe: Non-Institutional,,1,0,99708018,233531,1,0,65545071,281788,1,0,2939228,84396,1,0,30486580,227893,,,,,,,,,,,,,,,,,,,,,1,0,19363957,198634,1,0,32022931,238770,1,0,25134148,220706,1,0,23186983,211735,1,0,62988898,279361,1,0,36719120,247997,1,0,77096018,262855,1,0,11820843,169807,1,0,7956251,144704,1,0,2027535,61687,1,0,544756,32223,1,0,262614,21960,,,,,,,,,1,0,22028761,205377,1,0,76906846,276277,1,0,772412,33224,1,0,57413314,274817,1,0,42294704,261268,1,0,77980419,272126,1,0,20947215,201077,1,0,1668675,64610,1,0,208595,8703,1,0,1610328,66338,1,0,952203,38109,1,0,11432345,183904,1,0,1435110,54865,1,0,1285592,59907,1,0,274567,12469,1,0,267663,11741,1,0,5657096,114019,1,0,2673673,93007,1,0,408744,21611,1,0,411766,16444,1,0,4430662,94123,1,0,2158646,97176,1,0,1102615,43842,1,0,980574,38385,1,0,1532080,62608,1,0,1571716,65019,1,0,473056,20971,1,0,1992430,87926,1,0,2371069,50359,1,0,3693788,76573,1,0,1809317,72880,1,0,960381,37944,1,0,1979502,86630,1,0,338956,12623,1,0,643136,25782,1,0,583798,25198,1,0,427922,21431,1,0,2967575,61496,1,0,615782,24278,1,0,7191198,117114,1,0,2697423,57899,1,0,244686,9299,1,0,4326601,86790,1,0,1272568,47202,1,0,1190960,49935,1,0,4666061,91618,1,0,401432,17902,1,0,1373450,59692,1,0,271275,10676,1,0,2072162,81868,1,0,6784229,140299,1,0,605439,25452,1,0,226026,10819,1,0,2411191,81512,1,0,2201373,88238,1,0,752405,29310,1,0,1916700,70578,1,0,183476,7697,, +Nov 1994,isPerson,Person in Universe: Ages 3+ Not Active-Duty Military,,1,0,248509799,723058,1,0,125490400,504535,1,0,7179758,143172,1,0,68668835,393838,,,,,,,,,,,,,,,,,,,,,1,0,46369423,395315,1,0,65052491,394637,1,0,50470784,357177,1,0,39446296,326841,1,0,120802232,483934,1,0,127707566,450756,1,0,184215059,701920,1,0,30738364,486256,1,0,25209523,470567,1,0,6143270,193764,1,0,1378818,72272,1,0,824764,60676,,,,,,,,,1,0,55674976,392390,1,0,190903150,655830,1,0,1931673,45446,1,0,109877267,576760,1,0,138632531,929922,1,0,167584914,486431,1,0,26413736,227527,1,0,4073843,87296,1,0,549019,14384,1,0,3838642,88963,1,0,2349494,54603,1,0,29833154,292045,1,0,3478339,71427,1,0,3121902,81282,1,0,668702,16783,1,0,546349,15437,1,0,13402421,157053,1,0,6763053,164422,1,0,1044237,37069,1,0,1133715,28467,1,0,11226180,146314,1,0,5591779,153418,1,0,2588227,56586,1,0,2345988,53414,1,0,3713636,81949,1,0,3945503,91966,1,0,1248847,32678,1,0,4614652,112726,1,0,5668208,70660,1,0,9044709,109105,1,0,4314360,98722,1,0,2613567,66232,1,0,5008637,127530,1,0,824168,17370,1,0,1494363,33579,1,0,1405781,36197,1,0,1129251,30839,1,0,7531215,90506,1,0,1604886,38339,1,0,17582458,183641,1,0,6684073,88704,1,0,593901,13870,1,0,10715484,119904,1,0,3173878,67894,1,0,2914641,65976,1,0,11367737,130852,1,0,923103,24058,1,0,3586678,95533,1,0,667201,14932,1,0,4895851,104056,1,0,17789235,234245,1,0,1791096,44620,1,0,560296,15154,1,0,6238561,155582,1,0,5103906,118284,1,0,1770883,45537,1,0,4984995,107785,1,0,448994,10524,,CivilPerson +Oct 1997,isPerson,Person in Universe: Ages 3+ Not Active-Duty Military,,1,0,255688666,729457,1,0,131223454,517307,1,0,6216600,142124,1,0,70287866,414018,,,,,,,,,,,,,,,,,,,,,1,0,45498078,401484,1,0,66160685,416486,1,0,52443686,378081,1,0,43625471,355669,1,0,124589570,496001,1,0,131099096,466445,1,0,184295047,714273,1,0,31786358,519450,1,0,28233368,493380,1,0,9225313,281116,1,0,2148580,112520,,,,,,,,,,,,,1,0,49531801,360347,1,0,205602786,683493,1,0,554079,38408,1,0,139844631,597346,1,0,115844035,990991,1,0,174815907,492270,1,0,25000578,232948,1,0,4173177,90500,1,0,575943,14322,1,0,4459177,96664,1,0,2356740,50475,1,0,30876689,306021,1,0,3790222,79104,1,0,3106854,79762,1,0,710302,18834,1,0,492315,14384,1,0,13836974,163158,1,0,7198558,132181,1,0,1106079,29966,1,0,1185423,28335,1,0,11237775,146657,1,0,5666981,117206,1,0,2694457,57647,1,0,2463028,53886,1,0,3754725,77921,1,0,4127461,98831,1,0,1199227,27553,1,0,4591025,104034,1,0,5920355,97721,1,0,9183915,131591,1,0,4357149,91743,1,0,2563708,60606,1,0,5095040,118117,1,0,858799,17814,1,0,1601636,35053,1,0,1626033,41472,1,0,1138643,30867,1,0,7635428,115102,1,0,1761451,39370,1,0,17589762,191271,1,0,6939885,110061,1,0,609288,13483,1,0,10932484,161295,1,0,3128742,63307,1,0,3240172,76136,1,0,11513445,149038,1,0,908536,22816,1,0,3580164,79819,1,0,700970,16486,1,0,5108403,107613,1,0,18602147,218517,1,0,1905461,45925,1,0,569660,14074,1,0,6470097,126814,1,0,5359857,117741,1,0,1729260,35740,1,0,4988102,106477,1,0,466942,10367,,CivilPerson +Dec 1998,isHouseholder,Household Reference Person in Universe: Non-Institutional,,1,0,104174641,231608,1,0,69116853,295641,1,0,2336082,79846,1,0,32206410,250072,,,,,,,,,,,,,,,,,,,,,1,0,18080409,205826,1,0,32580534,254768,1,0,26989220,240077,1,0,26524478,236425,1,0,60138023,293452,1,0,44036618,278855,1,0,78791846,272885,1,0,12447727,187806,1,0,8964828,158842,1,0,3181099,91879,1,0,789142,42036,,,,,,,,,,,,,1,0,20263377,209021,1,0,83651922,284621,1,0,259342,24917,1,0,75199428,290030,1,0,28975213,245399,1,0,85304356,279421,1,0,18290905,201181,1,0,1688438,66738,1,0,222611,9780,1,0,1764439,65217,1,0,1030387,39938,1,0,11814611,193347,1,0,1572832,58073,1,0,1254539,56440,1,0,292247,13049,1,0,235068,10324,1,0,6047390,121236,1,0,2951817,104465,1,0,417473,20419,1,0,445591,17556,1,0,4508956,104327,1,0,2337591,91746,1,0,1161753,45110,1,0,1057912,42660,1,0,1600328,61807,1,0,1607131,64690,1,0,519433,21926,1,0,2108180,92247,1,0,2406815,69403,1,0,3794831,94510,1,0,1865172,70198,1,0,1087393,46446,1,0,2202645,92880,1,0,353818,13397,1,0,658708,26135,1,0,667406,26903,1,0,453179,19858,1,0,2936614,77473,1,0,652882,25760,1,0,6986513,124487,1,0,2956634,82619,1,0,248322,9705,1,0,4491630,108419,1,0,1308497,48194,1,0,1313233,53563,1,0,4595305,101706,1,0,374914,16230,1,0,1493081,63667,1,0,280994,11611,1,0,2230004,89837,1,0,7339846,151712,1,0,677519,28370,1,0,240669,10628,1,0,2687098,98186,1,0,2252857,89391,1,0,751793,28282,1,0,2042075,78460,1,0,185468,7378,, +Dec 1998,workInternetUser,Uses the Internet at Work,isAdult,0.11791,0.001281,24780653,268255,0.1856,0.00194,24780653,268255,0,0,0,0,0,0,0,0,,,,,,,,,,,,,,,,,,,,,0.008087,0.000693,372491,32028,0.054561,0.001549,3578606,104056,0.124225,0.002456,6597272,137957,0.313444,0.003808,14232285,207891,0.133186,0.001826,13490849,186650,0.103697,0.001539,11289804,168956,0.132578,0.001534,20470472,244648,0.080422,0.003458,1966908,88570,0.052187,0.00292,1144634,65352,0.137306,0.007512,1061533,64071,0.083557,0.011229,137105,19401,,,,,,,,,,,,,0.078538,0.002335,3173660,95613,0.127444,0.001491,21570762,251208,0.072575,0.018963,36231,9539,0.124355,0.001657,16970261,232739,0.105977,0.002019,7810392,159739,0.123719,0.001414,21968264,252856,0.112954,0.003359,2792788,87561,0.099873,0.009364,349039,32555,0.170989,0.012674,75473,5557,0.122169,0.009325,436936,33342,0.094781,0.008854,187765,17790,0.121053,0.004303,3043617,106415,0.164759,0.010815,510768,33589,0.124641,0.011746,319945,29800,0.120537,0.011197,70955,6484,0.25092,0.017831,102762,7492,0.093753,0.004699,1111718,55721,0.1183,0.00898,697995,52745,0.093667,0.011012,84340,9843,0.139088,0.010107,131142,9493,0.129559,0.006078,1187547,55198,0.104859,0.009797,482196,45238,0.105577,0.009944,233550,22277,0.118522,0.010181,238525,20387,0.095499,0.008945,299719,27909,0.079544,0.008597,268391,29038,0.089377,0.009286,89501,9371,0.21142,0.014791,848997,60523,0.118996,0.007324,577696,35136,0.109811,0.006033,844212,46737,0.163749,0.010878,597058,39778,0.058401,0.008498,123605,18023,0.117841,0.010678,489916,44137,0.131687,0.01044,91610,7415,0.122641,0.010866,156251,13834,0.073562,0.008113,101010,10985,0.135389,0.011375,127252,11004,0.117721,0.00661,750143,41692,0.138913,0.010317,187146,13870,0.095447,0.00419,1355945,58740,0.089461,0.00633,521969,36676,0.131652,0.01077,64727,5360,0.114993,0.006186,1005934,54848,0.108652,0.008922,277774,22669,0.144566,0.011903,380979,31441,0.106515,0.005211,1006897,49827,0.115325,0.010499,88282,8079,0.08639,0.009353,258525,27882,0.099411,0.009298,56418,5293,0.109823,0.009767,477348,42485,0.12245,0.005373,1842767,79716,0.155555,0.011502,240014,17473,0.141192,0.011787,66293,5530,0.157899,0.010258,831601,54404,0.172117,0.01202,770823,53517,0.077264,0.008066,114086,12002,0.137511,0.010315,553342,41120,0.134076,0.010379,50146,3942,Adult, +Dec 1998,homeEverOnline,Household Used to Go Online from Home,isHouseholder,0.039539,0.001005,4118937,105238,0.050399,0.001386,3483396,97048,0.051021,0.007714,119190,18506,0.015288,0.001135,492385,36774,,,,,,,,,,,,,,,,,,,,,0.01144,0.001317,206835,23939,0.03485,0.00169,1135431,55835,0.048799,0.002193,1317057,60433,0.055029,0.002322,1459614,63065,0.039778,0.001318,2392166,80199,0.039212,0.001553,1726771,69391,0.04202,0.00117,3310812,93031,0.026036,0.002546,324089,32088,0.032409,0.003327,290539,30320,0.047263,0.006669,150347,21799,0.05468,0.013242,43151,10798,,,,,,,,,,,,,0.028991,0.001901,587461,39000,0.042094,0.001162,3521220,98086,0.039548,0.017404,10257,4589,0.037759,0.001159,2839447,87988,0.044158,0.002,1279490,59029,0.041329,0.001136,3525556,97739,0.031131,0.002123,569414,39401,0.031083,0.006896,52481,11825,0.046634,0.009325,10381,2126,0.041822,0.007573,73793,13656,0.019874,0.005518,20478,5743,0.047503,0.003704,561232,44821,0.063595,0.009106,100025,14796,0.034356,0.008237,43101,10515,0.039686,0.00898,11598,2680,0.059891,0.010831,14078,2631,0.038731,0.003995,234221,24625,0.054958,0.008347,162227,25373,0.030751,0.008213,12838,3478,0.069796,0.010237,31101,4732,0.03506,0.004449,158083,20426,0.033001,0.007197,77144,17117,0.031075,0.007048,36101,8327,0.046472,0.008243,49163,8910,0.041797,0.007939,66889,12988,0.033815,0.007219,54346,11794,0.039317,0.008307,20423,4403,0.03529,0.008268,74398,17754,0.038848,0.005637,93500,13831,0.047366,0.005378,179746,20892,0.042176,0.007905,78666,15086,0.021181,0.005859,23033,6430,0.038417,0.008142,84618,18284,0.041891,0.007672,14822,2773,0.027939,0.006722,18404,4493,0.036079,0.007494,24079,5092,0.035704,0.008322,16180,3842,0.022965,0.004048,67439,12025,0.047757,0.008361,31180,5588,0.039552,0.003646,276334,25983,0.034272,0.005174,101329,15561,0.03668,0.007598,9109,1924,0.037207,0.00467,167120,21369,0.049244,0.008077,64435,10839,0.06098,0.009698,80081,13126,0.028756,0.003773,132142,17584,0.031462,0.007611,11795,2899,0.019263,0.006114,28762,9224,0.022341,0.006131,6278,1742,0.03411,0.00729,76066,16532,0.039105,0.004069,287027,30432,0.046697,0.008866,31638,6150,0.027706,0.007391,6668,1804,0.03617,0.006808,97192,18621,0.066373,0.009964,149529,23226,0.024679,0.005777,18553,4395,0.03462,0.007013,70698,14567,0.045362,0.008571,8413,1629,, +Sep 2001,internetAtHome,Anyone in Household Uses Internet at Home,internetAtHome,1,0,55003519,182527,1,0,42303419,208990,1,0,1406983,59787,1,0,10913805,152504,,,,,,,,,,,,,,,,,,,,,1,0,3141856,89138,1,0,13277139,163297,1,0,16968871,180999,1,0,21615654,195419,1,0,32114406,212096,1,0,22889113,198229,1,0,44559845,196107,1,0,4014824,104663,1,0,3369169,97882,1,0,2713792,90869,1,0,345890,29990,,,,,,,,,,,,,1,0,8504745,129630,1,0,46358764,210212,1,0,140010,13803,1,0,36095090,214705,1,0,18908429,185839,1,0,46365266,204556,1,0,8213853,133312,1,0,649130,35884,1,0,145666,6701,1,0,1021638,54699,1,0,397036,25553,1,0,6560828,147657,1,0,1008670,38471,1,0,742974,32161,1,0,160979,8465,1,0,106775,6360,1,0,3489404,96024,1,0,1464945,77325,1,0,225128,11827,1,0,261118,13619,1,0,2225098,73343,1,0,1174240,51588,1,0,592274,26286,1,0,544902,24618,1,0,719418,38311,1,0,678846,42127,1,0,287308,12606,1,0,1175286,52461,1,0,1384594,56209,1,0,1984312,69572,1,0,1122623,45577,1,0,406984,27917,1,0,1111041,52614,1,0,175951,9783,1,0,308815,14900,1,0,417991,18076,1,0,304923,12455,1,0,1884736,66009,1,0,302916,18787,1,0,3678092,94638,1,0,1442653,61027,1,0,121812,6004,1,0,2322329,78678,1,0,603727,32102,1,0,781529,33917,1,0,2366993,75184,1,0,229576,9780,1,0,694068,39411,1,0,144780,6953,1,0,1038960,58933,1,0,3608547,109187,1,0,397201,21011,1,0,138159,6482,1,0,1496782,63918,1,0,1429182,58064,1,0,304743,16240,1,0,1064365,45222,1,0,103476,4993,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.csv new file mode 100644 index 0000000000..f840a92bc3 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.csv @@ -0,0 +1,22 @@ +observationAbout,observationDate,value,variableMeasured +country/USA,1994-11,5573616,dcid:Count_Household_HouseholderAge15To24Years_NonInstitutionalized +country/USA,1994-11,43123860,dcid:Count_Household_HouseholderAge25To44Years_NonInstitutionalized +country/USA,1994-11,29830455,dcid:Count_Household_HouseholderAge45To64Years_NonInstitutionalized +country/USA,1994-11,21180087,dcid:Count_Household_HouseholderAge65OrMoreYears_NonInstitutionalized +country/USA,1994-11,1021876,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge15To24Years_NonInstitutionalized +country/USA,1994-11,13039306,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge25To44Years_NonInstitutionalized +country/USA,1994-11,8525086,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge45To64Years_NonInstitutionalized +country/USA,1994-11,1622202,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge65OrMoreYears_NonInstitutionalized +country/USA,1997-10,1594328,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge15To24Years_NonInstitutionalized +country/USA,1997-10,19680204,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge25To44Years_NonInstitutionalized +country/USA,1997-10,13716027,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge45To64Years_NonInstitutionalized +country/USA,1997-10,2789026,dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge65OrMoreYears_NonInstitutionalized +country/USA,1998-12,48287009,dcid:Count_Person_3To14Years_Civilian +country/USA,1998-12,37477648,dcid:Count_Person_15To24Years_Civilian +country/USA,1998-12,82833709,dcid:Count_Person_25To44Years_Civilian +country/USA,1998-12,57582133,dcid:Count_Person_45To64Years_Civilian +country/USA,1998-12,32272407,dcid:Count_Person_65OrMoreYears_Civilian +country/USA,1998-12,1798778,dcid:Count_Person_15To24Years_Civilian_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,14741701,dcid:Count_Person_25To44Years_Civilian_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,7969110,dcid:Count_Person_45To64Years_Civilian_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,271065,dcid:Count_Person_65OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.tmcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.tmcf new file mode 100644 index 0000000000..0eafc0c7ba --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output.tmcf @@ -0,0 +1,6 @@ +Node: E:ntia_age_output->E0 +observationAbout: C:ntia_age_output->observationAbout +observationDate: C:ntia_age_output->observationDate +value: C:ntia_age_output->value +variableMeasured: C:ntia_age_output->variableMeasured +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars.mcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars.mcf new file mode 100644 index 0000000000..c33b1a2e12 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars.mcf @@ -0,0 +1,89 @@ +Node: dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge15To24Years_NonInstitutionalized +typeOf: dcid:StatisticalVariable +name: "Anyone in Household Uses a Desktop","age between 15 and 24 years","dcid:Laptop","or Tablet at Home" +populationType: dcid:Household +measuredProperty: dcid:count +statType: dcid:measuredValue +computerUsageLocation: dcid:Home +hasComputer: dcid:True +householderAge: [15 24 Years] +institutionalization: dcid:USC_NonInstitutionalized + +Node: dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge25To44Years_NonInstitutionalized +typeOf: dcid:StatisticalVariable +name: "Anyone in Household Uses a Desktop","age between 25 and 44 years","dcid:Laptop","or Tablet at Home" +populationType: dcid:Household +measuredProperty: dcid:count +statType: dcid:measuredValue +computerUsageLocation: dcid:Home +hasComputer: dcid:True +householderAge: [25 44 Years] +institutionalization: dcid:USC_NonInstitutionalized + +Node: dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge45To64Years_NonInstitutionalized +typeOf: dcid:StatisticalVariable +name: "Anyone in Household Uses a Desktop","age between 45 and 64 years","dcid:Laptop","or Tablet at Home" +populationType: dcid:Household +measuredProperty: dcid:count +statType: dcid:measuredValue +computerUsageLocation: dcid:Home +hasComputer: dcid:True +householderAge: [45 64 Years] +institutionalization: dcid:USC_NonInstitutionalized + +Node: dcid:Count_Household_ComputerUsageAtHome_HasComputer_HouseholderAge65OrMoreYears_NonInstitutionalized +typeOf: dcid:StatisticalVariable +name: "Anyone in Household Uses a Desktop","age 65 years and above","dcid:Laptop","or Tablet at Home" +populationType: dcid:Household +measuredProperty: dcid:count +statType: dcid:measuredValue +computerUsageLocation: dcid:Home +hasComputer: dcid:True +householderAge: [65 - Years] +institutionalization: dcid:USC_NonInstitutionalized + +Node: dcid:Count_Person_15To24Years_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","age between 15 and 24 years" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [15 24 Years] +armedForcesStatus: dcid:Civilian + +Node: dcid:Count_Person_25To44Years_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","age between 25 and 44 years" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [25 44 Years] +armedForcesStatus: dcid:Civilian + +Node: dcid:Count_Person_3To14Years_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","age between 3 and 14 years" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 14 Years] +armedForcesStatus: dcid:Civilian + +Node: dcid:Count_Person_45To64Years_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","age between 45 and 64 years" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [45 64 Years] +armedForcesStatus: dcid:Civilian + +Node: dcid:Count_Person_65OrMoreYears_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","age 65 years and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [65 - Years] +armedForcesStatus: dcid:Civilian + diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars_schema.mcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars_schema.mcf new file mode 100644 index 0000000000..ae7953261e --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_age_output_stat_vars_schema.mcf @@ -0,0 +1,10 @@ +Node: dcid:computerUsageLocation +typeOf: dcs:Property +name: "computerUsageLocation" +domainIncludes: dcid:Household + +Node: dcid:householderAge +typeOf: dcs:Property +name: "householderAge" +rangeIncludes: dcid:QuantityRange + diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.csv new file mode 100644 index 0000000000..de76dd7352 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.csv @@ -0,0 +1,441 @@ +observationAbout,observationDate,value,variableMeasured +country/USA,1994-11,99708018,dcid:Count_Household_NonInstitutionalized +country/USA,1994-11,65545071,dcid:Count_Household_Employed_NonInstitutionalized +country/USA,1994-11,2939228,dcid:Count_Household_Unemployed_NonInstitutionalized +country/USA,1994-11,30486580,dcid:Count_Household_NotInLaborForce_NonInstitutionalized +country/USA,1994-11,19363957,dcid:Count_Household_HouseholderEducationalAttainmentNoDiploma_NonInstitutionalized +country/USA,1994-11,32022931,dcid:Count_Household_HouseholderEducationalAttainmentHighSchoolGraduate_NonInstitutionalized +country/USA,1994-11,25134148,dcid:Count_Household_HouseholderEducationalAttainmentSomeCollegeOrAssociatesDegree_NonInstitutionalized +country/USA,1994-11,23186983,dcid:Count_Household_HouseholderEducationalAttainmentCollegeGraduate_NonInstitutionalized +country/USA,1994-11,62988898,dcid:Count_Household_Male_NonInstitutionalized +country/USA,1994-11,36719120,dcid:Count_Household_Female_NonInstitutionalized +country/USA,1994-11,77096018,dcid:Count_Household_HouseholderRaceWhiteAloneNotHispanicOrLatino_NonInstitutionalized +country/USA,1994-11,11820843,dcid:Count_Household_HouseholderRaceBlackOrAfricanAmericanAlone_NonInstitutionalized +country/USA,1994-11,7956251,dcid:Count_Household_HispanicOrLatino_NonInstitutionalized +country/USA,1994-11,2027535,dcid:Count_Household_HouseholderRaceAsian_NonInstitutionalized +country/USA,1994-11,544756,dcid:Count_Household_HouseholderRaceAmericanIndianOrAlaskaNative_NonInstitutionalized +country/USA,1994-11,262614,dcid:Count_Household_NotHispanicOrLatino_HouseholderRaceNTIAOtherRace_NonInstitutionalized +country/USA,1994-11,22028761,dcid:Count_Household_NonInstitutionalized_Rural +country/USA,1994-11,76906846,dcid:Count_Household_NonInstitutionalized_Urban +country/USA,1994-11,772412,dcid:Count_Household_NonInstitutionalized_UnknownPlaceOfResidence +country/USA,1994-11,57413314,dcid:Count_Household_NonInstitutionalized_NotEnrolledInSchool +country/USA,1994-11,42294704,dcid:Count_Household_NonInstitutionalized_EnrolledInSchool +country/USA,1994-11,77980419,dcid:Count_Household_NonInstitutionalized_Nonveteran +country/USA,1994-11,20947215,dcid:Count_Household_NonInstitutionalized_Veteran +geoId/01,1994-11,1668675,dcid:Count_Household_NonInstitutionalized +geoId/02,1994-11,208595,dcid:Count_Household_NonInstitutionalized +geoId/04,1994-11,1610328,dcid:Count_Household_NonInstitutionalized +geoId/05,1994-11,952203,dcid:Count_Household_NonInstitutionalized +geoId/06,1994-11,11432345,dcid:Count_Household_NonInstitutionalized +geoId/08,1994-11,1435110,dcid:Count_Household_NonInstitutionalized +geoId/09,1994-11,1285592,dcid:Count_Household_NonInstitutionalized +geoId/10,1994-11,274567,dcid:Count_Household_NonInstitutionalized +geoId/11,1994-11,267663,dcid:Count_Household_NonInstitutionalized +geoId/12,1994-11,5657096,dcid:Count_Household_NonInstitutionalized +geoId/13,1994-11,2673673,dcid:Count_Household_NonInstitutionalized +geoId/15,1994-11,408744,dcid:Count_Household_NonInstitutionalized +geoId/16,1994-11,411766,dcid:Count_Household_NonInstitutionalized +geoId/17,1994-11,4430662,dcid:Count_Household_NonInstitutionalized +geoId/18,1994-11,2158646,dcid:Count_Household_NonInstitutionalized +geoId/19,1994-11,1102615,dcid:Count_Household_NonInstitutionalized +geoId/20,1994-11,980574,dcid:Count_Household_NonInstitutionalized +geoId/21,1994-11,1532080,dcid:Count_Household_NonInstitutionalized +geoId/22,1994-11,1571716,dcid:Count_Household_NonInstitutionalized +geoId/23,1994-11,473056,dcid:Count_Household_NonInstitutionalized +geoId/24,1994-11,1992430,dcid:Count_Household_NonInstitutionalized +geoId/25,1994-11,2371069,dcid:Count_Household_NonInstitutionalized +geoId/26,1994-11,3693788,dcid:Count_Household_NonInstitutionalized +geoId/27,1994-11,1809317,dcid:Count_Household_NonInstitutionalized +geoId/28,1994-11,960381,dcid:Count_Household_NonInstitutionalized +geoId/29,1994-11,1979502,dcid:Count_Household_NonInstitutionalized +geoId/30,1994-11,338956,dcid:Count_Household_NonInstitutionalized +geoId/31,1994-11,643136,dcid:Count_Household_NonInstitutionalized +geoId/32,1994-11,583798,dcid:Count_Household_NonInstitutionalized +geoId/33,1994-11,427922,dcid:Count_Household_NonInstitutionalized +geoId/34,1994-11,2967575,dcid:Count_Household_NonInstitutionalized +geoId/35,1994-11,615782,dcid:Count_Household_NonInstitutionalized +geoId/36,1994-11,7191198,dcid:Count_Household_NonInstitutionalized +geoId/37,1994-11,2697423,dcid:Count_Household_NonInstitutionalized +geoId/38,1994-11,244686,dcid:Count_Household_NonInstitutionalized +geoId/39,1994-11,4326601,dcid:Count_Household_NonInstitutionalized +geoId/40,1994-11,1272568,dcid:Count_Household_NonInstitutionalized +geoId/41,1994-11,1190960,dcid:Count_Household_NonInstitutionalized +geoId/42,1994-11,4666061,dcid:Count_Household_NonInstitutionalized +geoId/44,1994-11,401432,dcid:Count_Household_NonInstitutionalized +geoId/45,1994-11,1373450,dcid:Count_Household_NonInstitutionalized +geoId/46,1994-11,271275,dcid:Count_Household_NonInstitutionalized +geoId/47,1994-11,2072162,dcid:Count_Household_NonInstitutionalized +geoId/48,1994-11,6784229,dcid:Count_Household_NonInstitutionalized +geoId/49,1994-11,605439,dcid:Count_Household_NonInstitutionalized +geoId/50,1994-11,226026,dcid:Count_Household_NonInstitutionalized +geoId/51,1994-11,2411191,dcid:Count_Household_NonInstitutionalized +geoId/53,1994-11,2201373,dcid:Count_Household_NonInstitutionalized +geoId/54,1994-11,752405,dcid:Count_Household_NonInstitutionalized +geoId/55,1994-11,1916700,dcid:Count_Household_NonInstitutionalized +geoId/56,1994-11,183476,dcid:Count_Household_NonInstitutionalized +country/USA,1994-11,248509799,dcid:Count_Person_3OrMoreYears_Civilian +country/USA,1994-11,125490400,dcid:Count_Person_Civilian_Employed +country/USA,1994-11,7179758,dcid:Count_Person_Civilian_Unemployed +country/USA,1994-11,68668835,dcid:Count_Person_Civilian_NotInLaborForce +country/USA,1994-11,46369423,dcid:Count_Person_Civilian_EducationalAttainmentNoDiploma +country/USA,1994-11,65052491,dcid:Count_Person_Civilian_EducationalAttainmentHighSchoolGraduate +country/USA,1994-11,50470784,dcid:Count_Person_Civilian_EducationalAttainmentSomeCollegeOrAssociatesDegree +country/USA,1994-11,39446296,dcid:Count_Person_Civilian_EducationalAttainmentCollegeGraduate +country/USA,1994-11,120802232,dcid:Count_Person_3OrMoreYears_Civilian_Male +country/USA,1994-11,127707566,dcid:Count_Person_3OrMoreYears_Civilian_Female +country/USA,1994-11,184215059,dcid:Count_Person_3OrMoreYears_Civilian_WhiteAloneNotHispanicOrLatino +country/USA,1994-11,30738364,dcid:Count_Person_3OrMoreYears_Civilian_BlackOrAfricanAmericanAlone +country/USA,1994-11,25209523,dcid:Count_Person_3OrMoreYears_Civilian_HispanicOrLatino +country/USA,1994-11,6143270,dcid:Count_Person_3OrMoreYears_Civilian_Asian +country/USA,1994-11,1378818,dcid:Count_Person_3OrMoreYears_Civilian_AmericanIndianOrAlaskaNative +country/USA,1994-11,824764,dcid:Count_Person_3OrMoreYears_Civilian_NotHispanicOrLatino_NTIAOtherRace +country/USA,1994-11,55674976,dcid:Count_Person_3OrMoreYears_Civilian_Rural +country/USA,1994-11,190903150,dcid:Count_Person_3OrMoreYears_Civilian_Urban +country/USA,1994-11,1931673,dcid:Count_Person_3OrMoreYears_Civilian_UnknownPlaceOfResidence +country/USA,1994-11,109877267,dcid:Count_Person_3OrMoreYears_Civilian_NotEnrolledInSchool +country/USA,1994-11,138632531,dcid:Count_Person_3OrMoreYears_Civilian_EnrolledInSchool +country/USA,1994-11,167584914,dcid:Count_Person_3OrMoreYears_Civilian_Nonveteran +country/USA,1994-11,26413736,dcid:Count_Person_3OrMoreYears_Civilian_Veteran +geoId/01,1994-11,4073843,dcid:Count_Person_3OrMoreYears_Civilian +geoId/02,1994-11,549019,dcid:Count_Person_3OrMoreYears_Civilian +geoId/04,1994-11,3838642,dcid:Count_Person_3OrMoreYears_Civilian +geoId/05,1994-11,2349494,dcid:Count_Person_3OrMoreYears_Civilian +geoId/06,1994-11,29833154,dcid:Count_Person_3OrMoreYears_Civilian +geoId/08,1994-11,3478339,dcid:Count_Person_3OrMoreYears_Civilian +geoId/09,1994-11,3121902,dcid:Count_Person_3OrMoreYears_Civilian +geoId/10,1994-11,668702,dcid:Count_Person_3OrMoreYears_Civilian +geoId/11,1994-11,546349,dcid:Count_Person_3OrMoreYears_Civilian +geoId/12,1994-11,13402421,dcid:Count_Person_3OrMoreYears_Civilian +geoId/13,1994-11,6763053,dcid:Count_Person_3OrMoreYears_Civilian +geoId/15,1994-11,1044237,dcid:Count_Person_3OrMoreYears_Civilian +geoId/16,1994-11,1133715,dcid:Count_Person_3OrMoreYears_Civilian +geoId/17,1994-11,11226180,dcid:Count_Person_3OrMoreYears_Civilian +geoId/18,1994-11,5591779,dcid:Count_Person_3OrMoreYears_Civilian +geoId/19,1994-11,2588227,dcid:Count_Person_3OrMoreYears_Civilian +geoId/20,1994-11,2345988,dcid:Count_Person_3OrMoreYears_Civilian +geoId/21,1994-11,3713636,dcid:Count_Person_3OrMoreYears_Civilian +geoId/22,1994-11,3945503,dcid:Count_Person_3OrMoreYears_Civilian +geoId/23,1994-11,1248847,dcid:Count_Person_3OrMoreYears_Civilian +geoId/24,1994-11,4614652,dcid:Count_Person_3OrMoreYears_Civilian +geoId/25,1994-11,5668208,dcid:Count_Person_3OrMoreYears_Civilian +geoId/26,1994-11,9044709,dcid:Count_Person_3OrMoreYears_Civilian +geoId/27,1994-11,4314360,dcid:Count_Person_3OrMoreYears_Civilian +geoId/28,1994-11,2613567,dcid:Count_Person_3OrMoreYears_Civilian +geoId/29,1994-11,5008637,dcid:Count_Person_3OrMoreYears_Civilian +geoId/30,1994-11,824168,dcid:Count_Person_3OrMoreYears_Civilian +geoId/31,1994-11,1494363,dcid:Count_Person_3OrMoreYears_Civilian +geoId/32,1994-11,1405781,dcid:Count_Person_3OrMoreYears_Civilian +geoId/33,1994-11,1129251,dcid:Count_Person_3OrMoreYears_Civilian +geoId/34,1994-11,7531215,dcid:Count_Person_3OrMoreYears_Civilian +geoId/35,1994-11,1604886,dcid:Count_Person_3OrMoreYears_Civilian +geoId/36,1994-11,17582458,dcid:Count_Person_3OrMoreYears_Civilian +geoId/37,1994-11,6684073,dcid:Count_Person_3OrMoreYears_Civilian +geoId/38,1994-11,593901,dcid:Count_Person_3OrMoreYears_Civilian +geoId/39,1994-11,10715484,dcid:Count_Person_3OrMoreYears_Civilian +geoId/40,1994-11,3173878,dcid:Count_Person_3OrMoreYears_Civilian +geoId/41,1994-11,2914641,dcid:Count_Person_3OrMoreYears_Civilian +geoId/42,1994-11,11367737,dcid:Count_Person_3OrMoreYears_Civilian +geoId/44,1994-11,923103,dcid:Count_Person_3OrMoreYears_Civilian +geoId/45,1994-11,3586678,dcid:Count_Person_3OrMoreYears_Civilian +geoId/46,1994-11,667201,dcid:Count_Person_3OrMoreYears_Civilian +geoId/47,1994-11,4895851,dcid:Count_Person_3OrMoreYears_Civilian +geoId/48,1994-11,17789235,dcid:Count_Person_3OrMoreYears_Civilian +geoId/49,1994-11,1791096,dcid:Count_Person_3OrMoreYears_Civilian +geoId/50,1994-11,560296,dcid:Count_Person_3OrMoreYears_Civilian +geoId/51,1994-11,6238561,dcid:Count_Person_3OrMoreYears_Civilian +geoId/53,1994-11,5103906,dcid:Count_Person_3OrMoreYears_Civilian +geoId/54,1994-11,1770883,dcid:Count_Person_3OrMoreYears_Civilian +geoId/55,1994-11,4984995,dcid:Count_Person_3OrMoreYears_Civilian +geoId/56,1994-11,448994,dcid:Count_Person_3OrMoreYears_Civilian +country/USA,1997-10,255688666,dcid:Count_Person_3OrMoreYears_Civilian +country/USA,1997-10,131223454,dcid:Count_Person_Civilian_Employed +country/USA,1997-10,6216600,dcid:Count_Person_Civilian_Unemployed +country/USA,1997-10,70287866,dcid:Count_Person_Civilian_NotInLaborForce +country/USA,1997-10,45498078,dcid:Count_Person_Civilian_EducationalAttainmentNoDiploma +country/USA,1997-10,66160685,dcid:Count_Person_Civilian_EducationalAttainmentHighSchoolGraduate +country/USA,1997-10,52443686,dcid:Count_Person_Civilian_EducationalAttainmentSomeCollegeOrAssociatesDegree +country/USA,1997-10,43625471,dcid:Count_Person_Civilian_EducationalAttainmentCollegeGraduate +country/USA,1997-10,124589570,dcid:Count_Person_3OrMoreYears_Civilian_Male +country/USA,1997-10,131099096,dcid:Count_Person_3OrMoreYears_Civilian_Female +country/USA,1997-10,184295047,dcid:Count_Person_3OrMoreYears_Civilian_WhiteAloneNotHispanicOrLatino +country/USA,1997-10,31786358,dcid:Count_Person_3OrMoreYears_Civilian_BlackOrAfricanAmericanAlone +country/USA,1997-10,28233368,dcid:Count_Person_3OrMoreYears_Civilian_HispanicOrLatino +country/USA,1997-10,9225313,dcid:Count_Person_3OrMoreYears_Civilian_Asian +country/USA,1997-10,2148580,dcid:Count_Person_3OrMoreYears_Civilian_AmericanIndianOrAlaskaNative +country/USA,1997-10,49531801,dcid:Count_Person_3OrMoreYears_Civilian_Rural +country/USA,1997-10,205602786,dcid:Count_Person_3OrMoreYears_Civilian_Urban +country/USA,1997-10,554079,dcid:Count_Person_3OrMoreYears_Civilian_UnknownPlaceOfResidence +country/USA,1997-10,139844631,dcid:Count_Person_3OrMoreYears_Civilian_NotEnrolledInSchool +country/USA,1997-10,115844035,dcid:Count_Person_3OrMoreYears_Civilian_EnrolledInSchool +country/USA,1997-10,174815907,dcid:Count_Person_3OrMoreYears_Civilian_Nonveteran +country/USA,1997-10,25000578,dcid:Count_Person_3OrMoreYears_Civilian_Veteran +geoId/01,1997-10,4173177,dcid:Count_Person_3OrMoreYears_Civilian +geoId/02,1997-10,575943,dcid:Count_Person_3OrMoreYears_Civilian +geoId/04,1997-10,4459177,dcid:Count_Person_3OrMoreYears_Civilian +geoId/05,1997-10,2356740,dcid:Count_Person_3OrMoreYears_Civilian +geoId/06,1997-10,30876689,dcid:Count_Person_3OrMoreYears_Civilian +geoId/08,1997-10,3790222,dcid:Count_Person_3OrMoreYears_Civilian +geoId/09,1997-10,3106854,dcid:Count_Person_3OrMoreYears_Civilian +geoId/10,1997-10,710302,dcid:Count_Person_3OrMoreYears_Civilian +geoId/11,1997-10,492315,dcid:Count_Person_3OrMoreYears_Civilian +geoId/12,1997-10,13836974,dcid:Count_Person_3OrMoreYears_Civilian +geoId/13,1997-10,7198558,dcid:Count_Person_3OrMoreYears_Civilian +geoId/15,1997-10,1106079,dcid:Count_Person_3OrMoreYears_Civilian +geoId/16,1997-10,1185423,dcid:Count_Person_3OrMoreYears_Civilian +geoId/17,1997-10,11237775,dcid:Count_Person_3OrMoreYears_Civilian +geoId/18,1997-10,5666981,dcid:Count_Person_3OrMoreYears_Civilian +geoId/19,1997-10,2694457,dcid:Count_Person_3OrMoreYears_Civilian +geoId/20,1997-10,2463028,dcid:Count_Person_3OrMoreYears_Civilian +geoId/21,1997-10,3754725,dcid:Count_Person_3OrMoreYears_Civilian +geoId/22,1997-10,4127461,dcid:Count_Person_3OrMoreYears_Civilian +geoId/23,1997-10,1199227,dcid:Count_Person_3OrMoreYears_Civilian +geoId/24,1997-10,4591025,dcid:Count_Person_3OrMoreYears_Civilian +geoId/25,1997-10,5920355,dcid:Count_Person_3OrMoreYears_Civilian +geoId/26,1997-10,9183915,dcid:Count_Person_3OrMoreYears_Civilian +geoId/27,1997-10,4357149,dcid:Count_Person_3OrMoreYears_Civilian +geoId/28,1997-10,2563708,dcid:Count_Person_3OrMoreYears_Civilian +geoId/29,1997-10,5095040,dcid:Count_Person_3OrMoreYears_Civilian +geoId/30,1997-10,858799,dcid:Count_Person_3OrMoreYears_Civilian +geoId/31,1997-10,1601636,dcid:Count_Person_3OrMoreYears_Civilian +geoId/32,1997-10,1626033,dcid:Count_Person_3OrMoreYears_Civilian +geoId/33,1997-10,1138643,dcid:Count_Person_3OrMoreYears_Civilian +geoId/34,1997-10,7635428,dcid:Count_Person_3OrMoreYears_Civilian +geoId/35,1997-10,1761451,dcid:Count_Person_3OrMoreYears_Civilian +geoId/36,1997-10,17589762,dcid:Count_Person_3OrMoreYears_Civilian +geoId/37,1997-10,6939885,dcid:Count_Person_3OrMoreYears_Civilian +geoId/38,1997-10,609288,dcid:Count_Person_3OrMoreYears_Civilian +geoId/39,1997-10,10932484,dcid:Count_Person_3OrMoreYears_Civilian +geoId/40,1997-10,3128742,dcid:Count_Person_3OrMoreYears_Civilian +geoId/41,1997-10,3240172,dcid:Count_Person_3OrMoreYears_Civilian +geoId/42,1997-10,11513445,dcid:Count_Person_3OrMoreYears_Civilian +geoId/44,1997-10,908536,dcid:Count_Person_3OrMoreYears_Civilian +geoId/45,1997-10,3580164,dcid:Count_Person_3OrMoreYears_Civilian +geoId/46,1997-10,700970,dcid:Count_Person_3OrMoreYears_Civilian +geoId/47,1997-10,5108403,dcid:Count_Person_3OrMoreYears_Civilian +geoId/48,1997-10,18602147,dcid:Count_Person_3OrMoreYears_Civilian +geoId/49,1997-10,1905461,dcid:Count_Person_3OrMoreYears_Civilian +geoId/50,1997-10,569660,dcid:Count_Person_3OrMoreYears_Civilian +geoId/51,1997-10,6470097,dcid:Count_Person_3OrMoreYears_Civilian +geoId/53,1997-10,5359857,dcid:Count_Person_3OrMoreYears_Civilian +geoId/54,1997-10,1729260,dcid:Count_Person_3OrMoreYears_Civilian +geoId/55,1997-10,4988102,dcid:Count_Person_3OrMoreYears_Civilian +geoId/56,1997-10,466942,dcid:Count_Person_3OrMoreYears_Civilian +country/USA,1998-12,104174641,dcid:Count_Household_NonInstitutionalized +country/USA,1998-12,69116853,dcid:Count_Household_Employed_NonInstitutionalized +country/USA,1998-12,2336082,dcid:Count_Household_Unemployed_NonInstitutionalized +country/USA,1998-12,32206410,dcid:Count_Household_NotInLaborForce_NonInstitutionalized +country/USA,1998-12,18080409,dcid:Count_Household_HouseholderEducationalAttainmentNoDiploma_NonInstitutionalized +country/USA,1998-12,32580534,dcid:Count_Household_HouseholderEducationalAttainmentHighSchoolGraduate_NonInstitutionalized +country/USA,1998-12,26989220,dcid:Count_Household_HouseholderEducationalAttainmentSomeCollegeOrAssociatesDegree_NonInstitutionalized +country/USA,1998-12,26524478,dcid:Count_Household_HouseholderEducationalAttainmentCollegeGraduate_NonInstitutionalized +country/USA,1998-12,60138023,dcid:Count_Household_Male_NonInstitutionalized +country/USA,1998-12,44036618,dcid:Count_Household_Female_NonInstitutionalized +country/USA,1998-12,78791846,dcid:Count_Household_HouseholderRaceWhiteAloneNotHispanicOrLatino_NonInstitutionalized +country/USA,1998-12,12447727,dcid:Count_Household_HouseholderRaceBlackOrAfricanAmericanAlone_NonInstitutionalized +country/USA,1998-12,8964828,dcid:Count_Household_HispanicOrLatino_NonInstitutionalized +country/USA,1998-12,3181099,dcid:Count_Household_HouseholderRaceAsian_NonInstitutionalized +country/USA,1998-12,789142,dcid:Count_Household_HouseholderRaceAmericanIndianOrAlaskaNative_NonInstitutionalized +country/USA,1998-12,20263377,dcid:Count_Household_NonInstitutionalized_Rural +country/USA,1998-12,83651922,dcid:Count_Household_NonInstitutionalized_Urban +country/USA,1998-12,259342,dcid:Count_Household_NonInstitutionalized_UnknownPlaceOfResidence +country/USA,1998-12,75199428,dcid:Count_Household_NonInstitutionalized_NotEnrolledInSchool +country/USA,1998-12,28975213,dcid:Count_Household_NonInstitutionalized_EnrolledInSchool +country/USA,1998-12,85304356,dcid:Count_Household_NonInstitutionalized_Nonveteran +country/USA,1998-12,18290905,dcid:Count_Household_NonInstitutionalized_Veteran +geoId/01,1998-12,1688438,dcid:Count_Household_NonInstitutionalized +geoId/02,1998-12,222611,dcid:Count_Household_NonInstitutionalized +geoId/04,1998-12,1764439,dcid:Count_Household_NonInstitutionalized +geoId/05,1998-12,1030387,dcid:Count_Household_NonInstitutionalized +geoId/06,1998-12,11814611,dcid:Count_Household_NonInstitutionalized +geoId/08,1998-12,1572832,dcid:Count_Household_NonInstitutionalized +geoId/09,1998-12,1254539,dcid:Count_Household_NonInstitutionalized +geoId/10,1998-12,292247,dcid:Count_Household_NonInstitutionalized +geoId/11,1998-12,235068,dcid:Count_Household_NonInstitutionalized +geoId/12,1998-12,6047390,dcid:Count_Household_NonInstitutionalized +geoId/13,1998-12,2951817,dcid:Count_Household_NonInstitutionalized +geoId/15,1998-12,417473,dcid:Count_Household_NonInstitutionalized +geoId/16,1998-12,445591,dcid:Count_Household_NonInstitutionalized +geoId/17,1998-12,4508956,dcid:Count_Household_NonInstitutionalized +geoId/18,1998-12,2337591,dcid:Count_Household_NonInstitutionalized +geoId/19,1998-12,1161753,dcid:Count_Household_NonInstitutionalized +geoId/20,1998-12,1057912,dcid:Count_Household_NonInstitutionalized +geoId/21,1998-12,1600328,dcid:Count_Household_NonInstitutionalized +geoId/22,1998-12,1607131,dcid:Count_Household_NonInstitutionalized +geoId/23,1998-12,519433,dcid:Count_Household_NonInstitutionalized +geoId/24,1998-12,2108180,dcid:Count_Household_NonInstitutionalized +geoId/25,1998-12,2406815,dcid:Count_Household_NonInstitutionalized +geoId/26,1998-12,3794831,dcid:Count_Household_NonInstitutionalized +geoId/27,1998-12,1865172,dcid:Count_Household_NonInstitutionalized +geoId/28,1998-12,1087393,dcid:Count_Household_NonInstitutionalized +geoId/29,1998-12,2202645,dcid:Count_Household_NonInstitutionalized +geoId/30,1998-12,353818,dcid:Count_Household_NonInstitutionalized +geoId/31,1998-12,658708,dcid:Count_Household_NonInstitutionalized +geoId/32,1998-12,667406,dcid:Count_Household_NonInstitutionalized +geoId/33,1998-12,453179,dcid:Count_Household_NonInstitutionalized +geoId/34,1998-12,2936614,dcid:Count_Household_NonInstitutionalized +geoId/35,1998-12,652882,dcid:Count_Household_NonInstitutionalized +geoId/36,1998-12,6986513,dcid:Count_Household_NonInstitutionalized +geoId/37,1998-12,2956634,dcid:Count_Household_NonInstitutionalized +geoId/38,1998-12,248322,dcid:Count_Household_NonInstitutionalized +geoId/39,1998-12,4491630,dcid:Count_Household_NonInstitutionalized +geoId/40,1998-12,1308497,dcid:Count_Household_NonInstitutionalized +geoId/41,1998-12,1313233,dcid:Count_Household_NonInstitutionalized +geoId/42,1998-12,4595305,dcid:Count_Household_NonInstitutionalized +geoId/44,1998-12,374914,dcid:Count_Household_NonInstitutionalized +geoId/45,1998-12,1493081,dcid:Count_Household_NonInstitutionalized +geoId/46,1998-12,280994,dcid:Count_Household_NonInstitutionalized +geoId/47,1998-12,2230004,dcid:Count_Household_NonInstitutionalized +geoId/48,1998-12,7339846,dcid:Count_Household_NonInstitutionalized +geoId/49,1998-12,677519,dcid:Count_Household_NonInstitutionalized +geoId/50,1998-12,240669,dcid:Count_Household_NonInstitutionalized +geoId/51,1998-12,2687098,dcid:Count_Household_NonInstitutionalized +geoId/53,1998-12,2252857,dcid:Count_Household_NonInstitutionalized +geoId/54,1998-12,751793,dcid:Count_Household_NonInstitutionalized +geoId/55,1998-12,2042075,dcid:Count_Household_NonInstitutionalized +geoId/56,1998-12,185468,dcid:Count_Household_NonInstitutionalized +country/USA,1998-12,24780653,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,24780653,dcid:Count_Person_Civilian_Employed_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,0,dcid:Count_Person_Civilian_Unemployed_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,0,dcid:Count_Person_Civilian_NotInLaborForce_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,372491,dcid:Count_Person_Civilian_EducationalAttainmentNoDiploma_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,3578606,dcid:Count_Person_Civilian_EducationalAttainmentHighSchoolGraduate_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,6597272,dcid:Count_Person_Civilian_EducationalAttainmentSomeCollegeOrAssociatesDegree_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,14232285,dcid:Count_Person_Civilian_EducationalAttainmentCollegeGraduate_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,13490849,dcid:Count_Person_15OrMoreYears_Civilian_Male_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,11289804,dcid:Count_Person_15OrMoreYears_Civilian_Female_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,20470472,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_WhiteAloneNotHispanicOrLatino +country/USA,1998-12,1966908,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_BlackOrAfricanAmericanAlone +country/USA,1998-12,1144634,dcid:Count_Person_15OrMoreYears_Civilian_HispanicOrLatino_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,1061533,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_Asian +country/USA,1998-12,137105,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_AmericanIndianOrAlaskaNative +country/USA,1998-12,3173660,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_Rural +country/USA,1998-12,21570762,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_Urban +country/USA,1998-12,36231,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_UnknownPlaceOfResidence +country/USA,1998-12,16970261,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_NotEnrolledInSchool +country/USA,1998-12,7810392,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_EnrolledInSchool +country/USA,1998-12,21968264,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_Nonveteran +country/USA,1998-12,2792788,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser_Veteran +geoId/01,1998-12,349039,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/02,1998-12,75473,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/04,1998-12,436936,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/05,1998-12,187765,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/06,1998-12,3043617,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/08,1998-12,510768,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/09,1998-12,319945,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/10,1998-12,70955,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/11,1998-12,102762,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/12,1998-12,1111718,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/13,1998-12,697995,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/15,1998-12,84340,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/16,1998-12,131142,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/17,1998-12,1187547,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/18,1998-12,482196,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/19,1998-12,233550,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/20,1998-12,238525,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/21,1998-12,299719,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/22,1998-12,268391,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/23,1998-12,89501,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/24,1998-12,848997,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/25,1998-12,577696,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/26,1998-12,844212,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/27,1998-12,597058,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/28,1998-12,123605,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/29,1998-12,489916,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/30,1998-12,91610,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/31,1998-12,156251,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/32,1998-12,101010,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/33,1998-12,127252,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/34,1998-12,750143,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/35,1998-12,187146,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/36,1998-12,1355945,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/37,1998-12,521969,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/38,1998-12,64727,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/39,1998-12,1005934,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/40,1998-12,277774,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/41,1998-12,380979,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/42,1998-12,1006897,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/44,1998-12,88282,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/45,1998-12,258525,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/46,1998-12,56418,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/47,1998-12,477348,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/48,1998-12,1842767,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/49,1998-12,240014,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/50,1998-12,66293,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/51,1998-12,831601,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/53,1998-12,770823,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/54,1998-12,114086,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/55,1998-12,553342,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +geoId/56,1998-12,50146,dcid:Count_Person_15OrMoreYears_Civilian_InternetUsageAtWork_IsInternetUser +country/USA,1998-12,4118937,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,3483396,dcid:Count_Household_Employed_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,119190,dcid:Count_Household_Unemployed_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,492385,dcid:Count_Household_NotInLaborForce_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,206835,dcid:Count_Household_HouseholderEducationalAttainmentNoDiploma_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,1135431,dcid:Count_Household_HouseholderEducationalAttainmentHighSchoolGraduate_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,1317057,dcid:Count_Household_HouseholderEducationalAttainmentSomeCollegeOrAssociatesDegree_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,1459614,dcid:Count_Household_HouseholderEducationalAttainmentCollegeGraduate_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,2392166,dcid:Count_Household_Male_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,1726771,dcid:Count_Household_Female_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,3310812,dcid:Count_Household_HouseholderRaceWhiteAloneNotHispanicOrLatino_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,324089,dcid:Count_Household_HouseholderRaceBlackOrAfricanAmericanAlone_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,290539,dcid:Count_Household_HispanicOrLatino_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,150347,dcid:Count_Household_HouseholderRaceAsian_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,43151,dcid:Count_Household_HouseholderRaceAmericanIndianOrAlaskaNative_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +country/USA,1998-12,587461,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_Rural +country/USA,1998-12,3521220,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_Urban +country/USA,1998-12,10257,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_UnknownPlaceOfResidence +country/USA,1998-12,2839447,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_NotEnrolledInSchool +country/USA,1998-12,1279490,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_EnrolledInSchool +country/USA,1998-12,3525556,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_Nonveteran +country/USA,1998-12,569414,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser_Veteran +geoId/01,1998-12,52481,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/02,1998-12,10381,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/04,1998-12,73793,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/05,1998-12,20478,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/06,1998-12,561232,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/08,1998-12,100025,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/09,1998-12,43101,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/10,1998-12,11598,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/11,1998-12,14078,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/12,1998-12,234221,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/13,1998-12,162227,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/15,1998-12,12838,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/16,1998-12,31101,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/17,1998-12,158083,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/18,1998-12,77144,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/19,1998-12,36101,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/20,1998-12,49163,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/21,1998-12,66889,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/22,1998-12,54346,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/23,1998-12,20423,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/24,1998-12,74398,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/25,1998-12,93500,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/26,1998-12,179746,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/27,1998-12,78666,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/28,1998-12,23033,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/29,1998-12,84618,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/30,1998-12,14822,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/31,1998-12,18404,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/32,1998-12,24079,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/33,1998-12,16180,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/34,1998-12,67439,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/35,1998-12,31180,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/36,1998-12,276334,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/37,1998-12,101329,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/38,1998-12,9109,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/39,1998-12,167120,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/40,1998-12,64435,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/41,1998-12,80081,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/42,1998-12,132142,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/44,1998-12,11795,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/45,1998-12,28762,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/46,1998-12,6278,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/47,1998-12,76066,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/48,1998-12,287027,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/49,1998-12,31638,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/50,1998-12,6668,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/51,1998-12,97192,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/53,1998-12,149529,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/54,1998-12,18553,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/55,1998-12,70698,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser +geoId/56,1998-12,8413,dcid:Count_Household_NonInstitutionalized_InternetUsageAtHome_IsInternetUser diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.tmcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.tmcf new file mode 100644 index 0000000000..3378b4685a --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output.tmcf @@ -0,0 +1,6 @@ +Node: E:ntia_output->E0 +observationAbout: C:ntia_output->observationAbout +observationDate: C:ntia_output->observationDate +value: C:ntia_output->value +variableMeasured: C:ntia_output->variableMeasured +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars.mcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars.mcf new file mode 100644 index 0000000000..cdbb499770 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars.mcf @@ -0,0 +1,223 @@ +Node: dcid:Count_Person_3OrMoreYears_Civilian +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian + +Node: dcid:Count_Person_3OrMoreYears_Civilian_AmericanIndianOrAlaskaNative +typeOf: dcid:StatisticalVariable +name: "American Indian or Alaska Native","Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +race: dcid:AmericanIndianOrAlaskaNative + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Asian +typeOf: dcid:StatisticalVariable +name: "Asian American","Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +race: dcid:Asian + +Node: dcid:Count_Person_3OrMoreYears_Civilian_BlackOrAfricanAmericanAlone +typeOf: dcid:StatisticalVariable +name: "African American","Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +race: dcid:BlackOrAfricanAmericanAlone + +Node: dcid:Count_Person_3OrMoreYears_Civilian_EnrolledInSchool +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","School-Aged Child at Home" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +schoolEnrollment: dcid:EnrolledInSchool + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Female +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","female" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +gender: dcid:Female + +Node: dcid:Count_Person_3OrMoreYears_Civilian_HispanicOrLatino +typeOf: dcid:StatisticalVariable +name: "Hispanic","Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +ethnicity: dcid:HispanicOrLatino + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Male +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","male" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +gender: dcid:Male + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Nonveteran +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Not Veteran","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +veteranStatus: dcid:Nonveteran + +Node: dcid:Count_Person_3OrMoreYears_Civilian_NotEnrolledInSchool +typeOf: dcid:StatisticalVariable +name: "No School-Aged Child at Home","Not Active-Duty Military","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +schoolEnrollment: dcid:NotEnrolledInSchool + +Node: dcid:Count_Person_3OrMoreYears_Civilian_NotHispanicOrLatino_NTIAOtherRace +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Other non-Hispanic","Person Aged 3 and above" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +ethnicity: dcid:NotHispanicOrLatino +race: dcid:NTIA_OtherRace + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Rural +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","Rural" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +placeOfResidenceClassification: dcid:Rural + +Node: dcid:Count_Person_3OrMoreYears_Civilian_UnknownPlaceOfResidence +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","Unknown place of residence" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +placeOfResidenceClassification: dcid:UnknownPlaceOfResidence + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Urban +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","Urban" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +placeOfResidenceClassification: dcid:Urban + +Node: dcid:Count_Person_3OrMoreYears_Civilian_Veteran +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","Veteran" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +veteranStatus: dcid:Veteran + +Node: dcid:Count_Person_3OrMoreYears_Civilian_WhiteAloneNotHispanicOrLatino +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","White non-Hispanic" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +age: [3 - Years] +armedForcesStatus: dcid:Civilian +race: dcid:WhiteAloneNotHispanicOrLatino + +Node: dcid:Count_Person_Civilian_EducationalAttainmentCollegeGraduate +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","education up to college Graduate" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +educationalAttainment: dcid:CollegeGraduate + +Node: dcid:Count_Person_Civilian_EducationalAttainmentHighSchoolGraduate +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","education up to Higher Secondary Graduate" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +educationalAttainment: dcid:HighSchoolGraduate + +Node: dcid:Count_Person_Civilian_EducationalAttainmentNoDiploma +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","education without Diploma" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +educationalAttainment: dcid:NoDiploma + +Node: dcid:Count_Person_Civilian_EducationalAttainmentSomeCollegeOrAssociatesDegree +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","education up to some college" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +educationalAttainment: dcid:SomeCollegeOrAssociatesDegree + +Node: dcid:Count_Person_Civilian_Employed +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","employed" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +employmentStatus: dcid:Employed + +Node: dcid:Count_Person_Civilian_NotInLaborForce +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","not in Labour Force" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +employmentStatus: dcid:NotInLaborForce + +Node: dcid:Count_Person_Civilian_Unemployed +typeOf: dcid:StatisticalVariable +name: "Not Active-Duty Military","Person Aged 3 and above","unemployed" +populationType: dcid:Person +measuredProperty: dcid:count +statType: dcid:measuredValue +armedForcesStatus: dcid:Civilian +employmentStatus: dcid:Unemployed + diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars_schema.mcf b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars_schema.mcf new file mode 100644 index 0000000000..eb6401e955 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/testdata/ntia_output_stat_vars_schema.mcf @@ -0,0 +1,5 @@ +Node: dcid:placeOfResidenceClassification +typeOf: dcs:Property +name: "placeOfResidenceClassification" +rangeIncludes: dcid:HumanSettlementClassificationEnum,dcid:LocationClassificationEnum + diff --git a/util/statvar_dcid_generator.py b/util/statvar_dcid_generator.py index 0dcae20ef1..11e04544d5 100644 --- a/util/statvar_dcid_generator.py +++ b/util/statvar_dcid_generator.py @@ -350,6 +350,9 @@ 'toCurrency': { 'prepend': 'ToCurrency_', }, + 'internetUsageLocation': { + 'prepend': 'InternetUsageAt', + } } # This is a list of boolean properties From 63151e031edfbc2f303ad20d459774d94207bad4 Mon Sep 17 00:00:00 2001 From: Harish <102517942+HarishC727@users.noreply.github.com> Date: Tue, 24 Jun 2025 18:37:43 +0530 Subject: [PATCH 4/5] India Life expectancy autorefresh (#1408) * India Life expectancy autorefresh * added test data * reverted to comments * reverted to comments * reverted to comments * reverted to comments * reverted to comments --- .../ndap/India_LifeExpectancy_metadata.csv | 10 ++ .../India_LifeExpectancy_placesresolved.csv | 24 ++++ .../ndap/India_LifeExpectancy_pvmap.csv | 29 +++++ statvar_imports/india_ndap/ndap/README.md | 19 +++ statvar_imports/india_ndap/ndap/config.py | 2 + .../india_ndap/ndap/download_script.py | 82 +++++++++++++ statvar_imports/india_ndap/ndap/manifest.json | 26 ++++ .../ndap/test_data/India_LifeExpectancy.csv | 96 +++++++++++++++ .../ndap/test_data/India_LifeExpectancy.tmcf | 7 ++ .../test_data/India_LifeExpectancy_input.csv | 115 ++++++++++++++++++ 10 files changed, 410 insertions(+) create mode 100644 statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv create mode 100644 statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv create mode 100644 statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv create mode 100644 statvar_imports/india_ndap/ndap/README.md create mode 100644 statvar_imports/india_ndap/ndap/config.py create mode 100644 statvar_imports/india_ndap/ndap/download_script.py create mode 100644 statvar_imports/india_ndap/ndap/manifest.json create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf create mode 100644 statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv diff --git a/statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv new file mode 100644 index 0000000000..f3ee245407 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv @@ -0,0 +1,10 @@ +config,value +provenance_url,https://ndap.niti.gov.in/dataset/7375 +header_rows,1 +output_columns,"observationAbout,observationDate,value,variableMeasured,unit" +#places_within,country/IND +#place_types,"AdministrativeArea,AdministrativeArea1,AdministrativeArea2,State" +mapped_columns,3 +mapped_rows,1 +#input_rows,50 +#debug,1 diff --git a/statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv new file mode 100644 index 0000000000..8ac24a7313 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_placesresolved.csv @@ -0,0 +1,24 @@ +place_name,observationAbout,dcid +jammu and kashmir,observationAbout,wikidataId/Q1180 +himachal pradesh,observationAbout,wikidataId/Q1177 +punjab,observationAbout,wikidataId/Q22424 +uttarakhand,observationAbout,wikidataId/Q1499 +haryana,observationAbout,wikidataId/Q1174 +delhi,observationAbout,wikidataId/Q1353 +rajasthan,observationAbout,wikidataId/Q1437 +uttar pradesh,observationAbout,wikidataId/Q1498 +bihar,observationAbout,wikidataId/Q1165 +assam,observationAbout,wikidataId/Q1164 +West Bengal,observationAbout,wikidataId/Q1356 +jharkhand,observationAbout,wikidataId/Q1184 +odisha,observationAbout,wikidataId/Q22048 +chhattisgarh,observationAbout,wikidataId/Q1168 +madhya pradesh,observationAbout,wikidataId/Q1188 +gujarat,observationAbout,wikidataId/Q1061 +maharashtra,observationAbout,wikidataId/Q1191 +kerala,observationAbout,wikidataId/Q1186 +andhra pradesh,observationAbout,wikidataId/Q1159 +karnataka,observationAbout,wikidataId/Q1185 +tamil nadu,observationAbout,wikidataId/Q1445 +telangana,observationAbout,wikidataId/Q677037 +ladakh,observationAbout,wikidataId/Q200667 \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv new file mode 100644 index 0000000000..0b79b22af4 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv @@ -0,0 +1,29 @@ +key,prop,val,,,,,, +life expectancy,measuredProperty,lifeExpectancy,value,{Number},populationType,Person,unit,Year +male,gender,Male,,,,,, +female,gender,Female,,,,,, +Total,gender,"""""",,,,,, +srcyear,observationDate,{Number},,,,,, +jammu and kashmir,observationAbout,wikidataId/Q1180 +himachal pradesh,observationAbout,wikidataId/Q1177 +punjab,observationAbout,wikidataId/Q22424 +uttarakhand,observationAbout,wikidataId/Q1499 +haryana,observationAbout,wikidataId/Q1174 +delhi,observationAbout,wikidataId/Q1353 +rajasthan,observationAbout,wikidataId/Q1437 +uttar pradesh,observationAbout,wikidataId/Q1498 +bihar,observationAbout,wikidataId/Q1165 +assam,observationAbout,wikidataId/Q1164 +West Bengal,observationAbout,wikidataId/Q1356 +jharkhand,observationAbout,wikidataId/Q1184 +odisha,observationAbout,wikidataId/Q22048 +chhattisgarh,observationAbout,wikidataId/Q1168 +madhya pradesh,observationAbout,wikidataId/Q1188 +gujarat,observationAbout,wikidataId/Q1061 +maharashtra,observationAbout,wikidataId/Q1191 +kerala,observationAbout,wikidataId/Q1186 +andhra pradesh,observationAbout,wikidataId/Q1159 +karnataka,observationAbout,wikidataId/Q1185 +tamil nadu,observationAbout,wikidataId/Q1445 +telangana,observationAbout,wikidataId/Q677037 +ladakh,observationAbout,wikidataId/Q200667 \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/README.md b/statvar_imports/india_ndap/ndap/README.md new file mode 100644 index 0000000000..4461c726d3 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/README.md @@ -0,0 +1,19 @@ +# NDAP - India Life Expectancy + +- source: `https://ndap.niti.gov.in/dataset/7375`, + +- how to download data: The files can be downloaded using `python3 scripts/world_bank/worldbank_ids/download.py`. + +- type of place: State. + +- statvars: Health + +- years: 1997 to 2020 + +- place_resolution: Places are resolved based on name. + +### How to run: + +`python3 donwload_script.py` + +`python3 stat_var_processor.py --input_data=../../statvar_imports/india_ndap/ndap/input_files/India_LifeExpectancy_input.csv --pv_map=../../statvar_imports/india_ndap/ndap/India_LifeExpectancy_pvmap.csv --config_file=../../statvar_imports/india_ndap/ndap/India_LifeExpectancy_metadata.csv --output_path=../../statvar_imports/india_ndap/ndap/output/Life_expectancy` diff --git a/statvar_imports/india_ndap/ndap/config.py b/statvar_imports/india_ndap/ndap/config.py new file mode 100644 index 0000000000..f7fee18e27 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/config.py @@ -0,0 +1,2 @@ +url='https://loadqa.ndapapi.com/v1/openapi?API_Key=gAAAAABnrzyB1tiwHrggYMoK81fqeUFDhVWacu2CFsVRzMJ4A23xSK4Ov6JLZYHpVi3fQK2ikFB65Xo56PpgeISlZB0OEYfPltYSRIYRfPqb27TNDZd_AK-gdnEIbkyqVuaZCkO9RQYh44TG2mGZwaRC3Yo16s8Ks8-FXtTf2RsV5c-L243hAfGRI984Bed6UTUJMWmDe4Fh&ind=I7375_4&dim=Country,StateName,StateCode,Year,GENDER' +input_files='input_files/' \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/download_script.py b/statvar_imports/india_ndap/ndap/download_script.py new file mode 100644 index 0000000000..fba0242ef2 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/download_script.py @@ -0,0 +1,82 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 20 ('License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How to run the script to download the files: +# python3 download_script.py +import json +import os +import pandas as pd +import sys +from absl import app +from absl import flags +from absl import logging +from google.cloud import storage +flags.DEFINE_string( + 'config_file_path', + 'gs://datcom-import-test/statvar_imports/india_ndap/ndap/config.json', + 'Input directory where config files downloaded.') + +_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(_SCRIPT_PATH, '../../../util/')) +from download_util_script import _retry_method + + +def main(_): + all_data = [] + _FLAGS = flags.FLAGS + config_file_path = _FLAGS.config_file_path + storage_client = storage.Client() + bucket_name = config_file_path.split('/')[2] + bucket = storage_client.bucket(bucket_name) + blob_name = '/'.join(config_file_path.split('/')[3:]) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + try: + file_config = json.loads(file_contents) + url = file_config.get('url') + input_files = file_config.get('input_files') + except json.JSONDecodeError: + logging.fatal("Cannot extract url and input files path.") + page_num = 1 + while True: + api_url = f"{url}&pageno={page_num}" + response = _retry_method(api_url, None, 3, 5, 2) + response_data = response.json() + if response_data and 'Data' in response_data and len(response_data['Data']) > 0: + keys = response_data['Data'] + try: + if not isinstance(keys, list): + logging.fatal(f"Value for key 'Data' is not a list.") + except KeyError as e: + logging.fatal(f"Missing expected key '{e}' in the API response.") + break + # Considering the table id I7375_4 which is specific to the import. + for i in keys: + a = a=i['StateName'],i['Year'].split(",")[-1].strip(),i['GENDER'],i['I7375_4']['TotalPopulationWeight'],i['Year'].split(",")[-1].strip(),i['Year'] + all_data.append(a) + page_num += 1 + else: + logging.error(f"failed to retrieve data from page {page_num}") + break + if all_data: + df = pd.DataFrame(all_data, columns=['srcStateName', 'srcYear', 'GENDER', 'Life Expectancy', 'YearCode', 'Year']) + os.makedirs(input_files, exist_ok=True) + input_filename = os.path.join(input_files, 'India_LifeExpectancy_input.csv') + df.to_csv(input_filename, index=False) + logging.info("Data saved to India_LifeExpectancy_input.csv") + else: + logging.info("No data was retrieved from the API.") + +if __name__=="__main__": + app.run(main) \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/manifest.json b/statvar_imports/india_ndap/ndap/manifest.json new file mode 100644 index 0000000000..822bc405ec --- /dev/null +++ b/statvar_imports/india_ndap/ndap/manifest.json @@ -0,0 +1,26 @@ +{ + "import_specifications": [ + { + "import_name": "India_LifeExpectancy", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://ndap.niti.gov.in/dataset/7375", + "provenance_description": "Life expectancy of a person in India based on gender at state level.", + "scripts": [ + "download_script.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/India_LifeExpectancy_input.csv --pv_map='India_LifeExpectancy_pvmap.csv' --config_file=India_LifeExpectancy_metadata.csv --output_path=output/Life_expectancy" + ], + "import_inputs": [ + { + "template_mcf": "output/Life_expectancy.tmcf", + "cleaned_csv": "output/Life_expectancy.csv" + } + ], + "source_files": [ + "input_files/India_LifeExpectancy_input.csv" + ], + "cron_schedule": "00 11 1,15 * *" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv new file mode 100644 index 0000000000..00aa94e15b --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.csv @@ -0,0 +1,96 @@ +observationAbout,observationDate,value,variableMeasured,unit +wikidataId/Q1180,2002,67.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2002,64.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2002,65.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2003,68,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2003,64.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2003,66,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2004,68.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2004,65.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2004,67.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2005,69.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2005,67.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2005,68.4,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2006,71,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2006,69.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2006,70,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2007,70.8,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2007,68.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2007,69.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2008,70.6,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2008,69,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2008,69.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2009,71.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2009,68.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2009,70,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2010,71.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2010,69.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2010,70.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2011,71.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2011,69.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2011,70.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2012,72.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2012,69.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2012,71,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2013,74,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2013,70.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2013,72,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2014,74.9,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2014,70.9,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2014,72.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2015,76.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2015,71.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2015,73.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2016,68.5,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2016,71.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2016,73.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2017,76.7,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2017,72.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2017,74.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2018,76.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2018,72.2,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2018,74,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2019,76.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2019,72.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2019,74.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1180,2020,76.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1180,2020,72.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1180,2020,74.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1997,65.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1997,64.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1997,64.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1998,65.5,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1998,64.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1998,65.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,1999,67.7,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,1999,64.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,1999,66.2,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2000,68.3,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2000,65.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2000,67,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2001,69.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2001,66.3,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2001,67.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2002,70.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2002,66.6,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2002,68.3,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2003,71.4,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2003,67.1,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2003,69.1,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2004,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2004,67.3,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2004,69.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2005,71.8,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2005,67.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2005,69.5,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2006,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2006,67.4,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2006,69.6,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2007,72.2,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2007,67.8,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2007,69.9,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2008,72.1,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2008,67.7,dcid:LifeExpectancy_Person_Male,Year +wikidataId/Q1177,2008,69.8,dcid:LifeExpectancy_Person,Year +wikidataId/Q1177,2009,72,dcid:LifeExpectancy_Person_Female,Year +wikidataId/Q1177,2009,67.7,dcid:LifeExpectancy_Person_Male,Year diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf new file mode 100644 index 0000000000..4bea313414 --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy.tmcf @@ -0,0 +1,7 @@ +Node: E:India_LifeExpectancy->E0 +observationAbout: C:India_LifeExpectancy->observationAbout +observationDate: C:India_LifeExpectancy->observationDate +value: C:India_LifeExpectancy->value +variableMeasured: C:India_LifeExpectancy->variableMeasured +unit: C:India_LifeExpectancy->unit +typeOf: dcs:StatVarObservation diff --git a/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv new file mode 100644 index 0000000000..d1d1db809a --- /dev/null +++ b/statvar_imports/india_ndap/ndap/test_data/India_LifeExpectancy_input.csv @@ -0,0 +1,115 @@ +srcStateName,srcYear,GENDER,Life Expectancy,YearCode,Year +Jammu And Kashmir,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Jammu And Kashmir,1997,Female,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1997,Male,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1997,Total,,1997,"Calendar Year (Jan - Dec), 1997" +Jammu And Kashmir,1998,Female,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1998,Male,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1998,Total,,1998,"Calendar Year (Jan - Dec), 1998" +Jammu And Kashmir,1999,Female,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,1999,Male,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,1999,Total,,1999,"Calendar Year (Jan - Dec), 1999" +Jammu And Kashmir,2000,Female,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2000,Male,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2000,Total,,2000,"Calendar Year (Jan - Dec), 2000" +Jammu And Kashmir,2001,Female,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2001,Male,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2001,Total,,2001,"Calendar Year (Jan - Dec), 2001" +Jammu And Kashmir,2002,Female,67.3,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2002,Male,64.7,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2002,Total,65.9,2002,"Calendar Year (Jan - Dec), 2002" +Jammu And Kashmir,2003,Female,68.0,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2003,Male,64.4,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2003,Total,66.0,2003,"Calendar Year (Jan - Dec), 2003" +Jammu And Kashmir,2004,Female,68.9,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2004,Male,65.9,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2004,Total,67.3,2004,"Calendar Year (Jan - Dec), 2004" +Jammu And Kashmir,2005,Female,69.9,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2005,Male,67.1,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2005,Total,68.4,2005,"Calendar Year (Jan - Dec), 2005" +Jammu And Kashmir,2006,Female,71.0,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2006,Male,69.2,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2006,Total,70.0,2006,"Calendar Year (Jan - Dec), 2006" +Jammu And Kashmir,2007,Female,70.8,2007,"Calendar Year (Jan - Dec), 2007" +Jammu And Kashmir,2007,Male,68.6,2007,"Calendar Year (Jan - Dec), 2007" +Jammu And Kashmir,2007,Total,69.6,2007,"Calendar Year (Jan - Dec), 2007" +Uttarakhand,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Jammu And Kashmir,2008,Female,70.6,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2008,Male,69.0,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2008,Total,69.8,2008,"Calendar Year (Jan - Dec), 2008" +Jammu And Kashmir,2009,Female,71.2,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2009,Male,68.9,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2009,Total,70.0,2009,"Calendar Year (Jan - Dec), 2009" +Jammu And Kashmir,2010,Female,71.1,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2010,Male,69.2,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2010,Total,70.1,2010,"Calendar Year (Jan - Dec), 2010" +Jammu And Kashmir,2011,Female,71.9,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2011,Male,69.4,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2011,Total,70.5,2011,"Calendar Year (Jan - Dec), 2011" +Jammu And Kashmir,2012,Female,72.4,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2012,Male,69.9,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2012,Total,71.0,2012,"Calendar Year (Jan - Dec), 2012" +Jammu And Kashmir,2013,Female,74.0,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2013,Male,70.6,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2013,Total,72.0,2013,"Calendar Year (Jan - Dec), 2013" +Jammu And Kashmir,2014,Female,74.9,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2014,Male,70.9,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2014,Total,72.6,2014,"Calendar Year (Jan - Dec), 2014" +Jammu And Kashmir,2015,Female,76.1,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2015,Male,71.2,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2015,Total,73.2,2015,"Calendar Year (Jan - Dec), 2015" +Jammu And Kashmir,2016,Female,68.5,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2016,Male,71.6,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2016,Total,73.5,2016,"Calendar Year (Jan - Dec), 2016" +Jammu And Kashmir,2017,Female,76.7,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2017,Male,72.1,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2017,Total,74.1,2017,"Calendar Year (Jan - Dec), 2017" +Jammu And Kashmir,2018,Female,76.2,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2018,Male,72.2,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2018,Total,74.0,2018,"Calendar Year (Jan - Dec), 2018" +Jammu And Kashmir,2019,Female,76.1,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2019,Male,72.6,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2019,Total,74.2,2019,"Calendar Year (Jan - Dec), 2019" +Jammu And Kashmir,2020,Female,76.3,2020,"Calendar Year (Jan - Dec), 2020" +Jammu And Kashmir,2020,Male,72.6,2020,"Calendar Year (Jan - Dec), 2020" +Jammu And Kashmir,2020,Total,74.3,2020,"Calendar Year (Jan - Dec), 2020" +Himachal Pradesh,1995,Total,,1995,"Calendar Year (Jan - Dec), 1995" +Himachal Pradesh,1997,Female,65.2,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1997,Male,64.6,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1997,Total,64.9,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,1998,Female,65.5,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1998,Male,64.8,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1998,Total,65.2,1998,"Calendar Year (Jan - Dec), 1998" +Himachal Pradesh,1999,Female,67.7,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,1999,Male,64.8,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,1999,Total,66.2,1999,"Calendar Year (Jan - Dec), 1999" +Himachal Pradesh,2000,Female,68.3,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2000,Male,65.7,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2000,Total,67.0,2000,"Calendar Year (Jan - Dec), 2000" +Himachal Pradesh,2001,Female,69.4,2001,"Calendar Year (Jan - Dec), 2001" +Himachal Pradesh,2001,Male,66.3,2001,"Calendar Year (Jan - Dec), 2001" +Himachal Pradesh,2001,Total,67.8,2001,"Calendar Year (Jan - Dec), 2001" +Uttarakhand,1997,Female,,1997,"Calendar Year (Jan - Dec), 1997" +Himachal Pradesh,2002,Female,70.2,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2002,Male,66.6,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2002,Total,68.3,2002,"Calendar Year (Jan - Dec), 2002" +Himachal Pradesh,2003,Female,71.4,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2003,Male,67.1,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2003,Total,69.1,2003,"Calendar Year (Jan - Dec), 2003" +Himachal Pradesh,2004,Female,72.0,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2004,Male,67.3,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2004,Total,69.5,2004,"Calendar Year (Jan - Dec), 2004" +Himachal Pradesh,2005,Female,71.8,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2005,Male,67.4,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2005,Total,69.5,2005,"Calendar Year (Jan - Dec), 2005" +Himachal Pradesh,2006,Female,72.0,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2006,Male,67.4,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2006,Total,69.6,2006,"Calendar Year (Jan - Dec), 2006" +Himachal Pradesh,2007,Female,72.2,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2007,Male,67.8,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2007,Total,69.9,2007,"Calendar Year (Jan - Dec), 2007" +Himachal Pradesh,2008,Female,72.1,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2008,Male,67.7,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2008,Total,69.8,2008,"Calendar Year (Jan - Dec), 2008" +Himachal Pradesh,2009,Female,72.0,2009,"Calendar Year (Jan - Dec), 2009" +Himachal Pradesh,2009,Male,67.7,2009,"Calendar Year (Jan - Dec), 2009" \ No newline at end of file From a6378ef500f4311fc69a5a3306fb4d8890d89ac8 Mon Sep 17 00:00:00 2001 From: Avantika Singh Date: Thu, 26 Jun 2025 09:26:55 +0000 Subject: [PATCH 5/5] IndiaNSS Health Ailment autorefresh --- .../download_script.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 statvar_imports/india_ndap/indiaNSS_healthailments/download_script.py diff --git a/statvar_imports/india_ndap/indiaNSS_healthailments/download_script.py b/statvar_imports/india_ndap/indiaNSS_healthailments/download_script.py new file mode 100644 index 0000000000..7576f86b39 --- /dev/null +++ b/statvar_imports/india_ndap/indiaNSS_healthailments/download_script.py @@ -0,0 +1,82 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 20 ('License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How to run the script to download the files: +# python3 download_script.py +import json +import os +import pandas as pd +import sys +from absl import app +from absl import flags +from absl import logging +from google.cloud import storage +flags.DEFINE_string( + 'config_file_path', + 'gs://datcom-import-test/statvar_imports/india_ndap/indiaNSS_healthailments/download_config.json', + 'Input directory where config files downloaded.') + +_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(_SCRIPT_PATH, '../../../util/')) +from download_util_script import _retry_method + + +def main(_): + all_data = [] + _FLAGS = flags.FLAGS + config_file_path = _FLAGS.config_file_path + storage_client = storage.Client() + bucket_name = config_file_path.split('/')[2] + bucket = storage_client.bucket(bucket_name) + blob_name = '/'.join(config_file_path.split('/')[3:]) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + try: + file_config = json.loads(file_contents) + url = file_config.get('url') + input_files = file_config.get('input_files') + except json.JSONDecodeError: + logging.fatal("Cannot extract url and input files path.") + page_num = 1 + while True: + api_url = f"{url}&pageno={page_num}" + response = _retry_method(api_url, None, 3, 5, 2) + response_data = response.json() + if response_data and 'Data' in response_data and len(response_data['Data']) > 0: + keys = response_data['Data'] + try: + if not isinstance(keys, list): + logging.fatal(f"Value for key 'Data' is not a list.") + except KeyError as e: + logging.fatal(f"Missing expected key '{e}' in the API response.") + break + # Considering the table id I7375_4 which is specific to the import. + for i in keys: + a = (i['StateName'], i['TRU'],i['D7300_3'],i['D7300_4'],i['D7300_5'],i['I7300_6']['TotalPopulationWeight'],i['I7300_7']['avg'],i['I7300_8']['avg'],i['Year'].split(",")[-1].strip() + str(int(i['Year'].split(",")[-1].strip()) + 1),i['Year'].split(",")[-1].strip(), i['Year']) + all_data.append(a) + page_num += 1 + else: + logging.error(f"failed to retrieve data from page {page_num}") + break + if all_data: + df = pd.DataFrame(all_data, columns=['srcStateName','TRU','GENDER', 'Broad ailment category','Age group','Ailments reported for each Broad caliment category per 100000 persons during last 15 days by different age groups','Estimated number of ailments under broad ailment category', 'Sample number of ailments under broad ailment category', 'srcYear', 'YearCode', 'Year']) + os.makedirs(input_files, exist_ok=True) + input_filename = os.path.join(input_files, 'IndiaNSS_HealthAilments.csv') + df.to_csv(input_filename, index=False) + logging.info("Data saved to ndiaNSS_HealthAilments.csv") + else: + logging.info("No data was retrieved from the API.") + +if __name__=="__main__": + app.run(main) \ No newline at end of file