Define full table ID upfront.

Author: toddbirchard

Makefile

@@ -1,5 +1,5 @@
 SRCPATH := $(shell pwd)
-PROJECTNAME := $(shell basename $(CURDIR))
+PROJECTNAME := $(shell basename $CURDIR)
 ENTRYPOINT := $(PROJECTNAME).ini
 
 define HELP
@@ -46,7 +46,7 @@ restart: env
 .PHONY: deploy
 deploy:
 	make clean
-	$(shell . ./deploy.sh)
+	. ./deploy.sh
 
 
 .PHONY: update
@@ -73,10 +73,3 @@ clean:
 	find . -name '__pycache__' -delete
 	find . -name 'poetry.lock' -delete
 	find . -name 'Pipefile.lock' -delete
-	find . -name 'logs/*.json' -delete
-	find . -name '*.log' -delete
-	find . -name '*/.pytest_cache' -delete
-	find . -name '*/logs/*.json' -delete
-	rm -rf .pytest_cache
-	rm -rf tests/.pytest_cache
-	rm -rf clients/tests/.pytest_cache

bigquery_python_tutorial/__init__.py

@@ -10,6 +10,7 @@
     GCP_BIGQUERY_DATASET_ID,
     GCP_BIGQUERY_TABLE_ID,
     GCP_BUCKET_NAME,
+    GCP_BIGQUERY_FULL_TABLE_ID,
 )
 
 
@@ -19,9 +20,7 @@ def init_script():
     blob = upload_csv_data(LOCAL_CSV_FILEPATH, GCP_BUCKET_NAME, REMOTE_CSV_DESTINATION)
 
     # Insert CSV as new BigQuery table
-    table = gcs_csv_to_table(
-        GCP_PROJECT_ID, GCP_BIGQUERY_DATASET_ID, GCP_BIGQUERY_TABLE_ID, blob.name
-    )
+    table = gcs_csv_to_table(GCP_BIGQUERY_FULL_TABLE_ID, blob.name)
 
     # Print schema of newly created table
     get_table_schema(table)

bigquery_python_tutorial/insert.py

@@ -6,24 +6,17 @@
 from config import GCP_BUCKET_NAME
 
 
-def gcs_csv_to_table(
-    project_id: str, dataset_id: str, table_id: str, remote_csv_path: str
-) -> Table:
+def gcs_csv_to_table(full_table_id: str, remote_csv_path: str) -> Table:
     """
     Insert CSV from Google Storage to BigQuery Table.
 
-    :param project_id: Google Cloud project ID.
-    :type project_id: str
-    :param dataset_id: ID of Google BigQuery dataset.
-    :type dataset_id: str
-    :param table_id: ID of Google BigQuery table.
-    :type table_id: str
+    :param full_table_id: Full ID of a Google BigQuery table.
+    :type full_table_id: str
     :param remote_csv_path: Path to uploaded CSV.
     :type remote_csv_path: str
     :returns: str
     """
     try:
-        full_table_id = f"{project_id}.{dataset_id}.{table_id}"
         gcs_csv_uri = f"gs://{GCP_BUCKET_NAME}/{remote_csv_path}"
         job_config = LoadJobConfig(
             autodetect=True,
@@ -37,6 +30,6 @@ def gcs_csv_to_table(
         LOGGER.info(load_job.result())  # Waits for table load to complete.
         return gbq.get_table(full_table_id)
     except BadRequest as e:
-        LOGGER.error(f"Invalid GCP request when creating table `{table_id}`: {e}")
+        LOGGER.error(f"Invalid GCP request when creating table `{full_table_id}`: {e}")
     except Exception as e:
-        LOGGER.error(f"Unexpected error when creating table `{table_id}`: {e}")
+        LOGGER.error(f"Unexpected error when creating table `{full_table_id}`: {e}")

bigquery_python_tutorial/tables.py

@@ -18,33 +18,28 @@ def list_all_tables() -> List[Optional[Table]]:
     for dataset in gbq.list_datasets():
         for listed_table in gbq.list_tables(dataset.dataset_id):
             table = get_table(
-                listed_table.project, listed_table.dataset_id, listed_table.table_id
+                f"{listed_table.project}.{listed_table.dataset_id}.{listed_table.table_id}"
             )
             tables.append(table)
     return tables
 
 
-def get_table(project_id: str, dataset_id: str, table_id: str) -> Table:
+def get_table(full_table_id: str) -> Table:
     """
     Get a single Google BigQuery table.
 
-    :param project_id: Google Cloud project ID.
-    :type project_id: str
-    :param dataset_id: ID of Google BigQuery dataset.
-    :type dataset_id: str
-    :param table_id: ID of Google BigQuery table.
-    :type table_id: str
+    :param full_table_id: Full ID of a Google BigQuery table.
+    :type full_table_id: str
     :returns: Table
     """
     try:
-        full_table_id = f"{project_id}.{dataset_id}.{table_id}"
         table = gbq.get_table(full_table_id)
-        LOGGER.info(f"{dataset_id}.{table_id}: {table.num_rows} rows.")
+        LOGGER.info(f"{table.dataset_id}.{table.table_id}: {table.num_rows} rows.")
         return table
     except BadRequest as e:
-        LOGGER.error(f"Invalid GCP request when fetching table `{table_id}`: {e}")
+        LOGGER.error(f"Invalid GCP request when fetching table `{full_table_id}`: {e}")
     except Exception as e:
-        LOGGER.error(f"Unexpected error when fetching table `{table_id}`: {e}")
+        LOGGER.error(f"Unexpected error when fetching table `{full_table_id}`: {e}")
 
 
 def get_table_schema(table: Table) -> Sequence[Union[SchemaField, Mapping[str, Any]]]:

config.py

@@ -13,6 +13,9 @@
 # Google BigQuery
 GCP_BIGQUERY_TABLE_ID: str = getenv("GCP_BIGQUERY_TABLE_ID")
 GCP_BIGQUERY_DATASET_ID: str = getenv("GCP_BIGQUERY_DATASET_ID")
+GCP_BIGQUERY_FULL_TABLE_ID = (
+    f"{GCP_PROJECT_ID}.{GCP_BIGQUERY_DATASET_ID}.{GCP_BIGQUERY_TABLE_ID}"
+)
 
 # Google Cloud Storage
 GCP_BUCKET_URI: str = getenv("GCP_BUCKET_URI")