Use only client time and update results

Author: Aniket Mokashi

bigquery/results/result.json

@@ -9,52 +9,52 @@
 
     "tags": ["serverless", "column-oriented", "gcp", "managed"],
 
-    "load_time": 781.50,
+    "load_time": 776.91,
     "data_size": 8760000000,
 
     "result": [
-[0.714,0.503,0.405],
-[0.53,0.33,0.426],
-[0.73,0.408,0.432],
-[0.799,0.33,0.281],
-[0.623,0.494,0.437],
-[0.738,0.553,0.546],
-[0.344,0.359,0.407],
-[0.434,0.772,0.379],
-[0.993,0.654,0.63],
-[0.855,0.855,0.82],
-[0.494,0.384,0.411],
-[0.585,0.564,0.596],
-[0.669,0.516,0.536],
-[2.199,2.128,1.78],
-[0.791,0.635,0.703],
-[0.571,0.561,0.546],
-[0.774,0.725,0.742],
-[0.577,0.896,0.517],
-[1.704,1.341,1.182],
-[0.31,0.289,0.331],
-[1.033,0.499,0.536],
-[0.588,0.62,0.544],
-[1.015,0.689,0.784],
-[1.399,0.754,0.903],
-[0.37,0.491,0.363],
-[0.369,0.348,0.34],
-[0.395,0.404,0.391],
-[0.651,0.667,0.818],
-[1.304,1.163,1.158],
-[0.659,0.425,0.529],
-[0.684,0.553,0.542],
-[0.876,0.57,0.492],
-[1.024,1.022,1.119],
-[1.261,1.257,1.374],
-[1.057,0.918,0.893],
-[0.556,0.585,0.522],
-[0.57,0.498,0.532],
-[0.473,0.411,0.411],
-[0.387,0.453,0.38],
-[0.703,0.623,0.626],
-[0.442,0.408,0.337],
-[0.372,0.361,0.442],
-[0.403,0.358,0.388]
+[0.383933,0.402355,0.370758],
+[0.334439,0.433776,0.416341],
+[0.469506,0.359557,0.386433],
+[0.491417,0.333208,0.4758],
+[0.552464,0.652322,0.555889],
+[0.581302,0.603089,0.674999],
+[1.087835,0.639649,0.360542],
+[0.438221,0.759105,0.497731],
+[0.702109,0.712533,0.678109],
+[0.857454,0.968303,0.995039],
+[0.547042,0.479513,0.475109],
+[0.547026,0.549529,0.614708],
+[0.686315,0.580551,0.630673],
+[1.792573,2.034019,1.845895],
+[0.610674,0.677655,0.643796],
+[0.580303,0.729024,0.622044],
+[0.760401,0.809858,0.822725],
+[0.721757,0.611165,0.744566],
+[1.49368,1.372045,1.498892],
+[0.363523,0.383959,0.366856],
+[0.625735,0.49802,0.473233],
+[0.513777,0.508772,0.527258],
+[0.895406,0.874879,0.799704],
+[0.909036,0.679151,0.730413],
+[0.358434,0.509104,0.467827],
+[0.421586,0.428603,0.33761],
+[0.54752,0.364919,0.444499],
+[0.691434,0.674469,0.930067],
+[1.143579,1.034013,1.105913],
+[0.569294,0.444362,0.463864],
+[0.517151,0.53565,0.523663],
+[0.56208,0.573,0.543899],
+[1.409102,1.116484,1.295522],
+[1.413902,1.346194,1.406088],
+[1.068575,0.985308,1.194028],
+[0.781501,0.524615,0.664192],
+[0.678144,0.666519,0.548661],
+[0.477265,0.445584,0.469621],
+[0.554599,0.530927,0.551336],
+[0.777017,0.696796,0.810055],
+[0.427604,0.43113,0.449339],
+[0.434927,0.407959,0.435918],
+[0.478507,0.425838,0.541504]
 ]
 }

bigquery/run_queries.py

@@ -22,7 +22,7 @@ def log(*objects: Any, sep: str = ' ', end: str = '\n', file: TextIO = sys.stder
     message = sep.join(str(obj) for obj in objects)
 
     # 2. Prepare the log prefix
-    timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     prefix = f"[{timestamp}] [{severity.upper()}]: "
 
     # 3. Combine the prefix and the message
@@ -52,33 +52,17 @@ def log(*objects: Any, sep: str = ' ', end: str = '\n', file: TextIO = sys.stder
     log(f"\n[{i}]: {query}")
     try:
       client_start_time = datetime.now()
-      query_job = client.query(query, job_config=job_config)
-      results = query_job.result()
+      results = client.query_and_wait(query, job_config=job_config)
       client_end_time = datetime.now()
 
-      execution_time = query_job.ended - query_job.started
-      total_time = query_job.ended - query_job.created
-      total_time_secs = total_time.total_seconds()
+      client_time = client_end_time - client_start_time
+      client_time_secs = client_time.total_seconds()
       endstr = "],\n" if i == 2 else ","
-      print(f"{total_time_secs}", end=endstr)
+      print(f"{client_time_secs}", end=endstr)
 
-      client_time = client_end_time - client_start_time 
-      log(f"Job ID: **{query_job.job_id}**")
-      log(f"Query ID: **{query_job.query_id}**")
-      log(f"State: **{query_job.state}**")
-      log(f"Results Fetched from Cache: {query_job.cache_hit}")
-      log(f"Created Time: {query_job.created}")
-      log(f"Start Time: {query_job.started}")
-      log(f"End Time: {query_job.ended}")
-      log(f"Execution Time: {execution_time}")
-      log(f"Total Time: {total_time}")
-      log(f"Client Time: {client_time}")
-      log(f"Total Rows Returned: {results.total_rows}")
+      log(f"Job ID: **{results.job_id}**")
+      log(f"Query ID: **{results.query_id}**")
+      log(f"Client time: **{client_time}**")
 
     except Exception as e:
       log(f"Job failed with error: {e}", severity="ERROR")
-      # Print error details from the job itself
-      if query_job.error_result:
-        log("\n--- Job Error Details ---")
-        log(f"Reason: {query_job.error_result.get('reason')}")
-        log(f"Message: {query_job.error_result.get('message')}")