Merge pull request #13 from Highlanderix/master

Changes made as Google uses a new datetime format

Author: matthieuheitz

get_visit_times.py

@@ -16,12 +16,13 @@
 
 import json
 import numpy as np
-import datetime
+import datetime as dt
+from datetime import datetime
 
 
 # Helper functions
 def date_ymd_to_timestamp_ms(y,m,d):
-    return datetime.datetime(y,m,d).timestamp()*1e3
+    return dt.datetime(y,m,d).timestamp()*1e3
 
 
 def deg2rad(a):
@@ -38,10 +39,17 @@ def dist_btw_two_points(p1,p2):
     r = 6371*1e3    # Earth's radius: 6371kms in meters
     return 2*r*np.arcsin(np.sqrt(np.sin((phi2-phi1)/2)**2 + np.cos(phi1)*np.cos(phi2)*np.sin((lba2-lba1)/2)**2))
 
+def parse_rfc3339(datetime_str: str) -> datetime:
+    try:
+        return datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%fZ")
+    except ValueError:
+        # Perhaps the datetime has a whole number of seconds with no decimal
+        # point. In that case, this will work:
+        return datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%SZ")
 
 # LOAD DATA
 
-json_file = "location_history.json"
+json_file = "Records.json"
 # Read the file
 print("Loading '%s' ..."%json_file)
 main_dict = json.load(open(json_file))   # This can take a bit of time
@@ -63,15 +71,17 @@ def dist_btw_two_points(p1,p2):
 
 # Build arrays of basic data
 print("Extracting relevant data...")
-timestampMs = np.zeros(n)   # in milliseconds
+timestamp = np.empty(n, object)   # empty as timestamp has to be parsed first
 positions = np.zeros([n,2])  # in degrees
 accuracy = np.zeros(n)      # don't know the unit
 # activity = {}         # Don't store activity since we don't use it
 
 for i in range(n):
     point = data[i]
-    if 'timestampMs' in point:
-        timestampMs[i] = float(point['timestampMs'])
+    if 'timestamp' in point:
+        timestamp[i] = point['timestamp']
+        timestamp[i] = parse_rfc3339(timestamp[i])
+        timestamp[i] = float(datetime.timestamp(timestamp[i])*1e3)
     if ('latitudeE7' in point) and ('longitudeE7' in point):
         positions[i] = np.array([float(point['latitudeE7']),float(point['longitudeE7'])])/1e7
     if 'accuracy' in point:
@@ -92,16 +102,16 @@ def dist_btw_two_points(p1,p2):
 #############
 
 # Get first and last timestamps of interest
-begin_ts = date_ymd_to_timestamp_ms(2018,9,1)
-end_ts = date_ymd_to_timestamp_ms(2018,9,30)
-# end_ts = timestampMs[-1]    # Last one
+begin_ts = date_ymd_to_timestamp_ms(2013,12,1)
+end_ts = date_ymd_to_timestamp_ms(2023,6,11)
+# end_ts = timestamp[-1]    # Last one
 
 # Point of interest
-poi = np.array([45.773944,4.890715])    # in degrees
+poi = np.array([44.84176165939715, -0.5690463850963396])    # in degrees
 radius_max = 50                         # in meters
 
 # Define the interval of time below which timestamps should be grouped together
-group_size = datetime.timedelta(weeks=0, days=0, hours=1, minutes=0, seconds=0, milliseconds=0)
+group_size = dt.timedelta(weeks=0, days=0, hours=1, minutes=0, seconds=0, milliseconds=0)
 # Amount of info to show about each group:
 # 0: None
 # 1: Number of points in group
@@ -113,12 +123,12 @@ def dist_btw_two_points(p1,p2):
 ####################
 
 # Converter
-ts2datetime = lambda x: datetime.datetime.utcfromtimestamp(int(x/1e3)).strftime('%Y-%m-%d %H:%M:%S')
+ts2datetime = lambda x: dt.datetime.utcfromtimestamp(int(x/1e3)).strftime('%Y-%m-%d %H:%M:%S')
 
 # Get time boundary index
 # np.searchsorted is a fast way to find the first element that is larger than the threshold. 1 is for True
-begin_index = np.searchsorted(timestampMs >= begin_ts, 1)
-end_index = np.searchsorted(timestampMs >= end_ts, 1)
+begin_index = np.searchsorted(timestamp >= begin_ts, 1)
+end_index = np.searchsorted(timestamp >= end_ts, 1)
 
 # Get group_size in milliseconds
 grpsMs =  group_size.total_seconds()*1000
@@ -141,23 +151,23 @@ def dist_btw_two_points(p1,p2):
 prev = 0    # Keeps in memory the last timestamp displayed
 for i in range(close_points.size):
     # If the delta between timestamps is bigger than the group size, or if it's the beginning or the end.
-    if timestampMs[close_points[i]]-timestampMs[close_points[prev]] > grpsMs or i == 0 or i == close_points.size-1:
+    if timestamp[close_points[i]]-timestamp[close_points[prev]] > grpsMs or i == 0 or i == close_points.size-1:
         # If it's not the beginning and there are at least 2 points to make a group.
         if i>0 and i-prev > 2:
             if i == close_points.size-1: i=i+1
             if group_verbosity == 1:
                 print("\tGroup of %d points"%(i-prev-2))
             if group_verbosity == 2:
                 print("\n\tGroup of %d points: %d -> %d"%(i-prev-2,close_points[prev+1],close_points[i-1]))
-                pt_date_im1 = ts2datetime(timestampMs[close_points[i-1]])
-                pt_date_prevp1 = ts2datetime(timestampMs[close_points[prev+1]])
+                pt_date_im1 = ts2datetime(timestamp[close_points[i-1]])
+                pt_date_prevp1 = ts2datetime(timestamp[close_points[prev+1]])
                 print("\tFrom: %s"%pt_date_prevp1)
                 print("\tTo  : %s"%pt_date_im1)
                 print("\tMean dist to POI: %0.1fm\n"%np.mean(dist2poi[prev+1:i]))
 
         # if group_verbosity == 2: print()    # Add space between lines
         # Else, display the point, unless it's the end
         if i != close_points.size:
-            pt_date = ts2datetime(timestampMs[close_points[i]])
+            pt_date = ts2datetime(timestamp[close_points[i]])
             print("Point %d  --  Date: %s  --  Distance to POI: %dm"%(close_points[i],pt_date,dist2poi[i]))
             prev = i