create trips after import

models.py

@@ -44,15 +44,6 @@ class CensoredLocation(models.Model):
   radius = models.IntegerField(default=800)
   name = models.CharField(max_length=255)
 
-  def delete_markers(self) -> int:
-    matches = Marker.objects.filter(location__distance_lt=(
-      self.location,
-      distance_to_decimal_degrees(D(m=self.radius), self.location.y)
-    ))
-    count = len(matches)
-    matches.delete()
-    return count
-
 
 def distance_to_decimal_degrees(distance, latitude):
     """

views.py

@@ -1,5 +1,6 @@
 import json
 import gzip
+import pytz
 from datetime import datetime
 
 from django.shortcuts import render
@@ -10,6 +11,7 @@ from django.core.cache import cache
 from django.contrib.auth.decorators import permission_required
 from django.contrib.auth.mixins import PermissionRequiredMixin
 from django.contrib.gis.geos import Point
+from django.db.models import Q
 
 from .models import *
 from .workers.points2trips import TripConverter
@@ -25,14 +27,16 @@ class MarkerCreateView(PermissionRequiredMixin, View):
   permission_required = ("gps-logger.markers.change")
 
   def post(self, request):
+    utctz = pytz.timezone("UTC")
     insert_count = censor_count = replace_count = 0
     data = json.loads(request.body)
+    minDate = datetime.utcnow().replace(tzinfo=utctz)
     for file in data:
       if len(data[file]) == 0:
         continue
       objects = [Marker(
           timestamp=datetime.strptime(x['timestamp'].replace("Z","-0000"), r"%Y-%m-%dT%H:%M:%S.%f%z"),
-          location = Point(x['lng'], x['lat']),
+          location = Point(x['lng'], x['lat'], srid=4326),
           alt = x['alt'],
           hdop = x.get('hdop', None),
           speed = x.get('speed', None)
@@ -40,8 +44,10 @@ class MarkerCreateView(PermissionRequiredMixin, View):
       start_date = objects[0].timestamp
       end_date = objects[-1].timestamp
       oldMarkers = Marker.objects.filter(timestamp__range=(start_date, end_date)).all()
-      identical = len(oldMarkers) == len(objects)
-    
+      identical = oldMarkers.count() == len(objects)
+
+      minDate = min(minDate, start_date)
+
       if identical:
         for i, marker in enumerate(oldMarkers):
           if marker.timestamp != objects[i].timestamp:
@@ -52,27 +58,34 @@ class MarkerCreateView(PermissionRequiredMixin, View):
         for i, marker in enumerate(oldMarkers):
           objects[i].id = marker.id
         Marker.objects.bulk_update(objects, ['location', 'alt', 'hdop', 'speed'])
-        replace_count += len(oldMarkers)
+        replace_count += oldMarkers.count()
       else:
-        insert_count += len(objects) - len(oldMarkers)
+        insert_count += len(objects) - oldMarkers.count()
         oldMarkers.delete()
         Marker.objects.bulk_create(objects)
     
     censor_count = censor()
 
+    minDate = minDate.replace(hour=0, minute=0, second=0, microsecond=0)
+    print("regenerating trips after", minDate)
+    tc = TripConverter(minDate)
+    tc.run()
+    cache.delete('trips')
+
     return JsonResponse({
-        'censored': censor_count,
-        'replaced': replace_count,
-        'inserted': insert_count - censor_count
+        'markers_censored': censor_count,
+        'markers_replaced': replace_count,
+        'markers_inserted': insert_count - censor_count,
+        'trips_created': tc.created,
+        'trips_updated': tc.updated,
+        'trips_skipped': tc.skipped
       }, status=201)
 
 class CreateTripsView(PermissionRequiredMixin, View):
   permission_required = ("gps-logger.trips.change")
 
   def get(self, request):
-    data = list(Marker.objects.all())
-
-    tc = TripConverter(data)
+    tc = TripConverter()
     tc.run()
     cache.delete('trips')
     return JsonResponse({
@@ -123,10 +136,14 @@ class DeleteCensoredView(PermissionRequiredMixin, View):
     return JsonResponse({'censored': counter})
 
 def censor() -> int:
-  counter = 0
   locations = CensoredLocation.objects.all()
+  filter = None
   for location in locations:
-    counter += location.delete_markers()
-  if(cache.get('markers')):
-    cache.delete('markers')
+    f = Q(location__distance_lte=(location.location, distance_to_decimal_degrees(D(m=location.radius), location.location.y)))
+    filter = f if filter is None else filter | f
+
+  matches = Marker.objects.filter(filter)
+  counter, _ = matches.delete()
+
+  cache.delete('markers')
   return counter

workers/points2trips.py

@@ -22,32 +22,40 @@ class TripConverter:
   updated: int
   skipped: int
 
-  def __init__(self, markers: list[Marker]):
-    self.markers = markers
-    self.trips = []
+  def __init__(self, startDate: datetime=None):
+    if startDate:
+      self.markers = Marker.objects.filter(timestamp__gte=startDate).all()
+      self.trips = list(Trip.objects.filter(endTime__gte=startDate).only("startTime", "endTime", "totalTime"))
+    else:
+      self.markers = Marker.objects.all()
+      self.trips = list(Trip.objects.only("startTime", "endTime", "totalTime"))
     self.created = 0
     self.updated = 0
     self.skipped = 0
 
   def run(self):
-    first_index = 0
-    for i, point in enumerate(self.markers[1:]):
-      prev_point = self.markers[i-1]
-      if point.timestamp - prev_point.timestamp > max_time_diff or \
-          Distance(point.location, prev_point.location) > max_distance:
-        if i - first_index > 2:
-          self.trips.append(self.create_trip(self.markers[first_index:i]))
-        first_index = i
-    
-    if first_index < len(self.markers) - 2:
-      self.trips.append(self.create_trip(self.markers[first_index:]))
+    tmp: list[Marker] = []
+    for prev, marker in zip(self.markers, self.markers[1:]):
+      if marker.timestamp - prev.timestamp > max_time_diff or Distance(marker.location, prev.location) > max_distance:
+        if len(tmp) > 1:
+          self.trips.append(self.create_trip(tmp))
+        tmp = []
+      tmp.append(marker)
+    if len(tmp) > 1:
+      self.trips.append(self.create_trip(tmp))
 
   def create_trip(self, markers: list[Marker]) -> Trip:
     print(len(markers), markers[0].timestamp, markers[-1].timestamp)
-    trip = Trip.objects.filter(startTime__lte=markers[-1].timestamp, endTime__gte=markers[0].timestamp).first()
+    trip = None
+    for current in self.trips:
+      if current.startTime <= markers[-1].timestamp and current.endTime >= markers[0].timestamp:
+        trip = current
+        self.trips.remove(current)
+        break
+    
     center = markers[len(markers)//2].location
     if not trip:
-      trip = Trip.objects.create(
+      trip = Trip(
         startTime = markers[0].timestamp,
         endTime = markers[-1].timestamp,
         name = f"Trip {markers[0].timestamp}",