add location filtering

admin.py

@@ -1,15 +1,16 @@
 from django.contrib import admin
+from django.contrib.gis.admin import GeoModelAdmin
 
 from .models import *
 
 @admin.register(Marker)
 class MarkerAdmin(admin.ModelAdmin):
-  list_display = ["timestamp", "lat", "lng", "alt", "hdop", "speed"]
+  list_display = ["timestamp", "location", "hdop", "speed"]
 
 @admin.register(Trip)
 class TripAdmin(admin.ModelAdmin):
   list_display = ["startTime", "endTime", "name", "description"]
 
 @admin.register(CensoredLocation)
-class CensoredLocationAdmin(admin.ModelAdmin):
-  list_display = ["lat", "lng", "radius", "name"]
+class CensoredLocationAdmin(GeoModelAdmin):
+  list_display = ["location", "radius", "name"]

migrations/0005_auto_20220815_1630.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.13 on 2022-08-15 14:30
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gps_logger', '0004_alter_marker_hdop'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='censoredlocation',
+            name='lat',
+        ),
+        migrations.RemoveField(
+            model_name='censoredlocation',
+            name='lng',
+        ),
+        migrations.RemoveField(
+            model_name='marker',
+            name='alt',
+        ),
+        migrations.RemoveField(
+            model_name='marker',
+            name='lat',
+        ),
+        migrations.RemoveField(
+            model_name='marker',
+            name='lng',
+        ),
+        migrations.AddField(
+            model_name='censoredlocation',
+            name='location',
+            field=django.contrib.gis.db.models.fields.PointField(default=0, srid=4326),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='marker',
+            name='location',
+            field=django.contrib.gis.db.models.fields.PointField(default=0, dim=3, srid=4326),
+            preserve_default=False,
+        ),
+    ]

models.py

@@ -1,10 +1,10 @@
 from django.db import models
+from django.contrib.gis.db.models.fields import PointField
+from django.contrib.gis.db.models.functions import Distance
 
 class Marker(models.Model):
   timestamp = models.DateTimeField(unique=True)
-  lat = models.FloatField()
-  lng = models.FloatField()
-  alt = models.FloatField()
+  location = PointField(dim=3)
   hdop = models.IntegerField(null=True, blank=True)
   speed = models.IntegerField(null=True, blank=True)
 
@@ -15,7 +15,18 @@ class Trip(models.Model):
   description = models.TextField(null=True, blank=True)
 
 class CensoredLocation(models.Model):
-  lat = models.FloatField()
-  lng = models.FloatField()
+  location = PointField(dim=2)
   radius = models.IntegerField(default=800)
   name = models.CharField(max_length=255)
+
+  def delete_markers(self, markers = None) -> int:
+    # matches = Marker.objects.filter(location__distance_lt=(
+    #   self.location,
+    #   Distance(m=self.radius)
+    # ))
+
+    matches = Marker.objects.annotate(distance=Distance('location', self.location)
+       ).filter(distance__lte=self.radius)
+    count = len(matches)
+    matches.delete()
+    return count

static/gps_logger/js/preprocessor.js

@@ -1,6 +1,6 @@
 const options = {
   legend: {
-    position: "nw",
+    position: "ne",
     show: true,
     noColumns: 3,
   },
@@ -20,11 +20,11 @@ const options = {
   },
   xaxis: {
     tickDecimals: 2,
-    tickSize: 0.01
+    //tickSize: 0.01
   },
   yaxis: {
     tickDecimals: 2,
-    tickSize: 0.01
+    //tickSize: 0.01
   }
 }
 
@@ -52,10 +52,10 @@ window.onload = function() {
     importFiles(fileList)
   })
 
-  const button = document.getElementById("upload-btn");
+  const uploadBtn = document.getElementById("upload-btn");
   const responseP = document.getElementById("response");
-  button.addEventListener('click', (event) => {
-    button.disabled = true;
+  uploadBtn.addEventListener('click', (event) => {
+    uploadBtn.disabled = true;
     responseP.textContent = "waiting for response..."
     const xmlhttp = new XMLHttpRequest()
     xmlhttp.open("POST", "markers/create", true);
@@ -64,18 +64,42 @@ window.onload = function() {
     xmlhttp.setRequestHeader('X-CSRFToken', token)
     xmlhttp.onreadystatechange = () => {
       if(xmlhttp.readyState == 4) {
-        if(xmlhttp.status == 201)
+        if(xmlhttp.status < 300) {
           responseP.textContent = "Markers created!"
-        else if(xmlhttp.status == 204)
-          responseP.textContent = "Nothing to change!"
+          const body = JSON.parse(xmlhttp.responseText)
+          for(const field in body)
+            responseP.innerHTML += `<br/>${field}: ${body[field]}`
+        }
         else
           responseP.textContent = `Error code ${xmlhttp.status}`
-        button.disabled = false;
-
+        uploadBtn.disabled = false
       }
     }
     xmlhttp.send(JSON.stringify(markersToUpload))
   })
+
+  const censorBtn = document.getElementById("censor-btn");
+  censorBtn.addEventListener('click', (event) => {
+    censorBtn.disabled = true;
+    responseP.textContent = "waiting for response..."
+    const xmlhttp = new XMLHttpRequest()
+    xmlhttp.open("GET", "markers/censor", true)
+    xmlhttp.onreadystatechange = () => {
+      if(xmlhttp.readyState == 4) {
+        if(xmlhttp.status < 300) {
+          responseP.textContent = "Markers censored!"
+          const body = JSON.parse(xmlhttp.responseText)
+          for(const field in body)
+            responseP.innerHTML += `<br/>${field}: ${body[field]}`
+        }
+        else
+          responseP.textContent = `Error code ${xmlhttp.status}`
+          censorBtn.disabled = false
+      }
+    }
+    xmlhttp.send()
+  })
+
 }
 function importFiles(fileList) {
   d = []
@@ -87,7 +111,7 @@ function importFiles(fileList) {
     else if (file.name?.toLowerCase().endsWith(".nmea"))
       parseNmeaFile(file)
   }
-  document.getElementById("upload-btn").style.visibility = "visible";
+  document.getElementById("upload-btn").style.display = "inline";
 }
 
 function readTxtFile(file) {

templates/gps_logger/upload.html

@@ -38,7 +38,8 @@
   </div>
   {% csrf_token %}
   <p>
-    <input type="submit" id="upload-btn" style="visibility: hidden"/>
+    <input type="submit" id="upload-btn" style="display: none"/>
+    <input type="button" id="censor-btn" value="Delete censored"/>
   </p>
   <p id="response"></p>
 </div>

urls.py

@@ -6,5 +6,6 @@ urlpatterns = [
   path('', views.index, name='index'),
   path('upload', views.upload, name='upload'),
   path('markers/create', views.MarkerCreateView.as_view(), name='marker-create'),
-  path('markers', views.MarkerView.as_view(), name='marker')
+  path('markers/censor', views.DeleteCensoredView.as_view(), name='marker-censor'),
+  path('markers', views.marker_view, name='marker')
 ]

views.py

@@ -2,14 +2,15 @@ import json
 from datetime import datetime
 from django.shortcuts import render
 from django.http import JsonResponse, HttpResponse
+from django.core.serializers.json import DjangoJSONEncoder
 from django.views import View
+from django.core.cache import cache
 from django.contrib.auth.decorators import permission_required
 from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.contrib.gis.geos import Point
 
 from .models import *
 
-from pprint import pprint
-
 def index(request):
   return render(request, 'gps_logger/index.html')
 
@@ -21,16 +22,14 @@ class MarkerCreateView(View, PermissionRequiredMixin):
   permission_required = ("gps-logger.markers.change")
 
   def post(self, request):
-    status_code = 204
+    insert_count = censor_count = replace_count = 0
     data = json.loads(request.body)
     for file in data:
       if len(data[file]) == 0:
         continue
       objects = [Marker(
           timestamp=datetime.strptime(x['timestamp'].replace("Z","-0000"), r"%Y-%m-%dT%H:%M:%S.%f%z"),
-          lat = x['lat'],
-          lng = x['lng'],
-          alt = x['alt'],
+          location = Point(x['lng'], x['lat'], x['alt']),
           hdop = x.get('hdop', None),
           speed = x.get('speed', None)
         ) for x in data[file]]
@@ -38,21 +37,58 @@ class MarkerCreateView(View, PermissionRequiredMixin):
       end_date = objects[-1].timestamp
       oldMarkers = Marker.objects.filter(timestamp__range=(start_date, end_date)).all()
       identical = len(oldMarkers) == len(objects)
+    
       if identical:
         for i, marker in enumerate(oldMarkers):
           if marker.timestamp != objects[i].timestamp:
-            print(type(marker.timestamp), type(objects[i].timestamp))
             identical = False
             break
 
-      print(len(oldMarkers), len(objects), identical)      
-      if not identical:
+      if identical:
+        for i, marker in enumerate(oldMarkers):
+          objects[i].id = marker.id
+        Marker.objects.bulk_update(objects, ['timestamp', 'location', 'hdop', 'speed'])
+        replace_count += len(oldMarkers)
+      else:
+        insert_count += len(objects) - len(oldMarkers)
         oldMarkers.delete()
         Marker.objects.bulk_create(objects)
-        status_code = 201
-    return HttpResponse(status=status_code)
+    
+    locations = CensoredLocation.objects.all()
+    for location in locations:
+      censor_count += location.delete_markers()
 
+    if(cache.get('markers')):
+      cache.delete('markers')
 
-class MarkerView(View):
+    return JsonResponse({
+        'censored': censor_count,
+        'replaced': replace_count,
+        'inserted': insert_count - censor_count
+      }, status=201)
+
+def marker_view(request):
+  data = cache.get('markers')
+  if not data:
+    values = list(Marker.objects.values())
+    for marker in values:
+      location  = marker['location']
+      if location:
+        marker['lng'] = location.x
+        marker['lat'] = location.y
+        marker['alt'] = location.z
+      del marker['location']
+    data = json.dumps(values, cls=DjangoJSONEncoder)
+    cache.set('markers', data, 3600*24)
+  return HttpResponse(data, content_type='application/json')
+
+class DeleteCensoredView(View, PermissionRequiredMixin):
+  permission_required = ("gps-logger.markers.change")
   def get(self, request):
-    return JsonResponse(list(Marker.objects.values()), safe=False)
+    counter = 0
+    locations = CensoredLocation.objects.all()
+    for location in locations:
+      counter += location.delete_markers()
+    if(cache.get('markers')):
+      cache.delete('markers')
+    return JsonResponse({'censored': counter})