add series field to csv storage

box-pc/application/database/csvFile.py

@@ -5,18 +5,33 @@ import dataclasses
 import zipfile
 import logging
 
-from structures.measurement import Measurement24v
+class CSVStorage:
+  files = {}
+
+  def __init__(self, path) -> None:
+    self.path = path
+    self.zipname = os.path.join(self.path, F"logs_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.zip")
+  
+  def write(self, values: list):
+    try:
+      for meas in values:
+        if not meas.series in self.files:
+          self.files[meas.series] = CSVFile(self.path, meas.series, self.zipname)
+        self.files[meas.series].write(meas)
+    except Exception as ex:
+      logging.exception("CSV write failed")
 
 class CSVFile:
   file = None
   filename = None
   row_count = 0
 
-  def __init__(self, path) -> None:
+  def __init__(self, path, series, zipname) -> None:
     self.path = path
+    self.series = series
     if not os.path.exists(self.path):
       os.mkdir(self.path)
-    self.zipname = os.path.join(self.path, F"logs_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.zip")
+    self.zipname = zipname
     self.new_file()
 
   def new_file(self):
@@ -27,26 +42,24 @@ class CSVFile:
         zf.write(self.filename, os.path.basename(self.filename))
       os.remove(self.filename)
       
-    self.filename = os.path.join(self.path, F"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv")
+    self.filename = os.path.join(self.path, F"{self.series}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv")
     self.file = open(self.filename, "w", newline='')
     self.writer = csv.writer(self.file, delimiter=',')
 
-  def write(self, values: list):
-    try:
-      meas: Measurement24v
-      for i, meas in enumerate(values):
-        row = dataclass_to_dict(meas)
-        if self.row_count == 0:
-          self.writer.writerow(row)
-        self.writer.writerow(row.values())
-        self.row_count += 1
+  def write(self, meas):
+    row = dataclass_to_dict(meas)
+    if self.row_count == 0:
+      self.writer.writerow(row)
+    self.writer.writerow(row.values())
+    self.row_count += 1
+
+    if self.row_count % 1000 == 0:
       self.file.flush()
 
-      if self.row_count > 50000:
-        self.new_file()
-        self.row_count = 0
-    except Exception as ex:
-      logging.exception("CSV write failed")
+    if self.row_count > 50000:
+      self.new_file()
+      self.row_count = 0
+
 
 def dataclass_to_dict(dc):
   ret = {}

box-pc/application/main.py

@@ -19,7 +19,7 @@ sources = [
 
 sinks = [
   InfluxDB("http://localhost:8086"),
-  CSVFile("logs"),
+  CSVStorage("logs"),
 ]
 
 for source in sources: