Merge branch 'pi-hole-api-replacement' into main

.dockerignore

@@ -2,4 +2,5 @@
 .git
 .github
 *~
-README.md
+README.md
+*.json

.github/workflows/docker.yml

@@ -29,6 +29,6 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v5,linux/arm/v7
           push: true
           tags: chriscn/pihole-to-influxdb:latest

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bullseye
 
 LABEL maintainer="Christopher Nethercott" \
     description="PiHole to InfluxDB data bridge"

Pi-hole-grafana-dashboard.json

@@ -0,0 +1,960 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_INFLUXDB",
+      "label": "InfluxDB",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "influxdb",
+      "pluginName": "InfluxDB"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "9.1.4"
+    },
+    {
+      "type": "datasource",
+      "id": "influxdb",
+      "name": "InfluxDB",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"queries\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"queries\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: sum, createEmpty: false)\r\n  |> yield(name: \"sum\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Queries",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "purple",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 4,
+        "y": 0
+      },
+      "id": 10,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"domains\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"cached\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: sum, createEmpty: false)\r\n  |> yield(name: \"sum\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Queries Cached",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 3,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"queries\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"blocked\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: sum, createEmpty: false)\r\n  |> yield(name: \"sum\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Queries Blocked",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "yellow",
+            "mode": "fixed"
+          },
+          "decimals": 1,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"queries\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"ads_percentage\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n  |> yield(name: \"mean\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Percentage Blocked",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "red",
+            "mode": "fixed"
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 16,
+        "y": 0
+      },
+      "id": 5,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"domains\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"domain_count\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n  |> yield(name: \"last\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Domains on Adlists",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 20,
+        "y": 0
+      },
+      "id": 11,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "max"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"clients\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"queries\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: max, createEmpty: false)\r\n  |> group(columns: [\"client\"])\r\n  |> distinct(column: \"client\")\r\n  |> keep(columns: [\"_value\", \"_time\"])\r\n  |> count()\r\n  |> yield(name: \"max\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Clients",
+      "transformations": [],
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"clients\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"queries\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n  |> yield(name: \"mean\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Queries per Client",
+      "transformations": [
+        {
+          "id": "renameByRegex",
+          "options": {
+            "regex": ".+client=\"(.+?)\".+",
+            "renamePattern": "$1"
+          }
+        }
+      ],
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 5,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"clients\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"blocked\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n  |> yield(name: \"mean\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Blocked queries per Client",
+      "transformations": [
+        {
+          "id": "renameByRegex",
+          "options": {
+            "regex": ".+client=\"(.+?)\".+",
+            "renamePattern": "$1"
+          }
+        }
+      ],
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 63,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "percent"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 14
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"query_types\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n  |> yield(name: \"last\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Types",
+      "transformations": [
+        {
+          "id": "renameByRegex",
+          "options": {
+            "regex": ".+query_type=\"(.*)\".+",
+            "renamePattern": "$1"
+          }
+        }
+      ],
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 6,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 14
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"forward_destinations\")\r\n  |> filter(fn: (r) => r[\"destination\"] != \"\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n  |> yield(name: \"mean\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Upstream Queries",
+      "transformations": [
+        {
+          "id": "renameByRegex",
+          "options": {
+            "regex": ".+destination=\"(.+?)#?\\d*\".+",
+            "renamePattern": "$1"
+          }
+        }
+      ],
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "influxdb",
+        "uid": "${DS_INFLUXDB}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "red",
+            "mode": "fixed"
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "dateTimeAsLocalNoDateIfToday"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 21
+      },
+      "id": 8,
+      "interval": "10m",
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.1.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "influxdb",
+            "uid": "${DS_INFLUXDB}"
+          },
+          "query": "from(bucket: \"pihole\")\r\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n  |> filter(fn: (r) => r[\"_measurement\"] == \"other\")\r\n  |> filter(fn: (r) => r[\"_field\"] == \"gravity_last_update\")\r\n  |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n  |> map(fn: (r) => ({r with _value: r._value * 1000}))\r\n  |> yield(name: \"last\")",
+          "refId": "A"
+        }
+      ],
+      "title": "Adlists last Updated",
+      "transparent": true,
+      "type": "stat"
+    }
+  ],
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Pi-hole",
+  "uid": "4Ku5WznVk",
+  "version": 10,
+  "weekStart": ""
+}

README.md

@@ -1,49 +1,72 @@
 # pihole-to-influxdb
+
 ## Introduction
-Based slightly on my other project, [speedtest-to-influxdb](https://github.com/chriscn/speedtest-to-influxdb). This project leverages the [Pi-Hole](https://pi-hole.net/) API to gather data about your PiHole instance and store it inside of InfluxDB for your future projects.
+Based slightly on my other project, [speedtest-to-influxdb](https://github.com/chriscn/speedtest-to-influxdb). This project leverages the [Pi-Hole](https://pi-hole.net/) API to gather data about your PiHole instance and store it inside of InfluxDB for your future projects. An example dashoard can be found [here](Pi-hole-grafana-dashboard.json).
+
+This project is automatically built through GitHub actions and published to [DockerHub](https://hub.docker.com/r/chriscn/pihole-to-influxdb).
 
-This project is automatically built through GitHub actions and the DockerHub file can be found [here](https://hub.docker.com/r/chriscn/pihole-to-influxdb).
 ## Setup
+
 ### Configuring the script
 The InfluxDB connection settings can be configured as followed:
-- "INFLUX_DB_URL=http://192.168.xxx.xxx:8086"
-- "INFLUX_DB_ORG=\<your org name\>"
-- "INFLUX_DB_TOKEN=\<token\>"
-- "INFLUX_DB_BUCKET=pihole"
+- `INFLUX_DB_URL=http://192.168.xxx.xxx:8086`
+- `INFLUX_DB_ORG=<your org name>`
+- `INFLUX_DB_TOKEN=<token>`
+- `INFLUX_DB_BUCKET=pihole`
 
 The PiHole settings can be configured as followed:
-- PIHOLE_HOSTNAME=192.168.xxx.xxx
-- PIHOLE_INTERVAL=15 *Interval in seconds*
+- `PIHOLE_URL=http://192.168.xxx.xxx`
+- `PIHOLE_INTERVAL=15` *Interval in seconds*
+
+Optionally you can also configure the following:
+- `PIHOLE_AUTHENTICATION=<token>`
+- `LOG_LEVEL=DEBUG`
+- `APP_MODE=Totals`
+
 ### Authentication
-Certain parts of the API require you to be authenticated, this can be achieved by supplying the `PIHOLE_AUTHENTICATION` token with the token from the API settings page of the admin interface.  
-By doing this you'll gain access to two new measurements (tables): 
+Certain parts of the API require you to be authenticated, this can be achieved by supplying the `PIHOLE_AUTHENTICATION` environment variable with the token from the API settings page of the admin interface.
+
+By doing this you'll gain access to two new measurements (tables):
 - query_types
 - forward_destinations
+
+and the following app modes:
+- `APP_MODE=Totals` *This is the default mode*
+- `APP_MODE=Live`
+- `APP_MODE=Raw`
+
 #### Sidenote
-This does mean that your password is stored in plaintext as an environmental variable and as such as malicious actor could find it and access your PiHole instance. You are advised to use this at your own risk.
+This does mean that your token is stored in plaintext as an environment variable and as such as malicious actor could find it and access your PiHole instance. You are advised to use this at your own risk.
+
+### App Modes
+The `APP_MODE` changes the way the script works. The default mode is `Totals` which will only send the daily totals of the PiHole instance, as displayed in the GUI. Another mode is `Live` which will send a summary of the Pi-hole queries of the last `PIHOLE_INTERVAL` seconds. The last mode is `Raw` which will send the raw data of the Pi-hole queries.
+
 ### Docker Command
 ```
-    docker run -d --name pihole-to-influx \
-    -e 'INFLUX_DB_URL'='<influxdb url>' \
-    -e 'INFLUX_DB_ORG'='<influxdb org>' \
-    -e 'INFLUX_DB_TOKEN'='<influxdb token>' \
-    -e 'INFLUX_DB_BUCKET'='pihole' \
-    -e 'PIHOLE_INTERVAL'='1800' \
-    -e 'PIHOLE_HOSTNAME'='192.168.xxx.xxx'  \
-    chriscn/pihole-to-influxdb
+docker run -d --name pihole-to-influx \
+  -e 'INFLUX_DB_URL'='<influxdb url>' \
+  -e 'INFLUX_DB_ORG'='<influxdb org>' \
+  -e 'INFLUX_DB_TOKEN'='<influxdb token>' \
+  -e 'INFLUX_DB_BUCKET'='pihole' \
+  -e 'PIHOLE_INTERVAL'='1800' \
+  -e 'PIHOLE_URL'='192.168.xxx.xxx'  \
+  chriscn/pihole-to-influxdb
 ```
 ### docker-compose
 ```yaml
 version: '3'
 services:
-    pihole-to-influxdb:
-        image: chriscn/pihole-to-influxdb
-        container_name: pihole-to-influxdb
-        environment:
-        - "INFLUX_DB_URL=http://192.168.xxx.xxx:8086"
-        - "INFLUX_DB_ORG=<your org name>"
-        - "INFLUX_DB_TOKEN=<token>"
-        - "INFLUX_DB_BUCKET=pihole"
-        - "PIHOLE_HOSTNAME=192.168.xxx.xxx"
-        - "PIHOLE_INTERVAL=15"
+  pihole-to-influxdb:
+    image: chriscn/pihole-to-influxdb
+    container_name: pihole-to-influxdb
+    environment:
+    - "INFLUX_DB_URL=http://192.168.xxx.xxx:8086"
+    - "INFLUX_DB_ORG=myOrg"
+    - "INFLUX_DB_TOKEN=<token>"
+    - "INFLUX_DB_BUCKET=pihole"
+    - "PIHOLE_URL=http://192.168.xxx.xxx"
+    - "PIHOLE_INTERVAL=15"
+    - "PIHOLE_AUTHENTICATION=<token>"
+    - "LOG_LEVEL=DEBUG"
+    - "APP_MODE=Totals"
 ```

main.py

@@ -3,140 +3,90 @@ import datetime
 import os
 import sys
 import logging
+from enum import Enum
 
 from influxdb_client import InfluxDBClient, Point
 from influxdb_client.client.write_api import SYNCHRONOUS
-from pihole import PiHole # PiHole API Wrapper
+from pihole import PiHole
 
 logger = logging.Logger('pihole-to-influxdb')
 logger.addHandler(logging.StreamHandler(sys.stdout))
 
+class AppMode(Enum):
+  Live = 1
+  Totals = 2
+  Raw = 3
+
 try:
-    # optional Logger Settings
-    logging.basicConfig(level=os.getenv("LOGLEVEL", "DEBUG"))
+  # optional Logger Settings
+  logging.basicConfig(level=os.getenv("LOG_LEVEL", "DEBUG"))
+
+  # InfluxDB Settings
+  DB_URL = os.environ['INFLUX_DB_URL']
+  DB_ORG = os.environ['INFLUX_DB_ORG']
+  DB_TOKEN = os.environ['INFLUX_DB_TOKEN']
+  DB_BUCKET = os.environ['INFLUX_DB_BUCKET']
 
-    # InfluxDB Settings
-    DB_URL = os.environ['INFLUX_DB_URL']
-    DB_ORG = os.environ['INFLUX_DB_ORG']
-    DB_TOKEN = os.environ['INFLUX_DB_TOKEN']
-    DB_BUCKET = os.environ['INFLUX_DB_BUCKET']
+  # PiHole Settings
+  PIHOLE_URL = str(os.environ['PIHOLE_URL'])
+  QUERY_INTERVAL = int(os.environ['PIHOLE_INTERVAL'])
 
-    # PiHole Settings
-    PIHOLE_HOSTNAME = str(os.environ['PIHOLE_HOSTNAME'])
-    TEST_INTERVAL = int(os.environ['PIHOLE_INTERVAL'])
+  # optional Pi-hole authentication
+  AUTHENTICATION_TOKEN = os.getenv('PIHOLE_AUTHENTICATION', None)
 
-    # optional Pi-hole authentication
-    AUTHENTICATION_TOKEN = os.getenv('PIHOLE_AUTHENTICATION', None)
+  # optional App Mode
+  APP_MODE = AppMode(os.getenv('APP_MODE', 'Totals'))
 
 except KeyError as e:
-    logger.fatal('Missing environment variable: {}'.format(e))
-    sys.exit(1)
+  logger.fatal('Missing environment variable: {}'.format(e))
+  sys.exit(1)
+except ValueError as e:
+  logger.fatal('Invalid environment variable: {}'.format(e))
+  sys.exit(1)
 
-influxdb_client = InfluxDBClient(DB_URL, DB_TOKEN, org=DB_ORG)
+if APP_MODE != AppMode.Totals and not AUTHENTICATION_TOKEN:
+  logger.fatal('Pi-hole authentication token is required for live data')
+  sys.exit(1)
 
-def get_data_for_influxdb(pihole: PiHole, timestamp: datetime.datetime):
-    return [
-        Point("domains") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .field("domain_count", int(pihole.domain_count.replace(',',''))) \
-            .field("unique_domains", int(pihole.unique_domains.replace(',',''))) \
-            .field("forwarded", int(pihole.forwarded.replace(',',''))) \
-            .field("cached", int(pihole.cached.replace(',',''))),
-
-        Point("queries") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .field("queries", int(pihole.queries.replace(',',''))) \
-            .field("blocked", int(pihole.blocked.replace(',',''))) \
-            .field("ads_percentage", float(pihole.ads_percentage)),
-
-        Point("clients") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .field("total_clients", int(pihole.total_clients.replace(',',''))) \
-            .field("unique_clients", int(pihole.unique_clients.replace(',',''))) \
-            .field("total_queries", int(pihole.total_queries.replace(',',''))),
-
-        Point("other") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .field("status", pihole.status == 'enabled') \
-            .field("gravity_last_update", pihole.gravity_last_updated['absolute'])
-    ]
-
-def get_authenticated_data_for_influxdb(pihole: PiHole, timestamp: datetime.datetime):
-    for key, value in pihole.query_types.items():
-        yield Point("query_types") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .tag("query_type", key) \
-            .field("value", float(value))
-    
-    for key, value in pihole.forward_destinations['forward_destinations'].items():
-        yield Point("forward_destinations") \
-            .time(timestamp) \
-            .tag("hostname", PIHOLE_HOSTNAME) \
-            .tag("destination", key.split('|')[0]) \
-            .field("value", value)
-
-class Auth(object):
-    def __init__(self, token):
-        # PiHole's web token is just a double sha256 hash of the utf8 encoded password
-        self.token = token
-        self.auth_timestamp = time.time()
+influxdb_client = InfluxDBClient(DB_URL, DB_TOKEN, org=DB_ORG)
+pihole = PiHole(PIHOLE_URL, AUTHENTICATION_TOKEN)
 
 def main():
-    # pihole ctor has side effects, so we create it locally
-    pihole = PiHole(PIHOLE_HOSTNAME)
-
-    write_api = influxdb_client.write_api(write_options=SYNCHRONOUS)
-
-    USE_AUTHENTICATION = False if AUTHENTICATION_TOKEN == None else True
-
-    if USE_AUTHENTICATION:
-        try:
-            pihole.auth_data = Auth(AUTHENTICATION_TOKEN)
-            pihole.refresh()
-            logger.info('Pi-Hole authentication successful')
-        except Exception as e:
-            logger.exception('Pi-Hole authentication failed')
-            USE_AUTHENTICATION = False
-            raise
-    
-    next_update = time.monotonic()
-
-    while True:
-        try:
-            pihole.refresh()
-            timestamp = datetime.datetime.now()
-            data = get_data_for_influxdb(pihole, timestamp)
-
-            if USE_AUTHENTICATION:
-                authenticated_data = list(get_authenticated_data_for_influxdb(pihole, timestamp))
-                try:
-                    write_api.write(bucket=DB_BUCKET, record=authenticated_data)
-                except Exception as e:
-                    logger.exception('Failed to write authenticated data to InfluxDB')
-
-            try:
-                write_api.write(bucket=DB_BUCKET, record=data)
-                logger.debug('Wrote data to InfluxDB')
-                sleep_duration = TEST_INTERVAL
-            except Exception as e:
-                logger.exception('Failed to write data to InfluxDB')
-                # Sleep at most two minutes
-                sleep_duration = min(TEST_INTERVAL, 120)
-
-        except Exception as e:
-            logger.exception('Failed to get data from Pi-Hole')
-            # Sleep at most two minutes
-            sleep_duration = min(TEST_INTERVAL, 120)
-
-        next_update = next_update + sleep_duration
-        logger.debug("Now sleeping for {}".format(datetime.timedelta(seconds=sleep_duration)))
-        time.sleep(max(0, next_update - time.monotonic()))
+  write_api = influxdb_client.write_api(write_options=SYNCHRONOUS)
+  next_update = time.monotonic()
+
+  logger.info('Starting PiHole Data Logger to InfluxDB')
+  logger.info('AppMode: {}'.format(APP_MODE))
+
+  # Test Pi-hole connection
+  try:
+    pihole.request_summary()
+  except Exception as e:
+    logger.fatal('Unable to connect to Pi-hole: {}'.format(e))
+    sys.exit(1)
+
+  while True:
+    try:
+      if APP_MODE == AppMode.Live:
+        timestamp = datetime.datetime.now()
+        data = list(pihole.get_queries_for_influxdb(timestamp, QUERY_INTERVAL))
+      elif APP_MODE == AppMode.Totals:
+        data = list(pihole.get_totals_for_influxdb())
+      elif APP_MODE == AppMode.Raw:
+        data = list(pihole.get_query_logs_for_influxdb())
+
+      logger.debug('Writing {} points to InfluxDB'.format(len(data)))
+      write_api.write(bucket=DB_BUCKET, record=data)
+      sleep_duration = QUERY_INTERVAL
+
+    except Exception as e:
+      logger.exception('Failed to get data from Pi-Hole to InfluxDB')
+      # Sleep at most two minutes
+      sleep_duration = min(QUERY_INTERVAL, 120)
+
+    next_update = next_update + sleep_duration
+    logger.debug("Now sleeping for {}".format(datetime.timedelta(seconds=sleep_duration)))
+    time.sleep(max(0, next_update - time.monotonic()))
 
 if __name__ == '__main__':
-    logger.info('PiHole Data Logger to InfluxDB')
-    main()
+  main()

pihole.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import requests
+from datetime import datetime
+from enum import Enum
+from influxdb_client import Point
+from pandas import DataFrame
+from urllib.parse import urlparse
+
+class QueryStati(Enum):
+  Blocked = 1
+  Forwarded = 2
+  Cached = 3
+  Wildcard = 4
+  Unknown = 5
+
+class PiHole:
+  def __init__(self, url, token):
+    url = urlparse(url)
+    self.url = "{}://{}".format(url.scheme or "http", url.netloc)
+    self.token = token
+
+  def query(self, endpoint, params={}):
+    url = f"{self.url}/admin/{endpoint}.php"
+    return requests.get(url, params=params)
+  
+  def request_all_queries(self, start: float, end: float):
+    """
+    keys[]: time, query_type, domain, client, status, destination, reply_type, reply_time, dnssec
+    """
+    if not self.token:
+      raise Exception("Token required")
+    params = {
+      "getAllQueries": "",
+      "from": int(start),
+      "until": int(end),
+      "auth": self.token
+      }
+    json = self.query("api_db", params=params).json()
+    if json:
+      return json['data']
+    else:
+      return []
+
+  def request_summary(self):
+    """
+    keys: 
+      - domains_being_blocked
+      - dns_queries_today
+      - ads_blocked_today
+      - ads_percentage_today
+      - unique_domains
+      - queries_forwarded
+      - queries_cached
+      - clients_ever_seen
+      - unique_clients
+      - dns_queries_all_types
+      - reply_UNKNOWN
+      - reply_NODATA
+      - reply_NXDOMAIN
+      - reply_CNAME
+      - reply_IP
+      - reply_DOMAIN
+      - reply_RRNAME
+      - reply_SERVFAIL
+      - reply_REFUSED
+      - reply_NOTIMP
+      - reply_OTHER
+      - reply_DNSSEC
+      - reply_NONE
+      - reply_BLOB
+      - dns_queries_all_replies
+      - privacy_level
+      - status
+      - gravity_last_update: file_exists, absolute, relative
+    """
+    json = self.query("api").json()
+    return json
+  
+  def request_forward_destinations(self):
+    if not self.token:
+      raise Exception("Token required")
+    params = {
+      "getForwardDestinations": "",
+      "auth": self.token
+      }
+    json = self.query("api", params=params).json()
+    if json:
+      return json['forward_destinations']
+    else:
+      return {}
+
+  def request_query_types(self):
+    if not self.token:
+      raise Exception("Token required")
+    params = {
+      "getQueryTypes": "",
+      "auth": self.token
+      }
+    json = self.query("api", params=params).json()
+    if json:
+      return json['querytypes']
+    else:
+      return {}
+
+  def get_totals_for_influxdb(self):
+    summary = self.request_summary()
+    timestamp = datetime.now().astimezone()
+    yield Point("domains") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("domain_count", summary['domains_being_blocked']) \
+      .field("unique_domains", summary['unique_domains']) \
+      .field("forwarded", summary['queries_forwarded']) \
+      .field("cached", summary['queries_cached'])
+      
+    yield Point("queries") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("queries", summary['dns_queries_today']) \
+      .field("blocked", summary['ads_blocked_today']) \
+      .field("ads_percentage", summary['ads_percentage_today'])
+      
+    yield Point("clients") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("total_clients", summary['clients_ever_seen']) \
+      .field("unique_clients", summary['unique_clients']) \
+      .field("total_queries", summary['dns_queries_all_types'])
+      
+    yield Point("other") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("status", summary['status'] == 'enabled') \
+      .field("gravity_last_update", summary['gravity_last_updated']['absolute'])
+
+    if self.token:
+      query_types = self.request_query_types()
+      for key, value in query_types.items():
+        yield Point("query_types") \
+          .time(timestamp) \
+          .tag("hostname", self.host) \
+          .tag("query_type", key) \
+          .field("value", float(value))
+
+      forward_destinations = self.request_forward_destinations()
+      for key, value in forward_destinations.items():
+        yield Point("forward_destinations") \
+          .time(timestamp) \
+          .tag("hostname", self.host) \
+          .tag("destination", key.split('|')[0]) \
+          .field("value", float(value))
+  
+  def get_queries_for_influxdb(self, query_date: datetime, sample_period: int):
+    # Get all queries since last sample
+    end_time = query_date.timestamp()
+    start_time = end_time - sample_period + 1
+    queries = self.request_all_queries(start_time, end_time)
+    timestamp = datetime.now().astimezone()
+    df = DataFrame(queries, columns=['time', 'query_type', 'domain', 'client', 'status', 'destination', 'reply_type', 'reply_time', 'dnssec'])
+
+    # we still need some stats from the summary
+    summary = self.request_summary()
+
+    yield Point("domains") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("domain_count", summary['domains_being_blocked']) \
+      .field("unique_domains", len(df.groupby('domain'))) \
+      .field("forwarded", len(df[df['status'] == QueryStati.Forwarded.value])) \
+      .field("cached", len(df[df['status'] == QueryStati.Cached.value]))
+    
+    blocked_count = len(df[(df['status'] == QueryStati.Blocked.value) | (df['status'] == QueryStati.Wildcard.value)])
+    queries_point = Point("queries") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("queries", len(df)) \
+      .field("blocked", blocked_count) \
+      .field("ads_percentage", blocked_count * 100.0 / max(1, len(df)))
+    yield queries_point
+
+    for key, client_df in df.groupby('client'):
+      blocked_count = len(client_df[(client_df['status'] == QueryStati.Blocked.value) | (client_df['status'] == QueryStati.Wildcard.value)])
+      clients_point = Point("clients") \
+        .time(timestamp) \
+        .tag("hostname", self.host) \
+        .tag("client", key) \
+        .field("queries", len(client_df)) \
+        .field("blocked", blocked_count) \
+        .field("ads_percentage", blocked_count * 100.0 / max(1, len(client_df)))
+      yield clients_point
+
+    yield Point("other") \
+      .time(timestamp) \
+      .tag("hostname", self.host) \
+      .field("status", summary['status'] == 'enabled') \
+      .field("gravity_last_update", summary['gravity_last_updated']['absolute'])
+
+    for key, group_df in df.groupby('query_type'):
+      yield Point("query_types") \
+        .time(timestamp) \
+        .tag("hostname", self.host) \
+        .tag("query_type", key) \
+        .field("queries", len(group_df))
+
+    for key, group_df in df.groupby('destination'):
+      yield Point("forward_destinations") \
+        .time(timestamp) \
+        .tag("hostname", self.host) \
+        .tag("destination", key.split('|')[0]) \
+        .field("queries", len(group_df))
+
+  def get_query_logs_for_influxdb(self, query_date: datetime, sample_period: int):
+    end_time = query_date.timestamp()
+    start_time = end_time - sample_period + 1
+
+    for data in self.request_all_queries(start_time, end_time):
+      timestamp, query_type, domain, client, status, destination, reply_type, reply_time, dnssec = data
+      p = Point("logs") \
+        .time(datetime.fromtimestamp(timestamp)) \
+        .tag("hostname", self.host) \
+        .tag("query_type", query_type) \
+        .field("domain", domain) \
+        .tag("client", client) \
+        .tag("status", QueryStati(status)) \
+        .tag("dnssec", dnssec != 0) \
+        .field("reply_time", reply_time)
+      if destination:
+        p.tag("destination", destination)
+      yield p
+
+if __name__ == "__main__":
+  import argparse
+  parser = argparse.ArgumentParser(description='Export Pi-Hole statistics')
+  parser.add_argument('--host', required=True, type=str, help='Pi-Hole host')
+  parser.add_argument('--token', '-t', required=True, type=str, help='Pi-Hole API token')
+  args = parser.parse_args()
+  pihole = PiHole(host=args.host, token=args.token)
+
+  points = list(pihole.get_queries_for_influxdb(datetime.now(), 600))
+  for p in points:
+    print(p._time, p._name, p._tags, p._fields)

requirements.txt

@@ -1,2 +1,3 @@
 influxdb-client
-PiHole-api
+pandas
+requests