go mod tidy

BREAKING: Migrate opendtu to opendtu_address
Add authentication capability for locked-down opendtu's.
Updated setup examples with new variables.

.gitignore

@@ -71,4 +71,5 @@ go.work
 docs
 
 # Don't include compiled binary
-main
+main
+opendtu-logger

Dockerfile

@@ -1,5 +1,5 @@
 # Use buildx for multi-architecture support
-FROM --platform=${BUILDPLATFORM} golang:1.22 AS builder
+FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder
 
 WORKDIR /app
 
@@ -11,7 +11,10 @@ RUN go mod download
 COPY . .
 
 # Build the application for the specified target architecture
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger .
+# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
+# Declare TARGETOS and TARGETARCH in the local scope so they can be used in the build stage.
+ARG TARGETOS TARGETARCH
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger .
 
 # Create a minimal runtime image
 FROM --platform=${TARGETPLATFORM} scratch

README.md

@@ -1,23 +1,39 @@
 # OpenDTU logger
 
-OpenDTU logger connects to the OpenDTU livedata websocket and captures metrics.
+OpenDTU logger adds detailed, long-term storage and visualisation of Hoymiles solar inverter metrics monitored with OpenDTU:
-These metrics are inserted into a PostgreSQL database.
-Optionally, TimescaleDB can be used.
-Included in OpenDTU Logger are a number of Grafana dashboards, which can be used to provide useful visualisations and insights.
 
-## Install instructions
+- It  connects to the OpenDTU livedata websocket and captures metrics.
+- These metrics are inserted into a PostgreSQL database. Optionally, TimescaleDB can be used.
+- Included in OpenDTU Logger are a number of Grafana dashboards, which can be used to provide useful visualisations and insights.
+  - The image below shows the PV dashboard of a given day in dark mode. Light mode is also supported. Hovering over the graphs will result in a tooltip, providing the exact values recorded at a given time.
 
-Docker Compose is the preferred way to install OpenDTU Logger, but using the binary is also possible.
+![PV dashboard in dark mode](https://git.hollander.online/energy/opendtu-logger/raw/commit/a3debb78665641fe7101555d85df89eb0f0d2a79/screenshots/PV-overview-dark-mode.png)
 
-### Docker
+## OpenDTU logger installation instructions
 
-```sh
+OpenDTU logger can be installed in different ways:
-docker pull git.hollander.online/energy/opendtu-logger:0.0
-```
 
-Preferably, run the Docker image using the Docker compose examples provided in the `./docker` folder.
+1. Using the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons).
+2. Using [Docker Compose](#docker-compose)
+3. Using the binary available on the [releases page](https://git.hollander.online/energy/opendtu-logger/releases)
+4. Compiling the code yourself.
 
-#### Docker Compose
+Using the Home Assistant add-on or Docker Compose is the preferred way to install OpenDTU Logger.
+
+Installation on Home Assistant can be done by adding the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons) to the Home Assistant add-on store and following the instructions provided. Other installation methods are described below.
+
+### Configuring OpenDTU
+
+In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used.
+
+- Within OpenDTU, go to `Settings` -> `Inverter settings` (<http://YOUR_OPENDTU_IP_ADDRESS/settings/inverter>).
+- For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`.
+- Enable `Zero daily yield at midnight` and `Clear Eventlog at midnight`.
+- Click `Save`
+- Repeat this procedure for every inverter.
+
+
+### Docker Compose
 
 The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments.
 
@@ -166,5 +182,9 @@ The `grafana` folder contains example dashboards which can be imported into exis
 
 ## Known limitations
 
-- The logger will currently log every event posted by OpenDTU to the websocket. Checks still need to be added to determine the uniqueness of an event. For more information, see also [this OpenDTU Github issue](https://github.com/tbnobody/OpenDTU/issues/1800).
+- The logger currently assumes that events reported by OpenDTU occurred on the current day. This works for the majority of users, as the inverters reset every night when the sun goes down and OpenDTU is configured to forget old events data at midnight. For micro inverters with battery storage added to the PV input side, such as Zendure or Anker Solix, checks still need to be added to account for the fact that these inverters don't necessarily reset every night. Event recording will work, but entries might be recorded multiple times. For more information, see also [this OpenDTU Github issue](https://github.com/tbnobody/OpenDTU/issues/1800).
-- Upon restart of the OpenDTU, the OpenDTU Logger binary will need to be restarted as well. When using the provided `compose.yml` files with Docker, or using the `systemd` service file when using the binary, this will happen automatically.
+- Upon restart of the OpenDTU, the OpenDTU Logger binary needs to be restarted as well. When using the provided `compose.yml` files with Docker, or using the `systemd` service file when using the binary, this happens automatically.
+
+## Feedback
+
+If you'd like to provide feedback about this project, or in case you'd like to ask questions, please refer to [this page](https://pieterhollander.nl/author/pieter-hollander/) for contact information (e-mail, Github or LinkedIn), or use the [contact form](https://pieterhollander.nl/#contact) on my website. You can also respond by placing your remarks in [this discussion on Github](https://github.com/tbnobody/OpenDTU/discussions/1842).

docker/compose.with-database-grafana.yml

@@ -22,13 +22,17 @@ services:
       retries: 20
 
   opendtu-logger:
-    image: git.hollander.online/energy/opendtu-logger:main
+    image: git.hollander.online/energy/opendtu-logger:latest
     restart: always
     environment:
       DB_URL: ${DB_URL}
-      REMOTE_URL: ${REMOTE_URL}
+      OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
+      OPENDTU_AUTH: ${OPENDTU_AUTH}
+      OPENDTU_USERNAME: ${OPENDTU_USERNAME}
+      OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
       TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
       TZ: ${TZ}
+      LOG_LEVEL: ${LOG_LEVEL}
     depends_on:
       timescaledb:
         condition: service_healthy

docker/compose.with-database.yml

@@ -23,12 +23,17 @@ services:
 
   opendtu-logger:
     restart: always
-    image: git.hollander.online/energy/opendtu-logger:main
+    image: git.hollander.online/energy/opendtu-logger:latest
     environment:
       DB_URL: ${DB_URL}
-      REMOTE_URL: ${REMOTE_URL}
+      OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
+      OPENDTU_AUTH: ${OPENDTU_AUTH}
+      OPENDTU_USERNAME: ${OPENDTU_USERNAME}
+      OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
       TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
       TZ: ${TZ}
+      LOG_LEVEL: ${LOG_LEVEL}
+
     depends_on:
       timescaledb:
         condition: service_healthy

docker/compose.yml

@@ -2,12 +2,16 @@ version: '3.8'
 services:
   opendtu-logger:
     restart: always
-    image: git.hollander.online/energy/opendtu-logger:main
+    image: git.hollander.online/energy/opendtu-logger:latest
     environment:
       DB_URL: ${DB_URL}
-      REMOTE_URL: ${REMOTE_URL}
+      OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
+      OPENDTU_AUTH: ${OPENDTU_AUTH}
+      OPENDTU_USERNAME: ${OPENDTU_USERNAME}
+      OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
       TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
       TZ: ${TZ}
+      LOG_LEVEL: ${LOG_LEVEL}
     depends_on:
       timescaledb:
         condition: service_healthy

docker/example.env

@@ -0,0 +1,11 @@
+# OpenDTU
+OPENDTU_ADDRESS="192.168.1.89:80"
+OPENDTU_AUTH=false
+OPENDTU_USERNAME=admin
+OPENDTU_PASSWORD=
+
+# OpenDTU Logger
+DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
+TIMESCALEDB_ENABLED=true
+TZ="Europe/Amsterdam"
+LOG_LEVEL=INFO"

docker/example.with-database.env

@@ -1,9 +1,16 @@
+# OpenDTU
+OPENDTU_ADDRESS="192.168.1.89:80"
+OPENDTU_AUTH=false
+OPENDTU_USERNAME=admin
+OPENDTU_PASSWORD=
+
 # OpenDTU Logger
-REMOTE_URL="192.168.1.89:80"
 DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
 TIMESCALEDB_ENABLED=true
 TZ="Europe/Amsterdam"
+LOG_LEVEL=INFO"
+
 # Database configuration
 PG_USER=postgres
 PG_PASSWORD=
-PG_DB=opendtu_logger
+PG_DB=opendtu_logger

go.mod

@@ -1,10 +1,16 @@
 module git.hollander.online/energy/opendtu-logger
 
-go 1.22
+go 1.23
 
 require (
-	github.com/gorilla/websocket v1.5.1
+	github.com/gorilla/websocket v1.5.3
 	github.com/lib/pq v1.10.9
+	github.com/pressly/goose/v3 v3.22.0
 )
 
-require golang.org/x/net v0.17.0 // indirect
+require (
+	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/sethvargo/go-retry v0.3.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+)

go.sum

@@ -1,6 +1,50 @@
-github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
+github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw=
+github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
+github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
+modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s=
+modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -1,25 +1,30 @@
 // TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring.
-// TODO: Use username and password provided using Basic Authentication.
 // TODO: Record Inverter struct data only on-change.
 // Idea: Make a full admin / config GUI and only configure through this utility.
 // Idea: Gather settings only on start-up.
 // TODO: Only update meter readings such as yieldday, yieldtotal on-change.
-// TODO: Implement proper DB migrations.
+// TODO: Add a health check endpoint, potentially log to it.
+// TODO: Add support for monitoring multiple OpenDTU's at once.
 package main
 
 import (
 	"database/sql"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"log"
 	"log/slog"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 	_ "time/tzdata"
 
+	"git.hollander.online/energy/opendtu-logger/migrations"
 	"github.com/gorilla/websocket"
 	_ "github.com/lib/pq"
+	"github.com/pressly/goose/v3"
 )
 
 // VUD contains three variables used for most metrics sent by OpenDTU:
@@ -103,10 +108,12 @@ type LiveData struct {
 
 // Events struct
 type Event struct {
-	MessageID int    `json:"message_id"`
+	MessageID      int    `json:"message_id"`
-	Message   string `json:"message"`
+	Message        string `json:"message"`
-	StartTime int    `json:"start_time"`
+	StartTime      int    `json:"start_time"`
-	EndTime   int    `json:"end_time"`
+	EndTime        int    `json:"end_time"`
+	StartTimestamp time.Time
+	EndTimestamp   time.Time
 }
 
 type EventsResponse struct {
@@ -141,14 +148,138 @@ type InverterSettingsData struct {
 	Inverters []InverterSettings `json:"inverter"`
 }
 
+// Config settings struct
+type Config struct {
+	DB              string `json:"db"`
+	OpenDTUAddress  string `json:"opendtu_address"`
+	OpenDTUAuth     bool   `json:"opendtu_auth"`
+	OpenDTUUser     string `json:"opendtu_username"`
+	OpenDTUPassword string `json:"opendtu_password"`
+	TimescaleDB     bool   `json:"timescaledb"`
+	TZ              string `json:"tz"`
+	LogLevel        string `json:"log_level"`
+}
+
 var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil))
+var config Config
+
+// LoadConfig attempts to read the configuration from options.json
+// If it fails, it falls back to using environment variables
+func loadConfig() Config {
+	configFilePath := os.Getenv("CONFIG_FILE")
+	if configFilePath == "" {
+		configFilePath = "/data/options.json"
+	}
+
+	data, err := os.ReadFile(configFilePath)
+	if err == nil {
+		// Successfully read the file, parse the JSON
+		err = json.Unmarshal(data, &config)
+		if err != nil {
+			log.Fatalf("Error parsing config file: %v", err)
+		}
+		if config.DB == "" {
+			log.Fatal("db connection settings are not set")
+		}
+		if config.OpenDTUAddress == "" {
+			log.Fatal("opendtu_address is not set")
+		}
+		if config.OpenDTUAuth {
+			if config.OpenDTUUser == "" {
+				log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username")
+			}
+			if config.OpenDTUPassword == "" {
+				log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password")
+			}
+		}
+	} else {
+		logger.Info("JSON config file not found. Falling back to environment variables.")
+		// Fallback to environment variables
+		config.DB = os.Getenv("DB_URL")
+		if config.DB == "" {
+			log.Fatal("DB_URL environment variable is not set.")
+		}
+		config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS")
+		if config.OpenDTUAddress == "" {
+			log.Fatal("OPENDTU_ADDRESS environment variable is not set.")
+		}
+
+		openDTUAuthStr := os.Getenv("OPENDTU_AUTH")
+		if openDTUAuthStr != "" {
+			openDTUAuth, err := strconv.ParseBool(openDTUAuthStr)
+			if err != nil {
+				log.Fatalf("Error parsing OPENDTU_AUTH: %v", err)
+			}
+			config.OpenDTUAuth = openDTUAuth
+		}
+		if config.OpenDTUAuth {
+			config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME")
+			if config.OpenDTUUser == "" {
+				log.Fatal("OPENDTU_USERNAME environment variable is not set.")
+			}
+			config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD")
+			if config.OpenDTUPassword == "" {
+				log.Fatal("OPENDTU_PASSWORD environment variable is not set.")
+			}
+
+		}
+
+		timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED")
+		if timescaleDBStr != "" {
+			timescaleDB, err := strconv.ParseBool(timescaleDBStr)
+			if err != nil {
+				log.Fatalf("Error parsing TIMESCALEDB_ENABLED: %v", err)
+			}
+			config.TimescaleDB = timescaleDB
+		}
+		config.TZ = os.Getenv("TZ")
+		config.LogLevel = os.Getenv("LOG_LEVEL")
+	}
+	_, err = time.LoadLocation(config.TZ)
+	if err != nil {
+		logger.Warn("invalid timezone")
+	}
+
+	return config
+}
+
+// Helper function to map environment variable to slog.Level
+func getLogLevel(defaultLevel slog.Level) slog.Level {
+	logLevelStr := config.LogLevel
+	switch logLevelStr {
+	case "DEBUG":
+		return slog.LevelDebug
+	case "INFO":
+		return slog.LevelInfo
+	case "WARN":
+		return slog.LevelWarn
+	case "ERROR":
+		return slog.LevelError
+	default:
+		return defaultLevel
+	}
+}
+
+// Function to create a new logger with a specified log level
+func createLoggerWithLevel(level slog.Level) *slog.Logger {
+	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: level,
+	}))
+}
 
 // Main program
 func main() {
 	// Initial logger setup
 	slog.SetDefault(logger)
 
-	dbConnStr := (os.Getenv("DB_URL"))
+	// Load the configuration
+	config := loadConfig()
+
+	// Set the logLevel
+	logLevel := getLogLevel(slog.LevelInfo) // Default to info level
+	logger = createLoggerWithLevel(logLevel)
+
+	dbConnStr := config.DB
 	// Connect to PostgreSQL
 	db, err := sql.Open("postgres", dbConnStr)
 	if err != nil {
@@ -157,17 +288,21 @@ func main() {
 	defer db.Close()
 
 	// Create tables if they don't exist
-	createTables(db)
+	migrateDB(db)
 
-	// Get WebSocket URL from environment variable
+	// Create WebSocket URL from config variable
-	RemoteURL := os.Getenv("REMOTE_URL")
+	wsURL := "ws://" + config.OpenDTUAddress + "/livedata"
-	wsURL := "ws://" + RemoteURL + "/livedata"
+
-	if wsURL == "" {
+	logger.Debug(wsURL)
-		log.Fatal("WEBSOCKET_URL environment variable is not set.")
+
+	// Create headers with optional Basic Auth
+	headers := http.Header{}
+	if config.OpenDTUAuth {
+		headers.Set("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
 	}
 
 	// Establish WebSocket connection
-	c, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
+	c, _, err := websocket.DefaultDialer.Dial(wsURL, headers)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -178,6 +313,8 @@ func main() {
 		log.Fatal(err)
 	}
 
+	logger.Info("OpenDTU Logger has been successfully initialised. Starting data recording...")
+
 	// Start listening for WebSocket messages
 	go func() {
 		for {
@@ -212,7 +349,9 @@ func handleMessage(message []byte, db *sql.DB) {
 	// Check conditions for recording data
 	for _, inverter := range liveData.Inverters {
 		// Record events data
-		if inverter.Events > 0 && inverter.Events > getPreviousEventsCount(db, inverter.Serial) {
+		// Optional debugging code:
+		// log.Println("Events data:", "age", inverter.DataAge, "event number", inverter.Events, "previous event count", getPreviousEventsCount(db, inverter.Serial))
+		if inverter.DataAge == 0 && inverter.Events > 0 && inverter.Events > getPreviousEventsCount(db, inverter.Serial) {
 			// Query the endpoint for events
 			events, err := queryEventsEndpoint(inverter.Serial)
 			if err != nil {
@@ -232,103 +371,16 @@ func handleMessage(message []byte, db *sql.DB) {
 	}
 }
 
-func createTables(db *sql.DB) {
+func migrateDB(db *sql.DB) {
-	// Execute SQL statements to create tables if they don't exist
-	// inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
-	// An additional advantage is that it makes plotting in Grafana easier.
 	// TODO: Foreign keys commented out as TimescaleDB hypertables don't support them.
-	createTableSQL := `
-	CREATE TABLE IF NOT EXISTS opendtu_log (
-		timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
-		power NUMERIC,
-		yieldday NUMERIC,
-		yieldtotal NUMERIC
-	);
-	
-	CREATE TABLE IF NOT EXISTS opendtu_inverters (
-		timestamp TIMESTAMPTZ,
-		-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-		inverter_serial TEXT,
-		name TEXT,
-		producing BOOL,
-		limit_relative NUMERIC,
-		limit_absolute NUMERIC
-	);
-	
-	CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
-		timestamp TIMESTAMPTZ,
-		-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-		inverter_serial TEXT,
-		ac_number INT,
-		power NUMERIC,
-		voltage NUMERIC,
-		current NUMERIC,
-		frequency NUMERIC,
-		powerfactor NUMERIC,
-		reactivepower NUMERIC
-	);
-	
-	CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
-		-- id SERIAL PRIMARY KEY,
-		timestamp TIMESTAMPTZ,
-		-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-		inverter_serial TEXT,
-     	dc_number INT,
-		name TEXT,
-		power NUMERIC,
-		voltage NUMERIC,
-		current NUMERIC,
-		yieldday NUMERIC,
-		yieldtotal NUMERIC,
-		irradiation NUMERIC
-	);
-	
-	CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
-		-- id SERIAL PRIMARY KEY,
-		timestamp TIMESTAMPTZ,
-		-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-		inverter_serial TEXT,
-		temperature NUMERIC,
-		power_dc NUMERIC,
-		yieldday NUMERIC,
-		yieldtotal NUMERIC,
-		efficiency NUMERIC
-	);
 
-	CREATE TABLE IF NOT EXISTS opendtu_events (
+	// Perform DB migrations
-		-- id SERIAL PRIMARY KEY,
+	err := migrateFS(db, migrations.FS, ".")
-		timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
-		inverter_serial TEXT,
-		message_id INT,
-		message TEXT,
-		start_time INT,
-		end_time INT
-	);
 
-	CREATE TABLE IF NOT EXISTS opendtu_hints (
-		-- id SERIAL PRIMARY KEY,
-		timestamp TIMESTAMPTZ,
-		-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-		time_sync BOOL,
-		radio_problem BOOL,
-		default_password BOOL
-	);
-	
-	CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
-	CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
-
-	`
-
-	_, err := db.Exec(createTableSQL)
 	if err != nil {
-		log.Fatal("Error creating tables: ", err)
+		log.Fatal("Error performing database migrations: ", err)
 	}
-	timescaleEnabled := os.Getenv("TIMESCALEDB_ENABLED")
+	timescaleEnabled := config.TimescaleDB
 
 	enableTimescaleDB := `
 	-- CREATE EXTENSION IF NOT EXISTS timescaledb;
@@ -340,7 +392,7 @@ func createTables(db *sql.DB) {
 	SELECT create_hypertable('opendtu_events', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);
 	SELECT create_hypertable('opendtu_hints', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);
 	`
-	if timescaleEnabled == "true" {
+	if timescaleEnabled {
 		_, err := db.Exec(enableTimescaleDB)
 		if err != nil {
 			log.Fatal("Error enabling TimescaleDB: ", err)
@@ -349,7 +401,7 @@ func createTables(db *sql.DB) {
 }
 
 func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
-	timeZone := os.Getenv("TZ")
+	timeZone := config.TZ
 	loc, _ := time.LoadLocation(timeZone)
 	timestamp := time.Now().In(loc)
 
@@ -426,16 +478,60 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
 
 }
 
-func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
+func migrate(db *sql.DB, dir string) error {
-	remoteURL := os.Getenv("REMOTE_URL")
+	err := goose.SetDialect("postgres")
-	endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial)
+	if err != nil {
+		return fmt.Errorf("migrate: %w", err)
+	}
+	err = goose.Up(db, dir)
+	if err != nil {
+		return fmt.Errorf("migrate: %w", err)
+	}
+	return nil
 
-	resp, err := http.Get(endpoint)
+}
+
+func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error {
+	// In case the dir is an empty string, they probably meant the current directory and goose wants a period for that.
+	if dir == "" {
+		dir = "."
+	}
+	goose.SetBaseFS(migrationFS)
+	defer func() {
+		// Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS.
+		goose.SetBaseFS(nil)
+	}()
+	return migrate(db, dir)
+}
+
+func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
+	endpoint := fmt.Sprintf("http://"+config.OpenDTUAddress+"/api/eventlog/status?inv=%s", inverterSerial)
+
+	// Create a new HTTP request
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if config.OpenDTUAuth {
+		// Add Basic Auth header
+		req.Header.Add("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
+	}
+
+	// Send the request
+	client := &http.Client{}
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
+	// Check for HTTP errors
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
+	}
+
+	// Decode the response
 	var eventsResponse EventsResponse
 	if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil {
 		return nil, err
@@ -445,6 +541,11 @@ func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
 }
 
 // The events counter reported by OpenDTU resets every day.
+// However, this assumes that the inverters from which the events are pulled are reset every day, during the night.
+// Additionally, this function requires OpenDTU to be set to "Clear Eventlog at midnight" for each inverter.
+// "Clear Eventlog at midnight" should be set to ON in "Inverter settings" -> "pencil" -> "Advanced".
+// To account for possible time drifts, the first and last 10 minutes of the day are excluded.
+//
 // Longest day NL: sun up 4:16, sun down 22:50
 // Shortest day NL: sun up 8:44, sun down 16:25
 func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
@@ -453,8 +554,8 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
 	SELECT COUNT(*) 
 	FROM opendtu_events 
 	WHERE inverter_serial = $1
-	AND timestamp >= CURRENT_DATE + INTERVAL '9 hours'
+    AND timestamp >= CURRENT_DATE + INTERVAL '10 minutes'
-	AND timestamp < CURRENT_DATE + INTERVAL '23 hours'
+	AND timestamp < CURRENT_DATE + INTERVAL '23 hours 50 minutes'
 	`, inverterSerial).Scan(&count)
 	if err != nil && err != sql.ErrNoRows {
 		logger.Error("Error querying previous events count", "error", err)
@@ -463,7 +564,9 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
 }
 
 func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
-	timestamp := time.Now()
+	timeZone := config.TZ
+	loc, _ := time.LoadLocation(timeZone)
+	timestamp := time.Now().In(loc)
 
 	for _, event := range events.Events {
 		// Insert events data into the events table
@@ -477,10 +580,10 @@ func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
 	}
 
 	if len(events.Events) > 0 && events.Events[0].EndTime == 0 {
-		// If end_time is 0, schedule a job to update the corresponding message row every 30 minutes
+		// If end_time is 0, schedule a job to update the corresponding message row every 10 minutes
 		go func() {
 			for {
-				time.Sleep(30 * time.Minute)
+				time.Sleep(10 * time.Minute)
 				updatedEvents, err := queryEventsEndpoint(inverterSerial)
 				if err != nil {
 					logger.Error("Error querying events endpoint for updates", "error", err)
@@ -506,6 +609,12 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
 	}
 }
 
+// basicAuth generates the Basic Auth header value
+func basicAuth(username, password string) string {
+	credentials := username + ":" + password
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials))
+}
+
 // TODO: finish this function.
 // func updateInverterConfig(db *sql.DB) {
 // 	// Periodically query the /api/inverter/list
@@ -524,8 +633,8 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
 // }
 
 // func queryConfigEndpoint() (*InverterSettingsData, error) {
-// 	remoteURL := os.Getenv("REMOTE_URL")
+// 	openDTUAddress := os.Getenv("OPENDTU_ADDRESS")
-// 	endpoint := fmt.Sprintf("http://" + remoteURL + "/api/inverter/list")
+// 	endpoint := fmt.Sprintf("http://" + openDTUAddress + "/api/inverter/list")
 
 // 	resp, err := http.Get(endpoint)
 // 	if err != nil {

migrations/00001_log.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_log (
+    timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
+    power NUMERIC,
+    yieldday NUMERIC,
+    yieldtotal NUMERIC
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00002_inverters.sql

@@ -0,0 +1,19 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_inverters (
+    timestamp TIMESTAMPTZ,
+    -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
+    -- 	inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
+    -- 	An additional advantage is that it makes plotting in Grafana easier.
+    inverter_serial TEXT,
+    name TEXT,
+    producing BOOL,
+    limit_relative NUMERIC,
+    limit_absolute NUMERIC
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00003_inverters_ac.sql

@@ -0,0 +1,20 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
+    timestamp TIMESTAMPTZ,
+    -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
+    inverter_serial TEXT,
+    ac_number INT,
+    power NUMERIC,
+    voltage NUMERIC,
+    current NUMERIC,
+    frequency NUMERIC,
+    powerfactor NUMERIC,
+    reactivepower NUMERIC
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00004_inverters_dc.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
+    -- id SERIAL PRIMARY KEY,
+    timestamp TIMESTAMPTZ,
+    -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
+    inverter_serial TEXT,
+    dc_number INT,
+    name TEXT,
+    power NUMERIC,
+    voltage NUMERIC,
+    current NUMERIC,
+    yieldday NUMERIC,
+    yieldtotal NUMERIC,
+    irradiation NUMERIC
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00005_inverters_inv.sql

@@ -0,0 +1,19 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
+    -- id SERIAL PRIMARY KEY,
+    timestamp TIMESTAMPTZ,
+    -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
+    inverter_serial TEXT,
+    temperature NUMERIC,
+    power_dc NUMERIC,
+    yieldday NUMERIC,
+    yieldtotal NUMERIC,
+    efficiency NUMERIC
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00006_events.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_events (
+	-- id SERIAL PRIMARY KEY,
+	timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+	inverter_serial TEXT,
+	message_id INT,
+	message TEXT,
+	start_time INT,
+	end_time INT
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00007_events.sql

@@ -0,0 +1,26 @@
+-- +goose Up
+-- +goose StatementBegin
+DO $$
+BEGIN
+    -- Check if start_timestamp column exists
+    IF NOT EXISTS (SELECT 1 FROM information_schema.columns 
+                WHERE table_name='opendtu_events' 
+                AND column_name='start_timestamp') THEN
+        -- Add start_timestamp column
+        ALTER TABLE opendtu_events 
+        ADD COLUMN start_timestamp TIMESTAMPTZ;
+    END IF;
+
+    -- Check if end_timestamp column exists
+    IF NOT EXISTS (SELECT 1 FROM information_schema.columns 
+                WHERE table_name='opendtu_events' 
+                AND column_name='end_timestamp') THEN
+        -- Add end_timestamp column
+        ALTER TABLE opendtu_events 
+        ADD COLUMN end_timestamp TIMESTAMPTZ;
+    END IF;
+END $$;
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/00008_hints.sql

@@ -0,0 +1,16 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS opendtu_hints (
+    -- id SERIAL PRIMARY KEY,
+    timestamp TIMESTAMPTZ,
+    -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
+    time_sync BOOL,
+    radio_problem BOOL,
+    default_password BOOL
+);
+
+CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+-- +goose StatementEnd

migrations/fs.go

@@ -0,0 +1,6 @@
+package migrations
+
+import "embed"
+
+//go:embed *.sql
+var FS embed.FS

systemd/opendtu-logger.service

@@ -16,10 +16,15 @@ Type=simple
 User=opendtu-logger
 Group=opendtu-logger
 
-Environment="REMOTE_URL=opendtu.local:80"
+Environment="OPENDTU_ADDRESS=opendtu.local:80"
+Environment="OPENDTU_AUTH=false"
+Environment="OPENDTU_USERNAME=admin"
+Environment="OPENDTU_PASSWORD=your_super_secret_password"
 Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable"
 Environment="TIMESCALEDB_ENABLED=true"
 Environment="TZ=Europe/Amsterdam"
+Environment="LOG_LEVEL=INFO"
+
 
 WorkingDirectory=/opt/opendtu-logger/
 ExecStart=/opt/opendtu-logger/opendtu-logger