Compare commits

..

No commits in common. "main" and "v0.0.8" have entirely different histories.
main ... v0.0.8

20 changed files with 164 additions and 444 deletions

View file

@ -1,5 +1,5 @@
# Use buildx for multi-architecture support # Use buildx for multi-architecture support
FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder FROM --platform=${BUILDPLATFORM} golang:1.22 AS builder
WORKDIR /app WORKDIR /app
@ -11,10 +11,7 @@ RUN go mod download
COPY . . COPY . .
# Build the application for the specified target architecture # Build the application for the specified target architecture
# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger .
# Declare TARGETOS and TARGETARCH in the local scope so they can be used in the build stage.
ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger .
# Create a minimal runtime image # Create a minimal runtime image
FROM --platform=${TARGETPLATFORM} scratch FROM --platform=${TARGETPLATFORM} scratch

View file

@ -9,20 +9,7 @@ OpenDTU logger adds detailed, long-term storage and visualisation of Hoymiles so
![PV dashboard in dark mode](https://git.hollander.online/energy/opendtu-logger/raw/commit/a3debb78665641fe7101555d85df89eb0f0d2a79/screenshots/PV-overview-dark-mode.png) ![PV dashboard in dark mode](https://git.hollander.online/energy/opendtu-logger/raw/commit/a3debb78665641fe7101555d85df89eb0f0d2a79/screenshots/PV-overview-dark-mode.png)
## OpenDTU logger installation instructions ## Configuring OpenDTU
OpenDTU logger can be installed in different ways:
1. Using the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons).
2. Using [Docker Compose](#docker-compose)
3. Using the binary available on the [releases page](https://git.hollander.online/energy/opendtu-logger/releases)
4. Compiling the code yourself.
Using the Home Assistant add-on or Docker Compose is the preferred way to install OpenDTU Logger.
Installation on Home Assistant can be done by adding the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons) to the Home Assistant add-on store and following the instructions provided. Other installation methods are described below.
### Configuring OpenDTU
In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used. In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used.
@ -32,8 +19,19 @@ In order for OpenDTU Logger to work properly, it is required to ensure the follo
- Click `Save` - Click `Save`
- Repeat this procedure for every inverter. - Repeat this procedure for every inverter.
## OpenDTU logger installation instructions
### Docker Compose Docker Compose is the preferred way to install OpenDTU Logger, but using the binary is also possible.
### Docker
```sh
docker pull git.hollander.online/energy/opendtu-logger:0.0
```
Preferably, run the Docker image using the Docker compose examples provided in the `./docker` folder.
#### Docker Compose
The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments. The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments.

View file

@ -22,17 +22,13 @@ services:
retries: 20 retries: 20
opendtu-logger: opendtu-logger:
image: git.hollander.online/energy/opendtu-logger:latest image: git.hollander.online/energy/opendtu-logger:main
restart: always restart: always
environment: environment:
DB_URL: ${DB_URL} DB_URL: ${DB_URL}
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} REMOTE_URL: ${REMOTE_URL}
OPENDTU_AUTH: ${OPENDTU_AUTH}
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
TZ: ${TZ} TZ: ${TZ}
LOG_LEVEL: ${LOG_LEVEL}
depends_on: depends_on:
timescaledb: timescaledb:
condition: service_healthy condition: service_healthy

View file

@ -23,17 +23,12 @@ services:
opendtu-logger: opendtu-logger:
restart: always restart: always
image: git.hollander.online/energy/opendtu-logger:latest image: git.hollander.online/energy/opendtu-logger:main
environment: environment:
DB_URL: ${DB_URL} DB_URL: ${DB_URL}
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} REMOTE_URL: ${REMOTE_URL}
OPENDTU_AUTH: ${OPENDTU_AUTH}
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
TZ: ${TZ} TZ: ${TZ}
LOG_LEVEL: ${LOG_LEVEL}
depends_on: depends_on:
timescaledb: timescaledb:
condition: service_healthy condition: service_healthy

View file

@ -2,16 +2,12 @@ version: '3.8'
services: services:
opendtu-logger: opendtu-logger:
restart: always restart: always
image: git.hollander.online/energy/opendtu-logger:latest image: git.hollander.online/energy/opendtu-logger:main
environment: environment:
DB_URL: ${DB_URL} DB_URL: ${DB_URL}
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} REMOTE_URL: ${REMOTE_URL}
OPENDTU_AUTH: ${OPENDTU_AUTH}
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
TZ: ${TZ} TZ: ${TZ}
LOG_LEVEL: ${LOG_LEVEL}
depends_on: depends_on:
timescaledb: timescaledb:
condition: service_healthy condition: service_healthy

View file

@ -1,11 +0,0 @@
# OpenDTU
OPENDTU_ADDRESS="192.168.1.89:80"
OPENDTU_AUTH=false
OPENDTU_USERNAME=admin
OPENDTU_PASSWORD=
# OpenDTU Logger
DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
TIMESCALEDB_ENABLED=true
TZ="Europe/Amsterdam"
LOG_LEVEL=INFO"

View file

@ -1,15 +1,8 @@
# OpenDTU
OPENDTU_ADDRESS="192.168.1.89:80"
OPENDTU_AUTH=false
OPENDTU_USERNAME=admin
OPENDTU_PASSWORD=
# OpenDTU Logger # OpenDTU Logger
REMOTE_URL="192.168.1.89:80"
DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
TIMESCALEDB_ENABLED=true TIMESCALEDB_ENABLED=true
TZ="Europe/Amsterdam" TZ="Europe/Amsterdam"
LOG_LEVEL=INFO"
# Database configuration # Database configuration
PG_USER=postgres PG_USER=postgres
PG_PASSWORD= PG_PASSWORD=

12
go.mod
View file

@ -1,16 +1,10 @@
module git.hollander.online/energy/opendtu-logger module git.hollander.online/energy/opendtu-logger
go 1.23 go 1.22
require ( require (
github.com/gorilla/websocket v1.5.3 github.com/gorilla/websocket v1.5.1
github.com/lib/pq v1.10.9 github.com/lib/pq v1.10.9
github.com/pressly/goose/v3 v3.22.0
) )
require ( require golang.org/x/net v0.17.0 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sync v0.8.0 // indirect
)

52
go.sum
View file

@ -1,50 +1,6 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw=
github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s=
modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

296
main.go
View file

@ -1,18 +1,16 @@
// TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring. // TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring.
// TODO: Use username and password provided using Basic Authentication.
// TODO: Record Inverter struct data only on-change. // TODO: Record Inverter struct data only on-change.
// Idea: Make a full admin / config GUI and only configure through this utility. // Idea: Make a full admin / config GUI and only configure through this utility.
// Idea: Gather settings only on start-up. // Idea: Gather settings only on start-up.
// TODO: Only update meter readings such as yieldday, yieldtotal on-change. // TODO: Only update meter readings such as yieldday, yieldtotal on-change.
// TODO: Add a health check endpoint, potentially log to it. // TODO: Implement proper DB migrations.
// TODO: Add support for monitoring multiple OpenDTU's at once.
package main package main
import ( import (
"database/sql" "database/sql"
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/fs"
"log" "log"
"log/slog" "log/slog"
"net/http" "net/http"
@ -21,10 +19,8 @@ import (
"time" "time"
_ "time/tzdata" _ "time/tzdata"
"git.hollander.online/energy/opendtu-logger/migrations"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/pressly/goose/v3"
) )
// VUD contains three variables used for most metrics sent by OpenDTU: // VUD contains three variables used for most metrics sent by OpenDTU:
@ -150,14 +146,10 @@ type InverterSettingsData struct {
// Config settings struct // Config settings struct
type Config struct { type Config struct {
DB string `json:"db"` DB string `json:"db"`
OpenDTUAddress string `json:"opendtu_address"` OpenDTU string `json:"opendtu"`
OpenDTUAuth bool `json:"opendtu_auth"` TimescaleDB bool `json:"timescaledb"`
OpenDTUUser string `json:"opendtu_username"` TZ string `json:"tz"`
OpenDTUPassword string `json:"opendtu_password"`
TimescaleDB bool `json:"timescaledb"`
TZ string `json:"tz"`
LogLevel string `json:"log_level"`
} }
var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil))
@ -165,7 +157,7 @@ var config Config
// LoadConfig attempts to read the configuration from options.json // LoadConfig attempts to read the configuration from options.json
// If it fails, it falls back to using environment variables // If it fails, it falls back to using environment variables
func loadConfig() Config { func LoadConfig() Config {
configFilePath := os.Getenv("CONFIG_FILE") configFilePath := os.Getenv("CONFIG_FILE")
if configFilePath == "" { if configFilePath == "" {
configFilePath = "/data/options.json" configFilePath = "/data/options.json"
@ -178,50 +170,15 @@ func loadConfig() Config {
if err != nil { if err != nil {
log.Fatalf("Error parsing config file: %v", err) log.Fatalf("Error parsing config file: %v", err)
} }
if config.DB == "" {
log.Fatal("db connection settings are not set")
}
if config.OpenDTUAddress == "" {
log.Fatal("opendtu_address is not set")
}
if config.OpenDTUAuth {
if config.OpenDTUUser == "" {
log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username")
}
if config.OpenDTUPassword == "" {
log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password")
}
}
} else { } else {
logger.Info("JSON config file not found. Falling back to environment variables.")
// Fallback to environment variables // Fallback to environment variables
config.DB = os.Getenv("DB_URL") config.DB = os.Getenv("DB_URL")
if config.DB == "" { if config.DB == "" {
log.Fatal("DB_URL environment variable is not set.") log.Fatal("DB_URL environment variable is not set.")
} }
config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS") config.OpenDTU = os.Getenv("REMOTE_URL")
if config.OpenDTUAddress == "" { if config.OpenDTU == "" {
log.Fatal("OPENDTU_ADDRESS environment variable is not set.") log.Fatal("REMOTE_URL environment variable is not set.")
}
openDTUAuthStr := os.Getenv("OPENDTU_AUTH")
if openDTUAuthStr != "" {
openDTUAuth, err := strconv.ParseBool(openDTUAuthStr)
if err != nil {
log.Fatalf("Error parsing OPENDTU_AUTH: %v", err)
}
config.OpenDTUAuth = openDTUAuth
}
if config.OpenDTUAuth {
config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME")
if config.OpenDTUUser == "" {
log.Fatal("OPENDTU_USERNAME environment variable is not set.")
}
config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD")
if config.OpenDTUPassword == "" {
log.Fatal("OPENDTU_PASSWORD environment variable is not set.")
}
} }
timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED") timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED")
@ -233,51 +190,18 @@ func loadConfig() Config {
config.TimescaleDB = timescaleDB config.TimescaleDB = timescaleDB
} }
config.TZ = os.Getenv("TZ") config.TZ = os.Getenv("TZ")
config.LogLevel = os.Getenv("LOG_LEVEL")
}
_, err = time.LoadLocation(config.TZ)
if err != nil {
logger.Warn("invalid timezone")
} }
return config return config
} }
// Helper function to map environment variable to slog.Level
func getLogLevel(defaultLevel slog.Level) slog.Level {
logLevelStr := config.LogLevel
switch logLevelStr {
case "DEBUG":
return slog.LevelDebug
case "INFO":
return slog.LevelInfo
case "WARN":
return slog.LevelWarn
case "ERROR":
return slog.LevelError
default:
return defaultLevel
}
}
// Function to create a new logger with a specified log level
func createLoggerWithLevel(level slog.Level) *slog.Logger {
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
Level: level,
}))
}
// Main program // Main program
func main() { func main() {
// Initial logger setup // Initial logger setup
slog.SetDefault(logger) slog.SetDefault(logger)
// Load the configuration // Load the configuration
config := loadConfig() config := LoadConfig()
// Set the logLevel
logLevel := getLogLevel(slog.LevelInfo) // Default to info level
logger = createLoggerWithLevel(logLevel)
dbConnStr := config.DB dbConnStr := config.DB
// Connect to PostgreSQL // Connect to PostgreSQL
@ -288,21 +212,13 @@ func main() {
defer db.Close() defer db.Close()
// Create tables if they don't exist // Create tables if they don't exist
migrateDB(db) createTables(db)
// Create WebSocket URL from config variable // Create WebSocket URL from config variable
wsURL := "ws://" + config.OpenDTUAddress + "/livedata" wsURL := "ws://" + config.OpenDTU + "/livedata"
logger.Debug(wsURL)
// Create headers with optional Basic Auth
headers := http.Header{}
if config.OpenDTUAuth {
headers.Set("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
}
// Establish WebSocket connection // Establish WebSocket connection
c, _, err := websocket.DefaultDialer.Dial(wsURL, headers) c, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -313,8 +229,6 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
logger.Info("OpenDTU Logger has been successfully initialised. Starting data recording...")
// Start listening for WebSocket messages // Start listening for WebSocket messages
go func() { go func() {
for { for {
@ -371,14 +285,122 @@ func handleMessage(message []byte, db *sql.DB) {
} }
} }
func migrateDB(db *sql.DB) { func createTables(db *sql.DB) {
// Execute SQL statements to create tables if they don't exist
// inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
// An additional advantage is that it makes plotting in Grafana easier.
// TODO: Foreign keys commented out as TimescaleDB hypertables don't support them. // TODO: Foreign keys commented out as TimescaleDB hypertables don't support them.
createTableSQL := `
CREATE TABLE IF NOT EXISTS opendtu_log (
timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
power NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC
);
// Perform DB migrations CREATE TABLE IF NOT EXISTS opendtu_inverters (
err := migrateFS(db, migrations.FS, ".") timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
name TEXT,
producing BOOL,
limit_relative NUMERIC,
limit_absolute NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
ac_number INT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
frequency NUMERIC,
powerfactor NUMERIC,
reactivepower NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
dc_number INT,
name TEXT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
irradiation NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
temperature NUMERIC,
power_dc NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
efficiency NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_events (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
inverter_serial TEXT,
message_id INT,
message TEXT,
start_time INT,
end_time INT
);
DO $$
BEGIN
-- Check if start_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='start_timestamp') THEN
-- Add start_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN start_timestamp TIMESTAMPTZ;
END IF;
-- Check if end_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='end_timestamp') THEN
-- Add end_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN end_timestamp TIMESTAMPTZ;
END IF;
END $$;
CREATE TABLE IF NOT EXISTS opendtu_hints (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
time_sync BOOL,
radio_problem BOOL,
default_password BOOL
);
CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
`
_, err := db.Exec(createTableSQL)
if err != nil { if err != nil {
log.Fatal("Error performing database migrations: ", err) log.Fatal("Error creating tables: ", err)
} }
timescaleEnabled := config.TimescaleDB timescaleEnabled := config.TimescaleDB
@ -401,7 +423,7 @@ func migrateDB(db *sql.DB) {
} }
func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
timeZone := config.TZ timeZone := os.Getenv("TZ")
loc, _ := time.LoadLocation(timeZone) loc, _ := time.LoadLocation(timeZone)
timestamp := time.Now().In(loc) timestamp := time.Now().In(loc)
@ -478,60 +500,16 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
} }
func migrate(db *sql.DB, dir string) error {
err := goose.SetDialect("postgres")
if err != nil {
return fmt.Errorf("migrate: %w", err)
}
err = goose.Up(db, dir)
if err != nil {
return fmt.Errorf("migrate: %w", err)
}
return nil
}
func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error {
// In case the dir is an empty string, they probably meant the current directory and goose wants a period for that.
if dir == "" {
dir = "."
}
goose.SetBaseFS(migrationFS)
defer func() {
// Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS.
goose.SetBaseFS(nil)
}()
return migrate(db, dir)
}
func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) { func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
endpoint := fmt.Sprintf("http://"+config.OpenDTUAddress+"/api/eventlog/status?inv=%s", inverterSerial) remoteURL := os.Getenv("REMOTE_URL")
endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial)
// Create a new HTTP request resp, err := http.Get(endpoint)
req, err := http.NewRequest("GET", endpoint, nil)
if err != nil {
return nil, err
}
if config.OpenDTUAuth {
// Add Basic Auth header
req.Header.Add("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
}
// Send the request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer resp.Body.Close() defer resp.Body.Close()
// Check for HTTP errors
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
}
// Decode the response
var eventsResponse EventsResponse var eventsResponse EventsResponse
if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil { if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil {
return nil, err return nil, err
@ -564,9 +542,7 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
} }
func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
timeZone := config.TZ timestamp := time.Now()
loc, _ := time.LoadLocation(timeZone)
timestamp := time.Now().In(loc)
for _, event := range events.Events { for _, event := range events.Events {
// Insert events data into the events table // Insert events data into the events table
@ -609,12 +585,6 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
} }
} }
// basicAuth generates the Basic Auth header value
func basicAuth(username, password string) string {
credentials := username + ":" + password
return "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials))
}
// TODO: finish this function. // TODO: finish this function.
// func updateInverterConfig(db *sql.DB) { // func updateInverterConfig(db *sql.DB) {
// // Periodically query the /api/inverter/list // // Periodically query the /api/inverter/list
@ -633,8 +603,8 @@ func basicAuth(username, password string) string {
// } // }
// func queryConfigEndpoint() (*InverterSettingsData, error) { // func queryConfigEndpoint() (*InverterSettingsData, error) {
// openDTUAddress := os.Getenv("OPENDTU_ADDRESS") // remoteURL := os.Getenv("REMOTE_URL")
// endpoint := fmt.Sprintf("http://" + openDTUAddress + "/api/inverter/list") // endpoint := fmt.Sprintf("http://" + remoteURL + "/api/inverter/list")
// resp, err := http.Get(endpoint) // resp, err := http.Get(endpoint)
// if err != nil { // if err != nil {

View file

@ -1,14 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_log (
timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
power NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC
);
CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,19 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_inverters (
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
-- inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
-- An additional advantage is that it makes plotting in Grafana easier.
inverter_serial TEXT,
name TEXT,
producing BOOL,
limit_relative NUMERIC,
limit_absolute NUMERIC
);
CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,20 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
ac_number INT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
frequency NUMERIC,
powerfactor NUMERIC,
reactivepower NUMERIC
);
CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,22 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
dc_number INT,
name TEXT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
irradiation NUMERIC
);
CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,19 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
temperature NUMERIC,
power_dc NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
efficiency NUMERIC
);
CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,17 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_events (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
inverter_serial TEXT,
message_id INT,
message TEXT,
start_time INT,
end_time INT
);
CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,26 +0,0 @@
-- +goose Up
-- +goose StatementBegin
DO $$
BEGIN
-- Check if start_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='start_timestamp') THEN
-- Add start_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN start_timestamp TIMESTAMPTZ;
END IF;
-- Check if end_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='end_timestamp') THEN
-- Add end_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN end_timestamp TIMESTAMPTZ;
END IF;
END $$;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,16 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS opendtu_hints (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
time_sync BOOL,
radio_problem BOOL,
default_password BOOL
);
CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- +goose StatementEnd

View file

@ -1,6 +0,0 @@
package migrations
import "embed"
//go:embed *.sql
var FS embed.FS

View file

@ -16,15 +16,10 @@ Type=simple
User=opendtu-logger User=opendtu-logger
Group=opendtu-logger Group=opendtu-logger
Environment="OPENDTU_ADDRESS=opendtu.local:80" Environment="REMOTE_URL=opendtu.local:80"
Environment="OPENDTU_AUTH=false"
Environment="OPENDTU_USERNAME=admin"
Environment="OPENDTU_PASSWORD=your_super_secret_password"
Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable" Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable"
Environment="TIMESCALEDB_ENABLED=true" Environment="TIMESCALEDB_ENABLED=true"
Environment="TZ=Europe/Amsterdam" Environment="TZ=Europe/Amsterdam"
Environment="LOG_LEVEL=INFO"
WorkingDirectory=/opt/opendtu-logger/ WorkingDirectory=/opt/opendtu-logger/
ExecStart=/opt/opendtu-logger/opendtu-logger ExecStart=/opt/opendtu-logger/opendtu-logger