diff --git a/Dockerfile b/Dockerfile index f8e11a4..d05c839 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use buildx for multi-architecture support -FROM --platform=${BUILDPLATFORM} golang:1.24 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.22 AS builder WORKDIR /app @@ -11,10 +11,7 @@ RUN go mod download COPY . . # Build the application for the specified target architecture -# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ -# Declare TARGETOS and TARGETARCH in the local scope so they can be used in the build stage. -ARG TARGETOS TARGETARCH -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger . # Create a minimal runtime image FROM --platform=${TARGETPLATFORM} scratch diff --git a/README.md b/README.md index 3139d93..b29bfe3 100644 --- a/README.md +++ b/README.md @@ -9,23 +9,9 @@ OpenDTU logger adds detailed, long-term storage and visualisation of Hoymiles so ![PV dashboard in dark mode](https://git.hollander.online/energy/opendtu-logger/raw/commit/a3debb78665641fe7101555d85df89eb0f0d2a79/screenshots/PV-overview-dark-mode.png) -## OpenDTU logger installation instructions - -OpenDTU logger can be installed in different ways: - -1. Using the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons). -2. Using [Docker Compose](#docker-compose) -3. Using the binary available on the [releases page](https://git.hollander.online/energy/opendtu-logger/releases) -4. Compiling the code yourself. - -Using the Home Assistant add-on or Docker Compose is the preferred way to install OpenDTU Logger. - -Installation on Home Assistant can be done by adding the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons) to the Home Assistant add-on store and following the instructions provided. Other installation methods are described below. - -### Configuring OpenDTU +## Configuring OpenDTU In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used. -OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.5.10. - Within OpenDTU, go to `Settings` -> `Inverter settings` (). - For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`. @@ -33,7 +19,19 @@ OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.5.10. - Click `Save` - Repeat this procedure for every inverter. -### Docker Compose +## OpenDTU logger installation instructions + +Docker Compose is the preferred way to install OpenDTU Logger, but using the binary is also possible. + +### Docker + +```sh +docker pull git.hollander.online/energy/opendtu-logger:0.0 +``` + +Preferably, run the Docker image using the Docker compose examples provided in the `./docker` folder. + +#### Docker Compose The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments. diff --git a/docker/compose.with-database-grafana.yml b/docker/compose.with-database-grafana.yml index 1833916..b7fa9d4 100644 --- a/docker/compose.with-database-grafana.yml +++ b/docker/compose.with-database-grafana.yml @@ -22,17 +22,13 @@ services: retries: 20 opendtu-logger: - image: git.hollander.online/energy/opendtu-logger:latest + image: git.hollander.online/energy/opendtu-logger:main restart: always environment: DB_URL: ${DB_URL} - OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} - OPENDTU_AUTH: ${OPENDTU_AUTH} - OPENDTU_USERNAME: ${OPENDTU_USERNAME} - OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} + REMOTE_URL: ${REMOTE_URL} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} - LOG_LEVEL: ${LOG_LEVEL} depends_on: timescaledb: condition: service_healthy diff --git a/docker/compose.with-database.yml b/docker/compose.with-database.yml index fdbb400..b0a9004 100644 --- a/docker/compose.with-database.yml +++ b/docker/compose.with-database.yml @@ -23,17 +23,12 @@ services: opendtu-logger: restart: always - image: git.hollander.online/energy/opendtu-logger:latest + image: git.hollander.online/energy/opendtu-logger:main environment: DB_URL: ${DB_URL} - OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} - OPENDTU_AUTH: ${OPENDTU_AUTH} - OPENDTU_USERNAME: ${OPENDTU_USERNAME} - OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} + REMOTE_URL: ${REMOTE_URL} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} - LOG_LEVEL: ${LOG_LEVEL} - depends_on: timescaledb: condition: service_healthy diff --git a/docker/compose.yml b/docker/compose.yml index 5d60048..d25a958 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -2,16 +2,12 @@ version: '3.8' services: opendtu-logger: restart: always - image: git.hollander.online/energy/opendtu-logger:latest + image: git.hollander.online/energy/opendtu-logger:main environment: DB_URL: ${DB_URL} - OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} - OPENDTU_AUTH: ${OPENDTU_AUTH} - OPENDTU_USERNAME: ${OPENDTU_USERNAME} - OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} + REMOTE_URL: ${REMOTE_URL} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} - LOG_LEVEL: ${LOG_LEVEL} depends_on: timescaledb: condition: service_healthy diff --git a/docker/example.env b/docker/example.env deleted file mode 100644 index f94605f..0000000 --- a/docker/example.env +++ /dev/null @@ -1,11 +0,0 @@ -# OpenDTU -OPENDTU_ADDRESS="192.168.1.89:80" -OPENDTU_AUTH=false -OPENDTU_USERNAME=admin -OPENDTU_PASSWORD= - -# OpenDTU Logger -DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" -TIMESCALEDB_ENABLED=true -TZ="Europe/Amsterdam" -LOG_LEVEL=INFO" \ No newline at end of file diff --git a/docker/example.with-database.env b/docker/example.with-database.env index b5d280f..a283d89 100644 --- a/docker/example.with-database.env +++ b/docker/example.with-database.env @@ -1,16 +1,9 @@ -# OpenDTU -OPENDTU_ADDRESS="192.168.1.89:80" -OPENDTU_AUTH=false -OPENDTU_USERNAME=admin -OPENDTU_PASSWORD= - # OpenDTU Logger +REMOTE_URL="192.168.1.89:80" DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" TIMESCALEDB_ENABLED=true TZ="Europe/Amsterdam" -LOG_LEVEL=INFO" - # Database configuration PG_USER=postgres PG_PASSWORD= -PG_DB=opendtu_logger +PG_DB=opendtu_logger \ No newline at end of file diff --git a/go.mod b/go.mod index 9e78730..07b4ae4 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,10 @@ module git.hollander.online/energy/opendtu-logger -go 1.24 +go 1.22 require ( - github.com/gorilla/websocket v1.5.3 + github.com/gorilla/websocket v1.5.1 github.com/lib/pq v1.10.9 - github.com/pressly/goose/v3 v3.24.3 ) -require ( - github.com/mfridman/interpolate v0.0.2 // indirect - github.com/sethvargo/go-retry v0.3.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.14.0 // indirect -) +require golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index 0a2bb1c..944767b 100644 --- a/go.sum +++ b/go.sum @@ -1,66 +1,6 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= -github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= -github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= -github.com/pressly/goose/v3 v3.24.1 h1:bZmxRco2uy5uu5Ng1MMVEfYsFlrMJI+e/VMXHQ3C4LY= -github.com/pressly/goose/v3 v3.24.1/go.mod h1:rEWreU9uVtt0DHCyLzF9gRcWiiTF/V+528DV+4DORug= -github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwWlM= -github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= -github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/libc v1.65.0 h1:e183gLDnAp9VJh6gWKdTy0CThL9Pt7MfcR/0bgb7Y1Y= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= -modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= -modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= -modernc.org/sqlite v1.34.1 h1:u3Yi6M0N8t9yKRDwhXcyp1eS5/ErhPTBggxWFuR6Hfk= -modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= diff --git a/main.go b/main.go index 295ed2a..abce7c6 100644 --- a/main.go +++ b/main.go @@ -1,30 +1,25 @@ // TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring. +// TODO: Use username and password provided using Basic Authentication. // TODO: Record Inverter struct data only on-change. // Idea: Make a full admin / config GUI and only configure through this utility. // Idea: Gather settings only on start-up. // TODO: Only update meter readings such as yieldday, yieldtotal on-change. -// TODO: Add a health check endpoint, potentially log to it. -// TODO: Add support for monitoring multiple OpenDTU's at once. +// TODO: Implement proper DB migrations. package main import ( "database/sql" - "encoding/base64" "encoding/json" "fmt" - "io/fs" "log" "log/slog" "net/http" "os" - "strconv" "time" _ "time/tzdata" - "git.hollander.online/energy/opendtu-logger/migrations" "github.com/gorilla/websocket" _ "github.com/lib/pq" - "github.com/pressly/goose/v3" ) // VUD contains three variables used for most metrics sent by OpenDTU: @@ -148,138 +143,14 @@ type InverterSettingsData struct { Inverters []InverterSettings `json:"inverter"` } -// Config settings struct -type Config struct { - DB string `json:"db"` - OpenDTUAddress string `json:"opendtu_address"` - OpenDTUAuth bool `json:"opendtu_auth"` - OpenDTUUser string `json:"opendtu_username"` - OpenDTUPassword string `json:"opendtu_password"` - TimescaleDB bool `json:"timescaledb"` - TZ string `json:"tz"` - LogLevel string `json:"log_level"` -} - var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) -var config Config - -// LoadConfig attempts to read the configuration from options.json -// If it fails, it falls back to using environment variables -func loadConfig() Config { - configFilePath := os.Getenv("CONFIG_FILE") - if configFilePath == "" { - configFilePath = "/data/options.json" - } - - data, err := os.ReadFile(configFilePath) - if err == nil { - // Successfully read the file, parse the JSON - err = json.Unmarshal(data, &config) - if err != nil { - log.Fatalf("Error parsing config file: %v", err) - } - if config.DB == "" { - log.Fatal("db connection settings are not set") - } - if config.OpenDTUAddress == "" { - log.Fatal("opendtu_address is not set") - } - if config.OpenDTUAuth { - if config.OpenDTUUser == "" { - log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username") - } - if config.OpenDTUPassword == "" { - log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password") - } - } - } else { - logger.Info("JSON config file not found. Falling back to environment variables.") - // Fallback to environment variables - config.DB = os.Getenv("DB_URL") - if config.DB == "" { - log.Fatal("DB_URL environment variable is not set.") - } - config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS") - if config.OpenDTUAddress == "" { - log.Fatal("OPENDTU_ADDRESS environment variable is not set.") - } - - openDTUAuthStr := os.Getenv("OPENDTU_AUTH") - if openDTUAuthStr != "" { - openDTUAuth, err := strconv.ParseBool(openDTUAuthStr) - if err != nil { - log.Fatalf("Error parsing OPENDTU_AUTH: %v", err) - } - config.OpenDTUAuth = openDTUAuth - } - if config.OpenDTUAuth { - config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME") - if config.OpenDTUUser == "" { - log.Fatal("OPENDTU_USERNAME environment variable is not set.") - } - config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD") - if config.OpenDTUPassword == "" { - log.Fatal("OPENDTU_PASSWORD environment variable is not set.") - } - - } - - timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED") - if timescaleDBStr != "" { - timescaleDB, err := strconv.ParseBool(timescaleDBStr) - if err != nil { - log.Fatalf("Error parsing TIMESCALEDB_ENABLED: %v", err) - } - config.TimescaleDB = timescaleDB - } - config.TZ = os.Getenv("TZ") - config.LogLevel = os.Getenv("LOG_LEVEL") - } - _, err = time.LoadLocation(config.TZ) - if err != nil { - logger.Warn("invalid timezone") - } - - return config -} - -// Helper function to map environment variable to slog.Level -func getLogLevel(defaultLevel slog.Level) slog.Level { - logLevelStr := config.LogLevel - switch logLevelStr { - case "DEBUG": - return slog.LevelDebug - case "INFO": - return slog.LevelInfo - case "WARN": - return slog.LevelWarn - case "ERROR": - return slog.LevelError - default: - return defaultLevel - } -} - -// Function to create a new logger with a specified log level -func createLoggerWithLevel(level slog.Level) *slog.Logger { - return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: level, - })) -} // Main program func main() { // Initial logger setup slog.SetDefault(logger) - // Load the configuration - config := loadConfig() - - // Set the logLevel - logLevel := getLogLevel(slog.LevelInfo) // Default to info level - logger = createLoggerWithLevel(logLevel) - - dbConnStr := config.DB + dbConnStr := (os.Getenv("DB_URL")) // Connect to PostgreSQL db, err := sql.Open("postgres", dbConnStr) if err != nil { @@ -288,21 +159,17 @@ func main() { defer db.Close() // Create tables if they don't exist - migrateDB(db) + createTables(db) - // Create WebSocket URL from config variable - wsURL := "ws://" + config.OpenDTUAddress + "/livedata" - - logger.Debug(wsURL) - - // Create headers with optional Basic Auth - headers := http.Header{} - if config.OpenDTUAuth { - headers.Set("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword)) + // Get WebSocket URL from environment variable + RemoteURL := os.Getenv("REMOTE_URL") + wsURL := "ws://" + RemoteURL + "/livedata" + if wsURL == "" { + log.Fatal("WEBSOCKET_URL environment variable is not set.") } // Establish WebSocket connection - c, _, err := websocket.DefaultDialer.Dial(wsURL, headers) + c, _, err := websocket.DefaultDialer.Dial(wsURL, nil) if err != nil { log.Fatal(err) } @@ -313,8 +180,6 @@ func main() { log.Fatal(err) } - logger.Info("OpenDTU Logger has been successfully initialised. Starting data recording...") - // Start listening for WebSocket messages go func() { for { @@ -371,16 +236,124 @@ func handleMessage(message []byte, db *sql.DB) { } } -func migrateDB(db *sql.DB) { +func createTables(db *sql.DB) { + // Execute SQL statements to create tables if they don't exist + // inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers. + // An additional advantage is that it makes plotting in Grafana easier. // TODO: Foreign keys commented out as TimescaleDB hypertables don't support them. + createTableSQL := ` + CREATE TABLE IF NOT EXISTS opendtu_log ( + timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP, + power NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC + ); + + CREATE TABLE IF NOT EXISTS opendtu_inverters ( + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + name TEXT, + producing BOOL, + limit_relative NUMERIC, + limit_absolute NUMERIC + ); + + CREATE TABLE IF NOT EXISTS opendtu_inverters_ac ( + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + ac_number INT, + power NUMERIC, + voltage NUMERIC, + current NUMERIC, + frequency NUMERIC, + powerfactor NUMERIC, + reactivepower NUMERIC + ); + + CREATE TABLE IF NOT EXISTS opendtu_inverters_dc ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + dc_number INT, + name TEXT, + power NUMERIC, + voltage NUMERIC, + current NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC, + irradiation NUMERIC + ); + + CREATE TABLE IF NOT EXISTS opendtu_inverters_inv ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + temperature NUMERIC, + power_dc NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC, + efficiency NUMERIC + ); - // Perform DB migrations - err := migrateFS(db, migrations.FS, ".") + CREATE TABLE IF NOT EXISTS opendtu_events ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + inverter_serial TEXT, + message_id INT, + message TEXT, + start_time INT, + end_time INT + ); + DO $$ + BEGIN + -- Check if start_timestamp column exists + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name='opendtu_events' + AND column_name='start_timestamp') THEN + -- Add start_timestamp column + ALTER TABLE opendtu_events + ADD COLUMN start_timestamp TIMESTAMPTZ; + END IF; + + -- Check if end_timestamp column exists + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name='opendtu_events' + AND column_name='end_timestamp') THEN + -- Add end_timestamp column + ALTER TABLE opendtu_events + ADD COLUMN end_timestamp TIMESTAMPTZ; + END IF; + END $$; + + CREATE TABLE IF NOT EXISTS opendtu_hints ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + time_sync BOOL, + radio_problem BOOL, + default_password BOOL + ); + + CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp); + CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp); + + ` + + _, err := db.Exec(createTableSQL) if err != nil { - log.Fatal("Error performing database migrations: ", err) + log.Fatal("Error creating tables: ", err) } - timescaleEnabled := config.TimescaleDB + timescaleEnabled := os.Getenv("TIMESCALEDB_ENABLED") enableTimescaleDB := ` -- CREATE EXTENSION IF NOT EXISTS timescaledb; @@ -392,7 +365,7 @@ func migrateDB(db *sql.DB) { SELECT create_hypertable('opendtu_events', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); SELECT create_hypertable('opendtu_hints', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); ` - if timescaleEnabled { + if timescaleEnabled == "true" { _, err := db.Exec(enableTimescaleDB) if err != nil { log.Fatal("Error enabling TimescaleDB: ", err) @@ -401,7 +374,7 @@ func migrateDB(db *sql.DB) { } func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { - timeZone := config.TZ + timeZone := os.Getenv("TZ") loc, _ := time.LoadLocation(timeZone) timestamp := time.Now().In(loc) @@ -478,60 +451,16 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { } -func migrate(db *sql.DB, dir string) error { - err := goose.SetDialect("postgres") - if err != nil { - return fmt.Errorf("migrate: %w", err) - } - err = goose.Up(db, dir) - if err != nil { - return fmt.Errorf("migrate: %w", err) - } - return nil - -} - -func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error { - // In case the dir is an empty string, they probably meant the current directory and goose wants a period for that. - if dir == "" { - dir = "." - } - goose.SetBaseFS(migrationFS) - defer func() { - // Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS. - goose.SetBaseFS(nil) - }() - return migrate(db, dir) -} - func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) { - endpoint := fmt.Sprintf("http://"+config.OpenDTUAddress+"/api/eventlog/status?inv=%s", inverterSerial) + remoteURL := os.Getenv("REMOTE_URL") + endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial) - // Create a new HTTP request - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return nil, err - } - - if config.OpenDTUAuth { - // Add Basic Auth header - req.Header.Add("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword)) - } - - // Send the request - client := &http.Client{} - resp, err := client.Do(req) + resp, err := http.Get(endpoint) if err != nil { return nil, err } defer resp.Body.Close() - // Check for HTTP errors - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status) - } - - // Decode the response var eventsResponse EventsResponse if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil { return nil, err @@ -564,9 +493,7 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int { } func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { - timeZone := config.TZ - loc, _ := time.LoadLocation(timeZone) - timestamp := time.Now().In(loc) + timestamp := time.Now() for _, event := range events.Events { // Insert events data into the events table @@ -609,12 +536,6 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { } } -// basicAuth generates the Basic Auth header value -func basicAuth(username, password string) string { - credentials := username + ":" + password - return "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials)) -} - // TODO: finish this function. // func updateInverterConfig(db *sql.DB) { // // Periodically query the /api/inverter/list @@ -633,8 +554,8 @@ func basicAuth(username, password string) string { // } // func queryConfigEndpoint() (*InverterSettingsData, error) { -// openDTUAddress := os.Getenv("OPENDTU_ADDRESS") -// endpoint := fmt.Sprintf("http://" + openDTUAddress + "/api/inverter/list") +// remoteURL := os.Getenv("REMOTE_URL") +// endpoint := fmt.Sprintf("http://" + remoteURL + "/api/inverter/list") // resp, err := http.Get(endpoint) // if err != nil { diff --git a/migrations/00001_log.sql b/migrations/00001_log.sql deleted file mode 100644 index 6a9d6b4..0000000 --- a/migrations/00001_log.sql +++ /dev/null @@ -1,14 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_log ( - timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP, - power NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC -); - -CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00002_inverters.sql b/migrations/00002_inverters.sql deleted file mode 100644 index 935eb63..0000000 --- a/migrations/00002_inverters.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_inverters ( - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - -- inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers. - -- An additional advantage is that it makes plotting in Grafana easier. - inverter_serial TEXT, - name TEXT, - producing BOOL, - limit_relative NUMERIC, - limit_absolute NUMERIC -); - -CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00003_inverters_ac.sql b/migrations/00003_inverters_ac.sql deleted file mode 100644 index cccfb2d..0000000 --- a/migrations/00003_inverters_ac.sql +++ /dev/null @@ -1,20 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_inverters_ac ( - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - ac_number INT, - power NUMERIC, - voltage NUMERIC, - current NUMERIC, - frequency NUMERIC, - powerfactor NUMERIC, - reactivepower NUMERIC -); - -CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00004_inverters_dc.sql b/migrations/00004_inverters_dc.sql deleted file mode 100644 index c958fef..0000000 --- a/migrations/00004_inverters_dc.sql +++ /dev/null @@ -1,22 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_inverters_dc ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - dc_number INT, - name TEXT, - power NUMERIC, - voltage NUMERIC, - current NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC, - irradiation NUMERIC -); - -CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00005_inverters_inv.sql b/migrations/00005_inverters_inv.sql deleted file mode 100644 index 677248f..0000000 --- a/migrations/00005_inverters_inv.sql +++ /dev/null @@ -1,19 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_inverters_inv ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - temperature NUMERIC, - power_dc NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC, - efficiency NUMERIC -); - -CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00006_events.sql b/migrations/00006_events.sql deleted file mode 100644 index 4f56cc7..0000000 --- a/migrations/00006_events.sql +++ /dev/null @@ -1,17 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_events ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - inverter_serial TEXT, - message_id INT, - message TEXT, - start_time INT, - end_time INT -); - -CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd \ No newline at end of file diff --git a/migrations/00007_events.sql b/migrations/00007_events.sql deleted file mode 100644 index d7ad456..0000000 --- a/migrations/00007_events.sql +++ /dev/null @@ -1,26 +0,0 @@ --- +goose Up --- +goose StatementBegin -DO $$ -BEGIN - -- Check if start_timestamp column exists - IF NOT EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name='opendtu_events' - AND column_name='start_timestamp') THEN - -- Add start_timestamp column - ALTER TABLE opendtu_events - ADD COLUMN start_timestamp TIMESTAMPTZ; - END IF; - - -- Check if end_timestamp column exists - IF NOT EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name='opendtu_events' - AND column_name='end_timestamp') THEN - -- Add end_timestamp column - ALTER TABLE opendtu_events - ADD COLUMN end_timestamp TIMESTAMPTZ; - END IF; -END $$; --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/00008_hints.sql b/migrations/00008_hints.sql deleted file mode 100644 index ed5e87b..0000000 --- a/migrations/00008_hints.sql +++ /dev/null @@ -1,16 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS opendtu_hints ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - time_sync BOOL, - radio_problem BOOL, - default_password BOOL -); - -CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp); --- +goose StatementEnd --- +goose Down --- +goose StatementBegin --- +goose StatementEnd diff --git a/migrations/fs.go b/migrations/fs.go deleted file mode 100644 index 91cca1c..0000000 --- a/migrations/fs.go +++ /dev/null @@ -1,6 +0,0 @@ -package migrations - -import "embed" - -//go:embed *.sql -var FS embed.FS diff --git a/systemd/opendtu-logger.service b/systemd/opendtu-logger.service index 88ad6ad..9c14fb6 100644 --- a/systemd/opendtu-logger.service +++ b/systemd/opendtu-logger.service @@ -16,15 +16,10 @@ Type=simple User=opendtu-logger Group=opendtu-logger -Environment="OPENDTU_ADDRESS=opendtu.local:80" -Environment="OPENDTU_AUTH=false" -Environment="OPENDTU_USERNAME=admin" -Environment="OPENDTU_PASSWORD=your_super_secret_password" +Environment="REMOTE_URL=opendtu.local:80" Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable" Environment="TIMESCALEDB_ENABLED=true" Environment="TZ=Europe/Amsterdam" -Environment="LOG_LEVEL=INFO" - WorkingDirectory=/opt/opendtu-logger/ ExecStart=/opt/opendtu-logger/opendtu-logger