From d7262164f642cbf74d4e9bc5b06ac91f2776400f Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Thu, 25 Jul 2024 22:46:34 +0200 Subject: [PATCH 01/34] loadConfig should not be exported. --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 0bf084c..9479234 100644 --- a/main.go +++ b/main.go @@ -157,7 +157,7 @@ var config Config // LoadConfig attempts to read the configuration from options.json // If it fails, it falls back to using environment variables -func LoadConfig() Config { +func loadConfig() Config { configFilePath := os.Getenv("CONFIG_FILE") if configFilePath == "" { configFilePath = "/data/options.json" @@ -201,7 +201,7 @@ func main() { slog.SetDefault(logger) // Load the configuration - config := LoadConfig() + config := loadConfig() dbConnStr := config.DB // Connect to PostgreSQL From 6902facab609bd674ddd6e498014f358ff10c11d Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 11:00:41 +0200 Subject: [PATCH 02/34] Bugfix: Migrate insertLiveData and queryEventsEndpoint to new config struct. --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9479234..4ea5cc7 100644 --- a/main.go +++ b/main.go @@ -425,7 +425,7 @@ func createTables(db *sql.DB) { } func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { - timeZone := os.Getenv("TZ") + timeZone := config.TZ loc, _ := time.LoadLocation(timeZone) timestamp := time.Now().In(loc) @@ -503,7 +503,7 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { } func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) { - remoteURL := os.Getenv("REMOTE_URL") + remoteURL := config.OpenDTU endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial) resp, err := http.Get(endpoint) From 444e5065a473cc1d405a3bfb9040f70cd4d7f539 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 22:35:30 +0200 Subject: [PATCH 03/34] Implement proper DB migrations using Goose. Upgrade go packages. --- go.mod | 11 ++- go.sum | 55 ++++++++++- main.go | 147 +++++++---------------------- migrations/00001_log.sql | 14 +++ migrations/00002_inverters.sql | 19 ++++ migrations/00003_inverters_ac.sql | 20 ++++ migrations/00004_inverters_dc.sql | 22 +++++ migrations/00005_inverters_inv.sql | 19 ++++ migrations/00006_events.sql | 17 ++++ migrations/00007_events.sql | 26 +++++ migrations/00008_hints.sql | 16 ++++ migrations/fs.go | 6 ++ 12 files changed, 255 insertions(+), 117 deletions(-) create mode 100644 migrations/00001_log.sql create mode 100644 migrations/00002_inverters.sql create mode 100644 migrations/00003_inverters_ac.sql create mode 100644 migrations/00004_inverters_dc.sql create mode 100644 migrations/00005_inverters_inv.sql create mode 100644 migrations/00006_events.sql create mode 100644 migrations/00007_events.sql create mode 100644 migrations/00008_hints.sql create mode 100644 migrations/fs.go diff --git a/go.mod b/go.mod index 07b4ae4..1d86d89 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,15 @@ module git.hollander.online/energy/opendtu-logger go 1.22 require ( - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/websocket v1.5.3 github.com/lib/pq v1.10.9 + github.com/pressly/goose/v3 v3.21.1 ) -require golang.org/x/net v0.17.0 // indirect +require ( + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/sethvargo/go-retry v0.2.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect +) diff --git a/go.sum b/go.sum index 944767b..f93ce7b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,57 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78buQ= +github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= +github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= +modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/main.go b/main.go index 4ea5cc7..e6fbfa2 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "database/sql" "encoding/json" "fmt" + "io/fs" "log" "log/slog" "net/http" @@ -19,8 +20,10 @@ import ( "time" _ "time/tzdata" + "git.hollander.online/energy/opendtu-logger/migrations" "github.com/gorilla/websocket" _ "github.com/lib/pq" + "github.com/pressly/goose/v3" ) // VUD contains three variables used for most metrics sent by OpenDTU: @@ -212,7 +215,7 @@ func main() { defer db.Close() // Create tables if they don't exist - createTables(db) + migrateDB(db) // Create WebSocket URL from config variable wsURL := "ws://" + config.OpenDTU + "/livedata" @@ -287,122 +290,14 @@ func handleMessage(message []byte, db *sql.DB) { } } -func createTables(db *sql.DB) { - // Execute SQL statements to create tables if they don't exist - // inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers. - // An additional advantage is that it makes plotting in Grafana easier. +func migrateDB(db *sql.DB) { // TODO: Foreign keys commented out as TimescaleDB hypertables don't support them. - createTableSQL := ` - CREATE TABLE IF NOT EXISTS opendtu_log ( - timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP, - power NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC - ); - - CREATE TABLE IF NOT EXISTS opendtu_inverters ( - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - name TEXT, - producing BOOL, - limit_relative NUMERIC, - limit_absolute NUMERIC - ); - - CREATE TABLE IF NOT EXISTS opendtu_inverters_ac ( - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - ac_number INT, - power NUMERIC, - voltage NUMERIC, - current NUMERIC, - frequency NUMERIC, - powerfactor NUMERIC, - reactivepower NUMERIC - ); - - CREATE TABLE IF NOT EXISTS opendtu_inverters_dc ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - dc_number INT, - name TEXT, - power NUMERIC, - voltage NUMERIC, - current NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC, - irradiation NUMERIC - ); - - CREATE TABLE IF NOT EXISTS opendtu_inverters_inv ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - inverter_serial TEXT, - temperature NUMERIC, - power_dc NUMERIC, - yieldday NUMERIC, - yieldtotal NUMERIC, - efficiency NUMERIC - ); - CREATE TABLE IF NOT EXISTS opendtu_events ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - inverter_serial TEXT, - message_id INT, - message TEXT, - start_time INT, - end_time INT - ); + // Perform DB migrations + err := migrateFS(db, migrations.FS, ".") - DO $$ - BEGIN - -- Check if start_timestamp column exists - IF NOT EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name='opendtu_events' - AND column_name='start_timestamp') THEN - -- Add start_timestamp column - ALTER TABLE opendtu_events - ADD COLUMN start_timestamp TIMESTAMPTZ; - END IF; - - -- Check if end_timestamp column exists - IF NOT EXISTS (SELECT 1 FROM information_schema.columns - WHERE table_name='opendtu_events' - AND column_name='end_timestamp') THEN - -- Add end_timestamp column - ALTER TABLE opendtu_events - ADD COLUMN end_timestamp TIMESTAMPTZ; - END IF; - END $$; - - CREATE TABLE IF NOT EXISTS opendtu_hints ( - -- id SERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ, - -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), - time_sync BOOL, - radio_problem BOOL, - default_password BOOL - ); - - CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp); - CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp); - - ` - - _, err := db.Exec(createTableSQL) if err != nil { - log.Fatal("Error creating tables: ", err) + log.Fatal("Error performing database migrations: ", err) } timescaleEnabled := config.TimescaleDB @@ -502,6 +397,32 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { } +func migrate(db *sql.DB, dir string) error { + err := goose.SetDialect("postgres") + if err != nil { + return fmt.Errorf("migrate: %w", err) + } + err = goose.Up(db, dir) + if err != nil { + return fmt.Errorf("migrate: %w", err) + } + return nil + +} + +func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error { + // In case the dir is an empty string, they probably meant the current directory and goose wants a period for that. + if dir == "" { + dir = "." + } + goose.SetBaseFS(migrationFS) + defer func() { + // Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS. + goose.SetBaseFS(nil) + }() + return migrate(db, dir) +} + func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) { remoteURL := config.OpenDTU endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial) diff --git a/migrations/00001_log.sql b/migrations/00001_log.sql new file mode 100644 index 0000000..6a9d6b4 --- /dev/null +++ b/migrations/00001_log.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_log ( + timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP, + power NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC +); + +CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00002_inverters.sql b/migrations/00002_inverters.sql new file mode 100644 index 0000000..935eb63 --- /dev/null +++ b/migrations/00002_inverters.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_inverters ( + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + -- inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers. + -- An additional advantage is that it makes plotting in Grafana easier. + inverter_serial TEXT, + name TEXT, + producing BOOL, + limit_relative NUMERIC, + limit_absolute NUMERIC +); + +CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00003_inverters_ac.sql b/migrations/00003_inverters_ac.sql new file mode 100644 index 0000000..cccfb2d --- /dev/null +++ b/migrations/00003_inverters_ac.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_inverters_ac ( + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + ac_number INT, + power NUMERIC, + voltage NUMERIC, + current NUMERIC, + frequency NUMERIC, + powerfactor NUMERIC, + reactivepower NUMERIC +); + +CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00004_inverters_dc.sql b/migrations/00004_inverters_dc.sql new file mode 100644 index 0000000..c958fef --- /dev/null +++ b/migrations/00004_inverters_dc.sql @@ -0,0 +1,22 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_inverters_dc ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + dc_number INT, + name TEXT, + power NUMERIC, + voltage NUMERIC, + current NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC, + irradiation NUMERIC +); + +CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00005_inverters_inv.sql b/migrations/00005_inverters_inv.sql new file mode 100644 index 0000000..677248f --- /dev/null +++ b/migrations/00005_inverters_inv.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_inverters_inv ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + inverter_serial TEXT, + temperature NUMERIC, + power_dc NUMERIC, + yieldday NUMERIC, + yieldtotal NUMERIC, + efficiency NUMERIC +); + +CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00006_events.sql b/migrations/00006_events.sql new file mode 100644 index 0000000..4f56cc7 --- /dev/null +++ b/migrations/00006_events.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_events ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + inverter_serial TEXT, + message_id INT, + message TEXT, + start_time INT, + end_time INT +); + +CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd \ No newline at end of file diff --git a/migrations/00007_events.sql b/migrations/00007_events.sql new file mode 100644 index 0000000..d7ad456 --- /dev/null +++ b/migrations/00007_events.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +DO $$ +BEGIN + -- Check if start_timestamp column exists + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name='opendtu_events' + AND column_name='start_timestamp') THEN + -- Add start_timestamp column + ALTER TABLE opendtu_events + ADD COLUMN start_timestamp TIMESTAMPTZ; + END IF; + + -- Check if end_timestamp column exists + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name='opendtu_events' + AND column_name='end_timestamp') THEN + -- Add end_timestamp column + ALTER TABLE opendtu_events + ADD COLUMN end_timestamp TIMESTAMPTZ; + END IF; +END $$; +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/00008_hints.sql b/migrations/00008_hints.sql new file mode 100644 index 0000000..ed5e87b --- /dev/null +++ b/migrations/00008_hints.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS opendtu_hints ( + -- id SERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ, + -- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp), + time_sync BOOL, + radio_problem BOOL, + default_password BOOL +); + +CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp); +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/migrations/fs.go b/migrations/fs.go new file mode 100644 index 0000000..91cca1c --- /dev/null +++ b/migrations/fs.go @@ -0,0 +1,6 @@ +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS From 9a0fb6ad1c24c5bc5d29a2ce15b5c5dd47dfa5a0 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 22:36:50 +0200 Subject: [PATCH 04/34] Remove completed DB migration TODO. --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index e6fbfa2..1184b82 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ // Idea: Make a full admin / config GUI and only configure through this utility. // Idea: Gather settings only on start-up. // TODO: Only update meter readings such as yieldday, yieldtotal on-change. -// TODO: Implement proper DB migrations. package main import ( From a7576cd0c06b5077dac1250bfeb1d912c4f91e67 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 22:37:29 +0200 Subject: [PATCH 05/34] Add health check TODO. --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 1184b82..8bd6657 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ // Idea: Make a full admin / config GUI and only configure through this utility. // Idea: Gather settings only on start-up. // TODO: Only update meter readings such as yieldday, yieldtotal on-change. +// TODO: Add a health check endpoint, potentially log to it. package main import ( From bf885fb84ce45ddbfabe4d000db64b400b46e394 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 22:42:31 +0200 Subject: [PATCH 06/34] Change default version in compose examples from main to latest. --- docker/compose.with-database-grafana.yml | 2 +- docker/compose.with-database.yml | 2 +- docker/compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/compose.with-database-grafana.yml b/docker/compose.with-database-grafana.yml index b7fa9d4..2a63282 100644 --- a/docker/compose.with-database-grafana.yml +++ b/docker/compose.with-database-grafana.yml @@ -22,7 +22,7 @@ services: retries: 20 opendtu-logger: - image: git.hollander.online/energy/opendtu-logger:main + image: git.hollander.online/energy/opendtu-logger:latest restart: always environment: DB_URL: ${DB_URL} diff --git a/docker/compose.with-database.yml b/docker/compose.with-database.yml index b0a9004..867bbd9 100644 --- a/docker/compose.with-database.yml +++ b/docker/compose.with-database.yml @@ -23,7 +23,7 @@ services: opendtu-logger: restart: always - image: git.hollander.online/energy/opendtu-logger:main + image: git.hollander.online/energy/opendtu-logger:latest environment: DB_URL: ${DB_URL} REMOTE_URL: ${REMOTE_URL} diff --git a/docker/compose.yml b/docker/compose.yml index d25a958..dc0fbe3 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: opendtu-logger: restart: always - image: git.hollander.online/energy/opendtu-logger:main + image: git.hollander.online/energy/opendtu-logger:latest environment: DB_URL: ${DB_URL} REMOTE_URL: ${REMOTE_URL} From 48d0382b0e15e9fbef12ebc54e0f10ea314fd8a1 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 26 Jul 2024 23:46:19 +0200 Subject: [PATCH 07/34] BREAKING: Migrate REMOTE_URL to OPENDTU_ADDRESS. BREAKING: Migrate opendtu to opendtu_address Add authentication capability for locked-down opendtu's. Updated setup examples with new variables. --- docker/compose.with-database-grafana.yml | 5 +- docker/compose.with-database.yml | 5 +- docker/compose.yml | 5 +- docker/example.with-database.env | 8 +- main.go | 102 +++++++++++++++++++---- systemd/opendtu-logger.service | 5 +- 6 files changed, 109 insertions(+), 21 deletions(-) diff --git a/docker/compose.with-database-grafana.yml b/docker/compose.with-database-grafana.yml index 2a63282..d72d7e7 100644 --- a/docker/compose.with-database-grafana.yml +++ b/docker/compose.with-database-grafana.yml @@ -26,7 +26,10 @@ services: restart: always environment: DB_URL: ${DB_URL} - REMOTE_URL: ${REMOTE_URL} + OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} + OPENDTU_AUTH: ${OPENDTU_AUTH} + OPENDTU_USERNAME: ${OPENDTU_USERNAME} + OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} depends_on: diff --git a/docker/compose.with-database.yml b/docker/compose.with-database.yml index 867bbd9..277ee63 100644 --- a/docker/compose.with-database.yml +++ b/docker/compose.with-database.yml @@ -26,7 +26,10 @@ services: image: git.hollander.online/energy/opendtu-logger:latest environment: DB_URL: ${DB_URL} - REMOTE_URL: ${REMOTE_URL} + OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} + OPENDTU_AUTH: ${OPENDTU_AUTH} + OPENDTU_USERNAME: ${OPENDTU_USERNAME} + OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} depends_on: diff --git a/docker/compose.yml b/docker/compose.yml index dc0fbe3..2c29835 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -5,7 +5,10 @@ services: image: git.hollander.online/energy/opendtu-logger:latest environment: DB_URL: ${DB_URL} - REMOTE_URL: ${REMOTE_URL} + OPENDTU_ADDRESS: ${OPENDTU_ADDRESS} + OPENDTU_AUTH: ${OPENDTU_AUTH} + OPENDTU_USERNAME: ${OPENDTU_USERNAME} + OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} depends_on: diff --git a/docker/example.with-database.env b/docker/example.with-database.env index a283d89..5131f51 100644 --- a/docker/example.with-database.env +++ b/docker/example.with-database.env @@ -1,9 +1,13 @@ +# OpenDTU +OPENDTU_ADDRESS="192.168.1.89:80" +OPENDTU_AUTH=false +OPENDTU_USERNAME= +OPENDTU_PASSWORD= # OpenDTU Logger -REMOTE_URL="192.168.1.89:80" DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" TIMESCALEDB_ENABLED=true TZ="Europe/Amsterdam" # Database configuration PG_USER=postgres PG_PASSWORD= -PG_DB=opendtu_logger \ No newline at end of file +PG_DB=opendtu_logger diff --git a/main.go b/main.go index 8bd6657..2d3921f 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,4 @@ // TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring. -// TODO: Use username and password provided using Basic Authentication. // TODO: Record Inverter struct data only on-change. // Idea: Make a full admin / config GUI and only configure through this utility. // Idea: Gather settings only on start-up. @@ -9,6 +8,7 @@ package main import ( "database/sql" + "encoding/base64" "encoding/json" "fmt" "io/fs" @@ -149,10 +149,13 @@ type InverterSettingsData struct { // Config settings struct type Config struct { - DB string `json:"db"` - OpenDTU string `json:"opendtu"` - TimescaleDB bool `json:"timescaledb"` - TZ string `json:"tz"` + DB string `json:"db"` + OpenDTUAddress string `json:"opendtu_address"` + OpenDTUAuth bool `json:"opendtu_auth"` + OpenDTUUser string `json:"opendtu_username"` + OpenDTUPassword string `json:"opendtu_password"` + TimescaleDB bool `json:"timescaledb"` + TZ string `json:"tz"` } var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) @@ -173,15 +176,50 @@ func loadConfig() Config { if err != nil { log.Fatalf("Error parsing config file: %v", err) } + if config.DB == "" { + log.Fatal("db connection settings are not set") + } + if config.OpenDTUAddress == "" { + log.Fatal("opendtu_address is not set") + } + if config.OpenDTUAuth { + if config.OpenDTUUser == "" { + log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username") + } + if config.OpenDTUPassword == "" { + log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password") + } + } } else { + logger.Info("JSON config file not found. Falling back to environment variables.") // Fallback to environment variables config.DB = os.Getenv("DB_URL") if config.DB == "" { log.Fatal("DB_URL environment variable is not set.") } - config.OpenDTU = os.Getenv("REMOTE_URL") - if config.OpenDTU == "" { - log.Fatal("REMOTE_URL environment variable is not set.") + config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS") + if config.OpenDTUAddress == "" { + log.Fatal("OPENDTU_ADDRESS environment variable is not set.") + } + + openDTUAuthStr := os.Getenv("OPENDTU_AUTH") + if openDTUAuthStr != "" { + openDTUAuth, err := strconv.ParseBool(openDTUAuthStr) + if err != nil { + log.Fatalf("Error parsing OPENDTU_AUTH: %v", err) + } + config.OpenDTUAuth = openDTUAuth + } + if config.OpenDTUAuth { + config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME") + if config.OpenDTUUser == "" { + log.Fatal("OPENDTU_USERNAME environment variable is not set.") + } + config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD") + if config.OpenDTUPassword == "" { + log.Fatal("OPENDTU_PASSWORD environment variable is not set.") + } + } timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED") @@ -194,6 +232,10 @@ func loadConfig() Config { } config.TZ = os.Getenv("TZ") } + _, err = time.LoadLocation(config.TZ) + if err != nil { + logger.Warn("invalid timezone") + } return config } @@ -218,10 +260,16 @@ func main() { migrateDB(db) // Create WebSocket URL from config variable - wsURL := "ws://" + config.OpenDTU + "/livedata" + wsURL := "ws://" + config.OpenDTUAddress + "/livedata" + + // Create headers with optional Basic Auth + headers := http.Header{} + if config.OpenDTUAuth { + headers.Set("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword)) + } // Establish WebSocket connection - c, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + c, _, err := websocket.DefaultDialer.Dial(wsURL, headers) if err != nil { log.Fatal(err) } @@ -424,15 +472,33 @@ func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error { } func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) { - remoteURL := config.OpenDTU - endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial) + endpoint := fmt.Sprintf("http://"+config.OpenDTUAddress+"/api/eventlog/status?inv=%s", inverterSerial) - resp, err := http.Get(endpoint) + // Create a new HTTP request + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + + if config.OpenDTUAuth { + // Add Basic Auth header + req.Header.Add("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword)) + } + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() + // Check for HTTP errors + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status) + } + + // Decode the response var eventsResponse EventsResponse if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil { return nil, err @@ -508,6 +574,12 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { } } +// basicAuth generates the Basic Auth header value +func basicAuth(username, password string) string { + credentials := username + ":" + password + return "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials)) +} + // TODO: finish this function. // func updateInverterConfig(db *sql.DB) { // // Periodically query the /api/inverter/list @@ -526,8 +598,8 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { // } // func queryConfigEndpoint() (*InverterSettingsData, error) { -// remoteURL := os.Getenv("REMOTE_URL") -// endpoint := fmt.Sprintf("http://" + remoteURL + "/api/inverter/list") +// openDTUAddress := os.Getenv("OPENDTU_ADDRESS") +// endpoint := fmt.Sprintf("http://" + openDTUAddress + "/api/inverter/list") // resp, err := http.Get(endpoint) // if err != nil { diff --git a/systemd/opendtu-logger.service b/systemd/opendtu-logger.service index 9c14fb6..60fa257 100644 --- a/systemd/opendtu-logger.service +++ b/systemd/opendtu-logger.service @@ -16,7 +16,10 @@ Type=simple User=opendtu-logger Group=opendtu-logger -Environment="REMOTE_URL=opendtu.local:80" +Environment="OPENDTU_ADDRESS=opendtu.local:80" +Environment="OPENDTU_AUTH=false" +Environment="OPENDTU_USERNAME=admin" +Environment="OPENDTU_PASSWORD=your_super_secret_password" Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable" Environment="TIMESCALEDB_ENABLED=true" Environment="TZ=Europe/Amsterdam" From b63c1e85d33b47812049a8526c743c8a1647baf4 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 27 Jul 2024 00:08:40 +0200 Subject: [PATCH 08/34] Add DEBUG log_level. --- main.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/main.go b/main.go index 2d3921f..60d76ae 100644 --- a/main.go +++ b/main.go @@ -156,6 +156,7 @@ type Config struct { OpenDTUPassword string `json:"opendtu_password"` TimescaleDB bool `json:"timescaledb"` TZ string `json:"tz"` + LogLevel string `json:"log_level"` } var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) @@ -240,6 +241,30 @@ func loadConfig() Config { return config } +// Helper function to map environment variable to slog.Level +func getLogLevel(defaultLevel slog.Level) slog.Level { + logLevelStr := config.LogLevel + switch logLevelStr { + case "DEBUG": + return slog.LevelDebug + case "INFO": + return slog.LevelInfo + case "WARN": + return slog.LevelWarn + case "ERROR": + return slog.LevelError + default: + return defaultLevel + } +} + +// Function to create a new logger with a specified log level +func createLoggerWithLevel(level slog.Level) *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + })) +} + // Main program func main() { // Initial logger setup @@ -248,6 +273,10 @@ func main() { // Load the configuration config := loadConfig() + // Set the logLevel + logLevel := getLogLevel(slog.LevelInfo) // Default to info level + logger = createLoggerWithLevel(logLevel) + dbConnStr := config.DB // Connect to PostgreSQL db, err := sql.Open("postgres", dbConnStr) @@ -262,6 +291,8 @@ func main() { // Create WebSocket URL from config variable wsURL := "ws://" + config.OpenDTUAddress + "/livedata" + logger.Debug(wsURL) + // Create headers with optional Basic Auth headers := http.Header{} if config.OpenDTUAuth { From 3556e401bc3fc0aebdfe91d196c604aa42ff49c9 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 27 Jul 2024 00:42:51 +0200 Subject: [PATCH 09/34] Add configurable LOG_LEVEL env vars. --- docker/compose.with-database-grafana.yml | 1 + docker/compose.with-database.yml | 2 ++ docker/compose.yml | 1 + docker/example.env | 11 +++++++++++ docker/example.with-database.env | 3 +++ main.go | 1 + systemd/opendtu-logger.service | 2 ++ 7 files changed, 21 insertions(+) create mode 100644 docker/example.env diff --git a/docker/compose.with-database-grafana.yml b/docker/compose.with-database-grafana.yml index d72d7e7..1833916 100644 --- a/docker/compose.with-database-grafana.yml +++ b/docker/compose.with-database-grafana.yml @@ -32,6 +32,7 @@ services: OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} + LOG_LEVEL: ${LOG_LEVEL} depends_on: timescaledb: condition: service_healthy diff --git a/docker/compose.with-database.yml b/docker/compose.with-database.yml index 277ee63..fdbb400 100644 --- a/docker/compose.with-database.yml +++ b/docker/compose.with-database.yml @@ -32,6 +32,8 @@ services: OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} + LOG_LEVEL: ${LOG_LEVEL} + depends_on: timescaledb: condition: service_healthy diff --git a/docker/compose.yml b/docker/compose.yml index 2c29835..5d60048 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -11,6 +11,7 @@ services: OPENDTU_PASSWORD: ${OPENDTU_PASSWORD} TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED} TZ: ${TZ} + LOG_LEVEL: ${LOG_LEVEL} depends_on: timescaledb: condition: service_healthy diff --git a/docker/example.env b/docker/example.env new file mode 100644 index 0000000..aa51cd1 --- /dev/null +++ b/docker/example.env @@ -0,0 +1,11 @@ +# OpenDTU +OPENDTU_ADDRESS="192.168.1.89:80" +OPENDTU_AUTH=false +OPENDTU_USERNAME= +OPENDTU_PASSWORD= + +# OpenDTU Logger +DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" +TIMESCALEDB_ENABLED=true +TZ="Europe/Amsterdam" +LOG_LEVEL=INFO" \ No newline at end of file diff --git a/docker/example.with-database.env b/docker/example.with-database.env index 5131f51..d6cd1d8 100644 --- a/docker/example.with-database.env +++ b/docker/example.with-database.env @@ -3,10 +3,13 @@ OPENDTU_ADDRESS="192.168.1.89:80" OPENDTU_AUTH=false OPENDTU_USERNAME= OPENDTU_PASSWORD= + # OpenDTU Logger DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable" TIMESCALEDB_ENABLED=true TZ="Europe/Amsterdam" +LOG_LEVEL=INFO" + # Database configuration PG_USER=postgres PG_PASSWORD= diff --git a/main.go b/main.go index 60d76ae..ac98c20 100644 --- a/main.go +++ b/main.go @@ -232,6 +232,7 @@ func loadConfig() Config { config.TimescaleDB = timescaleDB } config.TZ = os.Getenv("TZ") + config.LogLevel = os.Getenv("LOG_LEVEL") } _, err = time.LoadLocation(config.TZ) if err != nil { diff --git a/systemd/opendtu-logger.service b/systemd/opendtu-logger.service index 60fa257..88ad6ad 100644 --- a/systemd/opendtu-logger.service +++ b/systemd/opendtu-logger.service @@ -23,6 +23,8 @@ Environment="OPENDTU_PASSWORD=your_super_secret_password" Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable" Environment="TIMESCALEDB_ENABLED=true" Environment="TZ=Europe/Amsterdam" +Environment="LOG_LEVEL=INFO" + WorkingDirectory=/opt/opendtu-logger/ ExecStart=/opt/opendtu-logger/opendtu-logger From 32bebfdca2f4d4762d0cd54cfd8a85fb1c8198d2 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 27 Jul 2024 00:43:29 +0200 Subject: [PATCH 10/34] Add default admin user to OPENDTU_USERNAME --- docker/example.env | 2 +- docker/example.with-database.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/example.env b/docker/example.env index aa51cd1..f94605f 100644 --- a/docker/example.env +++ b/docker/example.env @@ -1,7 +1,7 @@ # OpenDTU OPENDTU_ADDRESS="192.168.1.89:80" OPENDTU_AUTH=false -OPENDTU_USERNAME= +OPENDTU_USERNAME=admin OPENDTU_PASSWORD= # OpenDTU Logger diff --git a/docker/example.with-database.env b/docker/example.with-database.env index d6cd1d8..b5d280f 100644 --- a/docker/example.with-database.env +++ b/docker/example.with-database.env @@ -1,7 +1,7 @@ # OpenDTU OPENDTU_ADDRESS="192.168.1.89:80" OPENDTU_AUTH=false -OPENDTU_USERNAME= +OPENDTU_USERNAME=admin OPENDTU_PASSWORD= # OpenDTU Logger From 014a5e85a2290b88ed93c0ee368b7a2dcfa40094 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Wed, 7 Aug 2024 18:55:26 +0200 Subject: [PATCH 11/34] Fix to take into account user-defined timezone for insertEvents. Add multi OpenDTU TODO. --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index ac98c20..295ed2a 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ // Idea: Gather settings only on start-up. // TODO: Only update meter readings such as yieldday, yieldtotal on-change. // TODO: Add a health check endpoint, potentially log to it. +// TODO: Add support for monitoring multiple OpenDTU's at once. package main import ( @@ -563,7 +564,9 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int { } func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) { - timestamp := time.Now() + timeZone := config.TZ + loc, _ := time.LoadLocation(timeZone) + timestamp := time.Now().In(loc) for _, event := range events.Events { // Insert events data into the events table From b8ea94bb61bd1d53e61e176f88ecd0be8ec19b32 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Thu, 8 Aug 2024 11:18:26 +0200 Subject: [PATCH 12/34] Add TARGETOS and TARGETARCH to fix multi-platform compilation issue. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d05c839..3859d1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ RUN go mod download COPY . . # Build the application for the specified target architecture +# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ +ARG TARGETOS TARGETARCH RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger . # Create a minimal runtime image From ce336b4f852970e333697daf1dd724d079c07e9d Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Thu, 8 Aug 2024 11:19:01 +0200 Subject: [PATCH 13/34] Make go build TARGETOS configurable. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3859d1c..f5c5bfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY . . # Build the application for the specified target architecture # https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ ARG TARGETOS TARGETARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger . +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger . # Create a minimal runtime image FROM --platform=${TARGETPLATFORM} scratch From c8b8211e3a29fe9495b028b4c53ac715790564d1 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 10 Aug 2024 09:54:57 +0200 Subject: [PATCH 14/34] Add documentation for ARG TARGET* --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f5c5bfa..e3370f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ COPY . . # Build the application for the specified target architecture # https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ +# Declare TARGETOS and TARGETARCH in the local scope so they can be used in the build stage. ARG TARGETOS TARGETARCH RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger . From ed1b6d69eced04280c500d3a5b50eed71f4270ca Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 10 Aug 2024 23:42:38 +0200 Subject: [PATCH 15/34] Go dependency upgrades --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1d86d89..56b9f3e 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( require ( github.com/mfridman/interpolate v0.0.2 // indirect - github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index f93ce7b..4b6f974 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -36,6 +38,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= From 626b7f9993856b3e76262a731ff178d96acb4814 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Wed, 14 Aug 2024 19:34:59 +0200 Subject: [PATCH 16/34] Update Go 1.22 -> 1.23 --- Dockerfile | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3370f4..0d495b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use buildx for multi-architecture support -FROM --platform=${BUILDPLATFORM} golang:1.22 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder WORKDIR /app diff --git a/go.mod b/go.mod index 56b9f3e..95f7e17 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.hollander.online/energy/opendtu-logger -go 1.22 +go 1.23 require ( github.com/gorilla/websocket v1.5.3 From 0052a9848b2e3d485c2e231d9f898608bfee4a2b Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Wed, 14 Aug 2024 23:18:54 +0200 Subject: [PATCH 17/34] go mod tidy --- go.mod | 2 +- go.sum | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 95f7e17..d12d80b 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,6 @@ require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 4b6f974..e94c10f 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -24,25 +22,16 @@ github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78b github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= -github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= From dc6f41eeb05b2a8bb9112042e5d3d48dc85db699 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 14 Sep 2024 18:15:52 +0200 Subject: [PATCH 18/34] Update go dependencies. --- go.mod | 2 +- go.sum | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d12d80b..bd0988c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23 require ( github.com/gorilla/websocket v1.5.3 github.com/lib/pq v1.10.9 - github.com/pressly/goose/v3 v3.21.1 + github.com/pressly/goose/v3 v3.22.0 ) require ( diff --git a/go.sum b/go.sum index e94c10f..01051c6 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78buQ= github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= +github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= +github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= @@ -38,12 +40,15 @@ modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQX modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 50d89748f7361163740e7ffb1eba35af45afb20d Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 14 Sep 2024 18:16:13 +0200 Subject: [PATCH 19/34] go mod tidy --- go.mod | 1 - go.sum | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bd0988c..aa9616d 100644 --- a/go.mod +++ b/go.mod @@ -13,5 +13,4 @@ require ( github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 01051c6..60e41cf 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78buQ= -github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -38,17 +36,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= +modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 40abe7309d3bbd7f7104c952867902f0e70d38de Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Fri, 17 Jan 2025 12:40:34 +0100 Subject: [PATCH 20/34] Add note about OpenDTU version compatibility --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index faa518d..a1221fb 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Installation on Home Assistant can be done by adding the [Home Assistant add-on ### Configuring OpenDTU In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used. +OpenDTU Logger 0.1.3 has been tested with OpenDTU versions v24.6.10 - v25.1.14. - Within OpenDTU, go to `Settings` -> `Inverter settings` (). - For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`. From af30019ba0226a0d2d913e90a3a107ac6f1bdcd4 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 15 Feb 2025 11:30:05 +0100 Subject: [PATCH 21/34] Upgrade dependencies. Upgrade go 1.23 > 1.24. --- go.mod | 6 +++--- go.sum | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aa9616d..292cd48 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module git.hollander.online/energy/opendtu-logger -go 1.23 +go 1.24 require ( github.com/gorilla/websocket v1.5.3 github.com/lib/pq v1.10.9 - github.com/pressly/goose/v3 v3.22.0 + github.com/pressly/goose/v3 v3.24.1 ) require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.8.0 // indirect + golang.org/x/sync v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index 60e41cf..de9e9dd 100644 --- a/go.sum +++ b/go.sum @@ -20,18 +20,24 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= +github.com/pressly/goose/v3 v3.24.1 h1:bZmxRco2uy5uu5Ng1MMVEfYsFlrMJI+e/VMXHQ3C4LY= +github.com/pressly/goose/v3 v3.24.1/go.mod h1:rEWreU9uVtt0DHCyLzF9gRcWiiTF/V+528DV+4DORug= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= @@ -44,6 +50,7 @@ modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/sqlite v1.34.1 h1:u3Yi6M0N8t9yKRDwhXcyp1eS5/ErhPTBggxWFuR6Hfk= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 7f5fa4962eee61be81c63559b40932fd60f49713 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 15 Feb 2025 11:33:16 +0100 Subject: [PATCH 22/34] Upgrade OpenDTU compatibiity to v25.2.3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1221fb..7a6b4d0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Installation on Home Assistant can be done by adding the [Home Assistant add-on ### Configuring OpenDTU In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used. -OpenDTU Logger 0.1.3 has been tested with OpenDTU versions v24.6.10 - v25.1.14. +OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.2.3. - Within OpenDTU, go to `Settings` -> `Inverter settings` (). - For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`. From f75fe8d72cb72664dbf5dd244376108ab3de9988 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 15 Feb 2025 11:41:19 +0100 Subject: [PATCH 23/34] Upgrade Docker golang 1.23 > 1.24 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0d495b7..f8e11a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use buildx for multi-architecture support -FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.24 AS builder WORKDIR /app From 39dd6ae0d61f750aa87bbe738e4c9feacddbc6b9 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Thu, 15 May 2025 14:04:48 +0200 Subject: [PATCH 24/34] Verify compatibility OpenDTU 25.5.10. Update dependencies. --- README.md | 2 +- go.mod | 4 ++-- go.sum | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a6b4d0..b90378a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Installation on Home Assistant can be done by adding the [Home Assistant add-on ### Configuring OpenDTU In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used. -OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.2.3. +OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.5.10. - Within OpenDTU, go to `Settings` -> `Inverter settings` (). - For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`. diff --git a/go.mod b/go.mod index 292cd48..9e78730 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.24 require ( github.com/gorilla/websocket v1.5.3 github.com/lib/pq v1.10.9 - github.com/pressly/goose/v3 v3.24.1 + github.com/pressly/goose/v3 v3.24.3 ) require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.11.0 // indirect + golang.org/x/sync v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index de9e9dd..0a2bb1c 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZ github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= github.com/pressly/goose/v3 v3.24.1 h1:bZmxRco2uy5uu5Ng1MMVEfYsFlrMJI+e/VMXHQ3C4LY= github.com/pressly/goose/v3 v3.24.1/go.mod h1:rEWreU9uVtt0DHCyLzF9gRcWiiTF/V+528DV+4DORug= +github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwWlM= +github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= @@ -35,22 +37,29 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/libc v1.65.0 h1:e183gLDnAp9VJh6gWKdTy0CThL9Pt7MfcR/0bgb7Y1Y= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= modernc.org/sqlite v1.34.1 h1:u3Yi6M0N8t9yKRDwhXcyp1eS5/ErhPTBggxWFuR6Hfk= +modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From a654dd802e6ff2908c83e53e2f79f76c23c35259 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Thu, 15 May 2025 14:19:06 +0200 Subject: [PATCH 25/34] Remove superfluous newline. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b90378a..3139d93 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ OpenDTU Logger 0.1.4 has been tested with OpenDTU versions v24.6.10 - v25.5.10. - Click `Save` - Repeat this procedure for every inverter. - ### Docker Compose The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments. From dfb7c0c35e6334f82088713803f74ae4b6d3504f Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 00:40:38 +0200 Subject: [PATCH 26/34] expand .gitignore. --- .gitignore | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 431cdac..5425bfa 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,32 @@ Temporary Items # .nfs files are created when an open file is removed but is still being accessed .nfs* +# ---> Windows +# thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + # ---> Go # Binaries for programs and plugins *.exe @@ -72,4 +98,11 @@ docs # Don't include compiled binary main -opendtu-logger \ No newline at end of file +opendtu-logger + +# OpenDTU reference repository for testing +opendtu/ + +# Test coverage files +coverage.out +coverage.html \ No newline at end of file From a6ddd2700beb09d09fe139fbe5fc14777e4aa044 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 00:43:11 +0200 Subject: [PATCH 27/34] Upgrade dependencies --- go.mod | 4 ++-- go.sum | 42 ++++++++++-------------------------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 9e78730..69fb927 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.hollander.online/energy/opendtu-logger -go 1.24 +go 1.25 require ( github.com/gorilla/websocket v1.5.3 @@ -12,5 +12,5 @@ require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.14.0 // indirect + golang.org/x/sync v0.16.0 // indirect ) diff --git a/go.sum b/go.sum index 0a2bb1c..0760f47 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -18,49 +16,29 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw= -github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg= -github.com/pressly/goose/v3 v3.24.1 h1:bZmxRco2uy5uu5Ng1MMVEfYsFlrMJI+e/VMXHQ3C4LY= -github.com/pressly/goose/v3 v3.24.1/go.mod h1:rEWreU9uVtt0DHCyLzF9gRcWiiTF/V+528DV+4DORug= github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwWlM= github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= modernc.org/libc v1.65.0 h1:e183gLDnAp9VJh6gWKdTy0CThL9Pt7MfcR/0bgb7Y1Y= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/libc v1.65.0/go.mod h1:7m9VzGq7APssBTydds2zBcxGREwvIGpuUBaKTXdm2Qs= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= -modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s= -modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= -modernc.org/sqlite v1.34.1 h1:u3Yi6M0N8t9yKRDwhXcyp1eS5/ErhPTBggxWFuR6Hfk= +modernc.org/memory v1.10.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= From 163e3672efc5744802b2b9c0de99775c2356cc1f Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 00:49:27 +0200 Subject: [PATCH 28/34] Rename LICENSE as it's not markdown. --- LICENSE.md => LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE From 851c6d01ecdcec79c2ee9bcf2a402c790d1fd184 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 01:03:49 +0200 Subject: [PATCH 29/34] Expand hints. Add RadioStatistics struct. Add new Order, DataAge and DataAgeMs. --- main.go | 61 +++++++++++++++++--------- migrations/00009_hints_pin_mapping.sql | 9 ++++ 2 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 migrations/00009_hints_pin_mapping.sql diff --git a/main.go b/main.go index 295ed2a..b6041cf 100644 --- a/main.go +++ b/main.go @@ -72,20 +72,34 @@ type InverterINV struct { YieldTotal VUD `json:"YieldTotal"` } +// RadioStatistics contains radio communication statistics +type RadioStatistics struct { + TxRequest int `json:"tx_request"` + TxReRequest int `json:"tx_re_request"` + RxSuccess int `json:"rx_success"` + RxFailNothing int `json:"rx_fail_nothing"` + RxFailPartial int `json:"rx_fail_partial"` + RxFailCorrupt int `json:"rx_fail_corrupt"` + RSSI float64 `json:"rssi"` +} + // Inverter struct type Inverter struct { Serial string `json:"serial"` Name string `json:"name"` + Order int `json:"order"` + DataAge int `json:"data_age"` + DataAgeMs int `json:"data_age_ms"` Producing bool `json:"producing"` LimitRelative float64 `json:"limit_relative"` LimitAbsolute float64 `json:"limit_absolute"` - AC map[string]InverterAC `json:"AC"` - DC map[string]InverterDC `json:"DC"` - Events int `json:"events"` PollEnabled bool `json:"poll_enabled"` Reachable bool `json:"reachable"` - DataAge int `json:"data_age"` + Events int `json:"events"` + AC map[string]InverterAC `json:"AC"` + DC map[string]InverterDC `json:"DC"` INV map[string]InverterINV `json:"INV"` + RadioStats RadioStatistics `json:"radio_stats"` } type Total struct { @@ -98,6 +112,7 @@ type Hints struct { TimeSync bool `json:"time_sync"` RadioProblem bool `json:"radio_problem"` DefaultPassword bool `json:"default_password"` + PinMappingIssue bool `json:"pin_mapping_issue"` } type LiveData struct { @@ -165,7 +180,7 @@ var config Config // LoadConfig attempts to read the configuration from options.json // If it fails, it falls back to using environment variables -func loadConfig() Config { +func loadConfig() (Config, error) { configFilePath := os.Getenv("CONFIG_FILE") if configFilePath == "" { configFilePath = "/data/options.json" @@ -176,20 +191,20 @@ func loadConfig() Config { // Successfully read the file, parse the JSON err = json.Unmarshal(data, &config) if err != nil { - log.Fatalf("Error parsing config file: %v", err) + return Config{}, fmt.Errorf("error parsing config file: %w", err) } if config.DB == "" { - log.Fatal("db connection settings are not set") + return Config{}, fmt.Errorf("db connection settings are not set") } if config.OpenDTUAddress == "" { - log.Fatal("opendtu_address is not set") + return Config{}, fmt.Errorf("opendtu_address is not set") } if config.OpenDTUAuth { if config.OpenDTUUser == "" { - log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username") + return Config{}, fmt.Errorf("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username") } if config.OpenDTUPassword == "" { - log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password") + return Config{}, fmt.Errorf("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password") } } } else { @@ -197,29 +212,29 @@ func loadConfig() Config { // Fallback to environment variables config.DB = os.Getenv("DB_URL") if config.DB == "" { - log.Fatal("DB_URL environment variable is not set.") + return Config{}, fmt.Errorf("DB_URL environment variable is not set") } config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS") if config.OpenDTUAddress == "" { - log.Fatal("OPENDTU_ADDRESS environment variable is not set.") + return Config{}, fmt.Errorf("OPENDTU_ADDRESS environment variable is not set") } openDTUAuthStr := os.Getenv("OPENDTU_AUTH") if openDTUAuthStr != "" { openDTUAuth, err := strconv.ParseBool(openDTUAuthStr) if err != nil { - log.Fatalf("Error parsing OPENDTU_AUTH: %v", err) + return Config{}, fmt.Errorf("error parsing OPENDTU_AUTH: %w", err) } config.OpenDTUAuth = openDTUAuth } if config.OpenDTUAuth { config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME") if config.OpenDTUUser == "" { - log.Fatal("OPENDTU_USERNAME environment variable is not set.") + return Config{}, fmt.Errorf("OPENDTU_USERNAME environment variable is not set") } config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD") if config.OpenDTUPassword == "" { - log.Fatal("OPENDTU_PASSWORD environment variable is not set.") + return Config{}, fmt.Errorf("OPENDTU_PASSWORD environment variable is not set") } } @@ -228,7 +243,7 @@ func loadConfig() Config { if timescaleDBStr != "" { timescaleDB, err := strconv.ParseBool(timescaleDBStr) if err != nil { - log.Fatalf("Error parsing TIMESCALEDB_ENABLED: %v", err) + return Config{}, fmt.Errorf("error parsing TIMESCALEDB_ENABLED: %w", err) } config.TimescaleDB = timescaleDB } @@ -240,7 +255,7 @@ func loadConfig() Config { logger.Warn("invalid timezone") } - return config + return config, nil } // Helper function to map environment variable to slog.Level @@ -273,7 +288,11 @@ func main() { slog.SetDefault(logger) // Load the configuration - config := loadConfig() + var err error + config, err = loadConfig() + if err != nil { + log.Fatal(err) + } // Set the logLevel logLevel := getLogLevel(slog.LevelInfo) // Default to info level @@ -468,9 +487,9 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { } // Insert data into hints table _, err = db.Exec(` - INSERT INTO opendtu_hints (timestamp, time_sync, radio_problem, default_password) - VALUES ($1, $2, $3, $4); - `, timestamp, hints.TimeSync, hints.RadioProblem, hints.DefaultPassword) + INSERT INTO opendtu_hints (timestamp, time_sync, radio_problem, default_password, pin_mapping_issue) + VALUES ($1, $2, $3, $4, $5); + `, timestamp, hints.TimeSync, hints.RadioProblem, hints.DefaultPassword, hints.PinMappingIssue) if err != nil { logger.Error("Error inserting into log table", "error", err) return diff --git a/migrations/00009_hints_pin_mapping.sql b/migrations/00009_hints_pin_mapping.sql new file mode 100644 index 0000000..73f5b74 --- /dev/null +++ b/migrations/00009_hints_pin_mapping.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE opendtu_hints ADD COLUMN IF NOT EXISTS pin_mapping_issue BOOLEAN NOT NULL DEFAULT FALSE; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE opendtu_hints DROP COLUMN IF EXISTS pin_mapping_issue; +-- +goose StatementEnd From 6b496a39ee38f5593a09799b7b9aafbb9f42746d Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 01:56:08 +0200 Subject: [PATCH 30/34] small refactor --- main.go | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/main.go b/main.go index b6041cf..c0a24d6 100644 --- a/main.go +++ b/main.go @@ -178,6 +178,17 @@ type Config struct { var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil)) var config Config +const timescaleHypertableSQL = ` +-- CREATE EXTENSION IF NOT EXISTS timescaledb; +SELECT create_hypertable('opendtu_log', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_inverters', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_inverters_ac', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_inverters_dc', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_inverters_inv', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_events', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +SELECT create_hypertable('opendtu_hints', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); +` + // LoadConfig attempts to read the configuration from options.json // If it fails, it falls back to using environment variables func loadConfig() (Config, error) { @@ -399,26 +410,18 @@ func migrateDB(db *sql.DB) { if err != nil { log.Fatal("Error performing database migrations: ", err) } - timescaleEnabled := config.TimescaleDB - - enableTimescaleDB := ` - -- CREATE EXTENSION IF NOT EXISTS timescaledb; - SELECT create_hypertable('opendtu_log', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_inverters', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_inverters_ac', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_inverters_dc', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_inverters_inv', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_events', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - SELECT create_hypertable('opendtu_hints', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE); - ` - if timescaleEnabled { - _, err := db.Exec(enableTimescaleDB) - if err != nil { + if config.TimescaleDB { + if err := enableTimescaleHypertables(db); err != nil { log.Fatal("Error enabling TimescaleDB: ", err) } } } +func enableTimescaleHypertables(db *sql.DB) error { + _, err := db.Exec(timescaleHypertableSQL) + return err +} + func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) { timeZone := config.TZ loc, _ := time.LoadLocation(timeZone) From 99959726c83144eb40457bbed70b7657beb10cdc Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 01:57:56 +0200 Subject: [PATCH 31/34] add testing, reverse engineered / generated from OpenDTU source to improve compatibility testing and detecting regressions early. --- config_test.go | 561 ++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 3 + integration_test.go | 466 +++++++++++++++++++++++++ main_persistence_test.go | 436 ++++++++++++++++++++++++ main_test.go | 566 +++++++++++++++++++++++++++++++ opendtu_compatibility_test.go | 500 +++++++++++++++++++++++++++ testdata/events_response.json | 23 ++ testdata/livedata_night.json | 112 ++++++ testdata/livedata_producing.json | 137 ++++++++ testdata/livedata_warnings.json | 73 ++++ testdata/test_config.json | 10 + 12 files changed, 2888 insertions(+) create mode 100644 config_test.go create mode 100644 integration_test.go create mode 100644 main_persistence_test.go create mode 100644 main_test.go create mode 100644 opendtu_compatibility_test.go create mode 100644 testdata/events_response.json create mode 100644 testdata/livedata_night.json create mode 100644 testdata/livedata_producing.json create mode 100644 testdata/livedata_warnings.json create mode 100644 testdata/test_config.json diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..660f988 --- /dev/null +++ b/config_test.go @@ -0,0 +1,561 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// TestConfigLoadFromJSONFile tests loading config from a valid JSON file +func TestConfigLoadFromJSONFile(t *testing.T) { + // Save original environment + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + + // Set CONFIG_FILE to our test data + testConfigPath := filepath.Join("testdata", "test_config.json") + absPath, err := filepath.Abs(testConfigPath) + if err != nil { + t.Fatalf("Failed to get absolute path: %v", err) + } + os.Setenv("CONFIG_FILE", absPath) + + // Load config from JSON file + cfg, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + // Verify all fields loaded correctly from JSON + if cfg.DB != "postgres://user:password@localhost:5432/opendtu" { + t.Errorf("Expected DB from JSON, got %v", cfg.DB) + } + if cfg.OpenDTUAddress != "192.168.1.100" { + t.Errorf("Expected OpenDTUAddress from JSON, got %v", cfg.OpenDTUAddress) + } + if !cfg.OpenDTUAuth { + t.Errorf("Expected OpenDTUAuth=true from JSON") + } + if cfg.OpenDTUUser != "admin" { + t.Errorf("Expected OpenDTUUser from JSON, got %v", cfg.OpenDTUUser) + } + if cfg.OpenDTUPassword != "secret123" { + t.Errorf("Expected OpenDTUPassword from JSON, got %v", cfg.OpenDTUPassword) + } + if !cfg.TimescaleDB { + t.Errorf("Expected TimescaleDB=true from JSON") + } + if cfg.TZ != "Europe/Amsterdam" { + t.Errorf("Expected TZ from JSON, got %v", cfg.TZ) + } + if cfg.LogLevel != "INFO" { + t.Errorf("Expected LogLevel from JSON, got %v", cfg.LogLevel) + } +} + +// TestConfigLoadFromJSONFile_NoAuth tests loading config from JSON without auth +func TestConfigLoadFromJSONFile_NoAuth(t *testing.T) { + // Create a temporary config file without auth + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "config_noauth.json") + + configContent := `{ + "db": "postgres://localhost/testdb", + "opendtu_address": "192.168.1.200", + "opendtu_auth": false, + "timescaledb": false, + "tz": "UTC", + "log_level": "DEBUG" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + // Save original environment + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + + os.Setenv("CONFIG_FILE", tmpFile) + + // Load config + cfg, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + // Verify fields + if cfg.DB != "postgres://localhost/testdb" { + t.Errorf("Expected DB from JSON, got %v", cfg.DB) + } + if cfg.OpenDTUAddress != "192.168.1.200" { + t.Errorf("Expected OpenDTUAddress from JSON, got %v", cfg.OpenDTUAddress) + } + if cfg.OpenDTUAuth { + t.Errorf("Expected OpenDTUAuth=false from JSON") + } + if cfg.TimescaleDB { + t.Errorf("Expected TimescaleDB=false from JSON") + } +} + +// TestConfigLoadFromJSONFile_InvalidTimezone tests the timezone validation +func TestConfigLoadFromJSONFile_InvalidTimezone(t *testing.T) { + // Create a temporary config file with invalid timezone + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "config_badtz.json") + + configContent := `{ + "db": "postgres://localhost/testdb", + "opendtu_address": "192.168.1.200", + "opendtu_auth": false, + "tz": "Invalid/Timezone" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + // Save original environment + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + + os.Setenv("CONFIG_FILE", tmpFile) + + // Load config - should not fatal, just log warning + cfg, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + // Should still load other fields successfully + if cfg.DB != "postgres://localhost/testdb" { + t.Errorf("Expected DB to load despite invalid timezone") + } +} + +// TestConfigLoadFromEnv_WithTimescaleDB tests env var path with TimescaleDB enabled +func TestConfigLoadFromEnv_WithTimescaleDB(t *testing.T) { + // Save original environment + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + "TIMESCALEDB_ENABLED": os.Getenv("TIMESCALEDB_ENABLED"), + "TZ": os.Getenv("TZ"), + "LOG_LEVEL": os.Getenv("LOG_LEVEL"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + // Set environment for non-existent config file + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://testhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "10.0.0.1") + os.Setenv("OPENDTU_AUTH", "false") + os.Setenv("TIMESCALEDB_ENABLED", "true") + os.Setenv("TZ", "America/New_York") + os.Setenv("LOG_LEVEL", "WARN") + + cfg, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + if cfg.DB != "postgres://testhost/testdb" { + t.Errorf("Expected DB from env, got %v", cfg.DB) + } + if !cfg.TimescaleDB { + t.Errorf("Expected TimescaleDB=true from env") + } + if cfg.TZ != "America/New_York" { + t.Errorf("Expected TZ from env, got %v", cfg.TZ) + } + if cfg.LogLevel != "WARN" { + t.Errorf("Expected LogLevel from env, got %v", cfg.LogLevel) + } +} + +// TestConfigLoadFromEnv_WithAuth tests env var path with auth enabled +func TestConfigLoadFromEnv_WithAuth(t *testing.T) { + // Save original environment + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + "OPENDTU_USERNAME": os.Getenv("OPENDTU_USERNAME"), + "OPENDTU_PASSWORD": os.Getenv("OPENDTU_PASSWORD"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + // Set environment with auth enabled + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://testhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "10.0.0.1") + os.Setenv("OPENDTU_AUTH", "true") + os.Setenv("OPENDTU_USERNAME", "testuser") + os.Setenv("OPENDTU_PASSWORD", "testpass") + + cfg, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + if !cfg.OpenDTUAuth { + t.Errorf("Expected OpenDTUAuth=true from env") + } + if cfg.OpenDTUUser != "testuser" { + t.Errorf("Expected OpenDTUUser from env, got %v", cfg.OpenDTUUser) + } + if cfg.OpenDTUPassword != "testpass" { + t.Errorf("Expected OpenDTUPassword from env, got %v", cfg.OpenDTUPassword) + } +} + +// TestConfigLoadFromJSONFile_InvalidJSON tests error when JSON is malformed +func TestConfigLoadFromJSONFile_InvalidJSON(t *testing.T) { + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "invalid.json") + + // Write invalid JSON + err := os.WriteFile(tmpFile, []byte("{invalid json}"), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + os.Setenv("CONFIG_FILE", tmpFile) + + _, err = loadConfig() + if err == nil { + t.Error("Expected error for invalid JSON, got nil") + } + if err != nil && !contains(err.Error(), "error parsing config file") { + t.Errorf("Expected 'error parsing config file' error, got: %v", err) + } +} + +// TestConfigLoadFromJSONFile_MissingDB tests error when DB is not set in JSON +func TestConfigLoadFromJSONFile_MissingDB(t *testing.T) { + // Reset global config to avoid test pollution + config = Config{} + + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "no_db.json") + + configContent := `{ + "opendtu_address": "192.168.1.100" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + os.Setenv("CONFIG_FILE", tmpFile) + + _, err = loadConfig() + if err == nil { + t.Error("Expected error for missing DB, got nil") + } + if err != nil && !contains(err.Error(), "db connection settings are not set") { + t.Errorf("Expected 'db connection settings' error, got: %v", err) + } +} + +// TestConfigLoadFromJSONFile_MissingOpenDTUAddress tests error when opendtu_address is not set +func TestConfigLoadFromJSONFile_MissingOpenDTUAddress(t *testing.T) { + // Reset global config to avoid test pollution + config = Config{} + + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "no_address.json") + + configContent := `{ + "db": "postgres://localhost/testdb" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + os.Setenv("CONFIG_FILE", tmpFile) + + _, err = loadConfig() + if err == nil { + t.Error("Expected error for missing OpenDTU address, got nil") + } + if err != nil && !contains(err.Error(), "opendtu_address is not set") { + t.Errorf("Expected 'opendtu_address is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromJSONFile_MissingUsername tests error when username is missing with auth enabled +func TestConfigLoadFromJSONFile_MissingUsername(t *testing.T) { + // Reset global config to avoid test pollution + config = Config{} + + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "no_username.json") + + configContent := `{ + "db": "postgres://localhost/testdb", + "opendtu_address": "192.168.1.100", + "opendtu_auth": true, + "opendtu_password": "secret" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + os.Setenv("CONFIG_FILE", tmpFile) + + _, err = loadConfig() + if err == nil { + t.Error("Expected error for missing username, got nil") + } + if err != nil && !contains(err.Error(), "opendtu_username is not set") { + t.Errorf("Expected 'opendtu_username is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromJSONFile_MissingPassword tests error when password is missing with auth enabled +func TestConfigLoadFromJSONFile_MissingPassword(t *testing.T) { + // Reset global config to avoid test pollution + config = Config{} + + tmpDir := t.TempDir() + tmpFile := filepath.Join(tmpDir, "no_password.json") + + configContent := `{ + "db": "postgres://localhost/testdb", + "opendtu_address": "192.168.1.100", + "opendtu_auth": true, + "opendtu_username": "admin" + }` + + err := os.WriteFile(tmpFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write temp config: %v", err) + } + + originalConfigFile := os.Getenv("CONFIG_FILE") + defer func() { os.Setenv("CONFIG_FILE", originalConfigFile) }() + os.Setenv("CONFIG_FILE", tmpFile) + + _, err = loadConfig() + if err == nil { + t.Error("Expected error for missing password, got nil") + } + if err != nil && !contains(err.Error(), "opendtu_password is not set") { + t.Errorf("Expected 'opendtu_password is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_MissingDBURL tests error when DB_URL env var is missing +func TestConfigLoadFromEnv_MissingDBURL(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Unsetenv("DB_URL") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for missing DB_URL, got nil") + } + if err != nil && !contains(err.Error(), "DB_URL environment variable is not set") { + t.Errorf("Expected 'DB_URL environment variable is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_MissingOpenDTUAddress tests error when OPENDTU_ADDRESS is missing +func TestConfigLoadFromEnv_MissingOpenDTUAddress(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://localhost/testdb") + os.Unsetenv("OPENDTU_ADDRESS") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for missing OPENDTU_ADDRESS, got nil") + } + if err != nil && !contains(err.Error(), "OPENDTU_ADDRESS environment variable is not set") { + t.Errorf("Expected 'OPENDTU_ADDRESS environment variable is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_InvalidAuthBool tests error when OPENDTU_AUTH has invalid boolean +func TestConfigLoadFromEnv_InvalidAuthBool(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://localhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + os.Setenv("OPENDTU_AUTH", "invalid") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for invalid OPENDTU_AUTH, got nil") + } + if err != nil && !contains(err.Error(), "error parsing OPENDTU_AUTH") { + t.Errorf("Expected 'error parsing OPENDTU_AUTH' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_MissingUsername tests error when OPENDTU_USERNAME is missing with auth +func TestConfigLoadFromEnv_MissingUsername(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + "OPENDTU_USERNAME": os.Getenv("OPENDTU_USERNAME"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://localhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + os.Setenv("OPENDTU_AUTH", "true") + os.Unsetenv("OPENDTU_USERNAME") + os.Setenv("OPENDTU_PASSWORD", "secret") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for missing OPENDTU_USERNAME, got nil") + } + if err != nil && !contains(err.Error(), "OPENDTU_USERNAME environment variable is not set") { + t.Errorf("Expected 'OPENDTU_USERNAME environment variable is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_MissingPassword tests error when OPENDTU_PASSWORD is missing with auth +func TestConfigLoadFromEnv_MissingPassword(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + "OPENDTU_USERNAME": os.Getenv("OPENDTU_USERNAME"), + "OPENDTU_PASSWORD": os.Getenv("OPENDTU_PASSWORD"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://localhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + os.Setenv("OPENDTU_AUTH", "true") + os.Setenv("OPENDTU_USERNAME", "admin") + os.Unsetenv("OPENDTU_PASSWORD") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for missing OPENDTU_PASSWORD, got nil") + } + if err != nil && !contains(err.Error(), "OPENDTU_PASSWORD environment variable is not set") { + t.Errorf("Expected 'OPENDTU_PASSWORD environment variable is not set' error, got: %v", err) + } +} + +// TestConfigLoadFromEnv_InvalidTimescaleDBBool tests error when TIMESCALEDB_ENABLED has invalid boolean +func TestConfigLoadFromEnv_InvalidTimescaleDBBool(t *testing.T) { + originalVars := map[string]string{ + "CONFIG_FILE": os.Getenv("CONFIG_FILE"), + "DB_URL": os.Getenv("DB_URL"), + "OPENDTU_ADDRESS": os.Getenv("OPENDTU_ADDRESS"), + "OPENDTU_AUTH": os.Getenv("OPENDTU_AUTH"), + "TIMESCALEDB_ENABLED": os.Getenv("TIMESCALEDB_ENABLED"), + } + defer func() { + for k, v := range originalVars { + os.Setenv(k, v) + } + }() + + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") + os.Setenv("DB_URL", "postgres://localhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + os.Setenv("OPENDTU_AUTH", "false") // Set auth to false so we don't need username/password + os.Setenv("TIMESCALEDB_ENABLED", "not-a-bool") + + _, err := loadConfig() + if err == nil { + t.Error("Expected error for invalid TIMESCALEDB_ENABLED, got nil") + } + if err != nil && !contains(err.Error(), "error parsing TIMESCALEDB_ENABLED") { + t.Errorf("Expected 'error parsing TIMESCALEDB_ENABLED' error, got: %v", err) + } +} + +// Helper function to check if a string contains a substring +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && + (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || + func() bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false + }())) +} diff --git a/go.mod b/go.mod index 69fb927..1ef8362 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index 0760f47..baa2f8b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -6,6 +8,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..a0c16c7 --- /dev/null +++ b/integration_test.go @@ -0,0 +1,466 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func loadTestData(filename string) ([]byte, error) { + return os.ReadFile(filepath.Join("testdata", filename)) +} + +func TestLiveDataFromFile(t *testing.T) { + // Test producing data + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal LiveData: %v", err) + } + + // Validate the structure + if len(liveData.Inverters) != 2 { + t.Errorf("Expected 2 inverters, got %v", len(liveData.Inverters)) + } + + inverter := liveData.Inverters[0] + if inverter.Serial != "114173123456" { + t.Errorf("Expected first inverter serial 114173123456, got %v", inverter.Serial) + } + if !inverter.Producing { + t.Errorf("Expected first inverter to be producing") + } + if !inverter.Reachable { + t.Errorf("Expected first inverter to be reachable") + } + + // Test new OpenDTU fields + if inverter.Order != 0 { + t.Errorf("Expected first inverter Order=0, got %v", inverter.Order) + } + if inverter.DataAge != 0 { + t.Errorf("Expected first inverter DataAge=0, got %v", inverter.DataAge) + } + if inverter.DataAgeMs != 124 { + t.Errorf("Expected first inverter DataAgeMs=124, got %v", inverter.DataAgeMs) + } + + // Test radio_stats from OpenDTU + if inverter.RadioStats.TxRequest != 12345 { + t.Errorf("Expected RadioStats.TxRequest=12345, got %v", inverter.RadioStats.TxRequest) + } + if inverter.RadioStats.RxSuccess != 12000 { + t.Errorf("Expected RadioStats.RxSuccess=12000, got %v", inverter.RadioStats.RxSuccess) + } + if inverter.RadioStats.RSSI != -65.5 { + t.Errorf("Expected RadioStats.RSSI=-65.5, got %v", inverter.RadioStats.RSSI) + } + + // Test second inverter has correct order + if liveData.Inverters[1].Order != 1 { + t.Errorf("Expected second inverter Order=1, got %v", liveData.Inverters[1].Order) + } + + // Test Hints with pin_mapping_issue + if !liveData.Hints.TimeSync { + t.Errorf("Expected Hints.TimeSync=true, got %v", liveData.Hints.TimeSync) + } + if liveData.Hints.RadioProblem { + t.Errorf("Expected Hints.RadioProblem=false, got %v", liveData.Hints.RadioProblem) + } + if liveData.Hints.PinMappingIssue { + t.Errorf("Expected Hints.PinMappingIssue=false, got %v", liveData.Hints.PinMappingIssue) + } + + // Check total power + if liveData.Total.Power.V != 1276.9 { + t.Errorf("Expected total power 1276.9W, got %v", liveData.Total.Power.V) + } + + // Test night data + data, err = loadTestData("livedata_night.json") + if err != nil { + t.Fatalf("Failed to load night test data: %v", err) + } + + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal night LiveData: %v", err) + } + + if len(liveData.Inverters) != 1 { + t.Errorf("Expected 1 inverter in night data, got %v", len(liveData.Inverters)) + } + + inverter = liveData.Inverters[0] + if inverter.Producing { + t.Errorf("Expected inverter not to be producing at night") + } + if liveData.Total.Power.V != 0.0 { + t.Errorf("Expected total power 0W at night, got %v", liveData.Total.Power.V) + } + if liveData.Total.YieldTotal.V != 16253.99 { + t.Errorf("Expected total YieldTotal 16253.99kWh, got %v", liveData.Total.YieldTotal.V) + } + + // Test that DataAge is old at night (no recent data) + if inverter.DataAge != 20840 { + t.Errorf("Expected DataAge=20840 (old data at night), got %v", inverter.DataAge) + } + if inverter.DataAgeMs != 20840477 { + t.Errorf("Expected DataAgeMs=20840477, got %v", inverter.DataAgeMs) + } + if inverter.LimitAbsolute != 2250 { + t.Errorf("Expected LimitAbsolute=2250, got %v", inverter.LimitAbsolute) + } + if inverter.PollEnabled { + t.Errorf("Expected PollEnabled=false at night") + } + if inverter.Reachable { + t.Errorf("Expected Reachable=false at night") + } + + if len(inverter.DC) != 6 { + t.Errorf("Expected 6 DC strings, got %d", len(inverter.DC)) + } + if inverter.DC["5"].Irradiation.Max != 440 { + t.Errorf("Expected Irradiation max 440 for string 5, got %d", inverter.DC["5"].Irradiation.Max) + } + if inverter.RadioStats.TxRequest != 147 { + t.Errorf("Expected TxRequest=147, got %v", inverter.RadioStats.TxRequest) + } + if inverter.RadioStats.RxFailNothing != 147 { + t.Errorf("Expected RxFailNothing=147, got %v", inverter.RadioStats.RxFailNothing) + } + if inverter.RadioStats.RSSI != -62 { + t.Errorf("Expected RSSI=-62, got %v", inverter.RadioStats.RSSI) + } + + // Test warning data with limit fallback and hints + data, err = loadTestData("livedata_warnings.json") + if err != nil { + t.Fatalf("Failed to load warning test data: %v", err) + } + + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal warning LiveData: %v", err) + } + + if len(liveData.Inverters) != 1 { + t.Fatalf("Expected 1 inverter in warning data, got %d", len(liveData.Inverters)) + } + + inverter = liveData.Inverters[0] + if inverter.LimitAbsolute != -1 { + t.Errorf("Expected LimitAbsolute=-1 fallback, got %v", inverter.LimitAbsolute) + } + if inverter.Events != -1 { + t.Errorf("Expected Events=-1 when event log missing, got %v", inverter.Events) + } + if !inverter.PollEnabled { + t.Errorf("Expected PollEnabled=true in warning data") + } + if inverter.Reachable { + t.Errorf("Expected Reachable=false in warning data") + } + if inverter.RadioStats.RxFailNothing != 12 { + t.Errorf("Expected RxFailNothing=12, got %v", inverter.RadioStats.RxFailNothing) + } + + if liveData.Hints.TimeSync { + t.Errorf("Expected Hints.TimeSync=false, got %v", liveData.Hints.TimeSync) + } + if !liveData.Hints.RadioProblem { + t.Errorf("Expected Hints.RadioProblem=true, got %v", liveData.Hints.RadioProblem) + } + if !liveData.Hints.DefaultPassword { + t.Errorf("Expected Hints.DefaultPassword=true, got %v", liveData.Hints.DefaultPassword) + } + if !liveData.Hints.PinMappingIssue { + t.Errorf("Expected Hints.PinMappingIssue=true, got %v", liveData.Hints.PinMappingIssue) + } + + var generic map[string]interface{} + if err := json.Unmarshal(data, &generic); err != nil { + t.Fatalf("Failed to unmarshal warning fixture generically: %v", err) + } + invertersValue, ok := generic["inverters"].([]interface{}) + if !ok || len(invertersValue) == 0 { + t.Fatalf("Generic inverter array missing") + } + invMap, ok := invertersValue[0].(map[string]interface{}) + if !ok { + t.Fatalf("Generic inverter object missing") + } + if _, exists := invMap["BAT"]; !exists { + t.Errorf("Expected additional channel map 'BAT' to be present in fixture") + } +} + +func TestEventsResponseFromFile(t *testing.T) { + data, err := loadTestData("events_response.json") + if err != nil { + t.Fatalf("Failed to load events test data: %v", err) + } + + var eventsResponse EventsResponse + err = json.Unmarshal(data, &eventsResponse) + if err != nil { + t.Fatalf("Failed to unmarshal EventsResponse: %v", err) + } + + if eventsResponse.Count != 3 { + t.Errorf("Expected 3 events, got %v", eventsResponse.Count) + } + + if len(eventsResponse.Events) != 3 { + t.Errorf("Expected 3 events in array, got %v", len(eventsResponse.Events)) + } + + // Check first event + firstEvent := eventsResponse.Events[0] + if firstEvent.MessageID != 1 { + t.Errorf("Expected first event MessageID 1, got %v", firstEvent.MessageID) + } + if firstEvent.Message != "Inverter start" { + t.Errorf("Expected first event message 'Inverter start', got %v", firstEvent.Message) + } + if firstEvent.StartTime != 1634567890 { + t.Errorf("Expected first event start time 1634567890, got %v", firstEvent.StartTime) + } + if firstEvent.EndTime != 1634567950 { + t.Errorf("Expected first event end time 1634567950, got %v", firstEvent.EndTime) + } + + // Check ongoing event (end_time = 0) + ongoingEvent := eventsResponse.Events[2] + if ongoingEvent.EndTime != 0 { + t.Errorf("Expected ongoing event end time 0, got %v", ongoingEvent.EndTime) + } +} + +func TestConfigFromJSONFile(t *testing.T) { + // Test config with auth + data, err := loadTestData("test_config.json") + if err != nil { + t.Fatalf("Failed to load config test data: %v", err) + } + + var config Config + err = json.Unmarshal(data, &config) + if err != nil { + t.Fatalf("Failed to unmarshal Config: %v", err) + } + + if config.DB != "postgres://user:password@localhost:5432/opendtu" { + t.Errorf("Expected specific DB connection string, got %v", config.DB) + } + if config.OpenDTUAddress != "192.168.1.100" { + t.Errorf("Expected OpenDTU address 192.168.1.100, got %v", config.OpenDTUAddress) + } + if !config.OpenDTUAuth { + t.Errorf("Expected OpenDTU auth to be enabled") + } + if config.OpenDTUUser != "admin" { + t.Errorf("Expected OpenDTU user 'admin', got %v", config.OpenDTUUser) + } + if !config.TimescaleDB { + t.Errorf("Expected TimescaleDB to be enabled") + } + if config.TZ != "Europe/Amsterdam" { + t.Errorf("Expected timezone Europe/Amsterdam, got %v", config.TZ) + } + if config.LogLevel != "INFO" { + t.Errorf("Expected log level INFO, got %v", config.LogLevel) + } +} + +func TestQueryEventsEndpointMock(t *testing.T) { + // Create test data + testResponse := EventsResponse{ + Count: 2, + Events: []Event{ + { + MessageID: 1, + Message: "Test event 1", + StartTime: 1634567890, + EndTime: 1634567950, + }, + { + MessageID: 2, + Message: "Test event 2", + StartTime: 1634568000, + EndTime: 0, // Ongoing event + }, + }, + } + + // Create a test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check that the URL contains the expected inverter serial + if r.URL.Query().Get("inv") != "123456789" { + t.Errorf("Expected inverter serial 123456789, got %v", r.URL.Query().Get("inv")) + } + + // Check the endpoint path + if r.URL.Path != "/api/eventlog/status" { + t.Errorf("Expected path /api/eventlog/status, got %v", r.URL.Path) + } + + // Return test data + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(testResponse) + })) + defer server.Close() + + // Set config to use test server + originalAddress := config.OpenDTUAddress + config.OpenDTUAddress = server.URL[7:] // Remove "http://" prefix + defer func() { config.OpenDTUAddress = originalAddress }() + + // Test the function + response, err := queryEventsEndpoint("123456789") + if err != nil { + t.Fatalf("queryEventsEndpoint failed: %v", err) + } + + if response.Count != 2 { + t.Errorf("Expected 2 events, got %v", response.Count) + } + if len(response.Events) != 2 { + t.Errorf("Expected 2 events in array, got %v", len(response.Events)) + } + if response.Events[0].Message != "Test event 1" { + t.Errorf("Expected first event message 'Test event 1', got %v", response.Events[0].Message) + } +} + +func TestQueryEventsEndpointWithAuth(t *testing.T) { + // Create a test server that checks auth + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check for Authorization header + auth := r.Header.Get("Authorization") + expectedAuth := basicAuth("testuser", "testpass") + if auth != expectedAuth { + t.Errorf("Expected auth header %v, got %v", expectedAuth, auth) + w.WriteHeader(http.StatusUnauthorized) + return + } + + // Return empty response for this test + response := EventsResponse{Count: 0, Events: []Event{}} + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + })) + defer server.Close() + + // Set config to use test server with auth + originalAddress := config.OpenDTUAddress + originalAuth := config.OpenDTUAuth + originalUser := config.OpenDTUUser + originalPassword := config.OpenDTUPassword + + config.OpenDTUAddress = server.URL[7:] // Remove "http://" prefix + config.OpenDTUAuth = true + config.OpenDTUUser = "testuser" + config.OpenDTUPassword = "testpass" + + defer func() { + config.OpenDTUAddress = originalAddress + config.OpenDTUAuth = originalAuth + config.OpenDTUUser = originalUser + config.OpenDTUPassword = originalPassword + }() + + // Test the function + response, err := queryEventsEndpoint("123456789") + if err != nil { + t.Fatalf("queryEventsEndpoint with auth failed: %v", err) + } + + if response.Count != 0 { + t.Errorf("Expected 0 events, got %v", response.Count) + } +} + +func TestQueryEventsEndpointHTTPError(t *testing.T) { + // Create a test server that returns an error + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + // Set config to use test server + originalAddress := config.OpenDTUAddress + config.OpenDTUAddress = server.URL[7:] // Remove "http://" prefix + defer func() { config.OpenDTUAddress = originalAddress }() + + // Test the function + _, err := queryEventsEndpoint("123456789") + if err == nil { + t.Fatalf("Expected error from queryEventsEndpoint, got nil") + } +} + +func TestHandleMessageWithLiveData(t *testing.T) { + // Load test data + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + // Test that handleMessage doesn't crash with valid JSON + // Note: This will not actually insert into DB since we don't have a test DB setup + // handleMessage(data, nil) // Would panic without proper DB, so we skip this part + + // Instead, let's test the JSON parsing part + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("handleMessage would fail due to JSON parsing error: %v", err) + } + + // Verify the conditions that would trigger data recording + for _, inverter := range liveData.Inverters { + if inverter.DataAge == 0 && inverter.Reachable { + // This is the condition for recording data + t.Logf("Inverter %s would have data recorded: DataAge=%d, Reachable=%v", + inverter.Serial, inverter.DataAge, inverter.Reachable) + } + if inverter.DataAge == 0 && inverter.Events > 0 { + // This is the condition for recording events + t.Logf("Inverter %s would have events recorded: DataAge=%d, Events=%d", + inverter.Serial, inverter.DataAge, inverter.Events) + } + } +} + +func TestInvalidJSON(t *testing.T) { + invalidJSONs := []string{ + `{"invalid": json}`, + `{"inverters": [{"serial": }]}`, + `{"total": {"Power": {"v": "not a number"}}}`, + ``, + `null`, + } + + for i, jsonStr := range invalidJSONs { + t.Run(fmt.Sprintf("invalid_json_%d", i), func(t *testing.T) { + var liveData LiveData + err := json.Unmarshal([]byte(jsonStr), &liveData) + if err == nil && jsonStr != `` && jsonStr != `null` { + t.Errorf("Expected error parsing invalid JSON %q, but got none", jsonStr) + } + }) + } +} diff --git a/main_persistence_test.go b/main_persistence_test.go new file mode 100644 index 0000000..eee251f --- /dev/null +++ b/main_persistence_test.go @@ -0,0 +1,436 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestCreateLoggerWithLevel(t *testing.T) { + logger := createLoggerWithLevel(slog.LevelWarn) + if logger == nil { + t.Fatal("expected logger instance") + } + + handler := logger.Handler() + if handler.Enabled(context.Background(), slog.LevelInfo) { + t.Fatalf("expected info level to be disabled for warn handler") + } + if !handler.Enabled(context.Background(), slog.LevelError) { + t.Fatalf("expected error level to be enabled for warn handler") + } +} + +func TestInsertLiveDataInsertsAllTables(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + config = Config{TZ: "UTC"} + + inverter := Inverter{ + Serial: "INV01", + Name: "Dummy Inverter", + Producing: true, + LimitRelative: 90.5, + LimitAbsolute: 1500, + AC: map[string]InverterAC{ + "0": { + Power: VUD{V: 123.4}, + Voltage: VUD{V: 230.0}, + Current: VUD{V: 0.53}, + Frequency: VUD{V: 50.0}, + PowerFactor: VUD{V: 0.99}, + ReactivePower: VUD{ + V: 1.1, + }, + }, + }, + DC: map[string]InverterDC{ + "0": { + Name: struct { + U string `json:"u"` + }{U: "Dummy String"}, + Power: VUD{V: 111.1}, + Voltage: VUD{V: 36.5}, + Current: VUD{V: 3.05}, + YieldDay: VUD{V: 12.0}, + YieldTotal: VUD{V: 456.7}, + Irradiation: struct { + V float64 `json:"v"` + U string `json:"u"` + D int `json:"d"` + Max int `json:"max"` + }{V: 75.0, Max: 440}, + }, + }, + INV: map[string]InverterINV{ + "0": { + Temperature: VUD{V: 40.0}, + Efficiency: VUD{V: 97.5}, + PowerDC: VUD{V: 222.2}, + YieldDay: VUD{V: 15.0}, + YieldTotal: VUD{V: 789.0}, + }, + }, + } + + total := Total{ + Power: VUD{V: 321.0}, + YieldDay: VUD{V: 25.0}, + YieldTotal: VUD{ + V: 12345.6, + }, + } + + hints := Hints{ + TimeSync: true, + RadioProblem: true, + DefaultPassword: false, + PinMappingIssue: true, + } + + mock.ExpectExec("INSERT INTO opendtu_log"). + WithArgs(sqlmock.AnyArg(), total.Power.V, total.YieldDay.V, total.YieldTotal.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("INSERT INTO opendtu_inverters"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, inverter.Name, inverter.Producing, inverter.LimitRelative, inverter.LimitAbsolute). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("INSERT INTO opendtu_inverters_ac"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, "0", inverter.AC["0"].Power.V, inverter.AC["0"].Voltage.V, inverter.AC["0"].Current.V, inverter.AC["0"].Frequency.V, inverter.AC["0"].PowerFactor.V, inverter.AC["0"].ReactivePower.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("INSERT INTO opendtu_inverters_dc"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, "0", inverter.DC["0"].Name.U, inverter.DC["0"].Power.V, inverter.DC["0"].Voltage.V, inverter.DC["0"].Current.V, inverter.DC["0"].YieldDay.V, inverter.DC["0"].YieldTotal.V, inverter.DC["0"].Irradiation.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("INSERT INTO opendtu_inverters_inv"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, inverter.INV["0"].Temperature.V, inverter.INV["0"].Efficiency.V, inverter.INV["0"].PowerDC.V, inverter.INV["0"].YieldDay.V, inverter.INV["0"].YieldTotal.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("INSERT INTO opendtu_hints"). + WithArgs(sqlmock.AnyArg(), hints.TimeSync, hints.RadioProblem, hints.DefaultPassword, hints.PinMappingIssue). + WillReturnResult(sqlmock.NewResult(0, 1)) + + insertLiveData(db, inverter, total, hints) + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestGetPreviousEventsCount(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + rows := sqlmock.NewRows([]string{"count"}).AddRow(3) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs("INV01"). + WillReturnRows(rows) + + count := getPreviousEventsCount(db, "INV01") + if count != 3 { + t.Fatalf("expected count 3, got %d", count) + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestInsertEventsPersistsRows(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + config = Config{TZ: "UTC"} + + events := &EventsResponse{ + Events: []Event{ + { + MessageID: 10, + Message: "Test event", + StartTime: 111, + EndTime: 222, + }, + }, + } + + mock.ExpectExec("INSERT INTO opendtu_events"). + WithArgs(sqlmock.AnyArg(), "INV01", events.Events[0].MessageID, events.Events[0].Message, events.Events[0].StartTime, events.Events[0].EndTime). + WillReturnResult(sqlmock.NewResult(0, 1)) + + insertEvents(db, "INV01", events) + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestUpdateEventsUpdatesRows(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + events := &EventsResponse{ + Events: []Event{ + { + StartTime: 100, + EndTime: 200, + }, + { + StartTime: 300, + EndTime: 400, + }, + }, + } + + mock.ExpectExec("UPDATE opendtu_events SET end_time"). + WithArgs(events.Events[0].EndTime, "INV01", events.Events[0].StartTime). + WillReturnResult(sqlmock.NewResult(0, 1)) + + mock.ExpectExec("UPDATE opendtu_events SET end_time"). + WithArgs(events.Events[1].EndTime, "INV01", events.Events[1].StartTime). + WillReturnResult(sqlmock.NewResult(0, 1)) + + updateEvents(db, "INV01", events) + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestHandleMessageRecordsDataAndEvents(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + eventsResponse := EventsResponse{ + Count: 1, + Events: []Event{{MessageID: 1, Message: "Test event", StartTime: 50, EndTime: 60}}, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/eventlog/status" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + if got := r.URL.Query().Get("inv"); got != "INV01" { + t.Fatalf("unexpected inverter query: %s", got) + } + if err := json.NewEncoder(w).Encode(eventsResponse); err != nil { + t.Fatalf("failed to write response: %v", err) + } + })) + defer server.Close() + + previousConfig := config + defer func() { config = previousConfig }() + + address := strings.TrimPrefix(server.URL, "http://") + address = strings.TrimPrefix(address, "https://") + config = Config{ + TZ: "UTC", + OpenDTUAddress: address, + } + + rows := sqlmock.NewRows([]string{"count"}).AddRow(1) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs("INV01"). + WillReturnRows(rows) + + mock.ExpectExec("INSERT INTO opendtu_events"). + WithArgs(sqlmock.AnyArg(), "INV01", eventsResponse.Events[0].MessageID, eventsResponse.Events[0].Message, eventsResponse.Events[0].StartTime, eventsResponse.Events[0].EndTime). + WillReturnResult(sqlmock.NewResult(0, 1)) + + inverter := Inverter{ + Serial: "INV01", + Name: "Dummy Inverter", + Producing: true, + Reachable: true, + DataAge: 0, + Events: 2, + LimitRelative: 90.5, + LimitAbsolute: 1500, + AC: map[string]InverterAC{ + "0": { + Power: VUD{V: 123.4}, + Voltage: VUD{V: 230.0}, + Current: VUD{V: 0.53}, + Frequency: VUD{V: 50.0}, + PowerFactor: VUD{V: 0.99}, + ReactivePower: VUD{V: 1.1}, + }, + }, + DC: map[string]InverterDC{ + "0": { + Name: struct { + U string `json:"u"` + }{U: "Dummy String"}, + Power: VUD{V: 111.1}, + Voltage: VUD{V: 36.5}, + Current: VUD{V: 3.05}, + YieldDay: VUD{V: 12.0}, + YieldTotal: VUD{V: 456.7}, + Irradiation: struct { + V float64 `json:"v"` + U string `json:"u"` + D int `json:"d"` + Max int `json:"max"` + }{V: 75.0, Max: 440}, + }, + }, + INV: map[string]InverterINV{ + "0": { + Temperature: VUD{V: 40.0}, + Efficiency: VUD{V: 97.5}, + PowerDC: VUD{V: 222.2}, + YieldDay: VUD{V: 15.0}, + YieldTotal: VUD{V: 789.0}, + }, + }, + } + + total := Total{ + Power: VUD{V: 321.0}, + YieldDay: VUD{V: 25.0}, + YieldTotal: VUD{ + V: 12345.6, + }, + } + + hints := Hints{ + TimeSync: true, + RadioProblem: true, + DefaultPassword: false, + PinMappingIssue: true, + } + + mock.ExpectExec("INSERT INTO opendtu_log"). + WithArgs(sqlmock.AnyArg(), total.Power.V, total.YieldDay.V, total.YieldTotal.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO opendtu_inverters"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, inverter.Name, inverter.Producing, inverter.LimitRelative, inverter.LimitAbsolute). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO opendtu_inverters_ac"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, "0", inverter.AC["0"].Power.V, inverter.AC["0"].Voltage.V, inverter.AC["0"].Current.V, inverter.AC["0"].Frequency.V, inverter.AC["0"].PowerFactor.V, inverter.AC["0"].ReactivePower.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO opendtu_inverters_dc"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, "0", inverter.DC["0"].Name.U, inverter.DC["0"].Power.V, inverter.DC["0"].Voltage.V, inverter.DC["0"].Current.V, inverter.DC["0"].YieldDay.V, inverter.DC["0"].YieldTotal.V, inverter.DC["0"].Irradiation.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO opendtu_inverters_inv"). + WithArgs(sqlmock.AnyArg(), inverter.Serial, inverter.INV["0"].Temperature.V, inverter.INV["0"].Efficiency.V, inverter.INV["0"].PowerDC.V, inverter.INV["0"].YieldDay.V, inverter.INV["0"].YieldTotal.V). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO opendtu_hints"). + WithArgs(sqlmock.AnyArg(), hints.TimeSync, hints.RadioProblem, hints.DefaultPassword, hints.PinMappingIssue). + WillReturnResult(sqlmock.NewResult(0, 1)) + + liveData := LiveData{ + Inverters: []Inverter{inverter}, + Total: total, + Hints: hints, + } + + payload, err := json.Marshal(liveData) + if err != nil { + t.Fatalf("failed to marshal live data: %v", err) + } + + handleMessage(payload, db) + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestHandleMessageSkipsWhenDataStale(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + liveData := LiveData{ + Inverters: []Inverter{ + { + Serial: "INV01", + Reachable: false, + DataAge: 10, + Events: 2, + }, + }, + } + + payload, err := json.Marshal(liveData) + if err != nil { + t.Fatalf("failed to marshal live data: %v", err) + } + + handleMessage(payload, db) + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unexpected database calls: %v", err) + } +} + +func TestEnableTimescaleHypertablesExecutesStatements(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + mock.ExpectExec("(?s).*create_hypertable\\('opendtu_log'"). + WillReturnResult(sqlmock.NewResult(0, 1)) + + if err := enableTimescaleHypertables(db); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("not all expectations were met: %v", err) + } +} + +func TestEnableTimescaleHypertablesPropagatesError(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + mock.ExpectExec("(?s).*create_hypertable\\('opendtu_log'"). + WillReturnError(errors.New("boom")) + + err = enableTimescaleHypertables(db) + if err == nil { + t.Fatalf("expected error, got nil") + } + + if !strings.Contains(err.Error(), "boom") { + t.Fatalf("expected wrapped boom error, got %v", err) + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("expectations mismatch: %v", err) + } +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..ee58344 --- /dev/null +++ b/main_test.go @@ -0,0 +1,566 @@ +package main + +import ( + "encoding/json" + "log/slog" + "os" + "testing" + "time" +) + +func TestBasicAuth(t *testing.T) { + tests := []struct { + name string + username string + password string + expected string + }{ + { + name: "basic auth test", + username: "testuser", + password: "testpass", + expected: "Basic dGVzdHVzZXI6dGVzdHBhc3M=", + }, + { + name: "empty credentials", + username: "", + password: "", + expected: "Basic Og==", + }, + { + name: "special characters", + username: "user@domain.com", + password: "p@ssw0rd!", + expected: "Basic dXNlckBkb21haW4uY29tOnBAc3N3MHJkIQ==", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := basicAuth(tt.username, tt.password) + if result != tt.expected { + t.Errorf("basicAuth() = %v, want %v", result, tt.expected) + } + }) + } +} + +func TestGetLogLevel(t *testing.T) { + tests := []struct { + name string + logLevel string + defaultLevel slog.Level + expected slog.Level + }{ + { + name: "DEBUG level", + logLevel: "DEBUG", + defaultLevel: slog.LevelInfo, + expected: slog.LevelDebug, + }, + { + name: "INFO level", + logLevel: "INFO", + defaultLevel: slog.LevelDebug, + expected: slog.LevelInfo, + }, + { + name: "WARN level", + logLevel: "WARN", + defaultLevel: slog.LevelDebug, + expected: slog.LevelWarn, + }, + { + name: "ERROR level", + logLevel: "ERROR", + defaultLevel: slog.LevelDebug, + expected: slog.LevelError, + }, + { + name: "invalid level returns default", + logLevel: "INVALID", + defaultLevel: slog.LevelInfo, + expected: slog.LevelInfo, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Temporarily set the config.LogLevel + originalLogLevel := config.LogLevel + config.LogLevel = tt.logLevel + defer func() { config.LogLevel = originalLogLevel }() + + result := getLogLevel(tt.defaultLevel) + if result != tt.expected { + t.Errorf("getLogLevel() = %v, want %v", result, tt.expected) + } + }) + } +} + +func TestVUDStruct(t *testing.T) { + jsonData := `{"v": 123.45, "u": "W", "d": 2}` + + var vud VUD + err := json.Unmarshal([]byte(jsonData), &vud) + if err != nil { + t.Fatalf("Failed to unmarshal VUD: %v", err) + } + + if vud.V != 123.45 { + t.Errorf("Expected V=123.45, got %v", vud.V) + } + if vud.U != "W" { + t.Errorf("Expected U=W, got %v", vud.U) + } + if vud.D != 2 { + t.Errorf("Expected D=2, got %v", vud.D) + } +} + +func TestInverterStruct(t *testing.T) { + jsonData := `{ + "serial": "123456789", + "name": "Test Inverter", + "producing": true, + "limit_relative": 80.5, + "limit_absolute": 800.0, + "AC": { + "0": { + "Power": {"v": 250.5, "u": "W", "d": 1}, + "Voltage": {"v": 230.0, "u": "V", "d": 1}, + "Current": {"v": 1.09, "u": "A", "d": 2}, + "Frequency": {"v": 50.0, "u": "Hz", "d": 1}, + "PowerFactor": {"v": 1.0, "u": "", "d": 2}, + "ReactivePower": {"v": 0.0, "u": "var", "d": 1} + } + }, + "DC": { + "0": { + "Name": {"u": "String 1"}, + "Power": {"v": 260.0, "u": "W", "d": 1}, + "Voltage": {"v": 35.2, "u": "V", "d": 1}, + "Current": {"v": 7.4, "u": "A", "d": 1}, + "YieldDay": {"v": 2.5, "u": "kWh", "d": 3}, + "YieldTotal": {"v": 1250.8, "u": "kWh", "d": 3}, + "Irradiation": {"v": 85.2, "u": "%", "d": 1, "max": 100} + } + }, + "events": 2, + "poll_enabled": true, + "reachable": true, + "data_age": 0, + "INV": { + "0": { + "Temperature": {"v": 32.5, "u": "°C", "d": 1}, + "Efficiency": {"v": 96.2, "u": "%", "d": 1}, + "Power DC": {"v": 260.0, "u": "W", "d": 1}, + "YieldDay": {"v": 2.5, "u": "kWh", "d": 3}, + "YieldTotal": {"v": 1250.8, "u": "kWh", "d": 3} + } + } + }` + + var inverter Inverter + err := json.Unmarshal([]byte(jsonData), &inverter) + if err != nil { + t.Fatalf("Failed to unmarshal Inverter: %v", err) + } + + if inverter.Serial != "123456789" { + t.Errorf("Expected Serial=123456789, got %v", inverter.Serial) + } + if inverter.Name != "Test Inverter" { + t.Errorf("Expected Name=Test Inverter, got %v", inverter.Name) + } + if !inverter.Producing { + t.Errorf("Expected Producing=true, got %v", inverter.Producing) + } + if inverter.LimitRelative != 80.5 { + t.Errorf("Expected LimitRelative=80.5, got %v", inverter.LimitRelative) + } + + // Test AC data + if len(inverter.AC) != 1 { + t.Errorf("Expected 1 AC entry, got %v", len(inverter.AC)) + } + if ac, exists := inverter.AC["0"]; exists { + if ac.Power.V != 250.5 { + t.Errorf("Expected AC Power=250.5, got %v", ac.Power.V) + } + if ac.Voltage.V != 230.0 { + t.Errorf("Expected AC Voltage=230.0, got %v", ac.Voltage.V) + } + } else { + t.Error("Expected AC[0] to exist") + } + + // Test DC data + if len(inverter.DC) != 1 { + t.Errorf("Expected 1 DC entry, got %v", len(inverter.DC)) + } + if dc, exists := inverter.DC["0"]; exists { + if dc.Name.U != "String 1" { + t.Errorf("Expected DC Name=String 1, got %v", dc.Name.U) + } + if dc.Power.V != 260.0 { + t.Errorf("Expected DC Power=260.0, got %v", dc.Power.V) + } + } else { + t.Error("Expected DC[0] to exist") + } + + // Test INV data + if len(inverter.INV) != 1 { + t.Errorf("Expected 1 INV entry, got %v", len(inverter.INV)) + } + if inv, exists := inverter.INV["0"]; exists { + if inv.Temperature.V != 32.5 { + t.Errorf("Expected INV Temperature=32.5, got %v", inv.Temperature.V) + } + if inv.Efficiency.V != 96.2 { + t.Errorf("Expected INV Efficiency=96.2, got %v", inv.Efficiency.V) + } + } else { + t.Error("Expected INV[0] to exist") + } +} + +func TestEventStruct(t *testing.T) { + jsonData := `{ + "message_id": 1, + "message": "Test event message", + "start_time": 1634567890, + "end_time": 1634567950 + }` + + var event Event + err := json.Unmarshal([]byte(jsonData), &event) + if err != nil { + t.Fatalf("Failed to unmarshal Event: %v", err) + } + + if event.MessageID != 1 { + t.Errorf("Expected MessageID=1, got %v", event.MessageID) + } + if event.Message != "Test event message" { + t.Errorf("Expected Message=Test event message, got %v", event.Message) + } + if event.StartTime != 1634567890 { + t.Errorf("Expected StartTime=1634567890, got %v", event.StartTime) + } + if event.EndTime != 1634567950 { + t.Errorf("Expected EndTime=1634567950, got %v", event.EndTime) + } +} + +func TestLiveDataStruct(t *testing.T) { + jsonData := `{ + "inverters": [ + { + "serial": "123456789", + "name": "Test Inverter", + "producing": true, + "limit_relative": 80.5, + "limit_absolute": 800.0, + "AC": {}, + "DC": {}, + "events": 0, + "poll_enabled": true, + "reachable": true, + "data_age": 0, + "INV": {} + } + ], + "total": { + "Power": {"v": 250.5, "u": "W", "d": 1}, + "YieldDay": {"v": 2.5, "u": "kWh", "d": 3}, + "YieldTotal": {"v": 1250.8, "u": "kWh", "d": 3} + }, + "hints": { + "time_sync": true, + "radio_problem": false, + "default_password": false + } + }` + + var liveData LiveData + err := json.Unmarshal([]byte(jsonData), &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal LiveData: %v", err) + } + + if len(liveData.Inverters) != 1 { + t.Errorf("Expected 1 inverter, got %v", len(liveData.Inverters)) + } + + if liveData.Total.Power.V != 250.5 { + t.Errorf("Expected Total Power=250.5, got %v", liveData.Total.Power.V) + } + + if !liveData.Hints.TimeSync { + t.Errorf("Expected Hints.TimeSync=true, got %v", liveData.Hints.TimeSync) + } + if liveData.Hints.RadioProblem { + t.Errorf("Expected Hints.RadioProblem=false, got %v", liveData.Hints.RadioProblem) + } +} + +func TestConfigLoadFromEnv(t *testing.T) { + // Save original environment variables + originalDB := os.Getenv("DB_URL") + originalAddress := os.Getenv("OPENDTU_ADDRESS") + originalAuth := os.Getenv("OPENDTU_AUTH") + originalUser := os.Getenv("OPENDTU_USERNAME") + originalPassword := os.Getenv("OPENDTU_PASSWORD") + originalTimescale := os.Getenv("TIMESCALEDB_ENABLED") + originalTZ := os.Getenv("TZ") + originalLogLevel := os.Getenv("LOG_LEVEL") + originalConfigFile := os.Getenv("CONFIG_FILE") + + // Clean up after test + defer func() { + os.Setenv("DB_URL", originalDB) + os.Setenv("OPENDTU_ADDRESS", originalAddress) + os.Setenv("OPENDTU_AUTH", originalAuth) + os.Setenv("OPENDTU_USERNAME", originalUser) + os.Setenv("OPENDTU_PASSWORD", originalPassword) + os.Setenv("TIMESCALEDB_ENABLED", originalTimescale) + os.Setenv("TZ", originalTZ) + os.Setenv("LOG_LEVEL", originalLogLevel) + os.Setenv("CONFIG_FILE", originalConfigFile) + }() + + // Set test environment variables + os.Setenv("CONFIG_FILE", "/nonexistent/config.json") // Force env var fallback + os.Setenv("DB_URL", "postgres://testuser:testpass@localhost/testdb") + os.Setenv("OPENDTU_ADDRESS", "192.168.1.100") + os.Setenv("OPENDTU_AUTH", "true") + os.Setenv("OPENDTU_USERNAME", "admin") + os.Setenv("OPENDTU_PASSWORD", "secret") + os.Setenv("TIMESCALEDB_ENABLED", "true") + os.Setenv("TZ", "Europe/Amsterdam") + os.Setenv("LOG_LEVEL", "DEBUG") + + config, err := loadConfig() + if err != nil { + t.Fatalf("loadConfig() failed: %v", err) + } + + if config.DB != "postgres://testuser:testpass@localhost/testdb" { + t.Errorf("Expected DB from env, got %v", config.DB) + } + if config.OpenDTUAddress != "192.168.1.100" { + t.Errorf("Expected OpenDTUAddress from env, got %v", config.OpenDTUAddress) + } + if !config.OpenDTUAuth { + t.Errorf("Expected OpenDTUAuth=true from env, got %v", config.OpenDTUAuth) + } + if config.OpenDTUUser != "admin" { + t.Errorf("Expected OpenDTUUser from env, got %v", config.OpenDTUUser) + } + if config.OpenDTUPassword != "secret" { + t.Errorf("Expected OpenDTUPassword from env, got %v", config.OpenDTUPassword) + } + if !config.TimescaleDB { + t.Errorf("Expected TimescaleDB=true from env, got %v", config.TimescaleDB) + } + if config.TZ != "Europe/Amsterdam" { + t.Errorf("Expected TZ from env, got %v", config.TZ) + } + if config.LogLevel != "DEBUG" { + t.Errorf("Expected LogLevel from env, got %v", config.LogLevel) + } +} + +func TestTimeZoneValidation(t *testing.T) { + validTimezones := []string{ + "UTC", + "Europe/Amsterdam", + "America/New_York", + "Asia/Tokyo", + } + + for _, tz := range validTimezones { + t.Run(tz, func(t *testing.T) { + _, err := time.LoadLocation(tz) + if err != nil { + t.Errorf("Timezone %s should be valid but got error: %v", tz, err) + } + }) + } + + invalidTimezones := []string{ + "Invalid/Timezone", + "Not/A/Real/Place", + "", + } + + for _, tz := range invalidTimezones { + if tz == "" { + continue // Empty string is handled differently + } + t.Run(tz, func(t *testing.T) { + _, err := time.LoadLocation(tz) + if err == nil { + t.Errorf("Timezone %s should be invalid but was accepted", tz) + } + }) + } +} + +// Test RadioStatistics structure (from official OpenDTU) +func TestRadioStatisticsStruct(t *testing.T) { + jsonData := `{ + "tx_request": 12345, + "tx_re_request": 234, + "rx_success": 12000, + "rx_fail_nothing": 50, + "rx_fail_partial": 30, + "rx_fail_corrupt": 21, + "rssi": -65.5 + }` + + var stats RadioStatistics + err := json.Unmarshal([]byte(jsonData), &stats) + if err != nil { + t.Fatalf("Failed to unmarshal RadioStatistics: %v", err) + } + + if stats.TxRequest != 12345 { + t.Errorf("Expected TxRequest=12345, got %v", stats.TxRequest) + } + if stats.TxReRequest != 234 { + t.Errorf("Expected TxReRequest=234, got %v", stats.TxReRequest) + } + if stats.RxSuccess != 12000 { + t.Errorf("Expected RxSuccess=12000, got %v", stats.RxSuccess) + } + if stats.RxFailNothing != 50 { + t.Errorf("Expected RxFailNothing=50, got %v", stats.RxFailNothing) + } + if stats.RxFailPartial != 30 { + t.Errorf("Expected RxFailPartial=30, got %v", stats.RxFailPartial) + } + if stats.RxFailCorrupt != 21 { + t.Errorf("Expected RxFailCorrupt=21, got %v", stats.RxFailCorrupt) + } + if stats.RSSI != -65.5 { + t.Errorf("Expected RSSI=-65.5, got %v", stats.RSSI) + } +} + +// Test inverter struct with new OpenDTU fields: order, data_age_ms, radio_stats +func TestInverterWithOpenDTUFields(t *testing.T) { + jsonData := `{ + "serial": "114173123456", + "name": "Hoymiles HM-800", + "order": 0, + "data_age": 0, + "data_age_ms": 124, + "poll_enabled": true, + "reachable": true, + "producing": true, + "limit_relative": 100.0, + "limit_absolute": 800.0, + "events": 2, + "radio_stats": { + "tx_request": 12345, + "tx_re_request": 234, + "rx_success": 12000, + "rx_fail_nothing": 50, + "rx_fail_partial": 30, + "rx_fail_corrupt": 21, + "rssi": -65.5 + }, + "AC": {}, + "DC": {}, + "INV": {} + }` + + var inverter Inverter + err := json.Unmarshal([]byte(jsonData), &inverter) + if err != nil { + t.Fatalf("Failed to unmarshal Inverter with OpenDTU fields: %v", err) + } + + // Test new fields from official OpenDTU + if inverter.Order != 0 { + t.Errorf("Expected Order=0, got %v", inverter.Order) + } + if inverter.DataAge != 0 { + t.Errorf("Expected DataAge=0, got %v", inverter.DataAge) + } + if inverter.DataAgeMs != 124 { + t.Errorf("Expected DataAgeMs=124, got %v", inverter.DataAgeMs) + } + + // Test radio_stats + if inverter.RadioStats.TxRequest != 12345 { + t.Errorf("Expected RadioStats.TxRequest=12345, got %v", inverter.RadioStats.TxRequest) + } + if inverter.RadioStats.RxSuccess != 12000 { + t.Errorf("Expected RadioStats.RxSuccess=12000, got %v", inverter.RadioStats.RxSuccess) + } + if inverter.RadioStats.RSSI != -65.5 { + t.Errorf("Expected RadioStats.RSSI=-65.5, got %v", inverter.RadioStats.RSSI) + } + + // Test original fields still work + if inverter.Serial != "114173123456" { + t.Errorf("Expected Serial=114173123456, got %v", inverter.Serial) + } + if inverter.Name != "Hoymiles HM-800" { + t.Errorf("Expected Name=Hoymiles HM-800, got %v", inverter.Name) + } + if !inverter.Producing { + t.Errorf("Expected Producing=true, got %v", inverter.Producing) + } +} + +// Test Hints struct with new pin_mapping_issue field from OpenDTU +func TestHintsWithPinMappingIssue(t *testing.T) { + jsonData := `{ + "time_sync": true, + "radio_problem": false, + "default_password": false, + "pin_mapping_issue": false + }` + + var hints Hints + err := json.Unmarshal([]byte(jsonData), &hints) + if err != nil { + t.Fatalf("Failed to unmarshal Hints with pin_mapping_issue: %v", err) + } + + if !hints.TimeSync { + t.Errorf("Expected TimeSync=true, got %v", hints.TimeSync) + } + if hints.RadioProblem { + t.Errorf("Expected RadioProblem=false, got %v", hints.RadioProblem) + } + if hints.DefaultPassword { + t.Errorf("Expected DefaultPassword=false, got %v", hints.DefaultPassword) + } + if hints.PinMappingIssue { + t.Errorf("Expected PinMappingIssue=false, got %v", hints.PinMappingIssue) + } + + // Test when pin_mapping_issue is true + jsonDataWithIssue := `{ + "time_sync": true, + "radio_problem": false, + "default_password": false, + "pin_mapping_issue": true + }` + + err = json.Unmarshal([]byte(jsonDataWithIssue), &hints) + if err != nil { + t.Fatalf("Failed to unmarshal Hints with pin_mapping_issue=true: %v", err) + } + + if !hints.PinMappingIssue { + t.Errorf("Expected PinMappingIssue=true, got %v", hints.PinMappingIssue) + } +} diff --git a/opendtu_compatibility_test.go b/opendtu_compatibility_test.go new file mode 100644 index 0000000..2043421 --- /dev/null +++ b/opendtu_compatibility_test.go @@ -0,0 +1,500 @@ +package main + +import ( + "encoding/json" + "testing" +) + +// TestOpenDTUCompatibility_FullStructure validates that our Go structs +// can parse the exact JSON structure produced by OpenDTU WebSocket +// based on opendtu/src/WebApi_ws_live.cpp +func TestOpenDTUCompatibility_FullStructure(t *testing.T) { + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal LiveData: %v", err) + } + + // Validate top-level structure + if len(liveData.Inverters) == 0 { + t.Fatal("Expected at least one inverter") + } + + // Test first inverter completely + inv := liveData.Inverters[0] + + // Validate all root-level inverter fields (from generateInverterCommonJsonResponse) + t.Run("InverterCommonFields", func(t *testing.T) { + if inv.Serial == "" { + t.Error("serial field missing or empty") + } + if inv.Name == "" { + t.Error("name field missing or empty") + } + // order field (int, can be 0) + if inv.Order < 0 { + t.Error("order field has invalid value") + } + // data_age field (int, can be 0) + if inv.DataAge < 0 { + t.Error("data_age field has invalid value") + } + // data_age_ms field (int, can be 0) + if inv.DataAgeMs < 0 { + t.Error("data_age_ms field has invalid value") + } + // poll_enabled field (bool) + _ = inv.PollEnabled + // reachable field (bool) + _ = inv.Reachable + // producing field (bool) + _ = inv.Producing + // limit_relative field (float64) + if inv.LimitRelative < 0 { + t.Error("limit_relative field has invalid value") + } + // limit_absolute field (float64, can be -1) + _ = inv.LimitAbsolute + }) + + // Validate radio_stats structure (from generateInverterCommonJsonResponse lines 157-163) + t.Run("RadioStats", func(t *testing.T) { + rs := inv.RadioStats + // All fields from OpenDTU RadioStats + if rs.TxRequest < 0 { + t.Error("radio_stats.tx_request has invalid value") + } + if rs.TxReRequest < 0 { + t.Error("radio_stats.tx_re_request has invalid value") + } + if rs.RxSuccess < 0 { + t.Error("radio_stats.rx_success has invalid value") + } + if rs.RxFailNothing < 0 { + t.Error("radio_stats.rx_fail_nothing has invalid value") + } + if rs.RxFailPartial < 0 { + t.Error("radio_stats.rx_fail_partial has invalid value") + } + if rs.RxFailCorrupt < 0 { + t.Error("radio_stats.rx_fail_corrupt has invalid value") + } + // rssi can be negative (signal strength) + _ = rs.RSSI + }) + + // Validate AC channel structure (string-keyed map from WebApi_ws_live.cpp line 224) + t.Run("AC_Structure", func(t *testing.T) { + if len(inv.AC) == 0 { + t.Fatal("AC map is empty") + } + + // OpenDTU uses string keys: "0", "1", etc. + ac0, exists := inv.AC["0"] + if !exists { + t.Fatal("AC[\"0\"] does not exist - OpenDTU uses string-keyed maps") + } + + // Validate AC fields (from addField calls in WebApi_ws_live.cpp) + // Lines 184-188: FLD_PAC, FLD_UAC, FLD_IAC, FLD_F, FLD_PF, FLD_Q + validateVUD(t, "AC.Power", ac0.Power) + validateVUD(t, "AC.Voltage", ac0.Voltage) + validateVUD(t, "AC.Current", ac0.Current) + validateVUD(t, "AC.Frequency", ac0.Frequency) + validateVUD(t, "AC.PowerFactor", ac0.PowerFactor) + validateVUD(t, "AC.ReactivePower", ac0.ReactivePower) + }) + + // Validate DC channel structure (string-keyed map with name field) + t.Run("DC_Structure", func(t *testing.T) { + if len(inv.DC) == 0 { + t.Fatal("DC map is empty") + } + + // OpenDTU uses string keys: "0", "1", etc. + dc0, exists := inv.DC["0"] + if !exists { + t.Fatal("DC[\"0\"] does not exist - OpenDTU uses string-keyed maps") + } + + // Validate DC name field (from WebApi_ws_live.cpp line 182) + if dc0.Name.U == "" { + t.Error("DC channel name is empty") + } + + // Validate DC fields (from addField calls) + // Lines 184-200: FLD_PDC, FLD_UDC, FLD_IDC, FLD_YD, FLD_YT, FLD_IRR + validateVUD(t, "DC.Power", dc0.Power) + validateVUD(t, "DC.Voltage", dc0.Voltage) + validateVUD(t, "DC.Current", dc0.Current) + validateVUD(t, "DC.YieldDay", dc0.YieldDay) + validateVUD(t, "DC.YieldTotal", dc0.YieldTotal) + + // Irradiation has special max field (line 201-203) + if dc0.Irradiation.V < 0 || dc0.Irradiation.V > 100 { + t.Errorf("DC.Irradiation.V out of range: %v", dc0.Irradiation.V) + } + if dc0.Irradiation.Max <= 0 { + t.Error("DC.Irradiation.max field missing or invalid") + } + }) + + // Validate INV channel structure + t.Run("INV_Structure", func(t *testing.T) { + if len(inv.INV) == 0 { + t.Fatal("INV map is empty") + } + + // OpenDTU uses string keys: "0" + inv0, exists := inv.INV["0"] + if !exists { + t.Fatal("INV[\"0\"] does not exist - OpenDTU uses string-keyed maps") + } + + // Validate INV fields (from addField calls) + // Lines 184-200: FLD_T, FLD_EFF, FLD_PDC (as "Power DC"), FLD_YD, FLD_YT + validateVUD(t, "INV.Temperature", inv0.Temperature) + validateVUD(t, "INV.Efficiency", inv0.Efficiency) + validateVUD(t, "INV.PowerDC", inv0.PowerDC) + validateVUD(t, "INV.YieldDay", inv0.YieldDay) + validateVUD(t, "INV.YieldTotal", inv0.YieldTotal) + }) + + // Validate events field (from WebApi_ws_live.cpp lines 206-210) + t.Run("Events", func(t *testing.T) { + // events can be -1 if not available, or >= 0 + if inv.Events < -1 { + t.Errorf("events field has invalid value: %d", inv.Events) + } + }) + + // Validate total structure (from generateCommonJsonResponse lines 128-130) + t.Run("Total", func(t *testing.T) { + validateVUD(t, "Total.Power", liveData.Total.Power) + validateVUD(t, "Total.YieldDay", liveData.Total.YieldDay) + validateVUD(t, "Total.YieldTotal", liveData.Total.YieldTotal) + }) + + // Validate hints structure (from generateCommonJsonResponse lines 132-138) + t.Run("Hints", func(t *testing.T) { + // All fields are boolean + _ = liveData.Hints.TimeSync + _ = liveData.Hints.RadioProblem + _ = liveData.Hints.DefaultPassword + _ = liveData.Hints.PinMappingIssue + }) +} + +// TestOpenDTUCompatibility_StringKeyedMaps verifies that AC/DC/INV are string-keyed maps +// This is critical because OpenDTU C++ code uses String(channel) conversion (line 224) +func TestOpenDTUCompatibility_StringKeyedMaps(t *testing.T) { + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal LiveData: %v", err) + } + + inv := liveData.Inverters[0] + + // Test that maps use string keys "0", "1", NOT integer indices + t.Run("AC_StringKeys", func(t *testing.T) { + if _, exists := inv.AC["0"]; !exists { + t.Error("AC must use string key \"0\" not integer 0") + } + // Should NOT be accessible as integer + if len(inv.AC) > 0 { + // This is correct - we're using map[string] + t.Log("AC correctly uses map[string]InverterAC") + } + }) + + t.Run("DC_StringKeys", func(t *testing.T) { + if _, exists := inv.DC["0"]; !exists { + t.Error("DC must use string key \"0\" not integer 0") + } + if _, exists := inv.DC["1"]; len(inv.DC) > 1 && !exists { + t.Error("DC must use string key \"1\" not integer 1") + } + }) + + t.Run("INV_StringKeys", func(t *testing.T) { + if _, exists := inv.INV["0"]; !exists { + t.Error("INV must use string key \"0\" not integer 0") + } + }) +} + +// TestOpenDTUCompatibility_MultipleInverters validates proper handling of inverter arrays +// OpenDTU creates an array of inverter objects (WebApi_ws_live.cpp line 95) +func TestOpenDTUCompatibility_MultipleInverters(t *testing.T) { + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal LiveData: %v", err) + } + + if len(liveData.Inverters) < 2 { + t.Skip("Test requires at least 2 inverters") + } + + // Verify each inverter has unique serial and proper order + t.Run("UniqueSerials", func(t *testing.T) { + serials := make(map[string]bool) + for _, inv := range liveData.Inverters { + if serials[inv.Serial] { + t.Errorf("Duplicate serial found: %s", inv.Serial) + } + serials[inv.Serial] = true + } + }) + + t.Run("ProperOrdering", func(t *testing.T) { + for i, inv := range liveData.Inverters { + if inv.Order != i { + t.Logf("Warning: inverter at index %d has order %d (may be intentional)", i, inv.Order) + } + } + }) +} + +// TestOpenDTUCompatibility_NightMode validates zero-production scenario +// Tests that the structure is valid even when inverters are not producing +func TestOpenDTUCompatibility_NightMode(t *testing.T) { + data, err := loadTestData("livedata_night.json") + if err != nil { + t.Fatalf("Failed to load night test data: %v", err) + } + + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal night LiveData: %v", err) + } + + if len(liveData.Inverters) == 0 { + t.Fatal("Expected at least one inverter in night data") + } + + inv := liveData.Inverters[0] + + // During night, inverters should not be producing + if inv.Producing { + t.Error("Inverter should not be producing at night") + } + + // Data age should be higher (older data) + if inv.DataAge == 0 { + t.Error("Expected non-zero data_age at night (stale data)") + } + + // Total power should be zero or near-zero + if liveData.Total.Power.V > 1.0 { + t.Errorf("Expected near-zero total power at night, got %v W", liveData.Total.Power.V) + } + + // Structure should still be valid + if len(inv.AC) == 0 { + t.Error("AC structure should exist even at night") + } + if len(inv.DC) == 0 { + t.Error("DC structure should exist even at night") + } + if len(inv.INV) == 0 { + t.Error("INV structure should exist even at night") + } +} + +// TestOpenDTUCompatibility_FieldNames validates exact field naming from OpenDTU +// Field names come from getChannelFieldName in OpenDTU C++ code +func TestOpenDTUCompatibility_FieldNames(t *testing.T) { + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + // Parse as generic JSON to inspect field names + var jsonData map[string]interface{} + err = json.Unmarshal(data, &jsonData) + if err != nil { + t.Fatalf("Failed to unmarshal as generic JSON: %v", err) + } + + inverters := jsonData["inverters"].([]interface{}) + inv := inverters[0].(map[string]interface{}) + + // Check root inverter field names (exact names from C++ code) + expectedRootFields := []string{ + "serial", "name", "order", "data_age", "data_age_ms", + "poll_enabled", "reachable", "producing", + "limit_relative", "limit_absolute", "events", + "AC", "DC", "INV", "radio_stats", + } + + for _, field := range expectedRootFields { + if _, exists := inv[field]; !exists { + t.Errorf("Missing expected root field: %s", field) + } + } + + // Check radio_stats field names + radioStats := inv["radio_stats"].(map[string]interface{}) + expectedRadioFields := []string{ + "tx_request", "tx_re_request", "rx_success", + "rx_fail_nothing", "rx_fail_partial", "rx_fail_corrupt", "rssi", + } + + for _, field := range expectedRadioFields { + if _, exists := radioStats[field]; !exists { + t.Errorf("Missing expected radio_stats field: %s", field) + } + } + + // Check hints field names + hints := jsonData["hints"].(map[string]interface{}) + expectedHintFields := []string{ + "time_sync", "radio_problem", "default_password", "pin_mapping_issue", + } + + for _, field := range expectedHintFields { + if _, exists := hints[field]; !exists { + t.Errorf("Missing expected hints field: %s", field) + } + } +} + +// TestOpenDTUCompatibility_WarningFixture validates branches only hit when OpenDTU +// cannot determine inverter max power, event log, or when hints flag issues. +func TestOpenDTUCompatibility_WarningFixture(t *testing.T) { + data, err := loadTestData("livedata_warnings.json") + if err != nil { + t.Fatalf("Failed to load warning test data: %v", err) + } + + var liveData LiveData + if err := json.Unmarshal(data, &liveData); err != nil { + t.Fatalf("Failed to unmarshal warning fixture: %v", err) + } + + if len(liveData.Inverters) != 1 { + t.Fatalf("Expected 1 inverter in warning fixture, got %d", len(liveData.Inverters)) + } + + inv := liveData.Inverters[0] + if inv.LimitAbsolute != -1 { + t.Errorf("Expected LimitAbsolute fallback -1, got %v", inv.LimitAbsolute) + } + if inv.Events != -1 { + t.Errorf("Expected Events=-1 when event log channel missing, got %v", inv.Events) + } + if !inv.PollEnabled { + t.Errorf("Expected PollEnabled=true for warning fixture") + } + if inv.Reachable { + t.Errorf("Expected Reachable=false for warning fixture") + } + + if liveData.Hints.TimeSync { + t.Errorf("Expected TimeSync=false, got %v", liveData.Hints.TimeSync) + } + if !liveData.Hints.RadioProblem { + t.Errorf("Expected RadioProblem=true, got %v", liveData.Hints.RadioProblem) + } + if !liveData.Hints.DefaultPassword { + t.Errorf("Expected DefaultPassword=true, got %v", liveData.Hints.DefaultPassword) + } + if !liveData.Hints.PinMappingIssue { + t.Errorf("Expected PinMappingIssue=true, got %v", liveData.Hints.PinMappingIssue) + } + + var generic map[string]interface{} + if err := json.Unmarshal(data, &generic); err != nil { + t.Fatalf("Failed to unmarshal warning fixture generically: %v", err) + } + invertersRaw, ok := generic["inverters"].([]interface{}) + if !ok || len(invertersRaw) == 0 { + t.Fatalf("Generic inverter array missing") + } + invMap, ok := invertersRaw[0].(map[string]interface{}) + if !ok { + t.Fatalf("Generic inverter map missing") + } + if _, exists := invMap["BAT"]; !exists { + t.Errorf("Expected additional channel map 'BAT' to be present") + } +} + +// Helper function to validate VUD structure +func validateVUD(t *testing.T, fieldName string, vud VUD) { + t.Helper() + // Value can be any number (including 0 or negative) + _ = vud.V + // Unit can be empty string for dimensionless values + _ = vud.U + // Decimals should be non-negative + if vud.D < 0 { + t.Errorf("%s: decimals (d) should be non-negative, got %d", fieldName, vud.D) + } +} + +// TestOpenDTUCompatibility_EventsValue tests the events field edge cases +// From WebApi_ws_live.cpp lines 206-210: +// - Returns event count if available +// - Returns -1 if not available +func TestOpenDTUCompatibility_EventsValue(t *testing.T) { + testCases := []struct { + name string + eventsValue int + expectedValid bool + }{ + {"No events", 0, true}, + {"Some events", 5, true}, + {"Many events", 100, true}, + {"Not available", -1, true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Parse test data and modify events value + data, err := loadTestData("livedata_producing.json") + if err != nil { + t.Fatalf("Failed to load test data: %v", err) + } + + // Parse, modify, and re-encode + var liveData LiveData + err = json.Unmarshal(data, &liveData) + if err != nil { + t.Fatalf("Failed to unmarshal: %v", err) + } + + liveData.Inverters[0].Events = tc.eventsValue + + // Re-marshal and unmarshal to test the value + modifiedData, _ := json.Marshal(liveData) + var testData LiveData + err = json.Unmarshal(modifiedData, &testData) + if err != nil { + t.Fatalf("Failed to unmarshal modified data: %v", err) + } + + if testData.Inverters[0].Events != tc.eventsValue { + t.Errorf("Expected events=%d, got %d", tc.eventsValue, testData.Inverters[0].Events) + } + }) + } +} diff --git a/testdata/events_response.json b/testdata/events_response.json new file mode 100644 index 0000000..1e98030 --- /dev/null +++ b/testdata/events_response.json @@ -0,0 +1,23 @@ +{ + "count": 3, + "events": [ + { + "message_id": 1, + "message": "Inverter start", + "start_time": 1634567890, + "end_time": 1634567950 + }, + { + "message_id": 2, + "message": "Grid fault", + "start_time": 1634568000, + "end_time": 1634568120 + }, + { + "message_id": 3, + "message": "Communication error", + "start_time": 1634568200, + "end_time": 0 + } + ] +} diff --git a/testdata/livedata_night.json b/testdata/livedata_night.json new file mode 100644 index 0000000..d9d1397 --- /dev/null +++ b/testdata/livedata_night.json @@ -0,0 +1,112 @@ +{ + "inverters": [ + { + "serial": "987654321012", + "name": "Dummy 02", + "order": 1, + "data_age": 20840, + "data_age_ms": 20840477, + "poll_enabled": false, + "reachable": false, + "producing": false, + "limit_relative": 100, + "limit_absolute": 2250, + "events": 0, + "AC": { + "0": { + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "Frequency": {"v": 0, "u": "Hz", "d": 2}, + "PowerFactor": {"v": 0, "u": "", "d": 3}, + "ReactivePower": {"v": 0, "u": "var", "d": 1} + } + }, + "DC": { + "0": { + "name": {"u": "X1A DMY000000000000001ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 768.194, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + }, + "1": { + "name": {"u": "X2B DMY000000000000002ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 723.244, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + }, + "2": { + "name": {"u": "X3C DMY000000000000003ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 603.444, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + }, + "3": { + "name": {"u": "X4D DMY000000000000004ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 633.541, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + }, + "4": { + "name": {"u": "X5E DMY000000000000005ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 725.182, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + }, + "5": { + "name": {"u": "X6F DMY000000000000006ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 773.515, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0, "u": "%", "d": 3, "max": 440} + } + }, + "INV": { + "0": { + "Power DC": {"v": 0, "u": "W", "d": 1}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 4227.12, "u": "kWh", "d": 3}, + "Temperature": {"v": 0, "u": "°C", "d": 1}, + "Efficiency": {"v": 0, "u": "%", "d": 3} + } + }, + "radio_stats": { + "tx_request": 147, + "tx_re_request": 0, + "rx_success": 0, + "rx_fail_nothing": 147, + "rx_fail_partial": 0, + "rx_fail_corrupt": 0, + "rssi": -62 + } + } + ], + "total": { + "Power": {"v": 0, "u": "W", "d": 0}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 16253.99, "u": "kWh", "d": 3} + }, + "hints": { + "time_sync": false, + "radio_problem": false, + "default_password": false, + "pin_mapping_issue": false + } +} diff --git a/testdata/livedata_producing.json b/testdata/livedata_producing.json new file mode 100644 index 0000000..98bc6fd --- /dev/null +++ b/testdata/livedata_producing.json @@ -0,0 +1,137 @@ +{ + "inverters": [ + { + "serial": "114173123456", + "name": "Hoymiles HM-800", + "order": 0, + "data_age": 0, + "data_age_ms": 124, + "poll_enabled": true, + "reachable": true, + "producing": true, + "limit_relative": 100.0, + "limit_absolute": 800.0, + "events": 2, + "AC": { + "0": { + "Power": {"v": 734.2, "u": "W", "d": 1}, + "Voltage": {"v": 230.1, "u": "V", "d": 1}, + "Current": {"v": 3.19, "u": "A", "d": 2}, + "Frequency": {"v": 50.02, "u": "Hz", "d": 2}, + "PowerFactor": {"v": 1.0, "u": "", "d": 3}, + "ReactivePower": {"v": 0.0, "u": "var", "d": 1} + } + }, + "DC": { + "0": { + "name": {"u": "String 1"}, + "Power": {"v": 381.3, "u": "W", "d": 1}, + "Voltage": {"v": 36.7, "u": "V", "d": 1}, + "Current": {"v": 10.39, "u": "A", "d": 2}, + "YieldDay": {"v": 3847, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 1247.531, "u": "kWh", "d": 3}, + "Irradiation": {"v": 87.3, "u": "%", "d": 3, "max": 440} + }, + "1": { + "name": {"u": "String 2"}, + "Power": {"v": 367.8, "u": "W", "d": 1}, + "Voltage": {"v": 35.2, "u": "V", "d": 1}, + "Current": {"v": 10.45, "u": "A", "d": 2}, + "YieldDay": {"v": 3712, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 1203.847, "u": "kWh", "d": 3}, + "Irradiation": {"v": 84.2, "u": "%", "d": 3, "max": 440} + } + }, + "INV": { + "0": { + "Temperature": {"v": 34.2, "u": "°C", "d": 1}, + "Efficiency": {"v": 97.8, "u": "%", "d": 3}, + "Power DC": {"v": 749.1, "u": "W", "d": 1}, + "YieldDay": {"v": 7559, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 2451.378, "u": "kWh", "d": 3} + } + }, + "radio_stats": { + "tx_request": 12345, + "tx_re_request": 234, + "rx_success": 12000, + "rx_fail_nothing": 50, + "rx_fail_partial": 30, + "rx_fail_corrupt": 21, + "rssi": -65.5 + } + }, + { + "serial": "114173654321", + "name": "Hoymiles HM-600", + "order": 1, + "data_age": 0, + "data_age_ms": 235, + "poll_enabled": true, + "reachable": true, + "producing": true, + "limit_relative": 100.0, + "limit_absolute": 600.0, + "events": 0, + "AC": { + "0": { + "Power": {"v": 542.7, "u": "W", "d": 1}, + "Voltage": {"v": 229.8, "u": "V", "d": 1}, + "Current": {"v": 2.36, "u": "A", "d": 2}, + "Frequency": {"v": 50.01, "u": "Hz", "d": 2}, + "PowerFactor": {"v": 1.0, "u": "", "d": 3}, + "ReactivePower": {"v": 0.0, "u": "var", "d": 1} + } + }, + "DC": { + "0": { + "name": {"u": "String 1"}, + "Power": {"v": 281.4, "u": "W", "d": 1}, + "Voltage": {"v": 32.1, "u": "V", "d": 1}, + "Current": {"v": 8.77, "u": "A", "d": 2}, + "YieldDay": {"v": 2834, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 923.156, "u": "kWh", "d": 3}, + "Irradiation": {"v": 81.4, "u": "%", "d": 3, "max": 440} + }, + "1": { + "name": {"u": "String 2"}, + "Power": {"v": 273.9, "u": "W", "d": 1}, + "Voltage": {"v": 31.8, "u": "V", "d": 1}, + "Current": {"v": 8.61, "u": "A", "d": 2}, + "YieldDay": {"v": 2756, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 897.423, "u": "kWh", "d": 3}, + "Irradiation": {"v": 79.2, "u": "%", "d": 3, "max": 440} + } + }, + "INV": { + "0": { + "Temperature": {"v": 32.8, "u": "°C", "d": 1}, + "Efficiency": {"v": 97.2, "u": "%", "d": 3}, + "Power DC": {"v": 555.3, "u": "W", "d": 1}, + "YieldDay": {"v": 5590, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 1820.579, "u": "kWh", "d": 3} + } + }, + "radio_stats": { + "tx_request": 8765, + "tx_re_request": 123, + "rx_success": 8600, + "rx_fail_nothing": 20, + "rx_fail_partial": 15, + "rx_fail_corrupt": 7, + "rssi": -72.3 + } + } + ], + "total": { + "Power": {"v": 1276.9, "u": "W", "d": 1}, + "YieldDay": {"v": 13149, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 4271.957, "u": "kWh", "d": 3} + }, + "hints": { + "time_sync": true, + "radio_problem": false, + "default_password": false, + "pin_mapping_issue": false + } +} diff --git a/testdata/livedata_warnings.json b/testdata/livedata_warnings.json new file mode 100644 index 0000000..2b2cb8e --- /dev/null +++ b/testdata/livedata_warnings.json @@ -0,0 +1,73 @@ +{ + "inverters": [ + { + "serial": "555444333222", + "name": "Dummy Warning", + "order": 2, + "data_age": 45, + "data_age_ms": 45231, + "poll_enabled": true, + "reachable": false, + "producing": false, + "limit_relative": 100.0, + "limit_absolute": -1, + "events": -1, + "AC": { + "0": { + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 229.4, "u": "V", "d": 1}, + "Current": {"v": 0, "u": "A", "d": 2}, + "Frequency": {"v": 50.01, "u": "Hz", "d": 2}, + "PowerFactor": {"v": 0.0, "u": "", "d": 3}, + "ReactivePower": {"v": 0.0, "u": "var", "d": 1} + } + }, + "DC": { + "0": { + "name": {"u": "X7G DMY000000000000007ZX"}, + "Power": {"v": 0, "u": "W", "d": 1}, + "Voltage": {"v": 0.0, "u": "V", "d": 1}, + "Current": {"v": 0.0, "u": "A", "d": 2}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 123.456, "u": "kWh", "d": 3}, + "Irradiation": {"v": 0.0, "u": "%", "d": 3, "max": 350} + } + }, + "INV": { + "0": { + "Power DC": {"v": 0.0, "u": "W", "d": 1}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 222.222, "u": "kWh", "d": 3}, + "Temperature": {"v": 18.4, "u": "°C", "d": 1}, + "Efficiency": {"v": 0.0, "u": "%", "d": 3} + } + }, + "BAT": { + "0": { + "Power": {"v": -120.5, "u": "W", "d": 1}, + "StateOfCharge": {"v": 64.2, "u": "%", "d": 1} + } + }, + "radio_stats": { + "tx_request": 0, + "tx_re_request": 0, + "rx_success": 0, + "rx_fail_nothing": 12, + "rx_fail_partial": 0, + "rx_fail_corrupt": 0, + "rssi": -80.0 + } + } + ], + "total": { + "Power": {"v": 0.0, "u": "W", "d": 1}, + "YieldDay": {"v": 0, "u": "Wh", "d": 0}, + "YieldTotal": {"v": 222.222, "u": "kWh", "d": 3} + }, + "hints": { + "time_sync": false, + "radio_problem": true, + "default_password": true, + "pin_mapping_issue": true + } +} diff --git a/testdata/test_config.json b/testdata/test_config.json new file mode 100644 index 0000000..0f3f665 --- /dev/null +++ b/testdata/test_config.json @@ -0,0 +1,10 @@ +{ + "db": "postgres://user:password@localhost:5432/opendtu", + "opendtu_address": "192.168.1.100", + "opendtu_auth": true, + "opendtu_username": "admin", + "opendtu_password": "secret123", + "timescaledb": true, + "tz": "Europe/Amsterdam", + "log_level": "INFO" +} From cf63da919e943de77436ef13e3ea45a81a406512 Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sat, 4 Oct 2025 02:00:19 +0200 Subject: [PATCH 32/34] add testing --- .gitea/workflows/go.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitea/workflows/go.yml b/.gitea/workflows/go.yml index 01e8bcc..c6bc16e 100644 --- a/.gitea/workflows/go.yml +++ b/.gitea/workflows/go.yml @@ -10,7 +10,21 @@ on: - "main" jobs: + test: + runs-on: ubuntu-docker + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.22' + cache: true + - name: Run unit tests + run: go test -v ./... + release: + needs: test if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-docker steps: From c681678a59ca3ae4a07f0052f259927de3e5117a Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sun, 12 Oct 2025 17:23:35 +0200 Subject: [PATCH 33/34] Update golang version. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f8e11a4..6dd2615 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use buildx for multi-architecture support -FROM --platform=${BUILDPLATFORM} golang:1.24 AS builder +FROM --platform=${BUILDPLATFORM} golang:1 AS builder WORKDIR /app From b484eef170f81c13617563ee74aa6cb12c2bea9e Mon Sep 17 00:00:00 2001 From: Pieter Hollander Date: Sun, 12 Oct 2025 17:24:10 +0200 Subject: [PATCH 34/34] update dependencies --- go.mod | 4 ++-- go.sum | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1ef8362..85e4e2a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25 require ( github.com/gorilla/websocket v1.5.3 github.com/lib/pq v1.10.9 - github.com/pressly/goose/v3 v3.24.3 + github.com/pressly/goose/v3 v3.26.0 ) require ( @@ -13,5 +13,5 @@ require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.16.0 // indirect + golang.org/x/sync v0.17.0 // indirect ) diff --git a/go.sum b/go.sum index baa2f8b..a7e34a6 100644 --- a/go.sum +++ b/go.sum @@ -21,18 +21,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwWlM= github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E= +github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= +github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=