Implement proper DB migrations using Goose. Upgrade go packages.
All checks were successful
Build Docker image / build (push) Successful in 1m8s
Build Golang packages / release (push) Has been skipped

This commit is contained in:
Pieter Hollander 2024-07-26 22:35:30 +02:00
parent 6902facab6
commit 444e5065a4
Signed by: pieter
SSH key fingerprint: SHA256:HbX+9cBXsop9SuvL+mELd29sK+7DehFfdVweFVDtMSg
12 changed files with 255 additions and 117 deletions

147
main.go
View file

@ -11,6 +11,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"io/fs"
"log"
"log/slog"
"net/http"
@ -19,8 +20,10 @@ import (
"time"
_ "time/tzdata"
"git.hollander.online/energy/opendtu-logger/migrations"
"github.com/gorilla/websocket"
_ "github.com/lib/pq"
"github.com/pressly/goose/v3"
)
// VUD contains three variables used for most metrics sent by OpenDTU:
@ -212,7 +215,7 @@ func main() {
defer db.Close()
// Create tables if they don't exist
createTables(db)
migrateDB(db)
// Create WebSocket URL from config variable
wsURL := "ws://" + config.OpenDTU + "/livedata"
@ -287,122 +290,14 @@ func handleMessage(message []byte, db *sql.DB) {
}
}
func createTables(db *sql.DB) {
// Execute SQL statements to create tables if they don't exist
// inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
// An additional advantage is that it makes plotting in Grafana easier.
func migrateDB(db *sql.DB) {
// TODO: Foreign keys commented out as TimescaleDB hypertables don't support them.
createTableSQL := `
CREATE TABLE IF NOT EXISTS opendtu_log (
timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
power NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters (
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
name TEXT,
producing BOOL,
limit_relative NUMERIC,
limit_absolute NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
ac_number INT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
frequency NUMERIC,
powerfactor NUMERIC,
reactivepower NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
dc_number INT,
name TEXT,
power NUMERIC,
voltage NUMERIC,
current NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
irradiation NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
inverter_serial TEXT,
temperature NUMERIC,
power_dc NUMERIC,
yieldday NUMERIC,
yieldtotal NUMERIC,
efficiency NUMERIC
);
CREATE TABLE IF NOT EXISTS opendtu_events (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
inverter_serial TEXT,
message_id INT,
message TEXT,
start_time INT,
end_time INT
);
// Perform DB migrations
err := migrateFS(db, migrations.FS, ".")
DO $$
BEGIN
-- Check if start_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='start_timestamp') THEN
-- Add start_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN start_timestamp TIMESTAMPTZ;
END IF;
-- Check if end_timestamp column exists
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name='opendtu_events'
AND column_name='end_timestamp') THEN
-- Add end_timestamp column
ALTER TABLE opendtu_events
ADD COLUMN end_timestamp TIMESTAMPTZ;
END IF;
END $$;
CREATE TABLE IF NOT EXISTS opendtu_hints (
-- id SERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ,
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
time_sync BOOL,
radio_problem BOOL,
default_password BOOL
);
CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
`
_, err := db.Exec(createTableSQL)
if err != nil {
log.Fatal("Error creating tables: ", err)
log.Fatal("Error performing database migrations: ", err)
}
timescaleEnabled := config.TimescaleDB
@ -502,6 +397,32 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
}
func migrate(db *sql.DB, dir string) error {
err := goose.SetDialect("postgres")
if err != nil {
return fmt.Errorf("migrate: %w", err)
}
err = goose.Up(db, dir)
if err != nil {
return fmt.Errorf("migrate: %w", err)
}
return nil
}
func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error {
// In case the dir is an empty string, they probably meant the current directory and goose wants a period for that.
if dir == "" {
dir = "."
}
goose.SetBaseFS(migrationFS)
defer func() {
// Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS.
goose.SetBaseFS(nil)
}()
return migrate(db, dir)
}
func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
remoteURL := config.OpenDTU
endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial)