diff --git a/main.go b/main.go index 5665eba..d5a9428 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,8 @@ import ( // // All metrics are stored in PostgreSQL as INT and not NUMERIC or FLOAT to optimise their storage size. // -// Dt1, Dt2, Rt1, Rt2 and G are cumulative meter readings. +// Dt1, Dt2, Rt1, Rt2, G, F and Fl are cumulative meter readings. +// Therefore, they function as counters that can only go up. // By making them pointers to int, they can be set to 'nil' when the reading has not changed. // This way, they are logged as NULL in the database when no change has happened, saving storage capacity. // As an added benefit, this also make data retrieval, visualisation and analysis more light-weight. @@ -28,14 +29,11 @@ import ( // Gas consumption is updated once per minute. // Not saving intermittent values theoretically saves 4 bytes per value. // -// Storage for non-NULL values every second: -// 86,400 entries/day×4 bytes/entry=345,600 bytes/day86,400entries/day×4bytes/entry=345,600bytes/day -// Storage for non-NULL values every minute: -// 1,440 entries/day×4 bytes/entry=5,760 bytes/day1,440entries/day×4bytes/entry=5,760bytes/day -// Storage savings: -// 345,600 bytes/day−5,760 bytes/day=339,840 bytes/day345,600bytes/day−5,760bytes/day=339,840bytes/day -// (about 331.5 KB / day or 9.72 MB / month) -// In practice, savings will be even higher when gas is not being consumed continuously. +// An average month has 43 829.0639 minutes. +// 4*43829,0639 = 175,316.26 bytes = 0.18 MB +// +// In practice, storage will be even lower when gas is not being consumed continuously, +// as a new metric won't be available for every minute. // // When Tariff 1 is active, Tariff 2 isn't. // When energy is being imported, it is not being returned. @@ -50,16 +48,36 @@ import ( // In addition, many meters only update these metrics once every 10 seconds, // meaning even more storage capacity is saved: // 10.52 / 10 * 9 = 9,468 MB. +// +// For D, R, D1-3 and R1-3, either D or R will be active. +// Therefore, their storage requirements can be sliced in half, saving 4 * 10.52 = 42.08 MB +// +// Not saving failures when there is no change, will save around 10.52MB per month per metric, so an additional 21.04MB +// Theoretical storage requirements for a DSMR5 meter with metrics emitted every second: +// +// Column Column size (bytes) Per month (MB) X Total (MB) +// --------------------------------------------------------------------------- +// Timestamp 8 bytes 021.04 +// Dt1, Dt2, Rt1, Rt2 4 bytes 001.05 +// D, R 4 bytes 010.52 +// R 4 bytes 010.52 +// F 4 bytes 000.00 +// Fl 4 bytes 000.00 +// G 4 bytes 000.18 +// V1-3, C1-3 4 bytes 010.52 6 063.12 +// D1-3, R1-3 4 bytes 010.52 3 031.56 +// --------------------------------------------------------------------------- +// Total 137.99 type Payload struct { T string `json:"t"` // Timestamp Dt1 *int `json:"dt1"` // Delivered / imported meter reading tariff 1 (kWh) Dt2 *int `json:"dt2"` // Delivered / imported tariff 2 (kWh) Rt1 *int `json:"rt1"` // Returned / exported tariff 1 (kWh) Rt2 *int `json:"rt2"` // Returned / exported tariff 2 (kWh) - D int `json:"d"` // Delivering / importing (W) - R int `json:"r"` // Returning / exporting (W) - F int `json:"f"` // Failure (counter) - Fl int `json:"fl"` // Failure long duration (counter) + D *int `json:"d"` // Delivering / importing (W) + R *int `json:"r"` // Returning / exporting (W) + F *int `json:"f"` // Failure (counter) + Fl *int `json:"fl"` // Failure long duration (counter) G *int `json:"g"` // Gas meter reading (l) V1 int `json:"v1"` // Voltage L1 (V) V2 int `json:"v2"` // Voltage L2 (V) @@ -67,12 +85,12 @@ type Payload struct { C1 int `json:"c1"` // Current L1 (A) C2 int `json:"c2"` // Current L2 (A) C3 int `json:"c3"` // Current L3 (A) - D1 int `json:"d1"` // Delivering / importing L1 (W) - D2 int `json:"d2"` // Delivering / importing L2 (W) - D3 int `json:"d3"` // Delivering / importing L3 (W) - R1 int `json:"r1"` // Returning / exporting L1 (W) - R2 int `json:"r2"` // Returning / exporting L2 (W) - R3 int `json:"r3"` // Returning / exporting L3 (W) + D1 *int `json:"d1"` // Delivering / importing L1 (W) + D2 *int `json:"d2"` // Delivering / importing L2 (W) + D3 *int `json:"d3"` // Delivering / importing L3 (W) + R1 *int `json:"r1"` // Returning / exporting L1 (W) + R2 *int `json:"r2"` // Returning / exporting L2 (W) + R3 *int `json:"r3"` // Returning / exporting L3 (W) } var db *sql.DB @@ -152,6 +170,7 @@ func createLoggerWithLevel(level slog.Level) *slog.Logger { } // updateFieldIfChanged ensures that meter readings that haven't been updated aren't written to the database, in order to save storage space. +// If they haven't changed, they are set to nil, meaning they will be inserted as NULL into the database. func updateFieldIfChanged(currentValue *int, previousValue *int) (*int, bool) { if currentValue != nil && *currentValue == *previousValue { return nil, false // No change @@ -172,7 +191,9 @@ func safeDerefInt(ptr *int) string { return "nil" // Return a string indicating the value is nil } -var prevDt1, prevDt2, prevRt1, prevRt2, prevG int +var prevDt1, prevDt2, prevRt1, prevRt2, prevG, prevF, prevFl int +var prevD, prevR int +var prevD1, prevD2, prevD3, prevR1, prevR2, prevR3 int func mqttMessageHandler(client mqtt.Client, msg mqtt.Message) { // Parse JSON payload @@ -194,6 +215,7 @@ func mqttMessageHandler(client mqtt.Client, msg mqtt.Message) { // Update each field, directly updating `changed` if any change is detected var tempChanged bool // Used to capture the change status for each field + // Electricity meter readings payload.Dt1, tempChanged = updateFieldIfChanged(payload.Dt1, &prevDt1) changed = changed || tempChanged @@ -206,9 +228,39 @@ func mqttMessageHandler(client mqtt.Client, msg mqtt.Message) { payload.Rt2, tempChanged = updateFieldIfChanged(payload.Rt2, &prevRt2) changed = changed || tempChanged + // Faults + payload.F, tempChanged = updateFieldIfChanged(payload.F, &prevF) + changed = changed || tempChanged + + payload.Fl, tempChanged = updateFieldIfChanged(payload.Fl, &prevFl) + changed = changed || tempChanged + + // Gas payload.G, tempChanged = updateFieldIfChanged(payload.G, &prevG) changed = changed || tempChanged + // D, R + payload.D, tempChanged = updateFieldIfChanged(payload.D, &prevD) + changed = changed || tempChanged + payload.R, tempChanged = updateFieldIfChanged(payload.R, &prevR) + changed = changed || tempChanged + + // D1-3 + payload.D1, tempChanged = updateFieldIfChanged(payload.D1, &prevD1) + changed = changed || tempChanged + payload.D2, tempChanged = updateFieldIfChanged(payload.D2, &prevD2) + changed = changed || tempChanged + payload.D3, tempChanged = updateFieldIfChanged(payload.D3, &prevD3) + changed = changed || tempChanged + + // R1-3 + payload.R1, tempChanged = updateFieldIfChanged(payload.R1, &prevR1) + changed = changed || tempChanged + payload.R2, tempChanged = updateFieldIfChanged(payload.R2, &prevR2) + changed = changed || tempChanged + payload.R3, tempChanged = updateFieldIfChanged(payload.R3, &prevR3) + changed = changed || tempChanged + // If any value has changed, log all the relevant values if changed { logger.Debug("Values changed", @@ -216,7 +268,22 @@ func mqttMessageHandler(client mqtt.Client, msg mqtt.Message) { "dt2", safeDerefInt(payload.Dt2), "rt1", safeDerefInt(payload.Rt1), "rt2", safeDerefInt(payload.Rt2), - "g", safeDerefInt(payload.G)) + + "d", safeDerefInt(payload.D), + "r", safeDerefInt(payload.R), + + "f", safeDerefInt(payload.F), + "fl", safeDerefInt(payload.Fl), + "g", safeDerefInt(payload.G), + + "d1", safeDerefInt(payload.D1), + "d2", safeDerefInt(payload.D2), + "d3", safeDerefInt(payload.D3), + + "r1", safeDerefInt(payload.R1), + "r2", safeDerefInt(payload.R2), + "r3", safeDerefInt(payload.R3), + ) } // Insert data into PostgreSQL err = insertData(timestamp, payload)