Compare commits
No commits in common. "main" and "v0.0.5" have entirely different histories.
24 changed files with 153 additions and 752 deletions
|
|
@ -51,6 +51,6 @@ jobs:
|
|||
push: ${{ gitea.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/arm64, linux/amd64, linux/riscv64, linux/arm/v7, linux/arm/v6, linux/386, linux/mips64
|
||||
platforms: linux/arm64,linux/amd64
|
||||
- name: Cleanup old images
|
||||
run: docker system prune -f
|
||||
|
|
@ -23,20 +23,12 @@ jobs:
|
|||
cache: false
|
||||
- name: release-build-linux-amd64
|
||||
run: GOOS=linux GOARCH=amd64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-linux-amd64
|
||||
- name: release-build-linux-386
|
||||
run: GOOS=linux GOARCH=386 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-linux-386
|
||||
- name: release-build-linux-arm64
|
||||
run: GOOS=linux GOARCH=arm64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-linux-arm64
|
||||
- name: release-build-linux-arm
|
||||
run: GOOS=linux GOARCH=arm go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-linux-arm
|
||||
- name: release-build-linux-riscv64
|
||||
run: GOOS=linux GOARCH=riscv64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-linux-riscv64
|
||||
|
||||
- name: release-build-windows-amd64
|
||||
run: GOOS=windows GOARCH=amd64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-windows-amd64.exe
|
||||
- name: release-build-windows-arm64
|
||||
run: GOOS=windows GOARCH=arm64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-windows-arm64.exe
|
||||
|
||||
- name: release-build-darwin-amd64
|
||||
run: GOOS=darwin GOARCH=amd64 go build -v -ldflags="-s -w -X 'main.Version=${{ github.ref_name }}'" -o bin/opendtu-logger-${{ github.ref_name }}-darwin-amd64
|
||||
- name: release-build-darwin-arm64
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -72,4 +72,3 @@ docs
|
|||
|
||||
# Don't include compiled binary
|
||||
main
|
||||
opendtu-logger
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# Use buildx for multi-architecture support
|
||||
FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder
|
||||
FROM --platform=${BUILDPLATFORM} golang:1.22 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
@ -11,10 +11,7 @@ RUN go mod download
|
|||
COPY . .
|
||||
|
||||
# Build the application for the specified target architecture
|
||||
# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
|
||||
# Declare TARGETOS and TARGETARCH in the local scope so they can be used in the build stage.
|
||||
ARG TARGETOS TARGETARCH
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o opendtu-logger .
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -o opendtu-logger .
|
||||
|
||||
# Create a minimal runtime image
|
||||
FROM --platform=${TARGETPLATFORM} scratch
|
||||
|
|
|
|||
151
README.md
151
README.md
|
|
@ -1,145 +1,31 @@
|
|||
# OpenDTU logger
|
||||
|
||||
OpenDTU logger adds detailed, long-term storage and visualisation of Hoymiles solar inverter metrics monitored with OpenDTU:
|
||||
OpenDTU logger connects to the OpenDTU livedata websocket and captures metrics.
|
||||
These metrics are inserted into a PostgreSQL database.
|
||||
Optionally, TimescaleDB can be used.
|
||||
|
||||
- It connects to the OpenDTU livedata websocket and captures metrics.
|
||||
- These metrics are inserted into a PostgreSQL database. Optionally, TimescaleDB can be used.
|
||||
- Included in OpenDTU Logger are a number of Grafana dashboards, which can be used to provide useful visualisations and insights.
|
||||
- The image below shows the PV dashboard of a given day in dark mode. Light mode is also supported. Hovering over the graphs will result in a tooltip, providing the exact values recorded at a given time.
|
||||
## Install instructions
|
||||
|
||||
|
||||
|
||||
## OpenDTU logger installation instructions
|
||||
|
||||
OpenDTU logger can be installed in different ways:
|
||||
|
||||
1. Using the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons).
|
||||
2. Using [Docker Compose](#docker-compose)
|
||||
3. Using the binary available on the [releases page](https://git.hollander.online/energy/opendtu-logger/releases)
|
||||
4. Compiling the code yourself.
|
||||
|
||||
Using the Home Assistant add-on or Docker Compose is the preferred way to install OpenDTU Logger.
|
||||
|
||||
Installation on Home Assistant can be done by adding the [Home Assistant add-on repository](https://git.hollander.online/energy/home-assistant-addons) to the Home Assistant add-on store and following the instructions provided. Other installation methods are described below.
|
||||
|
||||
### Configuring OpenDTU
|
||||
|
||||
In order for OpenDTU Logger to work properly, it is required to ensure the following OpenDTU settings are used.
|
||||
|
||||
- Within OpenDTU, go to `Settings` -> `Inverter settings` (<http://YOUR_OPENDTU_IP_ADDRESS/settings/inverter>).
|
||||
- For each inverter in the inverter list, click on the pencil (Edit inverter) and go to `Advanced`.
|
||||
- Enable `Zero daily yield at midnight` and `Clear Eventlog at midnight`.
|
||||
- Click `Save`
|
||||
- Repeat this procedure for every inverter.
|
||||
|
||||
|
||||
### Docker Compose
|
||||
|
||||
The `docker` folder in this [repository](https://git.hollander.online/energy/opendtu-logger) contains example Docker compose files. The `compose.with-database-grafana.yml` file contains a full setup suitable for a standalone deployment. The other compose files are aimed at integration into existing environments.
|
||||
|
||||
To use the standalone deployment, take the following steps
|
||||
|
||||
Enter the `docker` folder
|
||||
### Docker
|
||||
|
||||
```sh
|
||||
cd docker
|
||||
docker pull git.hollander.online/energy/opendtu-logger:0.0
|
||||
```
|
||||
|
||||
Copy `example.with-database.env` to `.env`. E.g.:
|
||||
|
||||
```sh
|
||||
cp example.with-database.env .env
|
||||
```
|
||||
|
||||
In the `.env` file:
|
||||
|
||||
- Edit the `REMOTE_URL` to point to your OpenDTU.
|
||||
- In `DB_URL` alter the `password` field. Set the same value in. `PG_PASSWORD`
|
||||
|
||||
Deploy the Docker containers by running the following command from the `docker` folder.
|
||||
|
||||
```sh
|
||||
docker compose -f compose.with-database-grafana.yml up
|
||||
```
|
||||
|
||||
Docker should now start downloading the required containers and bring up OpenDTU Logger, TimescaleDB and Grafana. If everything seems ok, press `CTRL + C` to stop the containers. Next, execute
|
||||
|
||||
```sh
|
||||
docker compose -f compose.with-database-grafana.yml up -d
|
||||
```
|
||||
|
||||
The `-d` flag ensures that the containers are started in the background, so you can safely close your terminal window. If desired, containers running in the background can be stopped by running compose again with the `down` command.
|
||||
|
||||
```sh
|
||||
docker compose -f compose.with-database-grafana.yml down
|
||||
```
|
||||
|
||||
#### Setting up Grafana
|
||||
|
||||
After deploying OpenDTU Logger with Docker Compose, Grafana should be available at <http://localhost:3000>.
|
||||
|
||||
The initial login username and password are `admin:admin`. After the initial login, Grafana will request you to change the password to something more secure.
|
||||
|
||||
##### Configuring the OpenDTU Logger data source
|
||||
|
||||
After logging in, go to `Connections` -> `Data sources` -> `Add new data source`.
|
||||
|
||||
- Under `SQL` select `PostgreSQL`.
|
||||
- Change the following fields
|
||||
- `Host URL`: `timescaledb:5432`
|
||||
- `Database name`: `opendtu_logger`
|
||||
- `Username`: `postgres`
|
||||
- `Password`: The password you set in the `.env.` file in the previous step.
|
||||
- `TLS/SSL Mode`: `disable`
|
||||
- `TimescaleDB`: set the slider to `enable`
|
||||
- Click `Save and test`
|
||||
- You should get a green box with a `Database Connection OK` message.
|
||||
|
||||
##### Adding the OpenDTU Logger dashboards
|
||||
|
||||
Now that the data source has been configured, we can add our dashboards.
|
||||
|
||||
- Go to `Dashboards` -> `New` -> `Import`.
|
||||
- Upload `PV.json` from the `grafana` folder in this [repository](https://git.hollander.online/energy/opendtu-logger)
|
||||
- After uploading, Grafana requests you to configure a data source. Select the data source configured in the previous step. Click `Import`. The dashboard should now be ready.
|
||||
- If you want more dashboards, import the other dashboards from the `grafana` folder as well, or create your own.
|
||||
Preferably, run the Docker image using the Docker compose examples provided in the `./docker` folder.
|
||||
|
||||
### Binary
|
||||
|
||||
Go to the [releases page](https://git.hollander.online/energy/opendtu-logger/releases) and download the most recent stable release.
|
||||
|
||||
#### Running the binary as local user
|
||||
#### Running example
|
||||
|
||||
```sh
|
||||
REMOTE_URL="opendtu:80" DB_URL="host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable" TIMESCALEDB_ENABLED="true" TZ="Europe/Amsterdam" ./opendtu-logger
|
||||
```
|
||||
|
||||
#### Creating a more permanent setup on Linux using systemd
|
||||
|
||||
```sh
|
||||
sudo adduser --system --group --disabled-login --home /opt/opendtu-logger opendtu-logger
|
||||
|
||||
sudo mkdir /opt/opendtu-logger
|
||||
sudo mkdir /opt/opendtu-logger/data
|
||||
sudo mv ./opendtu-logger /opt/opendtu-logger/
|
||||
|
||||
sudo chown -R opendtu-logger:opendtu-logger /opt/opendtu-logger
|
||||
```
|
||||
|
||||
```sh
|
||||
sudo nano /etc/systemd/system/opendtu-logger.service
|
||||
```
|
||||
|
||||
Add the contents of `systemd/opendtu-logger.service` and alter the `REMOTE_URL` and `DB_URL` environment variables to match your setup.
|
||||
|
||||
### PostgreSQL setup
|
||||
|
||||
To install PostgreSQL on Debian / Ubuntu run
|
||||
|
||||
```sh
|
||||
sudo apt install postgresql
|
||||
```
|
||||
|
||||
The OpenDTU logger logs to a PostgreSQL database and optionally supports TimescaleDB. Create a separate database and user using the following commands.
|
||||
|
||||
```sql
|
||||
|
|
@ -149,7 +35,7 @@ GRANT ALL privileges ON DATABASE opendtu_logger TO opendtu_logger;
|
|||
GRANT ALL ON SCHEMA public TO opendtu_logger;
|
||||
```
|
||||
|
||||
Optional: enable the TimescaleDB extension. To make use of TimescaleDB, follow the steps outlined in [their documentation](https://docs.timescale.com/self-hosted/latest/install/installation-linux/). Then, enable TimescaleDB support for the `opendtu_logger` database by executing the following commands
|
||||
Optional: enable TimescaleDB extension
|
||||
|
||||
```sql
|
||||
\c opendtu_logger
|
||||
|
|
@ -167,24 +53,15 @@ TIMESCALEDB_ENABLED="true"
|
|||
TZ="Europe/Amsterdam"
|
||||
```
|
||||
|
||||
- `REMOTE_URL`is used to specify the IP address or hostname OpenDTU is running on. E.g.: `192.168.1.6` or `opendtu.internal`
|
||||
- `DB_URL` specifies the credentials required to connect to the PostgreSQL database. All [connection parameters documented by golang's `lib/pq` package](https://pkg.go.dev/github.com/lib/pq#hdr-Connection_String_Parameters) can be used.
|
||||
- `TIMESCALEDB_ENABLED` should be set to `true` or `false`.
|
||||
- `TZ` is used to ensure data is recorded with the right timestamp. Choose the timezone valid for your location from [this Wikipedia page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones).
|
||||
## Grafana
|
||||
|
||||
## Grafana dashboards
|
||||
|
||||
The `grafana` folder contains example dashboards which can be imported into existing Grafana installs, or into the provided Docker Compose with Grafana setup.
|
||||
The `grafana` folder contains example dashboards.
|
||||
|
||||
- The `opendtu_` dashboards contain visualisations for each table the OpenDTU logger records to.
|
||||
- `PV.json` is a dashboard which combines the visualisations of each table that I found to be most useful.
|
||||
- `Zonnepanelen.json` is the same as `PV.json`, but in Dutch language.
|
||||
|
||||
## Known limitations
|
||||
## Known issues
|
||||
|
||||
- The logger currently assumes that events reported by OpenDTU occurred on the current day. This works for the majority of users, as the inverters reset every night when the sun goes down and OpenDTU is configured to forget old events data at midnight. For micro inverters with battery storage added to the PV input side, such as Zendure or Anker Solix, checks still need to be added to account for the fact that these inverters don't necessarily reset every night. Event recording will work, but entries might be recorded multiple times. For more information, see also [this OpenDTU Github issue](https://github.com/tbnobody/OpenDTU/issues/1800).
|
||||
- Upon restart of the OpenDTU, the OpenDTU Logger binary needs to be restarted as well. When using the provided `compose.yml` files with Docker, or using the `systemd` service file when using the binary, this happens automatically.
|
||||
|
||||
## Feedback
|
||||
|
||||
If you'd like to provide feedback about this project, or in case you'd like to ask questions, please refer to [this page](https://pieterhollander.nl/author/pieter-hollander/) for contact information (e-mail, Github or LinkedIn), or use the [contact form](https://pieterhollander.nl/#contact) on my website. You can also respond by placing your remarks in [this discussion on Github](https://github.com/tbnobody/OpenDTU/discussions/1842).
|
||||
- The logger will currently log every event posted by OpenDTU to the websocket. Checks still need to be added to determine the uniqueness of an event. For more information, see also [this OpenDTU Github issue](https://github.com/tbnobody/OpenDTU/issues/1800).
|
||||
- Upon restart of the OpenDTU, OpenDTU Logger will need to be restarted manually as well. Automatic reconnection logic is a work in progress.
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
version: '3.8'
|
||||
services:
|
||||
timescaledb:
|
||||
image: timescale/timescaledb:latest-pg16
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_USER: ${PG_USER}
|
||||
POSTGRES_PASSWORD: ${PG_PASSWORD}
|
||||
POSTGRES_DB: ${PG_DB}
|
||||
ports:
|
||||
- "5432:5432"
|
||||
networks:
|
||||
- internal
|
||||
volumes:
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- opendtu_db:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${PG_USER}"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
opendtu-logger:
|
||||
image: git.hollander.online/energy/opendtu-logger:latest
|
||||
restart: always
|
||||
environment:
|
||||
DB_URL: ${DB_URL}
|
||||
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
|
||||
OPENDTU_AUTH: ${OPENDTU_AUTH}
|
||||
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
|
||||
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
|
||||
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
|
||||
TZ: ${TZ}
|
||||
LOG_LEVEL: ${LOG_LEVEL}
|
||||
depends_on:
|
||||
timescaledb:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- internal
|
||||
volumes:
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: grafana
|
||||
restart: always
|
||||
user: "1000:1000"
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- internal
|
||||
volumes:
|
||||
- /etc/timezone:/etc/timezone:ro
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- opendtu_logger_grafana:/var/lib/grafana:rw
|
||||
environment:
|
||||
GF_DATAPROXY_ROW_LIMIT: 5000000
|
||||
# GF_LOG_FILTERS: rendering:debug
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f localhost:3000/api/health && echo 'ready'"]
|
||||
interval: 10s
|
||||
retries: 30
|
||||
networks:
|
||||
internal:
|
||||
attachable: true
|
||||
volumes:
|
||||
opendtu_logger_grafana:
|
||||
opendtu_db:
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
version: '3.8'
|
||||
services:
|
||||
timescaledb:
|
||||
restart: always
|
||||
image: timescale/timescaledb:latest-pg16
|
||||
environment:
|
||||
POSTGRES_USER: ${PG_USER}
|
||||
|
|
@ -22,18 +21,12 @@ services:
|
|||
retries: 20
|
||||
|
||||
opendtu-logger:
|
||||
restart: always
|
||||
image: git.hollander.online/energy/opendtu-logger:latest
|
||||
image: git.hollander.online/energy/opendtu-logger:main
|
||||
environment:
|
||||
DB_URL: ${DB_URL}
|
||||
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
|
||||
OPENDTU_AUTH: ${OPENDTU_AUTH}
|
||||
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
|
||||
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
|
||||
REMOTE_URL: ${REMOTE_URL}
|
||||
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
|
||||
TZ: ${TZ}
|
||||
LOG_LEVEL: ${LOG_LEVEL}
|
||||
|
||||
depends_on:
|
||||
timescaledb:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
version: '3.8'
|
||||
services:
|
||||
opendtu-logger:
|
||||
restart: always
|
||||
image: git.hollander.online/energy/opendtu-logger:latest
|
||||
image: git.hollander.online/energy/opendtu-logger:main
|
||||
environment:
|
||||
DB_URL: ${DB_URL}
|
||||
OPENDTU_ADDRESS: ${OPENDTU_ADDRESS}
|
||||
OPENDTU_AUTH: ${OPENDTU_AUTH}
|
||||
OPENDTU_USERNAME: ${OPENDTU_USERNAME}
|
||||
OPENDTU_PASSWORD: ${OPENDTU_PASSWORD}
|
||||
REMOTE_URL: ${REMOTE_URL}
|
||||
TIMESCALEDB_ENABLED: ${TIMESCALEDB_ENABLED}
|
||||
TZ: ${TZ}
|
||||
LOG_LEVEL: ${LOG_LEVEL}
|
||||
depends_on:
|
||||
timescaledb:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
# OpenDTU
|
||||
OPENDTU_ADDRESS="192.168.1.89:80"
|
||||
OPENDTU_AUTH=false
|
||||
OPENDTU_USERNAME=admin
|
||||
OPENDTU_PASSWORD=
|
||||
|
||||
# OpenDTU Logger
|
||||
DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
|
||||
TIMESCALEDB_ENABLED=true
|
||||
TZ="Europe/Amsterdam"
|
||||
LOG_LEVEL=INFO"
|
||||
|
|
@ -1,16 +1,7 @@
|
|||
# OpenDTU
|
||||
OPENDTU_ADDRESS="192.168.1.89:80"
|
||||
OPENDTU_AUTH=false
|
||||
OPENDTU_USERNAME=admin
|
||||
OPENDTU_PASSWORD=
|
||||
|
||||
# OpenDTU Logger
|
||||
DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=opendtu_logger sslmode=disable"
|
||||
REMOTE_URL="opendtu:80"
|
||||
DB_URL="host=timescaledb port=5432 user=postgres password=secret dbname=dtu sslmode=disable"
|
||||
TIMESCALEDB_ENABLED=true
|
||||
TZ="Europe/Amsterdam"
|
||||
LOG_LEVEL=INFO"
|
||||
|
||||
# Database configuration
|
||||
PG_USER=postgres
|
||||
PG_USER=
|
||||
PG_PASSWORD=
|
||||
PG_DB=opendtu_logger
|
||||
PG_DB=
|
||||
12
go.mod
12
go.mod
|
|
@ -1,16 +1,10 @@
|
|||
module git.hollander.online/energy/opendtu-logger
|
||||
|
||||
go 1.23
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/gorilla/websocket v1.5.1
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/pressly/goose/v3 v3.22.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
)
|
||||
require golang.org/x/net v0.17.0 // indirect
|
||||
|
|
|
|||
52
go.sum
52
go.sum
|
|
@ -1,50 +1,6 @@
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.22.0 h1:wd/7kNiPTuNAztWun7iaB98DrhulbWPrzMAaw2DEZNw=
|
||||
github.com/pressly/goose/v3 v3.22.0/go.mod h1:yJM3qwSj2pp7aAaCvso096sguezamNb2OBgxCnh/EYg=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s=
|
||||
modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
|
|
|
|||
342
main.go
342
main.go
|
|
@ -1,30 +1,25 @@
|
|||
// TODO: Storage optimisation: Map inverter serial to shorter serial. Use that for referring.
|
||||
// TODO: Use username and password provided using Basic Authentication.
|
||||
// TODO: Record Inverter struct data only on-change.
|
||||
// Idea: Make a full admin / config GUI and only configure through this utility.
|
||||
// Idea: Gather settings only on start-up.
|
||||
// TODO: Only update meter readings such as yieldday, yieldtotal on-change.
|
||||
// TODO: Add a health check endpoint, potentially log to it.
|
||||
// TODO: Add support for monitoring multiple OpenDTU's at once.
|
||||
// TODO: Implement proper DB migrations.
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
_ "time/tzdata"
|
||||
|
||||
"git.hollander.online/energy/opendtu-logger/migrations"
|
||||
"github.com/gorilla/websocket"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
// VUD contains three variables used for most metrics sent by OpenDTU:
|
||||
|
|
@ -112,8 +107,6 @@ type Event struct {
|
|||
Message string `json:"message"`
|
||||
StartTime int `json:"start_time"`
|
||||
EndTime int `json:"end_time"`
|
||||
StartTimestamp time.Time
|
||||
EndTimestamp time.Time
|
||||
}
|
||||
|
||||
type EventsResponse struct {
|
||||
|
|
@ -148,138 +141,14 @@ type InverterSettingsData struct {
|
|||
Inverters []InverterSettings `json:"inverter"`
|
||||
}
|
||||
|
||||
// Config settings struct
|
||||
type Config struct {
|
||||
DB string `json:"db"`
|
||||
OpenDTUAddress string `json:"opendtu_address"`
|
||||
OpenDTUAuth bool `json:"opendtu_auth"`
|
||||
OpenDTUUser string `json:"opendtu_username"`
|
||||
OpenDTUPassword string `json:"opendtu_password"`
|
||||
TimescaleDB bool `json:"timescaledb"`
|
||||
TZ string `json:"tz"`
|
||||
LogLevel string `json:"log_level"`
|
||||
}
|
||||
|
||||
var logger = slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
var config Config
|
||||
|
||||
// LoadConfig attempts to read the configuration from options.json
|
||||
// If it fails, it falls back to using environment variables
|
||||
func loadConfig() Config {
|
||||
configFilePath := os.Getenv("CONFIG_FILE")
|
||||
if configFilePath == "" {
|
||||
configFilePath = "/data/options.json"
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(configFilePath)
|
||||
if err == nil {
|
||||
// Successfully read the file, parse the JSON
|
||||
err = json.Unmarshal(data, &config)
|
||||
if err != nil {
|
||||
log.Fatalf("Error parsing config file: %v", err)
|
||||
}
|
||||
if config.DB == "" {
|
||||
log.Fatal("db connection settings are not set")
|
||||
}
|
||||
if config.OpenDTUAddress == "" {
|
||||
log.Fatal("opendtu_address is not set")
|
||||
}
|
||||
if config.OpenDTUAuth {
|
||||
if config.OpenDTUUser == "" {
|
||||
log.Fatal("opendtu_username is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set username")
|
||||
}
|
||||
if config.OpenDTUPassword == "" {
|
||||
log.Fatal("opendtu_password is not set, while opendtu_auth is set to enabled. Set opendtu_auth to false or set password")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Info("JSON config file not found. Falling back to environment variables.")
|
||||
// Fallback to environment variables
|
||||
config.DB = os.Getenv("DB_URL")
|
||||
if config.DB == "" {
|
||||
log.Fatal("DB_URL environment variable is not set.")
|
||||
}
|
||||
config.OpenDTUAddress = os.Getenv("OPENDTU_ADDRESS")
|
||||
if config.OpenDTUAddress == "" {
|
||||
log.Fatal("OPENDTU_ADDRESS environment variable is not set.")
|
||||
}
|
||||
|
||||
openDTUAuthStr := os.Getenv("OPENDTU_AUTH")
|
||||
if openDTUAuthStr != "" {
|
||||
openDTUAuth, err := strconv.ParseBool(openDTUAuthStr)
|
||||
if err != nil {
|
||||
log.Fatalf("Error parsing OPENDTU_AUTH: %v", err)
|
||||
}
|
||||
config.OpenDTUAuth = openDTUAuth
|
||||
}
|
||||
if config.OpenDTUAuth {
|
||||
config.OpenDTUUser = os.Getenv("OPENDTU_USERNAME")
|
||||
if config.OpenDTUUser == "" {
|
||||
log.Fatal("OPENDTU_USERNAME environment variable is not set.")
|
||||
}
|
||||
config.OpenDTUPassword = os.Getenv("OPENDTU_PASSWORD")
|
||||
if config.OpenDTUPassword == "" {
|
||||
log.Fatal("OPENDTU_PASSWORD environment variable is not set.")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
timescaleDBStr := os.Getenv("TIMESCALEDB_ENABLED")
|
||||
if timescaleDBStr != "" {
|
||||
timescaleDB, err := strconv.ParseBool(timescaleDBStr)
|
||||
if err != nil {
|
||||
log.Fatalf("Error parsing TIMESCALEDB_ENABLED: %v", err)
|
||||
}
|
||||
config.TimescaleDB = timescaleDB
|
||||
}
|
||||
config.TZ = os.Getenv("TZ")
|
||||
config.LogLevel = os.Getenv("LOG_LEVEL")
|
||||
}
|
||||
_, err = time.LoadLocation(config.TZ)
|
||||
if err != nil {
|
||||
logger.Warn("invalid timezone")
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
// Helper function to map environment variable to slog.Level
|
||||
func getLogLevel(defaultLevel slog.Level) slog.Level {
|
||||
logLevelStr := config.LogLevel
|
||||
switch logLevelStr {
|
||||
case "DEBUG":
|
||||
return slog.LevelDebug
|
||||
case "INFO":
|
||||
return slog.LevelInfo
|
||||
case "WARN":
|
||||
return slog.LevelWarn
|
||||
case "ERROR":
|
||||
return slog.LevelError
|
||||
default:
|
||||
return defaultLevel
|
||||
}
|
||||
}
|
||||
|
||||
// Function to create a new logger with a specified log level
|
||||
func createLoggerWithLevel(level slog.Level) *slog.Logger {
|
||||
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: level,
|
||||
}))
|
||||
}
|
||||
|
||||
// Main program
|
||||
func main() {
|
||||
// Initial logger setup
|
||||
slog.SetDefault(logger)
|
||||
|
||||
// Load the configuration
|
||||
config := loadConfig()
|
||||
|
||||
// Set the logLevel
|
||||
logLevel := getLogLevel(slog.LevelInfo) // Default to info level
|
||||
logger = createLoggerWithLevel(logLevel)
|
||||
|
||||
dbConnStr := config.DB
|
||||
dbConnStr := (os.Getenv("DB_URL"))
|
||||
// Connect to PostgreSQL
|
||||
db, err := sql.Open("postgres", dbConnStr)
|
||||
if err != nil {
|
||||
|
|
@ -288,21 +157,17 @@ func main() {
|
|||
defer db.Close()
|
||||
|
||||
// Create tables if they don't exist
|
||||
migrateDB(db)
|
||||
createTables(db)
|
||||
|
||||
// Create WebSocket URL from config variable
|
||||
wsURL := "ws://" + config.OpenDTUAddress + "/livedata"
|
||||
|
||||
logger.Debug(wsURL)
|
||||
|
||||
// Create headers with optional Basic Auth
|
||||
headers := http.Header{}
|
||||
if config.OpenDTUAuth {
|
||||
headers.Set("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
|
||||
// Get WebSocket URL from environment variable
|
||||
RemoteURL := os.Getenv("REMOTE_URL")
|
||||
wsURL := "ws://" + RemoteURL + "/livedata"
|
||||
if wsURL == "" {
|
||||
log.Fatal("WEBSOCKET_URL environment variable is not set.")
|
||||
}
|
||||
|
||||
// Establish WebSocket connection
|
||||
c, _, err := websocket.DefaultDialer.Dial(wsURL, headers)
|
||||
c, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -313,8 +178,6 @@ func main() {
|
|||
log.Fatal(err)
|
||||
}
|
||||
|
||||
logger.Info("OpenDTU Logger has been successfully initialised. Starting data recording...")
|
||||
|
||||
// Start listening for WebSocket messages
|
||||
go func() {
|
||||
for {
|
||||
|
|
@ -348,10 +211,7 @@ func handleMessage(message []byte, db *sql.DB) {
|
|||
|
||||
// Check conditions for recording data
|
||||
for _, inverter := range liveData.Inverters {
|
||||
// Record events data
|
||||
// Optional debugging code:
|
||||
// log.Println("Events data:", "age", inverter.DataAge, "event number", inverter.Events, "previous event count", getPreviousEventsCount(db, inverter.Serial))
|
||||
if inverter.DataAge == 0 && inverter.Events > 0 && inverter.Events > getPreviousEventsCount(db, inverter.Serial) {
|
||||
if inverter.Events > 0 && inverter.Events > getPreviousEventsCount(db, inverter.Serial) {
|
||||
// Query the endpoint for events
|
||||
events, err := queryEventsEndpoint(inverter.Serial)
|
||||
if err != nil {
|
||||
|
|
@ -362,7 +222,7 @@ func handleMessage(message []byte, db *sql.DB) {
|
|||
// Insert events data into the opendtu_events table
|
||||
insertEvents(db, inverter.Serial, events)
|
||||
}
|
||||
// Record inverter data
|
||||
|
||||
if inverter.DataAge == 0 && inverter.Reachable {
|
||||
// Insert data into PostgreSQL tables
|
||||
insertLiveData(db, inverter, liveData.Total, liveData.Hints)
|
||||
|
|
@ -371,16 +231,103 @@ func handleMessage(message []byte, db *sql.DB) {
|
|||
}
|
||||
}
|
||||
|
||||
func migrateDB(db *sql.DB) {
|
||||
func createTables(db *sql.DB) {
|
||||
// Execute SQL statements to create tables if they don't exist
|
||||
// inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
|
||||
// An additional advantage is that it makes plotting in Grafana easier.
|
||||
// TODO: Foreign keys commented out as TimescaleDB hypertables don't support them.
|
||||
createTableSQL := `
|
||||
CREATE TABLE IF NOT EXISTS opendtu_log (
|
||||
timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
|
||||
power NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC
|
||||
);
|
||||
|
||||
// Perform DB migrations
|
||||
err := migrateFS(db, migrations.FS, ".")
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters (
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
name TEXT,
|
||||
producing BOOL,
|
||||
limit_relative NUMERIC,
|
||||
limit_absolute NUMERIC
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
ac_number INT,
|
||||
power NUMERIC,
|
||||
voltage NUMERIC,
|
||||
current NUMERIC,
|
||||
frequency NUMERIC,
|
||||
powerfactor NUMERIC,
|
||||
reactivepower NUMERIC
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
dc_number INT,
|
||||
name TEXT,
|
||||
power NUMERIC,
|
||||
voltage NUMERIC,
|
||||
current NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC,
|
||||
irradiation NUMERIC
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
temperature NUMERIC,
|
||||
power_dc NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC,
|
||||
efficiency NUMERIC
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opendtu_events (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
inverter_serial TEXT,
|
||||
message_id INT,
|
||||
message TEXT,
|
||||
start_time INT,
|
||||
end_time INT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS opendtu_hints (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
time_sync BOOL,
|
||||
radio_problem BOOL,
|
||||
default_password BOOL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
|
||||
CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
|
||||
|
||||
`
|
||||
|
||||
_, err := db.Exec(createTableSQL)
|
||||
if err != nil {
|
||||
log.Fatal("Error performing database migrations: ", err)
|
||||
log.Fatal("Error creating tables: ", err)
|
||||
}
|
||||
timescaleEnabled := config.TimescaleDB
|
||||
timescaleEnabled := os.Getenv("TIMESCALEDB_ENABLED")
|
||||
|
||||
enableTimescaleDB := `
|
||||
-- CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
|
|
@ -392,7 +339,7 @@ func migrateDB(db *sql.DB) {
|
|||
SELECT create_hypertable('opendtu_events', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);
|
||||
SELECT create_hypertable('opendtu_hints', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);
|
||||
`
|
||||
if timescaleEnabled {
|
||||
if timescaleEnabled == "true" {
|
||||
_, err := db.Exec(enableTimescaleDB)
|
||||
if err != nil {
|
||||
log.Fatal("Error enabling TimescaleDB: ", err)
|
||||
|
|
@ -401,7 +348,7 @@ func migrateDB(db *sql.DB) {
|
|||
}
|
||||
|
||||
func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
|
||||
timeZone := config.TZ
|
||||
timeZone := os.Getenv("TZ")
|
||||
loc, _ := time.LoadLocation(timeZone)
|
||||
timestamp := time.Now().In(loc)
|
||||
|
||||
|
|
@ -478,60 +425,16 @@ func insertLiveData(db *sql.DB, inverter Inverter, total Total, hints Hints) {
|
|||
|
||||
}
|
||||
|
||||
func migrate(db *sql.DB, dir string) error {
|
||||
err := goose.SetDialect("postgres")
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate: %w", err)
|
||||
}
|
||||
err = goose.Up(db, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate: %w", err)
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func migrateFS(db *sql.DB, migrationFS fs.FS, dir string) error {
|
||||
// In case the dir is an empty string, they probably meant the current directory and goose wants a period for that.
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
goose.SetBaseFS(migrationFS)
|
||||
defer func() {
|
||||
// Ensure that we remove the FS on the off chance some other part of our app uses goose for migrations and doesn't want to use our FS.
|
||||
goose.SetBaseFS(nil)
|
||||
}()
|
||||
return migrate(db, dir)
|
||||
}
|
||||
|
||||
func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
|
||||
endpoint := fmt.Sprintf("http://"+config.OpenDTUAddress+"/api/eventlog/status?inv=%s", inverterSerial)
|
||||
remoteURL := os.Getenv("REMOTE_URL")
|
||||
endpoint := fmt.Sprintf("http://"+remoteURL+"/api/eventlog/status?inv=%s", inverterSerial)
|
||||
|
||||
// Create a new HTTP request
|
||||
req, err := http.NewRequest("GET", endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if config.OpenDTUAuth {
|
||||
// Add Basic Auth header
|
||||
req.Header.Add("Authorization", basicAuth(config.OpenDTUUser, config.OpenDTUPassword))
|
||||
}
|
||||
|
||||
// Send the request
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
resp, err := http.Get(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check for HTTP errors
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status)
|
||||
}
|
||||
|
||||
// Decode the response
|
||||
var eventsResponse EventsResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&eventsResponse); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -541,11 +444,6 @@ func queryEventsEndpoint(inverterSerial string) (*EventsResponse, error) {
|
|||
}
|
||||
|
||||
// The events counter reported by OpenDTU resets every day.
|
||||
// However, this assumes that the inverters from which the events are pulled are reset every day, during the night.
|
||||
// Additionally, this function requires OpenDTU to be set to "Clear Eventlog at midnight" for each inverter.
|
||||
// "Clear Eventlog at midnight" should be set to ON in "Inverter settings" -> "pencil" -> "Advanced".
|
||||
// To account for possible time drifts, the first and last 10 minutes of the day are excluded.
|
||||
//
|
||||
// Longest day NL: sun up 4:16, sun down 22:50
|
||||
// Shortest day NL: sun up 8:44, sun down 16:25
|
||||
func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
|
||||
|
|
@ -554,8 +452,8 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
|
|||
SELECT COUNT(*)
|
||||
FROM opendtu_events
|
||||
WHERE inverter_serial = $1
|
||||
AND timestamp >= CURRENT_DATE + INTERVAL '10 minutes'
|
||||
AND timestamp < CURRENT_DATE + INTERVAL '23 hours 50 minutes'
|
||||
AND timestamp >= CURRENT_DATE + INTERVAL '9 hours'
|
||||
AND timestamp < CURRENT_DATE + INTERVAL '23 hours'
|
||||
`, inverterSerial).Scan(&count)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
logger.Error("Error querying previous events count", "error", err)
|
||||
|
|
@ -564,9 +462,7 @@ func getPreviousEventsCount(db *sql.DB, inverterSerial string) int {
|
|||
}
|
||||
|
||||
func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
|
||||
timeZone := config.TZ
|
||||
loc, _ := time.LoadLocation(timeZone)
|
||||
timestamp := time.Now().In(loc)
|
||||
timestamp := time.Now()
|
||||
|
||||
for _, event := range events.Events {
|
||||
// Insert events data into the events table
|
||||
|
|
@ -580,10 +476,10 @@ func insertEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
|
|||
}
|
||||
|
||||
if len(events.Events) > 0 && events.Events[0].EndTime == 0 {
|
||||
// If end_time is 0, schedule a job to update the corresponding message row every 10 minutes
|
||||
// If end_time is 0, schedule a job to update the corresponding message row every 30 minutes
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(10 * time.Minute)
|
||||
time.Sleep(30 * time.Minute)
|
||||
updatedEvents, err := queryEventsEndpoint(inverterSerial)
|
||||
if err != nil {
|
||||
logger.Error("Error querying events endpoint for updates", "error", err)
|
||||
|
|
@ -609,12 +505,6 @@ func updateEvents(db *sql.DB, inverterSerial string, events *EventsResponse) {
|
|||
}
|
||||
}
|
||||
|
||||
// basicAuth generates the Basic Auth header value
|
||||
func basicAuth(username, password string) string {
|
||||
credentials := username + ":" + password
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials))
|
||||
}
|
||||
|
||||
// TODO: finish this function.
|
||||
// func updateInverterConfig(db *sql.DB) {
|
||||
// // Periodically query the /api/inverter/list
|
||||
|
|
@ -633,8 +523,8 @@ func basicAuth(username, password string) string {
|
|||
// }
|
||||
|
||||
// func queryConfigEndpoint() (*InverterSettingsData, error) {
|
||||
// openDTUAddress := os.Getenv("OPENDTU_ADDRESS")
|
||||
// endpoint := fmt.Sprintf("http://" + openDTUAddress + "/api/inverter/list")
|
||||
// remoteURL := os.Getenv("REMOTE_URL")
|
||||
// endpoint := fmt.Sprintf("http://" + remoteURL + "/api/inverter/list")
|
||||
|
||||
// resp, err := http.Get(endpoint)
|
||||
// if err != nil {
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_log (
|
||||
timestamp TIMESTAMPTZ UNIQUE DEFAULT CURRENT_TIMESTAMP,
|
||||
power NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_log_timestamp_idx ON opendtu_log (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters (
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
-- inverter_serial is TEXT as some non-Hoymiles inverters use non-numeric serial numbers.
|
||||
-- An additional advantage is that it makes plotting in Grafana easier.
|
||||
inverter_serial TEXT,
|
||||
name TEXT,
|
||||
producing BOOL,
|
||||
limit_relative NUMERIC,
|
||||
limit_absolute NUMERIC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_timestamp_idx ON opendtu_inverters (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_ac (
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
ac_number INT,
|
||||
power NUMERIC,
|
||||
voltage NUMERIC,
|
||||
current NUMERIC,
|
||||
frequency NUMERIC,
|
||||
powerfactor NUMERIC,
|
||||
reactivepower NUMERIC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_ac_timestamp_idx ON opendtu_inverters_ac (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_dc (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
dc_number INT,
|
||||
name TEXT,
|
||||
power NUMERIC,
|
||||
voltage NUMERIC,
|
||||
current NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC,
|
||||
irradiation NUMERIC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_dc_timestamp_idx ON opendtu_inverters_dc (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_inverters_inv (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
inverter_serial TEXT,
|
||||
temperature NUMERIC,
|
||||
power_dc NUMERIC,
|
||||
yieldday NUMERIC,
|
||||
yieldtotal NUMERIC,
|
||||
efficiency NUMERIC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_inverters_inv_timestamp_idx ON opendtu_inverters_inv (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_events (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
inverter_serial TEXT,
|
||||
message_id INT,
|
||||
message TEXT,
|
||||
start_time INT,
|
||||
end_time INT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_events_timestamp_idx ON opendtu_events (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Check if start_timestamp column exists
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name='opendtu_events'
|
||||
AND column_name='start_timestamp') THEN
|
||||
-- Add start_timestamp column
|
||||
ALTER TABLE opendtu_events
|
||||
ADD COLUMN start_timestamp TIMESTAMPTZ;
|
||||
END IF;
|
||||
|
||||
-- Check if end_timestamp column exists
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name='opendtu_events'
|
||||
AND column_name='end_timestamp') THEN
|
||||
-- Add end_timestamp column
|
||||
ALTER TABLE opendtu_events
|
||||
ADD COLUMN end_timestamp TIMESTAMPTZ;
|
||||
END IF;
|
||||
END $$;
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE IF NOT EXISTS opendtu_hints (
|
||||
-- id SERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ,
|
||||
-- FOREIGN KEY (timestamp) REFERENCES opendtu_log(timestamp),
|
||||
time_sync BOOL,
|
||||
radio_problem BOOL,
|
||||
default_password BOOL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS opendtu_hints_timestamp_idx ON opendtu_hints (timestamp);
|
||||
-- +goose StatementEnd
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- +goose StatementEnd
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
package migrations
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed *.sql
|
||||
var FS embed.FS
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 741 KiB |
|
|
@ -1,44 +0,0 @@
|
|||
GNU nano 7.2 /etc/systemd/system/opendtu-logger.service
|
||||
[Unit]
|
||||
Description=OpenDTU Logger
|
||||
Documentation=https://git.hollander.online/energy/opendtu-logger
|
||||
After=network.target auditd.service
|
||||
|
||||
# If using a PostgreSQL DB on the local machine, uncomment the following additional "After" and "Require" sections.
|
||||
## PostgreSQL
|
||||
# After=postgresql.service
|
||||
# Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
RestartSec=10s
|
||||
Type=simple
|
||||
|
||||
User=opendtu-logger
|
||||
Group=opendtu-logger
|
||||
|
||||
Environment="OPENDTU_ADDRESS=opendtu.local:80"
|
||||
Environment="OPENDTU_AUTH=false"
|
||||
Environment="OPENDTU_USERNAME=admin"
|
||||
Environment="OPENDTU_PASSWORD=your_super_secret_password"
|
||||
Environment="DB_URL=host=localhost port=5432 user=postgres password=secret dbname=dtu sslmode=disable"
|
||||
Environment="TIMESCALEDB_ENABLED=true"
|
||||
Environment="TZ=Europe/Amsterdam"
|
||||
Environment="LOG_LEVEL=INFO"
|
||||
|
||||
|
||||
WorkingDirectory=/opt/opendtu-logger/
|
||||
ExecStart=/opt/opendtu-logger/opendtu-logger
|
||||
Restart=always
|
||||
|
||||
# Isolate OpenDTU Logger from the rest of the system
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectHome=true
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
|
||||
# Only allow writes to the following directories
|
||||
ReadWritePaths=/opt/opendtu-logger/data/
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in a new issue