From e7b53b6965e9ee3c5bacd82920fbb5bf982b873a Mon Sep 17 00:00:00 2001 From: Sven Luethi Date: Mon, 3 Aug 2026 15:04:38 +0200 Subject: [PATCH] modified: .env modified: README.md modified: compose.yml modified: configuration.py new file: haproxy.cfg new file: init-patroni-users.sql renamed: entrypoint.sh -> netbox-entrypoint.sh renamed: Containerfile -> netbox.Containerfile modified: nginx.conf new file: patroni-entrypoint.sh new file: patroni.Containerfile new file: patroni.yml --- .env | 2 + README.md | 131 ++++++++++++++++++++++++-- compose.yml | 129 +++++++++++++++++++------ configuration.py | 2 +- haproxy.cfg | 26 +++++ init-patroni-users.sql | 19 ++++ entrypoint.sh => netbox-entrypoint.sh | 2 +- Containerfile => netbox.Containerfile | 2 +- nginx.conf | 3 +- patroni-entrypoint.sh | 22 +++++ patroni.Containerfile | 34 +++++++ patroni.yml | 73 ++++++++++++++ 12 files changed, 404 insertions(+), 41 deletions(-) create mode 100644 haproxy.cfg create mode 100644 init-patroni-users.sql rename entrypoint.sh => netbox-entrypoint.sh (97%) rename Containerfile => netbox.Containerfile (97%) create mode 100644 patroni-entrypoint.sh create mode 100644 patroni.Containerfile create mode 100644 patroni.yml diff --git a/.env b/.env index 1b5fd26..f6bb83a 100644 --- a/.env +++ b/.env @@ -1,2 +1,4 @@ NETBOX_VERSION=4.6.5 PYTHON_VERSION=3.14 +POSTGRES_VERSION=16 +ETCD_VERSION=v3.5.15 diff --git a/README.md b/README.md index 35006c5..740dd51 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,21 @@ This project provides a fully working Netbox Installation connected to a BIND9 Server for DNS distribution. Further components like acme certificate management will be added eventually. -## Build netbox image -You can either build the netbox container yourself with following command or let -compose pull it from my own repository at git.luethi.co. Note that the reason +## Build images +You can either build the netbox and patroni-postgres containers yourself or let +compose pull them from the repository at git.luethi.co. Note that the reason i pushed all images to my repo that compose relies on is simply convenience and reducing load on dockerhub. You can modify `compose.yml` to source all but the -`netbox` images from dockerhub instead. +`netbox` and `patroni-postgres` images from dockerhub instead. ```bash -# Build the image (bakes in plugins) +# Build all images (bakes in netbox plugins + patroni) podman-compose --podman-build-args '\-\-network=host' build ``` ## Start the Stack via compose ```bash -# Start the full stack +# Start the full stack (etcd + patroni HA + netbox) podman compose up -d ``` @@ -25,11 +25,20 @@ podman compose up -d ## Building the image directly without compose ```bash +# NetBox podman build \ + -f netbox.Containerfile \ --build-arg PYTHON_VERSION=3.14 \ --build-arg NETBOX_VERSION=4.6.5 \ --tag localhost/netbox:4.6.5 \ . + +# Patroni-Postgres +podman build \ + -f patroni.Containerfile \ + --build-arg POSTGRES_VERSION=16 \ + --tag localhost/patroni-postgres:16 \ + . ``` --- @@ -41,3 +50,113 @@ podman build \ 3. Restart: `podman-compose up -d` Migrations run automatically on startup via the entrypoint. + +--- + +## PostgreSQL HA with Patroni + +PostgreSQL runs as a 3-node Patroni cluster (1 leader + 2 replicas) with etcd +as the Distributed Configuration Store (DCS). HAProxy routes write traffic to +the current leader via Patroni REST API health checks. + +### Architecture + +``` + ┌──────────────┐ + │ etcd │ :2379 + └──────┬───────┘ + │ + ┌───────────────────┼───────────────────┐ + │ │ │ + ┌──────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐ + │ patroni1 │ │ patroni2 │ │ patroni3 │ + │ pg :5432 │ │ pg :5433 │ │ pg :5434 │ + │ api :8008 │ │ api :8009 │ │ api :8010 │ + │ (leader) │◄───│ (replica) │◄───│ (replica) │ + └──────┬──────┘ └─────────────┘ └─────────────┘ + │ + ┌──────▼──────┐ + │ HAProxy │ :5000 → leader :7000 → stats + └──────┬──────┘ + │ + ┌──────▼──────┐ + │ NetBox │ connects to localhost:5000 + └─────────────┘ +``` + +### Ports + +| Service | PostgreSQL | Patroni API | +|----------|-----------|-------------| +| patroni1 | 5432 | 8008 | +| patroni2 | 5433 | 8009 | +| patroni3 | 5434 | 8010 | + +HAProxy listens on :5000 (write) and :7000 (stats UI). + +### Cluster management + +```bash +# Show cluster topology +podman exec patroni1 patronictl list + +# Switchover (planned leader change) +podman exec patroni1 patronictl switchover + +# Check cluster health +curl http://localhost:8008/health +curl http://localhost:8008/cluster | jq . + +# HAProxy stats UI +open http://localhost:7000 +``` + +### Converting existing standalone PostgreSQL to Patroni + +If you have an existing `postgres_data` volume from the old standalone postgres +service, follow these steps to convert it to a Patroni-managed cluster: + +1. **Stop the stack:** + ```bash + podman compose down + ``` + +2. **Start only the old postgres temporarily** to create Patroni users: + ```bash + podman compose run --rm --no-deps postgres \ + psql -U netbox -d netbox -f /init-patroni-users.sql + ``` + *(If the `postgres` service no longer exists in compose, use any postgres + 16-alpine container with the same volume mounted.)* + +3. **Update pg_hba.conf** in the existing data volume to allow replication: + ```bash + echo 'host replication replicator 0.0.0.0/0 md5' >> \ + /var/lib/containers/storage/volumes/netbox_postgres_data/_data/pg_hba.conf + ``` + +4. **Start the new Patroni stack:** + ```bash + podman compose up -d + ``` + + - patroni1 detects the existing data and takes over management without + re-initializing. + - patroni2 and patroni3 start with empty data dirs and are bootstrapped as + replicas via `pg_basebackup` from patroni1. + +5. **Verify:** + ```bash + podman exec patroni1 patronictl list + ``` + Should show 1 Leader + 2 Replicas in `running` / `streaming` state. + +### Fresh start (no existing data) + +If you don't need to preserve existing data, simply remove the old volume and +start fresh — Patroni will run `initdb` on patroni1 and bootstrap replicas: + +```bash +podman volume rm netbox_postgres_data +podman compose up -d +``` diff --git a/compose.yml b/compose.yml index 6e3aa0b..4e0aa7c 100644 --- a/compose.yml +++ b/compose.yml @@ -2,24 +2,100 @@ name: netbox services: - # __ PostgreSQL ______________________________________________ - postgres: - image: git.luethi.co/sven/netbox-stack/postgres:16-alpine + # __ etcd (DCS for Patroni) __________________________________ + etcd: + image: quay.io/coreos/etcd:${ETCD_VERSION} restart: unless-stopped network_mode: host - environment: - POSTGRES_DB: netbox - POSTGRES_USER: netbox - POSTGRES_PASSWORD: netbox + command: > + etcd + --data-dir /etcd-data + --listen-client-urls http://0.0.0.0:2379 + --advertise-client-urls http://127.0.0.1:2379 + --listen-peer-urls http://0.0.0.0:2380 + --initial-advertise-peer-urls http://127.0.0.1:2380 + --initial-cluster default=http://127.0.0.1:2380 + --initial-cluster-token netbox-etcd + --initial-cluster-state new volumes: - - postgres_data:/var/lib/postgresql/data:U + - etcd_data:/etcd-data + + # __ Patroni node 1 (leader candidate, existing data) ________ + patroni1: + image: git.luethi.co/sven/netbox-stack/patroni-postgres:${POSTGRES_VERSION} + build: + context: . + dockerfile: patroni.Containerfile + network: host + args: + POSTGRES_VERSION: ${POSTGRES_VERSION} + restart: unless-stopped + network_mode: host + depends_on: + - etcd + environment: + PATRONI_NAME: postgresql0 + PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432 + PATRONI_POSTGRESQL_CONNECT_ADDRESS: localhost:5432 + PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data + PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008 + PATRONI_RESTAPI_CONNECT_ADDRESS: localhost:8008 + volumes: + - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U netbox"] + test: ["CMD-SHELL", "curl -sf http://localhost:8008/health || exit 1"] interval: 10s timeout: 5s - retries: 5 + retries: 10 + start_period: 30s - # __ Redis ___________________________________________________ + # __ Patroni node 2 (replica) ________________________________ + patroni2: + image: git.luethi.co/sven/netbox-stack/patroni-postgres:${POSTGRES_VERSION} + restart: unless-stopped + network_mode: host + depends_on: + patroni1: + condition: service_healthy + environment: + PATRONI_NAME: postgresql1 + PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5433 + PATRONI_POSTGRESQL_CONNECT_ADDRESS: localhost:5433 + PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data + PATRONI_RESTAPI_LISTEN: 0.0.0.0:8009 + PATRONI_RESTAPI_CONNECT_ADDRESS: localhost:8009 + volumes: + - postgres_data2:/var/lib/postgresql/data + + # __ Patroni node 3 (replica) ________________________________ + patroni3: + image: git.luethi.co/sven/netbox-stack/patroni-postgres:${POSTGRES_VERSION} + restart: unless-stopped + network_mode: host + depends_on: + patroni1: + condition: service_healthy + environment: + PATRONI_NAME: postgresql2 + PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5434 + PATRONI_POSTGRESQL_CONNECT_ADDRESS: localhost:5434 + PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data + PATRONI_RESTAPI_LISTEN: 0.0.0.0:8010 + PATRONI_RESTAPI_CONNECT_ADDRESS: localhost:8010 + volumes: + - postgres_data3:/var/lib/postgresql/data + + # __ HAProxy (routes writes to leader) ________________________ + haproxy: + image: git.luethi.co/sven/netbox-stack/haproxy:alpine + restart: unless-stopped + network_mode: host + depends_on: + - patroni1 + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + + # __ Redis _____________________________________________________ redis: image: git.luethi.co/sven/netbox-stack/redis:7-alpine restart: unless-stopped @@ -27,18 +103,13 @@ services: command: redis-server --save 60 1 volumes: - redis_data:/data:U - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - # __ NetBox web (Gunicorn) ___________________________________ + # __ NetBox web (Gunicorn) _____________________________________ netbox: image: git.luethi.co/sven/netbox-stack/netbox:${NETBOX_VERSION} build: context: . - dockerfile: Containerfile + dockerfile: netbox.Containerfile network: host args: NETBOX_VERSION: ${NETBOX_VERSION} @@ -49,10 +120,8 @@ services: restart: unless-stopped network_mode: host depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy + - haproxy + - redis volumes: - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py:ro - netbox_custom_scripts:/opt/netbox/netbox/scripts:ro @@ -66,14 +135,9 @@ services: GUNICORN_MAX_REQUESTS: 5000 GUNICORN_MAX_REQUESTS_JITTER: 500 command: gunicorn - healthcheck: - test: ["CMD", "python", "/opt/netbox/netbox/manage.py", "check"] - interval: 60s - timeout: 30s - retries: 5 - # __ NetBox RQ worker ________________________________________ + # __ NetBox RQ worker __________________________________________ netbox-worker: image: git.luethi.co/sven/netbox-stack/netbox:${NETBOX_VERSION} restart: unless-stopped @@ -87,7 +151,7 @@ services: - netbox_static:/opt/netbox/netbox/static:U command: worker - # __ NetBox DNS transfer endpoint ____________________________ + # __ NetBox DNS transfer endpoint ______________________________ netbox-dns-transfer-endpoint: image: git.luethi.co/sven/netbox-stack/netbox:${NETBOX_VERSION} restart: unless-stopped @@ -98,7 +162,7 @@ services: - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py:ro command: dns-transfer-endpoint - # __ NGINX reverse proxy _____________________________________ + # __ NGINX reverse proxy _______________________________________ nginx: image: git.luethi.co/sven/netbox-stack/nginx:alpine restart: unless-stopped @@ -109,7 +173,7 @@ services: - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro - netbox_static:/static:ro - # __ BIND9 ___________________________________________________ + # __ BIND9 _____________________________________________________ bind9: image: git.luethi.co/sven/netbox-stack/bind9:9.20 restart: unless-stopped @@ -122,6 +186,9 @@ services: volumes: postgres_data: + postgres_data2: + postgres_data3: + etcd_data: redis_data: netbox_media: netbox_static: diff --git a/configuration.py b/configuration.py index cd9238f..88c20ca 100644 --- a/configuration.py +++ b/configuration.py @@ -23,7 +23,7 @@ DATABASE = { "USER": "netbox", "PASSWORD": "netbox", "HOST": "localhost", - "PORT": "5432", + "PORT": "5000", "CONN_MAX_AGE": 300, } diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 0000000..37463e7 --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,26 @@ +global + maxconn 100 + +defaults + log global + mode tcp + retries 2 + timeout client 30m + timeout connect 4s + timeout server 30m + timeout check 5s + +listen stats + mode http + bind *:7000 + stats enable + stats uri / + +listen postgres_write + bind *:5000 + option httpchk GET /primary + http-check expect status 200 + default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions + server patroni1 127.0.0.1:5432 check port 8008 + server patroni2 127.0.0.1:5433 check port 8009 + server patroni3 127.0.0.1:5434 check port 8010 diff --git a/init-patroni-users.sql b/init-patroni-users.sql new file mode 100644 index 0000000..1eb0f8a --- /dev/null +++ b/init-patroni-users.sql @@ -0,0 +1,19 @@ +-- ============================================================ +-- init-patroni-users.sql +-- Creates users required by Patroni on an existing standalone +-- PostgreSQL instance before Patroni takeover. +-- +-- Run this on the current primary BEFORE starting Patroni: +-- podman compose run --rm --no-deps postgres \ +-- psql -U netbox -d netbox -f /init-patroni-users.sql +-- ============================================================ + +-- Replication user (used by replicas for streaming replication) +CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator-pass'; + +-- Rewind user (used by pg_rewind when a former leader rejoins) +CREATE ROLE rewind_user WITH LOGIN PASSWORD 'rewind-pass'; +GRANT EXECUTE ON function pg_catalog.pg_ls_dir(text, boolean, boolean) TO rewind_user; +GRANT EXECUTE ON function pg_catalog.pg_stat_file(text, boolean) TO rewind_user; +GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO rewind_user; +GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) TO rewind_user; diff --git a/entrypoint.sh b/netbox-entrypoint.sh similarity index 97% rename from entrypoint.sh rename to netbox-entrypoint.sh index 1521dd9..116b502 100644 --- a/entrypoint.sh +++ b/netbox-entrypoint.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # ============================================================ -# /entrypoint.sh – NetBox container entrypoint +# /netbox-entrypoint.sh – NetBox container entrypoint # ============================================================ set -euo pipefail diff --git a/Containerfile b/netbox.Containerfile similarity index 97% rename from Containerfile rename to netbox.Containerfile index 5b31e99..36f8154 100644 --- a/Containerfile +++ b/netbox.Containerfile @@ -90,7 +90,7 @@ VOLUME ["/opt/netbox/netbox/media"] VOLUME ["/opt/netbox/netbox/static"] # Entrypoint script handles: collectstatic, migrate, then gunicorn/worker -COPY --chown=netbox:netbox ./entrypoint.sh /entrypoint.sh +COPY --chown=netbox:netbox ./netbox-entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh USER netbox diff --git a/nginx.conf b/nginx.conf index 831fff1..0c650e6 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,5 +1,6 @@ server { - listen 8084; + listen 127.0.0.1:8084; + listen [::1]:8084; server_name _; location /static/ { diff --git a/patroni-entrypoint.sh b/patroni-entrypoint.sh new file mode 100644 index 0000000..a4f3b2b --- /dev/null +++ b/patroni-entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# ============================================================ +# /patroni-entrypoint.sh – Patroni container entrypoint +# Ensures permissions then starts Patroni as the postgres user +# ============================================================ +set -euo pipefail + +DATA_DIR="${PATRONI_POSTGRESQL_DATA_DIR:-/var/lib/postgresql/data}" + +# Ensure data and socket directories exist with correct ownership +mkdir -p "$DATA_DIR" /var/run/postgresql +chown -R postgres:postgres "$DATA_DIR" /var/run/postgresql +# PostgreSQL requires data dir mode 0700 or 0750 +chmod 700 "$DATA_DIR" + +# Ensure pgpass file is writable by postgres +touch /tmp/pgpass +chown postgres:postgres /tmp/pgpass +chmod 600 /tmp/pgpass + +# Start Patroni (manages PostgreSQL lifecycle) as postgres user +exec su-exec postgres "$@" diff --git a/patroni.Containerfile b/patroni.Containerfile new file mode 100644 index 0000000..291f578 --- /dev/null +++ b/patroni.Containerfile @@ -0,0 +1,34 @@ +# ============================================================ +# Patroni + PostgreSQL 16 – Alpine +# HA PostgreSQL cluster managed by Patroni +# DCS: etcd v3 +# ============================================================ + +ARG POSTGRES_VERSION=16 + +FROM postgres:${POSTGRES_VERSION}-alpine + +LABEL org.opencontainers.image.title="patroni-postgres" \ + org.opencontainers.image.description="PostgreSQL 16 with Patroni HA (etcd DCS)" + +# Patroni runtime: Python 3, pip, su-exec (privilege drop), curl (healthchecks) +RUN apk add --no-cache \ + python3 \ + py3-pip \ + su-exec \ + bash \ + curl \ + && pip install --no-cache-dir --break-system-packages \ + "patroni[etcd3,psycopg3]" + +# Bake in base Patroni configuration +# Node-specific values overridden via PATRONI_* env vars at runtime +COPY patroni.yml /etc/patroni/patroni.yml +COPY patroni-entrypoint.sh /patroni-entrypoint.sh +RUN chmod +x /patroni-entrypoint.sh + +# Patroni manages postgres lifecycle; data dir is a volume +VOLUME ["/var/lib/postgresql/data"] + +ENTRYPOINT ["/patroni-entrypoint.sh"] +CMD ["patroni", "/etc/patroni/patroni.yml"] diff --git a/patroni.yml b/patroni.yml new file mode 100644 index 0000000..017433f --- /dev/null +++ b/patroni.yml @@ -0,0 +1,73 @@ +# ============================================================ +# patroni.yml – Base Patroni configuration +# Node-specific values overridden via PATRONI_* env vars +# ============================================================ + +scope: netbox-ha +namespace: /service +name: postgresql0 + +# ── REST API ────────────────────────────────────────────────── +# Used for health checks, leader race, and HAProxy routing +restapi: + listen: 0.0.0.0:8008 + connect_address: localhost:8008 + +# ── DCS: etcd v3 ────────────────────────────────────────────── +etcd3: + hosts: localhost:2379 + +# ── Bootstrap (only runs on first cluster init) ─────────────── +# WARNING: changes here are ignored after the cluster is initialized. +# Use `patronictl edit-config` to change dynamic config instead. +bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + primary_start_timeout: 300 + postgresql: + use_pg_rewind: true + use_slots: true + parameters: + wal_level: replica + hot_standby: "on" + max_wal_senders: 10 + max_replication_slots: 10 + wal_log_hints: "on" + pg_hba: + - host replication replicator 127.0.0.1/32 md5 + - host replication replicator 0.0.0.0/0 md5 + - host all all 0.0.0.0/0 md5 + initdb: + - encoding: UTF8 + - locale: UTF8 + - data-checksums + +# ── PostgreSQL ──────────────────────────────────────────────── +postgresql: + listen: 0.0.0.0:5432 + connect_address: localhost:5432 + data_dir: /var/lib/postgresql/data + bin_dir: /usr/local/bin + pgpass: /tmp/pgpass + authentication: + superuser: + username: netbox + password: netbox + replication: + username: replicator + password: replicator-pass + rewind: + username: rewind_user + password: rewind-pass + parameters: + unix_socket_directories: /var/run/postgresql + +# ── Tags ────────────────────────────────────────────────────── +tags: + nofailover: false + noloadbalance: false + clonefrom: false + nostream: false