✨ Adds TrueNAS Certificate Deployer with deployment logic and configuration

acme/compose.yaml

@@ -0,0 +1,29 @@
+name: image-builder-acme
+
+services:
+  truenas-certificate-deployer:
+    container_name: truenas-certificate-deployer
+    image: containers.kritikos.io/system/truenas-certificate-deployer:latest
+    build:
+      context: truenas-certificate-deployer
+      dockerfile: Containerfile
+      provenance: mode=max
+      sbom: true
+      args:
+        IMAGE_VERSION: ${IMAGE_VERSION:-0.0.0-dev}
+        IMAGE_REVISION: ${IMAGE_REVISION:-}
+        IMAGE_CREATED: ${IMAGE_CREATED:-}
+    restart: unless-stopped
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+    environment:
+      TRUENAS_URI: ws://localhost/api/current
+      TRUENAS_UI_RESTART: "true"
+    cap_drop:
+      - ALL
+    network_mode: host
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    tmpfs:
+      - /tmp:size=8M

acme/truenas-certificate-deployer/Containerfile

@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1
+ARG ALPINE_VERSION=3.20
+FROM alpine:${ALPINE_VERSION}
+
+# Build metadata (pass via --build-arg at build time)
+ARG IMAGE_VERSION=1.0.0
+ARG ALPINE_VERSION IMAGE_REVISION IMAGE_CREATED
+
+# OCI image annotations (https://github.com/opencontainers/image-spec/blob/main/annotations.md)
+LABEL org.opencontainers.image.title="TrueNAS Certificate Deployer" \
+  org.opencontainers.image.description="Sidecar for acme.sh that deploys renewed certificates to TrueNAS via the websocket API" \
+  org.opencontainers.image.vendor="kritikos.io" \
+  org.opencontainers.image.authors="Alexandros Kritikos" \
+  org.opencontainers.image.url="https://containers.kritikos.io/system/truenas-certificate-deployer" \
+  org.opencontainers.image.documentation="https://containers.kritikos.io/system/truenas-certificate-deployer" \
+  org.opencontainers.image.source="https://containers.kritikos.io/system/truenas-certificate-deployer" \
+  org.opencontainers.image.licenses="MIT" \
+  org.opencontainers.image.base.name="docker.io/library/alpine:${ALPINE_VERSION}" \
+  org.opencontainers.image.version="${IMAGE_VERSION}" \
+  org.opencontainers.image.revision="${IMAGE_REVISION}" \
+  org.opencontainers.image.created="${IMAGE_CREATED}"
+
+RUN apk add --no-cache \
+  python3 \
+  py3-pip \
+  py3-websocket-client \
+  inotify-tools \
+  tini \
+  && addgroup -g 12002 tls \
+  && adduser -D -H -u 65534 -G tls -s /sbin/nologin deployer 2>/dev/null || true
+
+COPY deploy.py /usr/local/bin/deploy.py
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod 0755 /usr/local/bin/deploy.py /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"]

acme/truenas-certificate-deployer/README.md

@@ -0,0 +1,75 @@
+TrueNAS Certificate Deployer
+===========================
+
+Imports acme.sh ECC certificates from `/certs/<domain>_ecc/` and selects them
+for the TrueNAS web interface. The entrypoint runs reconciliation at startup
+and whenever `/certs/.deploy-signal` changes.
+
+Certificate Rotation
+--------------------
+
+The final name is `acme_<domain with dots replaced by underscores>`, for example
+`acme_zagreus_ad_kritikos_io`. TrueNAS does not support replacing an imported
+certificate's PEM or private key in place, so each renewal receives a new ID.
+
+1. Query TrueNAS and reuse a managed record containing the current full chain,
+   preferring the record already selected for the UI.
+2. If no matching record exists, import one with a temporary name ending in
+   `_pending_<16-character full-chain hash>`.
+3. Select its ID for the UI and request a UI restart when needed. Connect directly
+   to HTTPS and require an exact match against the expected leaf certificate.
+4. Confirm the selected UI certificate ID, then delete superseded managed records
+   using non-forced deletion jobs. TrueNAS checks service dependencies and refuses
+   deletion of certificates still referenced by services.
+5. Rename the retained record to the stable name, request another UI restart to
+   refresh certificate paths, and check the served certificate again.
+6. Write the full-chain fingerprint to `/state/<domain>.fp` only after completion.
+
+Managed names are the exact stable name, that name plus a numeric timestamp from
+the previous deployer, or that name plus the temporary hash suffix. Only existing
+certificate records not managed by TrueNAS's native ACME implementation qualify.
+These names are reserved for this deployer. The default TrueNAS certificate,
+other domains, CSRs, and differently named records are not deleted.
+
+A failure leaves reconciliation incomplete and produces a nonzero deployer exit
+status. The next startup or signal retries from TrueNAS's actual records, even
+when the fingerprint is unchanged. It does not import another copy merely
+because the previous switch, deletion, or rename failed. There is no periodic
+retry loop beyond startup and certificate signals.
+
+If another service still references an older certificate, move that service to
+the new certificate separately. This deployer manages only the UI reference;
+it does not alter other services or force deletion. The temporary name and old
+records can remain until the dependency is removed and reconciliation reruns.
+Failed activation retains old records but does not automatically roll back the
+UI selection.
+
+Configuration
+-------------
+
+- `TRUENAS_URI`: direct middleware WebSocket URL, default `ws://localhost/api/current`.
+- `TRUENAS_API_KEY`: required credential supplied outside the image and repository.
+- `TRUENAS_DEPLOY_DOMAINS`: comma-separated domains matching the ECC directory names.
+- `TRUENAS_UI_RESTART`: defaults to `true`; required to finalize a new or legacy
+  certificate's stable name. With `false`, the deployer never requests a restart
+  and defers naming cleanup. Activation verification still applies.
+- `TRUENAS_JOB_TIMEOUT`: job and TLS-verification timeout in seconds, default `300`.
+
+TLS verification connects to the hostname in `TRUENAS_URI` and the configured
+TrueNAS `ui_httpsport`, using the certificate domain as SNI. That hostname must
+reach the TrueNAS UI directly, not a reverse proxy serving a different certificate.
+The exact leaf certificate is pinned to the local ACME output; the probe does not
+depend on the container's CA trust store. A connection failure or certificate
+mismatch prevents cleanup.
+
+TrueNAS has one UI certificate setting. Domains are processed in order, and the
+last successfully selected domain owns that setting. Normally configure just the
+UI certificate domain, including any additional names as SANs on that certificate.
+
+The API key needs certificate read/write, system-general read/write, and access
+to the jobs used by certificate create, delete, and update. The implementation
+targets TrueNAS 27. Its `system.general.config` and `system.general.update`
+responses expose `ui_certificate` as an integer ID or null, not an embedded
+certificate object. Both selection checks use that ID directly, as defined by
+the [TrueNAS 27 schema](https://github.com/truenas/middleware/blob/master/src/middlewared/middlewared/api/v27_0_0/system_general.py).
+Renaming preserves the new ID, not the old certificate's ID.

acme/truenas-certificate-deployer/deploy.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import pathlib
+import re
+import socket
+import ssl
+import sys
+import time
+from typing import Any, cast
+from urllib.parse import urlsplit
+
+import websocket  # type: ignore[import-untyped]
+
+
+TRUENAS_URI = os.environ.get("TRUENAS_URI", "ws://localhost/api/current")
+API_KEY = os.environ["TRUENAS_API_KEY"]
+DOMAINS = [d.strip() for d in os.environ.get("TRUENAS_DEPLOY_DOMAINS", "").split(",") if d.strip()]
+UI_RESTART = os.environ.get("TRUENAS_UI_RESTART", "true").lower() in ("1", "true", "yes")
+JOB_TIMEOUT = int(os.environ.get("TRUENAS_JOB_TIMEOUT", "300"))
+
+CERTS_ROOT = pathlib.Path("/certs")
+STATE_DIR = pathlib.Path("/state")
+
+
+def log(msg: str) -> None:
+    print(f"[{time.strftime('%Y-%m-%dT%H:%M:%S%z')}] {msg}", flush=True)
+
+
+class RPC:
+    def __init__(self, uri: str) -> None:
+        log(f"connecting {uri}")
+        self.uri = uri
+        self.ws = websocket.create_connection(uri, timeout=30)
+        self._id = 0
+
+    def reconnect(self) -> None:
+        self.close()
+        self.ws = websocket.create_connection(self.uri, timeout=30)
+        if not self.call("auth.login_with_api_key", [API_KEY]):
+            raise RuntimeError("authentication failed after UI restart")
+
+    def call(self, method: str, params: list[Any] | None = None) -> Any:
+        self._id += 1
+        req_id = str(self._id)
+        request: dict[str, Any] = {"jsonrpc": "2.0", "id": req_id, "method": method}
+        if params is not None:
+            request["params"] = params
+        self.ws.send(json.dumps(request))
+        while True:
+            msg = json.loads(self.ws.recv())
+            if str(msg.get("id")) != req_id:
+                # Discard unrelated notifications/events
+                continue
+            if "error" in msg:
+                err = msg["error"]
+                data = err.get("data") or {}
+                raise RuntimeError(
+                    f"RPC {method} failed: "
+                    f"{data.get('reason') or err.get('message')}"
+                )
+            return msg["result"]
+
+    def wait_job(self, job_id: int, timeout: int = JOB_TIMEOUT):
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            jobs = self.call("core.get_jobs", [[["id", "=", job_id]]])
+            if jobs:
+                state = jobs[0].get("state")
+                if state == "SUCCESS":
+                    return jobs[0].get("result")
+                if state in ("FAILED", "ABORTED"):
+                    raise RuntimeError(f"job {job_id} {state}: {jobs[0].get('error')}")
+            time.sleep(2)
+        raise TimeoutError(f"job {job_id} did not finish within {timeout}s")
+
+    def close(self) -> None:
+        try:
+            self.ws.close()
+        except Exception:
+            pass
+
+
+def cert_dir(domain: str) -> pathlib.Path:
+    return CERTS_ROOT / f"{domain}_ecc"
+
+
+def fingerprint(pem: str) -> str:
+    return hashlib.sha256(pem.encode()).hexdigest()
+
+
+def state_file(domain: str) -> pathlib.Path:
+    return STATE_DIR / f"{domain}.fp"
+
+
+def verify_ui_certificate(rpc: RPC, domain: str, fullchain: str, config: dict[str, Any],
+                          restart: bool = False) -> None:
+    leaf = fullchain.split("-----END CERTIFICATE-----", 1)[0] + "-----END CERTIFICATE-----\n"
+    expected = ssl.PEM_cert_to_DER_cert(leaf)
+    host = urlsplit(TRUENAS_URI).hostname
+    if not host:
+        raise ValueError("TRUENAS_URI must contain a hostname")
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+
+    def matches() -> bool:
+        with socket.create_connection((host, config["ui_httpsport"]), timeout=5) as connection:
+            with context.wrap_socket(connection, server_hostname=domain) as secure:
+                return secure.getpeercert(binary_form=True) == expected
+
+    if not restart:
+        try:
+            if matches():
+                return
+        except OSError:
+            pass
+    if UI_RESTART:
+        log(f"[{domain}] restarting UI before certificate verification")
+        rpc.call("system.general.ui_restart", [0])
+    deadline = time.monotonic() + JOB_TIMEOUT
+    while time.monotonic() < deadline:
+        try:
+            if matches():
+                if UI_RESTART:
+                    rpc.reconnect()
+                return
+        except OSError:
+            pass
+        time.sleep(2)
+    raise TimeoutError(f"[{domain}] UI did not serve the expected certificate; old certificates retained")
+
+
+def deploy_domain(rpc: RPC, domain: str) -> bool:
+    d = cert_dir(domain)
+    if not d.is_dir():
+        log(f"[{domain}] cert directory missing, skipping")
+        return False
+
+    try:
+        fullchain = (d / "fullchain.cer").read_text()
+        key = (d / f"{domain}.key").read_text()
+    except OSError as e:
+        log(f"[{domain}] cannot read cert files: {e}")
+        return False
+
+    fp = fingerprint(fullchain)
+    sf = state_file(domain)
+    restart = not sf.exists() or sf.read_text().strip() != fp
+    name = f"acme_{domain.replace('.', '_')}"
+    pending_name = f"{name}_pending_{fp[:16]}"
+    if len(pending_name) > 120:
+        raise ValueError(f"[{domain}] certificate name exceeds the TrueNAS 120-character limit")
+    owned_name = re.compile(rf"{re.escape(name)}(?:_[0-9]+|_pending_[0-9a-f]{{16}})?")
+    certificates: list[dict[str, Any]] = rpc.call("certificate.query", [])
+    managed = [
+        certificate for certificate in certificates
+        if owned_name.fullmatch(certificate["name"])
+        and certificate.get("cert_type_existing")
+        and not certificate.get("acme")
+    ]
+    config: dict[str, Any] = rpc.call("system.general.config", [])
+    selected_id: int | None = config["ui_certificate"]
+    matching = [
+        certificate for certificate in managed
+        if (certificate.get("certificate") or "").strip() == fullchain.strip()
+    ]
+    matching.sort(key=lambda certificate: (
+        certificate["id"] != selected_id, certificate["name"] != name, certificate["id"]
+    ))
+    if matching:
+        result = matching[0]
+        log(f"[{domain}] reusing cert id={result['id']}")
+    else:
+        sf.unlink(missing_ok=True)
+        log(f"[{domain}] importing certificate '{pending_name}'")
+        job_id = rpc.call("certificate.create", [{
+            "create_type": "CERTIFICATE_CREATE_IMPORTED",
+            "name": pending_name,
+            "certificate": fullchain,
+            "privatekey": key,
+        }])
+        result = rpc.wait_job(int(job_id))
+        if not isinstance(result, dict) or "id" not in result:
+            raise RuntimeError("unexpected certificate.create result")
+        result = cast(dict[str, Any], result)
+        restart = True
+    cert_id = result["id"]
+    if selected_id != cert_id:
+        sf.unlink(missing_ok=True)
+        log(f"[{domain}] setting UI certificate id={cert_id}")
+        rpc.call("system.general.update", [{"ui_certificate": cert_id}])
+        restart = True
+
+    verify_ui_certificate(rpc, domain, fullchain, config, restart=restart)
+    config = rpc.call("system.general.config", [])
+    if config["ui_certificate"] != cert_id:
+        raise RuntimeError(f"[{domain}] UI certificate selection changed; cleanup skipped")
+    if result["name"] != name and not UI_RESTART:
+        raise RuntimeError(f"[{domain}] enable TRUENAS_UI_RESTART to finalize certificate naming; cleanup skipped")
+
+    for certificate in managed:
+        if certificate["id"] != cert_id:
+            log(f"[{domain}] deleting superseded certificate '{certificate['name']}'")
+            job_id = rpc.call("certificate.delete", [certificate["id"], False])
+            if rpc.wait_job(int(job_id)) is not True:
+                raise RuntimeError(f"[{domain}] certificate deletion did not complete")
+
+    if result["name"] != name:
+        sf.unlink(missing_ok=True)
+        log(f"[{domain}] renaming certificate to '{name}'")
+        job_id = rpc.call("certificate.update", [cert_id, {"name": name}])
+        renamed = rpc.wait_job(int(job_id))
+        if not isinstance(renamed, dict) or cast(dict[str, Any], renamed).get("name") != name:
+            raise RuntimeError(f"[{domain}] certificate rename did not complete")
+        verify_ui_certificate(rpc, domain, fullchain, config, restart=True)
+
+    sf.write_text(fp)
+    log(f"[{domain}] deploy complete")
+    return True
+
+
+def main() -> int:
+    if not DOMAINS:
+        log("TRUENAS_DEPLOY_DOMAINS is empty, nothing to do")
+        return 0
+
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    log(f"target domains: {DOMAINS}")
+
+    rpc = RPC(TRUENAS_URI)
+    exit_code = 0
+    try:
+        if not rpc.call("auth.login_with_api_key", [API_KEY]):
+            log("authentication failed")
+            return 2
+        for domain in DOMAINS:
+            try:
+                deploy_domain(rpc, domain)
+            except Exception as e:
+                log(f"[{domain}] deploy error: {e}")
+                exit_code = 1
+    finally:
+        rpc.close()
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())

acme/truenas-certificate-deployer/entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+# Watch for acme.sh reload signals and invoke the TrueNAS deploy.
+#
+# The acme service (see compose.yaml) touches /certs/.deploy-signal after
+# every successful --issue / --renew via --reloadcmd. This script waits for
+# the signal file to change and re-runs the deploy.
+set -eu
+
+SIGNAL_DIR="/certs"
+SIGNAL_FILE=".deploy-signal"
+DEBOUNCE_SECONDS="${DEPLOY_DEBOUNCE_SECONDS:-3}"
+
+log() { printf '[%s] %s\n' "$(date -Iseconds)" "$*"; }
+
+# Prime: run once at startup so an initial deploy happens after a fresh
+# --issue (or after the sidecar itself is recreated).
+log "startup deploy"
+python3 /usr/local/bin/deploy.py || log "startup deploy returned non-zero"
+
+log "watching $SIGNAL_DIR/$SIGNAL_FILE"
+# --include filters the parent-directory events to just our signal file.
+# NB: anchored regex (`^...$`) does not fire reliably in inotifywait; the
+# file name is unique in the watched directory so a substring match is safe.
+inotifywait -m -q \
+    -e close_write -e create -e attrib -e moved_to \
+    --include "${SIGNAL_FILE}" \
+    "$SIGNAL_DIR" |
+while read -r _ _ _; do
+    # Debounce: coalesce bursts (acme issues + renews may touch several times).
+    sleep "$DEBOUNCE_SECONDS"
+    log "signal received, deploying"
+    python3 /usr/local/bin/deploy.py || log "deploy returned non-zero"
+done

compose.yaml

@@ -1,2 +1,3 @@
 include:
-  - database/compose.yaml
+  - database/compose.yaml
+  - acme/compose.yaml