diff --git a/kubernetes/README.md b/kubernetes/README.md deleted file mode 100644 index 8618e4f..0000000 --- a/kubernetes/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Kubernetes: sish on the edge - -Plain kustomize, applied by hand. `base/sish` is the generic deployment, each edge gets an -overlay (`cirrus-dev/`) with its domain, allowed hostnames, connector keys and host key. - -```sh -kubectl --context admin@cirrus_dev diff -k kubernetes/cirrus-dev # review first -kubectl --context admin@cirrus_dev apply -k kubernetes/cirrus-dev -kubectl --context admin@cirrus_dev -n sish logs deploy/sish -f -``` - -Config changes create a new ConfigMap/Secret name (kustomize hash), which rolls the pod. -Rollouts use `Recreate` (host ports cannot be shared), so connectors drop for a few seconds -and reconnect on their own. - -## sish - -- `hostNetwork`, non-root (uid 65534), no capabilities, read-only root filesystem. Binding - :22/:80/:443 relies on the Talos sysctl in `talos/patches/unprivileged-ports.yaml`. -- `config.yml`: SNI passthrough on :443 (sish never terminates TLS), HTTP by `Host` on :80, - raw TCP forwards, public-key auth only, no web consoles. -- `port-bind-range` (ports connectors may claim) must match the firewall rule in - `talos/patches/firewall.yaml`: 22, 443 and 20000-20099 for raw TCP forwards. -- Several connectors claiming the same host/port are load-balanced round-robin. - -## Overlay `cirrus-dev` - -| | | -|---|---| -| `SISH_DOMAIN` | Fallback domain sish prints for forwards | -| `SISH_BIND_HOSTS` | Parent domains connectors may claim hostnames under (exact match on everything after the first label) | -| `pubkeys/*.pub` | Authorized connector public keys, one file per connector | -| `.secrets/ssh_host_ed25519_key` | Edge SSH host key (gitignored). Connectors pin its public half (`SISH_HOST_KEY`) | - -Add a connector: put its public key into `pubkeys/`, list it under `sish-pubkeys` in -`kustomization.yaml`, then diff and apply. Remove a connector the same way; its sessions are -cut when the pod restarts. - -New host key (e.g. for a new edge): `ssh-keygen -t ed25519 -N '' -C sish-host@ -f -kubernetes//.secrets/ssh_host_ed25519_key`, then update `SISH_HOST_KEY` in every -connector. - -Connectors run `registry.traberph.de/public/sish-client` (see its repo README); the -`cumulus` cluster runs them in `infra/configs/base/networking/sish-client`. - -## sish gotchas - -- `port-bind-range` defaults to `0,1024-65535`, which rejects 22 and 443. -- A raw TCP forward must bind `0.0.0.0:`. With a hostname sish creates a TCP *alias*, - reachable only through sish itself, not as a public port. -- PROXY protocol is opt-in per connector (`proxy-protocol=…`); the server only fixes the version. -- `idle-connection-timeout` defaults to 5s; raised to 1h. -- `service-console-max-content-length` defaults to -1, which buffers every HTTP body in memory - (even with consoles off): a large upload OOM-kills sish. Set to 0 to stream. -- A name outside `bind-hosts` is not rejected: sish silently binds `.` instead. - Check the `HTTP:`/`TLS:` line the edge prints. -- HTTP (:80) and SNI (:443) forwards need separate connector sessions. -- `Can't read file ..data` log lines are harmless (Kubernetes volume symlinks). diff --git a/kubernetes/base/sish/config.yml b/kubernetes/base/sish/config.yml deleted file mode 100644 index 56a12f5..0000000 --- a/kubernetes/base/sish/config.yml +++ /dev/null @@ -1,52 +0,0 @@ -# sish configuration (keys mirror the CLI flags, see `sish --help`). -# Cluster-specific values (domain, bind-hosts) are injected as SISH_* env vars -# from the `sish-env` ConfigMap in each overlay. Env takes precedence over this file. - -# Listeners -ssh-address: ":2222" -http-address: ":80" -https: false # sish never terminates TLS; :443 is an SNI passthrough listener - -# SNI passthrough + multiple connectors per hostname -sni-proxy: true -sni-load-balancer: true -tcp-load-balancer: true -http-load-balancer: true - -# Connectors get exactly what they ask for, or the bind fails -bind-random-ports: false -bind-random-subdomains: false -bind-random-aliases: false -force-requested-subdomains: true -force-requested-ports: true -bind-wildcards: true -# Ports connectors may claim. Must match the sish-public rule in -# talos/patches/firewall.yaml, otherwise a claimed port is silently unreachable. -# 22 gitea ssh, 443 SNI, 20000-20099 reserved for raw tcp forwards. -# Ports below 80 also need talos/patches/unprivileged-ports.yaml. -port-bind-range: "22,443,20000-20099" - -# PROXY header version for connectors that request it (sish-client default: v2) -proxy-protocol: true -proxy-protocol-version: "2" - -# Default is 5s, which kills idle websockets/SSE/slow uploads -idle-connection-timeout: 1h - -# Auth: public keys only -authentication: true -authentication-keys-directory: /pubkeys -private-keys-directory: /keys - -# No web UI / consoles -redirect-root: false -admin-console: false -service-console: false -load-templates: false -# Default -1 makes the HTTP muxer io.ReadAll() every request/response body into memory -# (for the console), even with consoles disabled: large uploads OOM-kill sish. -# 0 = never buffer, stream bodies through. -service-console-max-content-length: 0 - -log-to-stdout: true -log-to-file: false diff --git a/kubernetes/base/sish/deployment.yaml b/kubernetes/base/sish/deployment.yaml deleted file mode 100644 index 94160c7..0000000 --- a/kubernetes/base/sish/deployment.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sish - labels: - app.kubernetes.io/name: sish -spec: - replicas: 1 - # Host ports cannot be shared, so the old pod must be gone before the new one starts. - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: sish - template: - metadata: - labels: - app.kubernetes.io/name: sish - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - enableServiceLinks: false - automountServiceAccountToken: false - # Binding :22/:80/:443 as non-root relies on the node sysctl - # net.ipv4.ip_unprivileged_port_start=22 (talos/patches/unprivileged-ports.yaml). - securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - fsGroup: 65534 - seccompProfile: - type: RuntimeDefault - containers: - - name: sish - image: docker.io/antoniomika/sish:v2.23.0 - args: - - --config=/config/config.yml - envFrom: - - configMapRef: - name: sish-env - optional: true - ports: - - name: ssh - containerPort: 2222 - - name: http - containerPort: 80 - - name: https - containerPort: 443 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - resources: - requests: - cpu: 20m - memory: 32Mi - limits: - memory: 256Mi - readinessProbe: - tcpSocket: - port: ssh - periodSeconds: 10 - livenessProbe: - tcpSocket: - port: ssh - initialDelaySeconds: 10 - periodSeconds: 20 - volumeMounts: - - name: config - mountPath: /config - readOnly: true - - name: hostkey - mountPath: /keys - readOnly: true - - name: pubkeys - mountPath: /pubkeys - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: config - configMap: - name: sish-config - - name: hostkey - secret: - secretName: sish-hostkey - defaultMode: 0440 - - name: pubkeys - configMap: - name: sish-pubkeys - - name: tmp - emptyDir: - sizeLimit: 16Mi diff --git a/kubernetes/base/sish/kustomization.yaml b/kubernetes/base/sish/kustomization.yaml deleted file mode 100644 index 80a05d7..0000000 --- a/kubernetes/base/sish/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sish -resources: - - namespace.yaml - - deployment.yaml -configMapGenerator: - - name: sish-config - files: - - config.yml -# Provided by each overlay: -# ConfigMap sish-env (SISH_DOMAIN, SISH_BIND_HOSTS) -# ConfigMap sish-pubkeys (authorized connector public keys) -# Secret sish-hostkey (pinned SSH host key) diff --git a/kubernetes/base/sish/namespace.yaml b/kubernetes/base/sish/namespace.yaml deleted file mode 100644 index 2931e57..0000000 --- a/kubernetes/base/sish/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: sish - labels: - # hostNetwork is only permitted by the privileged profile. - # The pod itself still runs non-root with all capabilities dropped. - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/audit: baseline diff --git a/kubernetes/cirrus/kustomization.yaml b/kubernetes/cirrus/kustomization.yaml deleted file mode 100644 index b898647..0000000 --- a/kubernetes/cirrus/kustomization.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sish -resources: - - ../base/sish -configMapGenerator: - - name: sish-env - literals: - # .test is a reserved TLD: fine for dev, replace with real domains on the edge. - - SISH_DOMAIN=tunnel.cirrus.test - # Parents a connector may bind under (exact match on everything after the first label). - # "sto" allows cirrus.sto itself (and any .sto); "cirrus.sto" allows .cirrus.sto - - SISH_BIND_HOSTS=cirrus.test,apps.cirrus.test,sto,cirrus.sto - - name: sish-pubkeys - files: - - pubkeys/connector-dev.pub - - pubkeys/connector-hello.pub -secretGenerator: - # Edge SSH host key (connectors pin its public half). Kept only locally in - # .secrets/ (gitignored), so back it up outside this folder. - - name: sish-hostkey - files: - - ssh_host_ed25519_key=.secrets/ssh_host_ed25519_key