mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(seaweedfs): serve the s3 internal gRPC over mTLS
weed s3 takes its gRPC server identity from [grpc.s3] in security.toml and serves plaintext when the section is missing, which it is in the 4.45 chart. Every peer dials that port with mTLS: the s3_lifecycle worker with its [grpc.worker] certificate, other s3 replicas with [grpc.client] for IAM cache propagation. Both fail the handshake, so lifecycle deletes cannot run and identity changes do not reach the other replicas. The section is backported from upstream (seaweedfs/seaweedfs#11157, first released in 4.46) as a chart patch. It uses the internal CA-signed client certificate the s3 pods already mount, not the public listener's tlsSecret, which internal peers would not trust. security.toml is mounted by subPath, so a running s3 pod never sees the new section. A pod annotation changes the s3 template in this release so the upgrade restarts it. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
c1ba05574f
commit
2cc90b18f5
5 files changed
+78
No files matched your search
@@ -61,6 +61,7 @@ update:
|
||||
patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch
|
||||
patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch
|
||||
patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch
|
||||
patch --no-backup-if-mismatch -p4 < patches/s3-grpc-tls.patch
|
||||
#patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml
|
||||
$(MAKE) update-image
|
||||
|
||||
|
||||
@@ -97,6 +97,19 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/worker/tls.key"
|
||||
|
||||
{{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000).
|
||||
Without this section, LoadServerTLS("grpc.s3") returns nil and the S3
|
||||
server serves its internal gRPC (IAM cache propagation, s3 lifecycle
|
||||
delete RPCs) as plaintext, while peers dial it with mTLS credentials:
|
||||
"tls: first record does not look like a TLS handshake".
|
||||
Always uses the internal CA-signed client certificate (already mounted
|
||||
on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that
|
||||
secret is for the public HTTPS listener and may be issued by a public
|
||||
CA which internal gRPC peers (trusting only grpc.ca) would reject. */}}
|
||||
[grpc.s3]
|
||||
cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
|
||||
# use this for any place needs a grpc client
|
||||
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
|
||||
[grpc.client]
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
|
||||
index a067e1de9..17b82cca0 100644
|
||||
--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
|
||||
+++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
|
||||
@@ -97,6 +97,19 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/worker/tls.key"
|
||||
|
||||
+ {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000).
|
||||
+ Without this section, LoadServerTLS("grpc.s3") returns nil and the S3
|
||||
+ server serves its internal gRPC (IAM cache propagation, s3 lifecycle
|
||||
+ delete RPCs) as plaintext, while peers dial it with mTLS credentials:
|
||||
+ "tls: first record does not look like a TLS handshake".
|
||||
+ Always uses the internal CA-signed client certificate (already mounted
|
||||
+ on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that
|
||||
+ secret is for the public HTTPS listener and may be issued by a public
|
||||
+ CA which internal gRPC peers (trusting only grpc.ca) would reject. */}}
|
||||
+ [grpc.s3]
|
||||
+ cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
+ key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
+
|
||||
# use this for any place needs a grpc client
|
||||
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
|
||||
[grpc.client]
|
||||
@@ -0,0 +1,32 @@
|
||||
suite: seaweedfs s3 gRPC listener speaks mTLS
|
||||
|
||||
templates:
|
||||
- charts/seaweedfs/templates/shared/security-configmap.yaml
|
||||
- charts/seaweedfs/templates/s3/s3-deployment.yaml
|
||||
|
||||
release:
|
||||
name: seaweedfs-system
|
||||
namespace: tenant-root
|
||||
|
||||
tests:
|
||||
# weed s3 serves its internal gRPC (the LifecycleDelete RPC among it) with
|
||||
# the [grpc.s3] identity and as plaintext without one, while the worker dials
|
||||
# it with [grpc.worker] mTLS credentials, so every delete failed the TLS
|
||||
# handshake. patches/s3-grpc-tls.patch backports the section from upstream.
|
||||
- it: gives the s3 gRPC server a TLS identity signed by the internal CA
|
||||
template: charts/seaweedfs/templates/shared/security-configmap.yaml
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["security.toml"]
|
||||
pattern: '\[grpc\.s3\]\s+cert = "/usr/local/share/ca-certificates/client/tls\.crt"\s+key = "/usr/local/share/ca-certificates/client/tls\.key"'
|
||||
|
||||
# security.toml reaches the s3 pods through a subPath mount, which a running
|
||||
# container never re-reads, so an upgrade that only changes the ConfigMap
|
||||
# leaves s3 serving plaintext gRPC until something restarts it. The pod
|
||||
# template has to change in the same release.
|
||||
- it: rolls the s3 pods so they load the new section
|
||||
template: charts/seaweedfs/templates/s3/s3-deployment.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.annotations["seaweedfs.cozystack.io/grpc-s3-tls"]
|
||||
value: "true"
|
||||
@@ -207,6 +207,14 @@ seaweedfs:
|
||||
# weed skip startLanceServer and drops the container port and Service port
|
||||
# with it.
|
||||
lancePort: 0
|
||||
# patches/s3-grpc-tls.patch adds [grpc.s3] to security.toml, which the s3
|
||||
# pods mount by subPath and so never re-read. This annotation changes the
|
||||
# pod template in the release that brings the section, so an upgrade
|
||||
# restarts s3 onto it instead of leaving its gRPC in plaintext. The
|
||||
# upstream s3 template carries no checksum of that ConfigMap, and adding
|
||||
# one would be another vendored patch for a change made once.
|
||||
podAnnotations:
|
||||
seaweedfs.cozystack.io/grpc-s3-tls: "true"
|
||||
# Keep S3 traffic on same-zone endpoints. Native chart support landed in 4.12
|
||||
# (replaces the former s3-traffic-distribution.patch); the chart gates the
|
||||
# field on Kubernetes >=1.31 and auto-converts to PreferSameZone on >=1.35.
|
||||
|
||||
Reference in new issue
Block a user