From 5c50c7c8460e4382ca8441a67294c00d9c20f5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20M=C3=BCller?= Date: Thu, 13 Aug 2026 20:38:20 +0200 Subject: [PATCH 1/5] chore: reproduce unnecessary statefulset restarts --- .../00-patch-ns.yaml.j2 | 9 +++++ .../10-assert.yaml | 14 +++++++ .../10-statefulset.yaml | 38 +++++++++++++++++++ .../20-assert.yaml | 23 +++++++++++ .../20-content-identical-write.yaml | 19 ++++++++++ tests/test-definition.yaml | 3 ++ 6 files changed, 106 insertions(+) create mode 100644 tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/restarter-content-identical-write/10-assert.yaml create mode 100644 tests/templates/kuttl/restarter-content-identical-write/10-statefulset.yaml create mode 100644 tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml create mode 100644 tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml diff --git a/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 b/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/restarter-content-identical-write/10-assert.yaml b/tests/templates/kuttl/restarter-content-identical-write/10-assert.yaml new file mode 100644 index 00000000..9941c1f5 --- /dev/null +++ b/tests/templates/kuttl/restarter-content-identical-write/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test + generation: 1 +status: + readyReplicas: 2 + replicas: 2 + observedGeneration: 1 diff --git a/tests/templates/kuttl/restarter-content-identical-write/10-statefulset.yaml b/tests/templates/kuttl/restarter-content-identical-write/10-statefulset.yaml new file mode 100644 index 00000000..453532be --- /dev/null +++ b/tests/templates/kuttl/restarter-content-identical-write/10-statefulset.yaml @@ -0,0 +1,38 @@ +--- +# A minimal StatefulSet opted in to the restarter, mounting a ConfigMap. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +data: + foo: bar +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test + labels: + restarter.stackable.tech/enabled: "true" +spec: + selector: + matchLabels: + app: test + serviceName: test + replicas: 2 + template: + metadata: + labels: + app: test + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: nginx + image: nginxinc/nginx-unprivileged:1.27 + volumeMounts: + - name: config + mountPath: /config + volumes: + - name: config + configMap: + name: test-config diff --git a/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml b/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml new file mode 100644 index 00000000..dbd23104 --- /dev/null +++ b/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +# The ConfigMap content never changed. +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-config +data: + foo: bar +--- +# This should stay on generation 1 +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test + generation: 1 +status: + readyReplicas: 2 + replicas: 2 + observedGeneration: 1 diff --git a/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml b/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml new file mode 100644 index 00000000..be86506e --- /dev/null +++ b/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + # Do a change that should (tm) not have an impact on a pod mounting/needing this cm. + - script: | + kubectl apply --server-side --field-manager=third-party-controller --force-conflicts -f - < Date: Thu, 13 Aug 2026 21:09:00 +0200 Subject: [PATCH 2/5] feat: create digest from content to avoid metadata-based restarts --- CHANGELOG.md | 4 + Cargo.lock | 96 ++++- Cargo.nix | 4 + Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + .../src/restart_controller/statefulset.rs | 342 ++++++++++++++++-- 6 files changed, 398 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6760aa0..38e502b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ All notable changes to this project will be documented in this file. - Extend Helm deployed RBAC permissions to list and watch customresourcedefinitions required for startup condition ([#461]). - The operator Deployment runs with a restricted security context, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#462]). - The operator Deployment runs with a read-only root filesystem ([#469]). +- The StatefulSet restarter now keys its annotations on a digest of the mounted ConfigMap/Secret + content instead of the object's `resourceVersion`, so a content-identical write by a third party + no longer triggers a rolling restart of the whole role group ([#449]). ### Fixed @@ -28,6 +31,7 @@ All notable changes to this project will be documented in this file. [#439]: https://github.com/stackabletech/commons-operator/pull/439 [#443]: https://github.com/stackabletech/commons-operator/pull/443 [#444]: https://github.com/stackabletech/commons-operator/pull/444 +[#449]: https://github.com/stackabletech/commons-operator/issues/449 [#461]: https://github.com/stackabletech/commons-operator/pull/461 [#462]: https://github.com/stackabletech/commons-operator/pull/462 [#466]: https://github.com/stackabletech/commons-operator/pull/466 diff --git a/Cargo.lock b/Cargo.lock index dbc3ef5c..e13f41c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,6 +292,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -348,7 +357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -491,6 +500,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -541,6 +559,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -699,15 +727,25 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -755,7 +793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der", - "digest", + "digest 0.11.3", "elliptic-curve", "rfc6979", "signature", @@ -789,8 +827,8 @@ checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" dependencies = [ "base16ct", "crypto-bigint", - "crypto-common", - "digest", + "crypto-common 0.2.2", + "digest 0.11.3", "ff", "group", "hybrid-array", @@ -1046,6 +1084,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1174,7 +1222,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -2123,7 +2171,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2", + "sha2 0.11.1", ] [[package]] @@ -2331,7 +2379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ "crypto-bigint", - "crypto-common", + "crypto-common 0.2.2", "ff", "rand_core 0.10.1", "subtle", @@ -2591,11 +2639,11 @@ dependencies = [ "const-oid", "crypto-bigint", "crypto-primes", - "digest", + "digest 0.11.3", "pkcs1", "pkcs8", "rand_core 0.10.1", - "sha2", + "sha2 0.11.1", "signature", "spki", "zeroize", @@ -2923,8 +2971,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -2934,8 +2993,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -2969,7 +3028,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ - "digest", + "digest 0.11.3", "rand_core 0.10.1", ] @@ -3088,7 +3147,7 @@ dependencies = [ "rand 0.10.3", "rand_core 0.10.1", "rsa", - "sha2", + "sha2 0.11.1", "signature", "snafu 0.9.2", "stackable-shared", @@ -3112,6 +3171,7 @@ dependencies = [ "json-patch", "serde", "serde_json", + "sha2 0.10.9", "snafu 0.9.2", "stackable-operator", "strum", @@ -3146,7 +3206,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.11.1", "snafu 0.9.2", "stackable-operator-derive", "stackable-shared", diff --git a/Cargo.nix b/Cargo.nix index 9a258385..f09d52af 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10633,6 +10633,10 @@ rec { name = "serde_json"; packageId = "serde_json"; } + { + name = "sha2"; + packageId = "sha2"; + } { name = "snafu"; packageId = "snafu 0.9.2"; diff --git a/Cargo.toml b/Cargo.toml index 4d2cbc1a..2d46dada 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ http = "1.5" json-patch = "4.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10" snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.54", features = ["full"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 12c1a69b..eb416144 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -19,6 +19,7 @@ futures.workspace = true json-patch.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true snafu.workspace = true strum.workspace = true tokio.workspace = true diff --git a/rust/operator-binary/src/restart_controller/statefulset.rs b/rust/operator-binary/src/restart_controller/statefulset.rs index 8e3e8749..8a07e61e 100644 --- a/rust/operator-binary/src/restart_controller/statefulset.rs +++ b/rust/operator-binary/src/restart_controller/statefulset.rs @@ -1,28 +1,34 @@ use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet}, future::Future, + marker::PhantomData, sync::Arc, time::Duration, }; -use futures::{Stream, StreamExt, TryStream, stream}; +use futures::{Stream, StreamExt, TryStream, TryStreamExt, stream}; use serde_json::json; +use sha2::{Digest, Sha256}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ client::Client, - k8s_openapi::api::{ - apps::v1::StatefulSet, - core::v1::{ConfigMap, EnvFromSource, EnvVar, PodSpec, Secret, Volume}, + k8s_openapi::{ + api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, EnvFromSource, EnvVar, PodSpec, Secret, Volume}, + }, + apimachinery::pkg::apis::meta::v1::ObjectMeta, }, kube::{ self, Resource, ResourceExt, - api::{PartialObjectMeta, Patch, PatchParams}, + api::{Patch, PatchParams}, core::{DeserializeGuard, DynamicObject, error_boundary}, runtime::{ Config, WatchStreamExt, applier, controller::{Action, ReconcileRequest, trigger_self, trigger_with}, events::{Recorder, Reporter}, - reflector::{self, ObjectRef, Store}, + reflector::{self, Lookup, ObjectRef, Store}, watcher::{self, watcher}, }, }, @@ -42,10 +48,72 @@ const CONFIGMAP_ANNOTATION_PREFIX: &str = "configmap.restarter.stackable.tech"; /// Prefix of the Pod annotations tracking the referenced Secrets. const SECRET_ANNOTATION_PREFIX: &str = "secret.restarter.stackable.tech"; +/// A watched ConfigMap or Secret, reduced to its identity and a digest of its content. +#[derive(Clone, Debug)] +pub struct ContentDigest { + name: Option, + namespace: Option, + uid: Option, + resource_version: Option, + digest: String, + _phantom: PhantomData, +} + +impl ContentDigest { + fn new(metadata: ObjectMeta, digest: String) -> Self { + Self { + name: metadata.name, + namespace: metadata.namespace, + uid: metadata.uid, + resource_version: metadata.resource_version, + digest, + _phantom: PhantomData, + } + } +} + +/// Lets a [`Store`] cache these objects, keyed by name and namespace - so a +/// `ContentDigest` is found under the same key as the ConfigMap it was made from. +impl Lookup for ContentDigest { + type DynamicType = K::DynamicType; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + K::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + K::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + K::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + K::plural(dt) + } + + fn name(&self) -> Option> { + self.name.as_deref().map(Cow::Borrowed) + } + + fn namespace(&self) -> Option> { + self.namespace.as_deref().map(Cow::Borrowed) + } + + fn resource_version(&self) -> Option> { + self.resource_version.as_deref().map(Cow::Borrowed) + } + + fn uid(&self) -> Option> { + self.uid.as_deref().map(Cow::Borrowed) + } +} + pub struct Ctx { client: Client, - cms: DelayedInit>>, - secrets: DelayedInit>>, + cms: DelayedInit>>, + secrets: DelayedInit>>, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -92,13 +160,76 @@ impl ReconcilerError for Error { } } +/// Adds one key/value pair to `hasher`, length-prefixed. +fn update_entry(hasher: &mut Sha256, key: &str, value: &[u8]) { + hasher.update((key.len() as u64).to_be_bytes()); + hasher.update(key.as_bytes()); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +/// Truncates the digest to 128 bits and hex-encodes it. +fn finish_digest(hasher: Sha256) -> String { + hasher.finalize()[..16] + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// Digest over the parts of a ConfigMap that a Pod can read. +fn config_map_content_digest(config_map: &ConfigMap) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"configmap/data"); + for (key, value) in config_map.data.iter().flatten() { + update_entry(&mut hasher, key, value.as_bytes()); + } + hasher.update(b"configmap/binaryData"); + for (key, value) in config_map.binary_data.iter().flatten() { + update_entry(&mut hasher, key, &value.0); + } + finish_digest(hasher) +} + +/// Digest over the parts of a Secret that a Pod can read. +fn secret_content_digest(secret: &Secret) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"secret/data"); + for (key, value) in secret.data.iter().flatten() { + update_entry(&mut hasher, key, &value.0); + } + finish_digest(hasher) +} + +/// Reduces a ConfigMap to what the restarter needs, dropping its content. +fn digest_config_map(config_map: ConfigMap) -> ContentDigest { + let digest = config_map_content_digest(&config_map); + ContentDigest::new(config_map.metadata, digest) +} + +/// Reduces a Secret to what the restarter needs, dropping its content. +fn digest_secret(secret: Secret) -> ContentDigest { + let digest = secret_content_digest(&secret); + ContentDigest::new(secret.metadata, digest) +} + +/// Maps the object contained in a watcher event, changing its type. +fn map_event(event: watcher::Event, f: impl FnOnce(K) -> L) -> watcher::Event { + match event { + watcher::Event::Apply(obj) => watcher::Event::Apply(f(obj)), + watcher::Event::Delete(obj) => watcher::Event::Delete(f(obj)), + watcher::Event::InitApply(obj) => watcher::Event::InitApply(f(obj)), + watcher::Event::Init => watcher::Event::Init, + watcher::Event::InitDone => watcher::Event::InitDone, + } +} + #[allow(clippy::type_complexity)] pub fn create_context( client: Client, ) -> ( Arc, - Initializer>>, - Initializer>>, + Initializer>>, + Initializer>>, ) { let (cm_store_tx, cm_store_delayed) = DelayedInit::new(); let (secret_store_tx, secret_store_delayed) = DelayedInit::new(); @@ -113,19 +244,19 @@ pub fn create_context( pub async fn start( ctx: Arc, - cm_store_tx: Initializer>>, - secret_store_tx: Initializer>>, + cm_store_tx: Initializer>>, + secret_store_tx: Initializer>>, watch_namespace: &WatchNamespace, shutdown_signal: F, ) where F: Future, { let stses = watch_namespace.get_api::>(&ctx.client); - let cms = watch_namespace.get_api::>(&ctx.client); - let secrets = watch_namespace.get_api::>(&ctx.client); + let cms = watch_namespace.get_api::(&ctx.client); + let secrets = watch_namespace.get_api::(&ctx.client); let sts_store = reflector::store::Writer::>::new(()); - let cm_store = reflector::store::Writer::>::new(()); - let secret_store = reflector::store::Writer::>::new(()); + let cm_store = reflector::store::Writer::>::new(()); + let secret_store = reflector::store::Writer::>::new(()); let mut cm_store_tx = Some(cm_store_tx); let mut secret_store_tx = Some(secret_store_tx); let ctx2 = ctx.clone(); @@ -153,7 +284,8 @@ pub async fn start( cms, watcher::Config::default() .labels("restarter.stackable.tech/ignore != true"), - ), + ) + .map_ok(|event| map_event(event, digest_config_map)), ) .inspect(move |_| { if let Some(tx) = cm_store_tx.take() { @@ -173,7 +305,8 @@ pub async fn start( secrets, watcher::Config::default() .labels("restarter.stackable.tech/ignore != true"), - ), + ) + .map_ok(|event| map_event(event, digest_secret)), ) .inspect(move |_| { if let Some(tx) = secret_store_tx.take() { @@ -233,7 +366,7 @@ where }) } -fn find_pod_refs<'a, K: Resource + 'a>( +fn find_pod_refs<'a, K: Lookup + 'a>( pod_spec: &'a PodSpec, volume_ref: impl Fn(&Volume) -> Option> + 'a, env_var_ref: impl Fn(&EnvVar) -> Option> + 'a, @@ -294,12 +427,12 @@ pub async fn get_updated_restarter_annotations( find_pod_refs( pod_spec, |volume| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &volume.config_map.as_ref()?.name, )) }, |env_var| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_var .value_from .as_ref()? @@ -309,7 +442,7 @@ pub async fn get_updated_restarter_annotations( )) }, |env_from| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_from.config_map_ref.as_ref()?.name, )) }, @@ -334,18 +467,18 @@ pub async fn get_updated_restarter_annotations( // It is intended that "changes-ignored" is also used if the ConfigMap is not in the store, // i.e. it doesn't exist (yet), e.g. because the StatefulSet was created before the // ConfigMap, or it has the label `restarter.stackable.tech/ignore=true` (filtered out by the - // watcher). Once the ConfigMap shows up, the value changes to its uid/resourceVersion + // watcher). Once the ConfigMap shows up, the value changes to its uid/content digest // (unless ignored via the StatefulSet annotations), which restarts the Pods so that they // pick it up. // Leaving out the annotation instead would not avoid any restart, but cause additional // ones, e.g. when an object ignored via a StatefulSet annotation is created after the // StatefulSet. let value = if let Some(cm) = cm - && let Some(uid) = &cm.metadata.uid - && let Some(resource_version) = &cm.metadata.resource_version + && let Some(uid) = &cm.uid && !ignored_cms.contains(cm_name) { - format!("{uid}/{resource_version}",) + let digest = &cm.digest; + format!("{uid}/{digest}") } else { "changes-ignored".to_owned() }; @@ -358,17 +491,17 @@ pub async fn get_updated_restarter_annotations( find_pod_refs( pod_spec, |volume| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( volume.secret.as_ref()?.secret_name.as_deref()?, )) }, |env_var| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_var.value_from.as_ref()?.secret_key_ref.as_ref()?.name, )) }, |env_from| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_from.secret_ref.as_ref()?.name, )) }, @@ -394,11 +527,11 @@ pub async fn get_updated_restarter_annotations( // Same as for ConfigMaps above, "changes-ignored" is intended for Secrets not in the store. let value = if let Some(secret) = secret - && let Some(uid) = &secret.metadata.uid - && let Some(resource_version) = &secret.metadata.resource_version + && let Some(uid) = &secret.uid && !ignored_secrets.contains(secret_name) { - format!("{uid}/{resource_version}",) + let digest = &secret.digest; + format!("{uid}/{digest}") } else { "changes-ignored".to_owned() }; @@ -471,8 +604,24 @@ fn error_policy(_obj: Arc>, error: &Error, _ctx: A #[cfg(test)] mod tests { + use stackable_operator::k8s_openapi::{ + ByteString, + apimachinery::pkg::apis::meta::v1::{ManagedFieldsEntry, ObjectMeta}, + }; + use super::*; + fn config_map(data: &[(&str, &str)]) -> ConfigMap { + ConfigMap { + data: Some( + data.iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + ), + ..ConfigMap::default() + } + } + #[test] fn test_annotation_names() { assert_eq!( @@ -497,4 +646,133 @@ mod tests { "secret.restarter.stackable.tech/hiverest-owner-user.stackable-postgres-cluster.credent-8221aa71" ); } + + #[test] + fn digest_is_stable_across_releases() { + // Changing the digest function rolls every StatefulSet labelled + // restarter.stackable.tech/enabled=true which we want to avoid + assert_eq!( + config_map_content_digest(&config_map(&[("foo", "bar")])), + "401bdc692ecd7b9bb6b02f186e2976c3" + ); + } + + #[test] + fn digest_ignores_metadata() { + let mut changed = config_map(&[("foo", "bar")]); + changed.metadata = ObjectMeta { + resource_version: Some("12345".to_owned()), + labels: Some([("cost-center".to_owned(), "irrelevant".to_owned())].into()), + annotations: Some([("probe".to_owned(), "1".to_owned())].into()), + managed_fields: Some(vec![ManagedFieldsEntry { + manager: Some("third-party-controller".to_owned()), + ..ManagedFieldsEntry::default() + }]), + ..ObjectMeta::default() + }; + + assert_eq!( + config_map_content_digest(&config_map(&[("foo", "bar")])), + config_map_content_digest(&changed) + ); + } + + #[test] + fn digest_changes_when_content_changes() { + assert_ne!( + config_map_content_digest(&config_map(&[("foo", "bar")])), + config_map_content_digest(&config_map(&[("foo", "baz")])) + ); + } + + #[test] + fn digest_distinguishes_ambiguous_entry_splits() { + // Without length prefixes these would hash identically. + assert_ne!( + config_map_content_digest(&config_map(&[("ab", "c")])), + config_map_content_digest(&config_map(&[("a", "bc")])) + ); + } + + #[test] + fn digest_distinguishes_data_from_binary_data() { + let binary = ConfigMap { + binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), + ..ConfigMap::default() + }; + + assert_ne!( + config_map_content_digest(&config_map(&[("foo", "bar")])), + config_map_content_digest(&binary) + ); + } + + fn identifying_metadata() -> ObjectMeta { + ObjectMeta { + name: Some("my-config".to_owned()), + namespace: Some("my-namespace".to_owned()), + uid: Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c".to_owned()), + managed_fields: Some(vec![ManagedFieldsEntry { + manager: Some("third-party-controller".to_owned()), + ..ManagedFieldsEntry::default() + }]), + annotations: Some( + [( + "kubectl.kubernetes.io/last-applied-configuration".to_owned(), + r#"{"data":{"password":"aHVudGVyMg=="}}"#.to_owned(), + )] + .into(), + ), + ..ObjectMeta::default() + } + } + + #[test] + fn digesting_keeps_the_identity() { + let config_map = ConfigMap { + metadata: identifying_metadata(), + ..config_map(&[("foo", "bar")]) + }; + let digest = config_map_content_digest(&config_map); + let digested = digest_config_map(config_map); + + assert_eq!(digested.digest, digest); + assert_eq!(digested.name.as_deref(), Some("my-config")); + assert_eq!(digested.namespace.as_deref(), Some("my-namespace")); + assert_eq!( + digested.uid.as_deref(), + Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c") + ); + } + + #[test] + fn digesting_a_secret_retains_no_content() { + let secret = Secret { + metadata: identifying_metadata(), + data: Some([("password".to_owned(), ByteString(b"hunter2".to_vec()))].into()), + ..Secret::default() + }; + let digest = secret_content_digest(&secret); + let digested = digest_secret(secret); + + assert_eq!(digested.digest, digest); + assert_eq!(digested.name.as_deref(), Some("my-config")); + + let debug = format!("{digested:?}"); + assert!(!debug.contains("hunter2"), "{debug}"); + assert!(!debug.contains("aHVudGVyMg=="), "{debug}"); + } + + #[test] + fn store_keys_match_the_refs_built_from_a_pod_template() { + let digested = digest_config_map(ConfigMap { + metadata: identifying_metadata(), + ..config_map(&[("foo", "bar")]) + }); + + assert_eq!( + ObjectRef::from_obj(&digested), + ObjectRef::>::new("my-config").within("my-namespace") + ); + } } From a1dda8511106f0f6b3da2730d6a53b9672695206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20M=C3=BCller?= Date: Fri, 20 Nov 2026 08:39:02 +0100 Subject: [PATCH 3/5] feat: rewrite to allow switch over and reduce memory overhead --- CHANGELOG.md | 11 +- Cargo.lock | 111 +--- Cargo.nix | 14 +- Cargo.toml | 2 +- .../commons-operator/templates/roles.yaml | 11 +- .../commons-operator/pages/restarter.adoc | 32 +- .../src/restart_controller/content_digest.rs | 347 +++++++++++++ .../src/restart_controller/mod.rs | 1 + .../src/restart_controller/statefulset.rs | 485 +++++++----------- .../src/webhooks/restarter_mutate_sts.rs | 23 +- .../00-patch-ns.yaml.j2 | 9 - .../20-assert.yaml | 23 - .../20-content-identical-write.yaml | 19 - .../03-assert.yaml} | 3 +- .../03-data-hash.yaml} | 17 +- .../04-assert.yaml | 6 + .../04-metadata-only-write.yaml | 7 + .../05-assert.yaml | 6 + .../05-content-change.yaml | 7 + tests/test-definition.yaml | 3 - 20 files changed, 670 insertions(+), 467 deletions(-) create mode 100644 rust/operator-binary/src/restart_controller/content_digest.rs delete mode 100644 tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 delete mode 100644 tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml delete mode 100644 tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml rename tests/templates/kuttl/{restarter-content-identical-write/10-assert.yaml => restarter-no-unneeded-restart/03-assert.yaml} (82%) rename tests/templates/kuttl/{restarter-content-identical-write/10-statefulset.yaml => restarter-no-unneeded-restart/03-data-hash.yaml} (58%) create mode 100644 tests/templates/kuttl/restarter-no-unneeded-restart/04-assert.yaml create mode 100644 tests/templates/kuttl/restarter-no-unneeded-restart/04-metadata-only-write.yaml create mode 100644 tests/templates/kuttl/restarter-no-unneeded-restart/05-assert.yaml create mode 100644 tests/templates/kuttl/restarter-no-unneeded-restart/05-content-change.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 38e502b0..c8087aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file. - Install and manage the OpenLineageConnection Custom Resource Definition ([#435]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#461]). +- Add the StatefulSet annotation `restarter.stackable.tech/change-tracking-mechanism`. With the + value `data-hash`, the restarter only restarts Pods when the content of a referenced ConfigMap + or Secret changes, so content-identical writes by third parties no longer trigger a rolling + restart. The default `resource-version` keeps the previous behavior ([#449]). ### Changed @@ -17,9 +21,8 @@ All notable changes to this project will be documented in this file. - Extend Helm deployed RBAC permissions to list and watch customresourcedefinitions required for startup condition ([#461]). - The operator Deployment runs with a restricted security context, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#462]). - The operator Deployment runs with a read-only root filesystem ([#469]). -- The StatefulSet restarter now keys its annotations on a digest of the mounted ConfigMap/Secret - content instead of the object's `resourceVersion`, so a content-identical write by a third party - no longer triggers a rolling restart of the whole role group ([#449]). +- Extend Helm deployed RBAC permissions to get configmaps and secrets, required to digest their + content for the `data-hash` change tracking mechanism ([#449]). ### Fixed @@ -31,7 +34,7 @@ All notable changes to this project will be documented in this file. [#439]: https://github.com/stackabletech/commons-operator/pull/439 [#443]: https://github.com/stackabletech/commons-operator/pull/443 [#444]: https://github.com/stackabletech/commons-operator/pull/444 -[#449]: https://github.com/stackabletech/commons-operator/issues/449 +[#449]: https://github.com/stackabletech/commons-operator/pull/449 [#461]: https://github.com/stackabletech/commons-operator/pull/461 [#462]: https://github.com/stackabletech/commons-operator/pull/462 [#466]: https://github.com/stackabletech/commons-operator/pull/466 diff --git a/Cargo.lock b/Cargo.lock index e13f41c6..80d172a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,15 +292,6 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.12.1" @@ -357,7 +348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures", "rand_core 0.10.1", ] @@ -500,15 +491,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.1" @@ -559,16 +541,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "crypto-common" version = "0.2.2" @@ -727,25 +699,15 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "crypto-common 0.1.7", -] - [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.1", + "block-buffer", "const-oid", - "crypto-common 0.2.2", + "crypto-common", "ctutils", ] @@ -793,7 +755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der", - "digest 0.11.3", + "digest", "elliptic-curve", "rfc6979", "signature", @@ -827,8 +789,8 @@ checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" dependencies = [ "base16ct", "crypto-bigint", - "crypto-common 0.2.2", - "digest 0.11.3", + "crypto-common", + "digest", "ff", "group", "hybrid-array", @@ -1084,16 +1046,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -1155,9 +1107,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" +checksum = "9bd04d0d9c7079b416b458d2317aea4aa964c81938cb7c20f559758c4f422dc8" dependencies = [ "arraydeque", "smallvec", @@ -1222,7 +1174,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest 0.11.3", + "digest", ] [[package]] @@ -1914,9 +1866,9 @@ checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" [[package]] name = "libz-sys" -version = "1.1.31" +version = "1.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" +checksum = "fee3f59f4321dab0c78d828942e7aa09d0bd8ad6e78a501977e6bf3d02938045" dependencies = [ "cc", "libc", @@ -2171,7 +2123,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2 0.11.1", + "sha2", ] [[package]] @@ -2379,7 +2331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ "crypto-bigint", - "crypto-common 0.2.2", + "crypto-common", "ff", "rand_core 0.10.1", "subtle", @@ -2639,11 +2591,11 @@ dependencies = [ "const-oid", "crypto-bigint", "crypto-primes", - "digest 0.11.3", + "digest", "pkcs1", "pkcs8", "rand_core 0.10.1", - "sha2 0.11.1", + "sha2", "signature", "spki", "zeroize", @@ -2862,12 +2814,12 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "969cde0349344dbbf58b5f9badbfb8f806db13969364eb8007c057ef687007d9" dependencies = [ "annotate-snippets", - "base64 0.23.1", + "base64 0.22.1", "encoding_rs_io", "granit-parser", "nohash-hasher", @@ -2971,19 +2923,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", - "digest 0.11.3", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", + "cpufeatures", + "digest", ] [[package]] @@ -2993,8 +2934,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", - "digest 0.11.3", + "cpufeatures", + "digest", ] [[package]] @@ -3028,7 +2969,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ - "digest 0.11.3", + "digest", "rand_core 0.10.1", ] @@ -3147,7 +3088,7 @@ dependencies = [ "rand 0.10.3", "rand_core 0.10.1", "rsa", - "sha2 0.11.1", + "sha2", "signature", "snafu 0.9.2", "stackable-shared", @@ -3171,7 +3112,7 @@ dependencies = [ "json-patch", "serde", "serde_json", - "sha2 0.10.9", + "sha2", "snafu 0.9.2", "stackable-operator", "strum", @@ -3206,7 +3147,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2 0.11.1", + "sha2", "snafu 0.9.2", "stackable-operator-derive", "stackable-shared", diff --git a/Cargo.nix b/Cargo.nix index f09d52af..ef30a2cb 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -3612,10 +3612,10 @@ rec { }; "granit-parser" = rec { crateName = "granit-parser"; - version = "2.1.0"; + version = "2.2.0"; edition = "2021"; crateBin = []; - sha256 = "1pq1ljzfc7n0sfx0f6la8f5p4lmj81n841rj293kvpylsj35bw05"; + sha256 = "1j1d897qqxarylh7rjrq37469aaax9x33ljqnhbb8ybhkh6lvl4v"; libName = "granit_parser"; license = "MIT OR Apache-2.0"; authors = [ @@ -6336,10 +6336,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.31"; + version = "1.1.33"; edition = "2018"; links = "z"; - sha256 = "1cjw8nisxy5iv8yms7dsxwhclb26djl38n6xq4nmlqxxra73087z"; + sha256 = "0ic0jc13vgz6fwcm12p7ss5bvl09mbkl52c2ip3v1ni18fgzbqzy"; libName = "libz_sys"; license = "MIT OR Apache-2.0"; authors = [ @@ -9678,10 +9678,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.2.0"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1n87f1lfymy00y0fnr4kjq9xn1pqp2zsv6szigsvnk9l941xx74n"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -9692,7 +9692,7 @@ rec { } { name = "base64"; - packageId = "base64 0.23.1"; + packageId = "base64 0.22.1"; optional = true; } { diff --git a/Cargo.toml b/Cargo.toml index 2d46dada..e14d2704 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ http = "1.5" json-patch = "4.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sha2 = "0.10" +sha2 = "0.11" snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.54", features = ["full"] } diff --git a/deploy/helm/commons-operator/templates/roles.yaml b/deploy/helm/commons-operator/templates/roles.yaml index 8ed0df31..40cf02d3 100644 --- a/deploy/helm/commons-operator/templates/roles.yaml +++ b/deploy/helm/commons-operator/templates/roles.yaml @@ -7,14 +7,23 @@ metadata: {{- include "commons-operator.labels" . | nindent 4 }} rules: # Watch pods to detect expired restart annotations. - # Watch configmaps and secrets to trigger rolling restarts of referencing StatefulSets. - apiGroups: - "" resources: - pods + verbs: + - list + - watch + # Watch configmaps and secrets to trigger rolling restarts of referencing StatefulSets. + # Get them to digest their content for StatefulSets using the change tracking mechanism + # data-hash (annotation restarter.stackable.tech/change-tracking-mechanism). + - apiGroups: + - "" + resources: - configmaps - secrets verbs: + - get - list - watch # For automatic cluster domain detection. diff --git a/docs/modules/commons-operator/pages/restarter.adoc b/docs/modules/commons-operator/pages/restarter.adoc index 36993b55..d2c5f4b9 100644 --- a/docs/modules/commons-operator/pages/restarter.adoc +++ b/docs/modules/commons-operator/pages/restarter.adoc @@ -61,13 +61,41 @@ spec: ... ---- +Annotation:: `restarter.stackable.tech/change-tracking-mechanism` + +This annotation selects what counts as a change of a referenced ConfigMap or Secret: + +`resource-version` (default):: Every write to the object counts as a change, including writes that only touch its metadata (labels, annotations, managed fields) or that write identical content. +`data-hash`:: Only a change of the content (`data` and `binaryData` of a ConfigMap, `data` of a Secret) counts as a change. +Writes by third parties that leave the content as it is do not restart the Pods. +To hash the content, the operator reads a referenced object whenever its resource version changes. +It only keeps the hash, not the content. +Anyone who can read the StatefulSet or its Pods can use the hash to verify guesses of a Secret's content, so keep low-entropy Secrets such as user-chosen passwords on `resource-version`. + +With any other value, the operator logs an error and stops updating the Pod template annotations of the StatefulSet until the value is fixed. +Switching between the mechanisms changes the values of the Pod template annotations described below and therefore rolling-restarts the StatefulSet once. + +[source,yaml] +---- +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: statefulset-with-content-based-restarts + labels: + restarter.stackable.tech/enabled: "true" + annotations: + restarter.stackable.tech/change-tracking-mechanism: data-hash +... +---- + ==== Pod template annotations To detect changes, the operator adds an annotation for every referenced ConfigMap and Secret to the Pod template of the StatefulSet. -Its key is `configmap.restarter.stackable.tech/\{name\}` or `secret.restarter.stackable.tech/\{name\}`, its value is the UID and resource version of the referenced object. +Its key is `configmap.restarter.stackable.tech/\{name\}` or `secret.restarter.stackable.tech/\{name\}`, its value is the UID of the referenced object, followed by its resource version or, with the `data-hash` mechanism, a hash of its content. The value is `changes-ignored` instead if the object is excluded, either by the StatefulSet annotations described above or by the `restarter.stackable.tech/ignore` label described below. The value is also `changes-ignored` if the referenced object does not exist (yet), for example because the StatefulSet was created before a mounted Secret. -This is intended: once the object is created, the value changes to its UID and resource version (unless it is excluded), and the Pods are restarted to pick it up. +This is intended: once the object is created, the value changes as described above (unless it is excluded), and the Pods are restarted to pick it up. Kubernetes rolling-restarts the Pods whenever one of these values changes. These annotations are managed by the operator, there is no need to set them yourself. diff --git a/rust/operator-binary/src/restart_controller/content_digest.rs b/rust/operator-binary/src/restart_controller/content_digest.rs new file mode 100644 index 00000000..ebcc1905 --- /dev/null +++ b/rust/operator-binary/src/restart_controller/content_digest.rs @@ -0,0 +1,347 @@ +//! Digests over the content of ConfigMaps and Secrets, for the `data-hash` change tracking +//! mechanism of the StatefulSet restarter. +//! +//! The restarter only watches the metadata of ConfigMaps and Secrets. The content of an object is +//! fetched on demand, only when a StatefulSet using `data-hash` references it, and only once per +//! resourceVersion: the [`DigestCache`] keeps the digest, never the content. + +use std::{ + collections::HashMap, + future::Future, + sync::{Mutex, MutexGuard}, +}; + +use sha2::{Digest, Sha256}; +use stackable_operator::{ + k8s_openapi::{ + api::core::v1::{ConfigMap, Secret}, + apimachinery::pkg::apis::meta::v1::ObjectMeta, + }, + kube::Resource, +}; + +/// An object whose content can be digested. +pub trait ContentDigest: Resource { + /// Digest over the parts of the object that a Pod can read. + fn content_digest(&self) -> String; +} + +impl ContentDigest for ConfigMap { + fn content_digest(&self) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"configmap/data"); + for (key, value) in self.data.iter().flatten() { + update_entry(&mut hasher, key, value.as_bytes()); + } + hasher.update(b"configmap/binaryData"); + for (key, value) in self.binary_data.iter().flatten() { + update_entry(&mut hasher, key, &value.0); + } + finish_digest(hasher) + } +} + +impl ContentDigest for Secret { + fn content_digest(&self) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"secret/data"); + for (key, value) in self.data.iter().flatten() { + update_entry(&mut hasher, key, &value.0); + } + finish_digest(hasher) + } +} + +/// Adds one key/value pair to `hasher`, length-prefixed. +fn update_entry(hasher: &mut Sha256, key: &str, value: &[u8]) { + hasher.update((key.len() as u64).to_be_bytes()); + hasher.update(key.as_bytes()); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +/// Truncates the digest to 128 bits and hex-encodes it. +fn finish_digest(hasher: Sha256) -> String { + hasher.finalize()[..16] + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// The content digest of one object at one resourceVersion. +/// +/// Deliberately holds no content, so no Secret data stays in memory after hashing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedDigest { + pub uid: String, + pub resource_version: String, + pub digest: String, +} + +/// Namespace and name of a cached object. +type CacheKey = (String, String); + +/// Content digests of the ConfigMaps or the Secrets referenced by StatefulSets using `data-hash`, +/// keyed by namespace and name. +/// +/// An entry is replaced when the uid or resourceVersion changes, and removed when a fetch finds +/// the object gone. Entries of objects deleted without a later fetch stay until the operator +/// restarts. They cannot be served for a recreated object, which has a new uid, and are only a +/// few hundred bytes each, so they are not tracked any further. +#[derive(Debug, Default)] +pub struct DigestCache { + entries: Mutex>, +} + +impl DigestCache { + /// Returns the digest of the object described by `metadata`, as seen by the metadata watch. + /// + /// The cached digest is used if it was computed for the same uid and resourceVersion, + /// otherwise the object is fetched by calling `fetch` with its name and namespace, and its + /// digest cached. Returns `None` if `metadata` lacks a name or namespace, or `fetch` does not + /// find the object. + pub async fn get_or_fetch<'a, K, E, Fut>( + &self, + metadata: &'a ObjectMeta, + fetch: impl FnOnce(&'a str, &'a str) -> Fut, + ) -> Result, E> + where + K: ContentDigest, + Fut: Future, E>>, + { + let (Some(namespace), Some(name)) = (&metadata.namespace, &metadata.name) else { + return Ok(None); + }; + let key = (namespace.clone(), name.clone()); + + if let Some(cached) = self.lock().get(&key) + && Some(&cached.uid) == metadata.uid.as_ref() + && Some(&cached.resource_version) == metadata.resource_version.as_ref() + { + return Ok(Some(cached.clone())); + } + + tracing::debug!( + k8s.object.namespace = namespace, + k8s.object.name = name, + "Fetching object to digest its content" + ); + let Some(object) = fetch(name, namespace).await? else { + self.lock().remove(&key); + return Ok(None); + }; + let (Some(uid), Some(resource_version)) = ( + object.meta().uid.clone(), + object.meta().resource_version.clone(), + ) else { + return Ok(None); + }; + let cached = CachedDigest { + uid, + resource_version, + digest: object.content_digest(), + }; + self.lock().insert(key, cached.clone()); + + Ok(Some(cached)) + } + + fn lock(&self) -> MutexGuard<'_, HashMap> { + self.entries + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use std::{cell::Cell, convert::Infallible}; + + use stackable_operator::k8s_openapi::{ + ByteString, apimachinery::pkg::apis::meta::v1::ManagedFieldsEntry, + }; + + use super::*; + + fn config_map(data: &[(&str, &str)]) -> ConfigMap { + ConfigMap { + data: Some( + data.iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + ), + ..ConfigMap::default() + } + } + + fn metadata(uid: &str, resource_version: &str) -> ObjectMeta { + ObjectMeta { + name: Some("my-config".to_owned()), + namespace: Some("my-namespace".to_owned()), + uid: Some(uid.to_owned()), + resource_version: Some(resource_version.to_owned()), + ..ObjectMeta::default() + } + } + + fn stored(uid: &str, resource_version: &str, data: &[(&str, &str)]) -> ConfigMap { + ConfigMap { + metadata: metadata(uid, resource_version), + ..config_map(data) + } + } + + /// Fetches `object` and counts the calls in `calls`. + async fn fetch_counted( + calls: &Cell, + object: Option, + ) -> Result, Infallible> { + calls.set(calls.get() + 1); + Ok(object) + } + + #[test] + fn digest_is_stable_across_releases() { + // Changing the digest function rolls every StatefulSet using data-hash + let binary_config_map = ConfigMap { + binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), + ..ConfigMap::default() + }; + let secret = Secret { + data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), + ..Secret::default() + }; + + assert_eq!( + config_map(&[("foo", "bar")]).content_digest(), + "401bdc692ecd7b9bb6b02f186e2976c3" + ); + assert_eq!( + binary_config_map.content_digest(), + "20feddae4a413582dab5c1c2698ef5b8" + ); + assert_eq!(secret.content_digest(), "fa14a91ea83afdfe42d96880fe813616"); + } + + #[test] + fn digest_ignores_metadata() { + let mut changed = config_map(&[("foo", "bar")]); + changed.metadata = ObjectMeta { + resource_version: Some("12345".to_owned()), + labels: Some([("cost-center".to_owned(), "irrelevant".to_owned())].into()), + annotations: Some([("probe".to_owned(), "1".to_owned())].into()), + managed_fields: Some(vec![ManagedFieldsEntry { + manager: Some("third-party-controller".to_owned()), + ..ManagedFieldsEntry::default() + }]), + ..ObjectMeta::default() + }; + + assert_eq!( + config_map(&[("foo", "bar")]).content_digest(), + changed.content_digest() + ); + } + + #[test] + fn digest_distinguishes_ambiguous_entry_splits() { + // Without length prefixes these would hash identically. + assert_ne!( + config_map(&[("ab", "c")]).content_digest(), + config_map(&[("a", "bc")]).content_digest() + ); + } + + #[test] + fn digest_distinguishes_data_from_binary_data() { + let binary = ConfigMap { + binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), + ..ConfigMap::default() + }; + + assert_ne!( + config_map(&[("foo", "bar")]).content_digest(), + binary.content_digest() + ); + } + + #[tokio::test] + async fn cache_fetches_once_per_resource_version() { + let cache = DigestCache::default(); + let calls = Cell::new(0); + let object = stored("uid-1", "100", &[("foo", "bar")]); + + for _ in 0..3 { + let cached = cache + .get_or_fetch(&metadata("uid-1", "100"), |_, _| { + fetch_counted(&calls, Some(object.clone())) + }) + .await + .expect("fetching is infallible") + .expect("the object exists"); + assert_eq!(cached.digest, object.content_digest()); + } + + assert_eq!(calls.get(), 1); + } + + #[tokio::test] + async fn cache_refetches_when_resource_version_or_uid_changes() { + let cache = DigestCache::default(); + let calls = Cell::new(0); + + for (uid, resource_version, content) in [ + ("uid-1", "100", "bar"), + ("uid-1", "101", "baz"), + ("uid-2", "101", "qux"), + ] { + let object = stored(uid, resource_version, &[("foo", content)]); + let cached = cache + .get_or_fetch(&metadata(uid, resource_version), |_, _| { + fetch_counted(&calls, Some(object.clone())) + }) + .await + .expect("fetching is infallible") + .expect("the object exists"); + + assert_eq!( + cached, + CachedDigest { + uid: uid.to_owned(), + resource_version: resource_version.to_owned(), + digest: object.content_digest(), + } + ); + } + + assert_eq!(calls.get(), 3); + } + + #[tokio::test] + async fn cache_forgets_objects_that_are_gone() { + let cache = DigestCache::default(); + let calls = Cell::new(0); + let object = stored("uid-1", "100", &[("foo", "bar")]); + let existing = metadata("uid-1", "100"); + let fetch_existing = |_, _| fetch_counted(&calls, Some(object.clone())); + + cache + .get_or_fetch(&existing, fetch_existing) + .await + .expect("fetching is infallible"); + let missing = cache + .get_or_fetch(&metadata("uid-1", "101"), |_, _| { + fetch_counted::(&calls, None) + }) + .await + .expect("fetching is infallible"); + + assert_eq!(missing, None); + + cache + .get_or_fetch(&existing, fetch_existing) + .await + .expect("fetching is infallible"); + assert_eq!(calls.get(), 3); + } +} diff --git a/rust/operator-binary/src/restart_controller/mod.rs b/rust/operator-binary/src/restart_controller/mod.rs index 37b64887..2238597f 100644 --- a/rust/operator-binary/src/restart_controller/mod.rs +++ b/rust/operator-binary/src/restart_controller/mod.rs @@ -1,2 +1,3 @@ +pub mod content_digest; pub mod pod; pub mod statefulset; diff --git a/rust/operator-binary/src/restart_controller/statefulset.rs b/rust/operator-binary/src/restart_controller/statefulset.rs index 8a07e61e..d8a6b796 100644 --- a/rust/operator-binary/src/restart_controller/statefulset.rs +++ b/rust/operator-binary/src/restart_controller/statefulset.rs @@ -1,19 +1,19 @@ use std::{ - borrow::Cow, collections::{BTreeMap, BTreeSet}, + fmt::Debug, future::Future, - marker::PhantomData, sync::Arc, time::Duration, }; -use futures::{Stream, StreamExt, TryStream, TryStreamExt, stream}; +use futures::{Stream, StreamExt, TryStream, stream}; +use serde::de::DeserializeOwned; use serde_json::json; -use sha2::{Digest, Sha256}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ client::Client, k8s_openapi::{ + NamespaceResourceScope, api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, EnvFromSource, EnvVar, PodSpec, Secret, Volume}, @@ -22,13 +22,13 @@ use stackable_operator::{ }, kube::{ self, Resource, ResourceExt, - api::{Patch, PatchParams}, + api::{PartialObjectMeta, Patch, PatchParams}, core::{DeserializeGuard, DynamicObject, error_boundary}, runtime::{ Config, WatchStreamExt, applier, controller::{Action, ReconcileRequest, trigger_self, trigger_with}, events::{Recorder, Reporter}, - reflector::{self, Lookup, ObjectRef, Store}, + reflector::{self, ObjectRef, Store}, watcher::{self, watcher}, }, }, @@ -36,9 +36,12 @@ use stackable_operator::{ logging::controller::{ReconcilerError, report_controller_reconciled}, namespace::WatchNamespace, }; -use strum::{EnumDiscriminants, IntoStaticStr}; +use strum::{Display, EnumDiscriminants, EnumString, IntoStaticStr}; -use crate::utils::delayed_init::{DelayedInit, InitDropped, Initializer}; +use crate::{ + restart_controller::content_digest::{ContentDigest, DigestCache}, + utils::delayed_init::{DelayedInit, InitDropped, Initializer}, +}; const FULL_CONTROLLER_NAME: &str = "statefulset.restarter.commons.stackable.tech"; @@ -48,72 +51,44 @@ const CONFIGMAP_ANNOTATION_PREFIX: &str = "configmap.restarter.stackable.tech"; /// Prefix of the Pod annotations tracking the referenced Secrets. const SECRET_ANNOTATION_PREFIX: &str = "secret.restarter.stackable.tech"; -/// A watched ConfigMap or Secret, reduced to its identity and a digest of its content. -#[derive(Clone, Debug)] -pub struct ContentDigest { - name: Option, - namespace: Option, - uid: Option, - resource_version: Option, - digest: String, - _phantom: PhantomData, +/// StatefulSet annotation selecting the [`ChangeTrackingMechanism`]. +const CHANGE_TRACKING_MECHANISM_ANNOTATION: &str = + "restarter.stackable.tech/change-tracking-mechanism"; + +/// How the restarter decides that a referenced ConfigMap or Secret has changed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, EnumString, Display)] +#[strum(serialize_all = "kebab-case")] +enum ChangeTrackingMechanism { + /// Every write to the object counts as a change, including metadata-only writes. + /// + /// Only needs the metadata of the object, which the restarter watches anyway. + #[default] + ResourceVersion, + + /// Only changes to the content (`data` and `binaryData`) count as a change. + /// + /// Fetches the object whenever its resourceVersion changes, to digest its content. + DataHash, } -impl ContentDigest { - fn new(metadata: ObjectMeta, digest: String) -> Self { - Self { - name: metadata.name, - namespace: metadata.namespace, - uid: metadata.uid, - resource_version: metadata.resource_version, - digest, - _phantom: PhantomData, +impl ChangeTrackingMechanism { + /// Reads the mechanism from the StatefulSet annotation, falling back to the default. + fn from_statefulset(sts: &StatefulSet) -> Result { + match sts.annotations().get(CHANGE_TRACKING_MECHANISM_ANNOTATION) { + Some(value) => value + .parse() + .context(InvalidChangeTrackingMechanismSnafu { value }), + None => Ok(Self::default()), } } } -/// Lets a [`Store`] cache these objects, keyed by name and namespace - so a -/// `ContentDigest` is found under the same key as the ConfigMap it was made from. -impl Lookup for ContentDigest { - type DynamicType = K::DynamicType; - - fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { - K::kind(dt) - } - - fn group(dt: &Self::DynamicType) -> Cow<'_, str> { - K::group(dt) - } - - fn version(dt: &Self::DynamicType) -> Cow<'_, str> { - K::version(dt) - } - - fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { - K::plural(dt) - } - - fn name(&self) -> Option> { - self.name.as_deref().map(Cow::Borrowed) - } - - fn namespace(&self) -> Option> { - self.namespace.as_deref().map(Cow::Borrowed) - } - - fn resource_version(&self) -> Option> { - self.resource_version.as_deref().map(Cow::Borrowed) - } - - fn uid(&self) -> Option> { - self.uid.as_deref().map(Cow::Borrowed) - } -} - pub struct Ctx { client: Client, - cms: DelayedInit>>, - secrets: DelayedInit>>, + cms: DelayedInit>>, + secrets: DelayedInit>>, + cm_digests: DigestCache, + secret_digests: DigestCache, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -142,6 +117,20 @@ pub enum Error { source: KeyError, object_name: String, }, + + #[snafu(display( + "invalid value {value:?} for StatefulSet annotation {CHANGE_TRACKING_MECHANISM_ANNOTATION:?}, expected one of \"resource-version\" or \"data-hash\"" + ))] + InvalidChangeTrackingMechanism { + source: strum::ParseError, + value: String, + }, + + #[snafu(display("failed to fetch {obj_ref} to digest its content"))] + FetchContentFailed { + source: stackable_operator::client::Error, + obj_ref: Box>, + }, } impl ReconcilerError for Error { @@ -156,80 +145,19 @@ impl ReconcilerError for Error { Error::ConfigMapsUninitialized { .. } => None, Error::SecretsUninitialized { .. } => None, Error::InvalidAnnotationKey { .. } => None, + Error::InvalidChangeTrackingMechanism { .. } => None, + Error::FetchContentFailed { obj_ref, .. } => Some(*obj_ref.clone()), } } } -/// Adds one key/value pair to `hasher`, length-prefixed. -fn update_entry(hasher: &mut Sha256, key: &str, value: &[u8]) { - hasher.update((key.len() as u64).to_be_bytes()); - hasher.update(key.as_bytes()); - hasher.update((value.len() as u64).to_be_bytes()); - hasher.update(value); -} - -/// Truncates the digest to 128 bits and hex-encodes it. -fn finish_digest(hasher: Sha256) -> String { - hasher.finalize()[..16] - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() -} - -/// Digest over the parts of a ConfigMap that a Pod can read. -fn config_map_content_digest(config_map: &ConfigMap) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"configmap/data"); - for (key, value) in config_map.data.iter().flatten() { - update_entry(&mut hasher, key, value.as_bytes()); - } - hasher.update(b"configmap/binaryData"); - for (key, value) in config_map.binary_data.iter().flatten() { - update_entry(&mut hasher, key, &value.0); - } - finish_digest(hasher) -} - -/// Digest over the parts of a Secret that a Pod can read. -fn secret_content_digest(secret: &Secret) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"secret/data"); - for (key, value) in secret.data.iter().flatten() { - update_entry(&mut hasher, key, &value.0); - } - finish_digest(hasher) -} - -/// Reduces a ConfigMap to what the restarter needs, dropping its content. -fn digest_config_map(config_map: ConfigMap) -> ContentDigest { - let digest = config_map_content_digest(&config_map); - ContentDigest::new(config_map.metadata, digest) -} - -/// Reduces a Secret to what the restarter needs, dropping its content. -fn digest_secret(secret: Secret) -> ContentDigest { - let digest = secret_content_digest(&secret); - ContentDigest::new(secret.metadata, digest) -} - -/// Maps the object contained in a watcher event, changing its type. -fn map_event(event: watcher::Event, f: impl FnOnce(K) -> L) -> watcher::Event { - match event { - watcher::Event::Apply(obj) => watcher::Event::Apply(f(obj)), - watcher::Event::Delete(obj) => watcher::Event::Delete(f(obj)), - watcher::Event::InitApply(obj) => watcher::Event::InitApply(f(obj)), - watcher::Event::Init => watcher::Event::Init, - watcher::Event::InitDone => watcher::Event::InitDone, - } -} - #[allow(clippy::type_complexity)] pub fn create_context( client: Client, ) -> ( Arc, - Initializer>>, - Initializer>>, + Initializer>>, + Initializer>>, ) { let (cm_store_tx, cm_store_delayed) = DelayedInit::new(); let (secret_store_tx, secret_store_delayed) = DelayedInit::new(); @@ -237,6 +165,8 @@ pub fn create_context( client, cms: cm_store_delayed, secrets: secret_store_delayed, + cm_digests: DigestCache::default(), + secret_digests: DigestCache::default(), }); (ctx, cm_store_tx, secret_store_tx) @@ -244,19 +174,19 @@ pub fn create_context( pub async fn start( ctx: Arc, - cm_store_tx: Initializer>>, - secret_store_tx: Initializer>>, + cm_store_tx: Initializer>>, + secret_store_tx: Initializer>>, watch_namespace: &WatchNamespace, shutdown_signal: F, ) where F: Future, { let stses = watch_namespace.get_api::>(&ctx.client); - let cms = watch_namespace.get_api::(&ctx.client); - let secrets = watch_namespace.get_api::(&ctx.client); + let cms = watch_namespace.get_api::>(&ctx.client); + let secrets = watch_namespace.get_api::>(&ctx.client); let sts_store = reflector::store::Writer::>::new(()); - let cm_store = reflector::store::Writer::>::new(()); - let secret_store = reflector::store::Writer::>::new(()); + let cm_store = reflector::store::Writer::>::new(()); + let secret_store = reflector::store::Writer::>::new(()); let mut cm_store_tx = Some(cm_store_tx); let mut secret_store_tx = Some(secret_store_tx); let ctx2 = ctx.clone(); @@ -284,8 +214,7 @@ pub async fn start( cms, watcher::Config::default() .labels("restarter.stackable.tech/ignore != true"), - ) - .map_ok(|event| map_event(event, digest_config_map)), + ), ) .inspect(move |_| { if let Some(tx) = cm_store_tx.take() { @@ -305,8 +234,7 @@ pub async fn start( secrets, watcher::Config::default() .labels("restarter.stackable.tech/ignore != true"), - ) - .map_ok(|event| map_event(event, digest_secret)), + ), ) .inspect(move |_| { if let Some(tx) = secret_store_tx.take() { @@ -366,7 +294,7 @@ where }) } -fn find_pod_refs<'a, K: Lookup + 'a>( +fn find_pod_refs<'a, K: Resource + 'a>( pod_spec: &'a PodSpec, volume_ref: impl Fn(&Volume) -> Option> + 'a, env_var_ref: impl Fn(&EnvVar) -> Option> + 'a, @@ -407,6 +335,46 @@ fn annotation_key(prefix: &str, object_name: &str) -> Result { Ok(key.to_string()) } +/// Value of the Pod template annotation tracking the object with the given `metadata` for +/// `ChangeTrackingMechanism::ResourceVersion`, or `None` if the metadata lacks uid or +/// resourceVersion. +fn resource_version_value(metadata: &ObjectMeta) -> Option { + let uid = metadata.uid.as_deref()?; + let resource_version = metadata.resource_version.as_deref()?; + Some(format!("{uid}/{resource_version}")) +} + +/// Value of the Pod template annotation tracking `object` according to `mechanism`, or `None` if +/// it cannot be determined, e.g. because the object was deleted in the meantime. +/// +/// For [`ChangeTrackingMechanism::DataHash`] this fetches the object, unless `digests` already +/// holds the digest for its current resourceVersion. +async fn tracking_value( + client: &Client, + digests: &DigestCache, + object: &PartialObjectMeta, + mechanism: ChangeTrackingMechanism, +) -> Result, Error> +where + K: ContentDigest + Resource + Clone + Debug + DeserializeOwned, +{ + let metadata = &object.metadata; + match mechanism { + ChangeTrackingMechanism::ResourceVersion => Ok(resource_version_value(metadata)), + ChangeTrackingMechanism::DataHash => { + let cached = digests + .get_or_fetch(metadata, |name, namespace| { + client.get_opt::(name, namespace) + }) + .await + .with_context(|_| FetchContentFailedSnafu { + obj_ref: ObjectRef::from_obj(object).erase(), + })?; + Ok(cached.map(|cached| format!("{}/{}", cached.uid, cached.digest))) + } + } +} + pub async fn get_updated_restarter_annotations( sts: &StatefulSet, ctx: Arc, @@ -414,6 +382,7 @@ pub async fn get_updated_restarter_annotations( let ns = sts.metadata.namespace.as_deref().expect( "A StatefulSet observed by a reflector (so send by Kubernetes) always has a namespace set", ); + let mechanism = ChangeTrackingMechanism::from_statefulset(sts)?; let mut annotations = BTreeMap::::new(); let pod_specs = sts @@ -427,12 +396,12 @@ pub async fn get_updated_restarter_annotations( find_pod_refs( pod_spec, |volume| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &volume.config_map.as_ref()?.name, )) }, |env_var| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_var .value_from .as_ref()? @@ -442,7 +411,7 @@ pub async fn get_updated_restarter_annotations( )) }, |env_from| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_from.config_map_ref.as_ref()?.name, )) }, @@ -467,21 +436,19 @@ pub async fn get_updated_restarter_annotations( // It is intended that "changes-ignored" is also used if the ConfigMap is not in the store, // i.e. it doesn't exist (yet), e.g. because the StatefulSet was created before the // ConfigMap, or it has the label `restarter.stackable.tech/ignore=true` (filtered out by the - // watcher). Once the ConfigMap shows up, the value changes to its uid/content digest - // (unless ignored via the StatefulSet annotations), which restarts the Pods so that they - // pick it up. + // watcher). Once the ConfigMap shows up, the value changes to its tracking value (see + // `ChangeTrackingMechanism`, unless ignored via the StatefulSet annotations), which + // restarts the Pods so that they pick it up. // Leaving out the annotation instead would not avoid any restart, but cause additional // ones, e.g. when an object ignored via a StatefulSet annotation is created after the // StatefulSet. - let value = if let Some(cm) = cm - && let Some(uid) = &cm.uid - && !ignored_cms.contains(cm_name) - { - let digest = &cm.digest; - format!("{uid}/{digest}") - } else { - "changes-ignored".to_owned() - }; + let value = match cm { + Some(cm) if !ignored_cms.contains(cm_name) => { + tracking_value(&ctx.client, &ctx.cm_digests, &cm, mechanism).await? + } + _ => None, + } + .unwrap_or_else(|| "changes-ignored".to_owned()); annotations.insert(annotation_key(CONFIGMAP_ANNOTATION_PREFIX, cm_name)?, value); } @@ -491,17 +458,17 @@ pub async fn get_updated_restarter_annotations( find_pod_refs( pod_spec, |volume| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( volume.secret.as_ref()?.secret_name.as_deref()?, )) }, |env_var| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_var.value_from.as_ref()?.secret_key_ref.as_ref()?.name, )) }, |env_from| { - Some(ObjectRef::>::new( + Some(ObjectRef::>::new( &env_from.secret_ref.as_ref()?.name, )) }, @@ -526,15 +493,13 @@ pub async fn get_updated_restarter_annotations( let secret = secrets.get(&secret_ref); // Same as for ConfigMaps above, "changes-ignored" is intended for Secrets not in the store. - let value = if let Some(secret) = secret - && let Some(uid) = &secret.uid - && !ignored_secrets.contains(secret_name) - { - let digest = &secret.digest; - format!("{uid}/{digest}") - } else { - "changes-ignored".to_owned() - }; + let value = match secret { + Some(secret) if !ignored_secrets.contains(secret_name) => { + tracking_value(&ctx.client, &ctx.secret_digests, &secret, mechanism).await? + } + _ => None, + } + .unwrap_or_else(|| "changes-ignored".to_owned()); annotations.insert( annotation_key(SECRET_ANNOTATION_PREFIX, secret_name)?, @@ -597,6 +562,7 @@ fn error_policy(_obj: Arc>, error: &Error, _ctx: A match error { // root object is invalid, will be requeued when modified anyway Error::InvalidStatefulSet { .. } => Action::await_change(), + Error::InvalidChangeTrackingMechanism { .. } => Action::await_change(), _ => Action::requeue(Duration::from_secs(5)), } @@ -604,24 +570,8 @@ fn error_policy(_obj: Arc>, error: &Error, _ctx: A #[cfg(test)] mod tests { - use stackable_operator::k8s_openapi::{ - ByteString, - apimachinery::pkg::apis::meta::v1::{ManagedFieldsEntry, ObjectMeta}, - }; - use super::*; - fn config_map(data: &[(&str, &str)]) -> ConfigMap { - ConfigMap { - data: Some( - data.iter() - .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) - .collect(), - ), - ..ConfigMap::default() - } - } - #[test] fn test_annotation_names() { assert_eq!( @@ -647,132 +597,67 @@ mod tests { ); } - #[test] - fn digest_is_stable_across_releases() { - // Changing the digest function rolls every StatefulSet labelled - // restarter.stackable.tech/enabled=true which we want to avoid - assert_eq!( - config_map_content_digest(&config_map(&[("foo", "bar")])), - "401bdc692ecd7b9bb6b02f186e2976c3" - ); - } - - #[test] - fn digest_ignores_metadata() { - let mut changed = config_map(&[("foo", "bar")]); - changed.metadata = ObjectMeta { - resource_version: Some("12345".to_owned()), - labels: Some([("cost-center".to_owned(), "irrelevant".to_owned())].into()), - annotations: Some([("probe".to_owned(), "1".to_owned())].into()), - managed_fields: Some(vec![ManagedFieldsEntry { - manager: Some("third-party-controller".to_owned()), - ..ManagedFieldsEntry::default() - }]), - ..ObjectMeta::default() - }; - - assert_eq!( - config_map_content_digest(&config_map(&[("foo", "bar")])), - config_map_content_digest(&changed) - ); - } - - #[test] - fn digest_changes_when_content_changes() { - assert_ne!( - config_map_content_digest(&config_map(&[("foo", "bar")])), - config_map_content_digest(&config_map(&[("foo", "baz")])) - ); - } - - #[test] - fn digest_distinguishes_ambiguous_entry_splits() { - // Without length prefixes these would hash identically. - assert_ne!( - config_map_content_digest(&config_map(&[("ab", "c")])), - config_map_content_digest(&config_map(&[("a", "bc")])) - ); + fn statefulset_with_mechanism(mechanism: Option<&str>) -> StatefulSet { + StatefulSet { + metadata: ObjectMeta { + annotations: mechanism.map(|mechanism| { + [( + CHANGE_TRACKING_MECHANISM_ANNOTATION.to_owned(), + mechanism.to_owned(), + )] + .into() + }), + ..ObjectMeta::default() + }, + ..StatefulSet::default() + } } #[test] - fn digest_distinguishes_data_from_binary_data() { - let binary = ConfigMap { - binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), - ..ConfigMap::default() - }; - - assert_ne!( - config_map_content_digest(&config_map(&[("foo", "bar")])), - config_map_content_digest(&binary) - ); - } - - fn identifying_metadata() -> ObjectMeta { - ObjectMeta { - name: Some("my-config".to_owned()), - namespace: Some("my-namespace".to_owned()), - uid: Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c".to_owned()), - managed_fields: Some(vec![ManagedFieldsEntry { - manager: Some("third-party-controller".to_owned()), - ..ManagedFieldsEntry::default() - }]), - annotations: Some( - [( - "kubectl.kubernetes.io/last-applied-configuration".to_owned(), - r#"{"data":{"password":"aHVudGVyMg=="}}"#.to_owned(), - )] - .into(), + fn change_tracking_mechanism_is_read_from_the_annotation() { + for (annotation, expected) in [ + (None, Some(ChangeTrackingMechanism::ResourceVersion)), + ( + Some("resource-version"), + Some(ChangeTrackingMechanism::ResourceVersion), ), - ..ObjectMeta::default() + (Some("data-hash"), Some(ChangeTrackingMechanism::DataHash)), + (Some("DataHash"), None), + ] { + let parsed = + ChangeTrackingMechanism::from_statefulset(&statefulset_with_mechanism(annotation)); + + match expected { + Some(expected) => { + assert_eq!( + parsed.expect("a missing or known mechanism must be accepted"), + expected + ); + if let Some(annotation) = annotation { + assert_eq!(expected.to_string(), annotation); + } + } + None => assert!(matches!( + parsed, + Err(Error::InvalidChangeTrackingMechanism { ref value, .. }) + if Some(value.as_str()) == annotation + )), + } } } #[test] - fn digesting_keeps_the_identity() { - let config_map = ConfigMap { - metadata: identifying_metadata(), - ..config_map(&[("foo", "bar")]) - }; - let digest = config_map_content_digest(&config_map); - let digested = digest_config_map(config_map); - - assert_eq!(digested.digest, digest); - assert_eq!(digested.name.as_deref(), Some("my-config")); - assert_eq!(digested.namespace.as_deref(), Some("my-namespace")); - assert_eq!( - digested.uid.as_deref(), - Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c") - ); - } - - #[test] - fn digesting_a_secret_retains_no_content() { - let secret = Secret { - metadata: identifying_metadata(), - data: Some([("password".to_owned(), ByteString(b"hunter2".to_vec()))].into()), - ..Secret::default() + fn resource_version_value_matches_previous_releases() { + let metadata = ObjectMeta { + uid: Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c".to_owned()), + resource_version: Some("12345".to_owned()), + ..ObjectMeta::default() }; - let digest = secret_content_digest(&secret); - let digested = digest_secret(secret); - - assert_eq!(digested.digest, digest); - assert_eq!(digested.name.as_deref(), Some("my-config")); - - let debug = format!("{digested:?}"); - assert!(!debug.contains("hunter2"), "{debug}"); - assert!(!debug.contains("aHVudGVyMg=="), "{debug}"); - } - - #[test] - fn store_keys_match_the_refs_built_from_a_pod_template() { - let digested = digest_config_map(ConfigMap { - metadata: identifying_metadata(), - ..config_map(&[("foo", "bar")]) - }); assert_eq!( - ObjectRef::from_obj(&digested), - ObjectRef::>::new("my-config").within("my-namespace") + resource_version_value(&metadata).as_deref(), + Some("f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c/12345") ); + assert_eq!(resource_version_value(&ObjectMeta::default()), None); } } diff --git a/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs b/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs index a00548ed..6aa310c9 100644 --- a/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs +++ b/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs @@ -23,7 +23,7 @@ use stackable_operator::{ use crate::{ FIELD_MANAGER, OPERATOR_NAME, - restart_controller::statefulset::{Ctx, get_updated_restarter_annotations}, + restart_controller::statefulset::{Ctx, Error, get_updated_restarter_annotations}, }; pub fn create_webhook( @@ -94,9 +94,11 @@ fn get_sts_restarter_mutating_webhook_configuration() -> MutatingWebhookConfigur // > Webhooks typically operate only on the content of the AdmissionReview sent to them. // > Some webhooks, however, make out-of-band changes as part of processing admission requests. // - // We read in the state of the world using the ConfigMap and Secret store. - // So, technically our outcome depends on external factors, *but* this webhook is not - // creating any external objects, so from our understanding it's side-effect free. + // We read in the state of the world using the ConfigMap and Secret store and, for the + // `data-hash` change tracking mechanism, by fetching the referenced ConfigMaps and + // Secrets. So, technically our outcome depends on external factors, *but* this webhook + // only reads and is not creating or changing any external objects, so from our + // understanding it's side-effect free. side_effects: "None".to_owned(), ..Default::default() }]), @@ -135,6 +137,19 @@ async fn add_sts_restarter_annotations_handler( let annotations = match get_updated_restarter_annotations(sts, ctx).await { Ok(annotations) => annotations, + // Rejecting the StatefulSet would block its creation over a typo in the change tracking + // mechanism or a transient API error while fetching a ConfigMap or Secret. Admitting it + // without the annotations instead has the same outcome as `failure_policy: Ignore`: the + // controller adds them later, which restarts the Pods once. + Err( + err @ (Error::InvalidChangeTrackingMechanism { .. } | Error::FetchContentFailed { .. }), + ) => { + tracing::warn!( + error = &err as &dyn std::error::Error, + "Admitting StatefulSet without restarter annotations" + ); + return AdmissionResponse::from(&request); + } Err(err) => { return AdmissionResponse::invalid(format!( "failed to get updated restarted annotations: {err:#}" diff --git a/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 b/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 deleted file mode 100644 index 67185acf..00000000 --- a/tests/templates/kuttl/restarter-content-identical-write/00-patch-ns.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{% if test_scenario['values']['openshift'] == 'true' %} -# see https://github.com/stackabletech/issues/issues/566 ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' - timeout: 120 -{% endif %} diff --git a/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml b/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml deleted file mode 100644 index dbd23104..00000000 --- a/tests/templates/kuttl/restarter-content-identical-write/20-assert.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 30 ---- -# The ConfigMap content never changed. -apiVersion: v1 -kind: ConfigMap -metadata: - name: test-config -data: - foo: bar ---- -# This should stay on generation 1 -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test - generation: 1 -status: - readyReplicas: 2 - replicas: 2 - observedGeneration: 1 diff --git a/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml b/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml deleted file mode 100644 index be86506e..00000000 --- a/tests/templates/kuttl/restarter-content-identical-write/20-content-identical-write.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 120 -commands: - # Do a change that should (tm) not have an impact on a pod mounting/needing this cm. - - script: | - kubectl apply --server-side --field-manager=third-party-controller --force-conflicts -f - < Date: Fri, 20 Nov 2026 15:29:16 +0100 Subject: [PATCH 4/5] feat: avoid brute forcing weak secrets --- CHANGELOG.md | 3 +- Cargo.lock | 16 +- Cargo.nix | 22 +- Cargo.toml | 2 + .../commons-operator/templates/roles.yaml | 32 +++ .../commons-operator/pages/restarter.adoc | 4 +- rust/operator-binary/Cargo.toml | 2 + rust/operator-binary/src/main.rs | 5 +- .../src/restart_controller/content_digest.rs | 258 +++++++++--------- .../src/restart_controller/digest_key.rs | 186 +++++++++++++ .../src/restart_controller/mod.rs | 1 + .../src/restart_controller/statefulset.rs | 59 ++-- .../src/webhooks/restarter_mutate_sts.rs | 19 +- 13 files changed, 423 insertions(+), 186 deletions(-) create mode 100644 rust/operator-binary/src/restart_controller/digest_key.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c8087aa4..84c92c20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,8 @@ All notable changes to this project will be documented in this file. - The operator Deployment runs with a restricted security context, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#462]). - The operator Deployment runs with a read-only root filesystem ([#469]). - Extend Helm deployed RBAC permissions to get configmaps and secrets, required to digest their - content for the `data-hash` change tracking mechanism ([#449]). + content for the `data-hash` change tracking mechanism, and to create secrets in the operator + namespace, required to store the key of these digests ([#449]). ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 80d172a3..b195dd3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1107,9 +1107,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd04d0d9c7079b416b458d2317aea4aa964c81938cb7c20f559758c4f422dc8" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" dependencies = [ "arraydeque", "smallvec", @@ -1866,9 +1866,9 @@ checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" [[package]] name = "libz-sys" -version = "1.1.33" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fee3f59f4321dab0c78d828942e7aa09d0bd8ad6e78a501977e6bf3d02938045" +checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" dependencies = [ "cc", "libc", @@ -2814,12 +2814,12 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969cde0349344dbbf58b5f9badbfb8f806db13969364eb8007c057ef687007d9" +checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" dependencies = [ "annotate-snippets", - "base64 0.22.1", + "base64 0.23.1", "encoding_rs_io", "granit-parser", "nohash-hasher", @@ -3108,8 +3108,10 @@ dependencies = [ "chrono", "clap", "futures 0.3.34", + "hmac", "http", "json-patch", + "rand 0.10.3", "serde", "serde_json", "sha2", diff --git a/Cargo.nix b/Cargo.nix index ef30a2cb..79c87ff6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -3612,10 +3612,10 @@ rec { }; "granit-parser" = rec { crateName = "granit-parser"; - version = "2.2.0"; + version = "2.1.0"; edition = "2021"; crateBin = []; - sha256 = "1j1d897qqxarylh7rjrq37469aaax9x33ljqnhbb8ybhkh6lvl4v"; + sha256 = "1pq1ljzfc7n0sfx0f6la8f5p4lmj81n841rj293kvpylsj35bw05"; libName = "granit_parser"; license = "MIT OR Apache-2.0"; authors = [ @@ -6336,10 +6336,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.33"; + version = "1.1.31"; edition = "2018"; links = "z"; - sha256 = "0ic0jc13vgz6fwcm12p7ss5bvl09mbkl52c2ip3v1ni18fgzbqzy"; + sha256 = "1cjw8nisxy5iv8yms7dsxwhclb26djl38n6xq4nmlqxxra73087z"; libName = "libz_sys"; license = "MIT OR Apache-2.0"; authors = [ @@ -9678,10 +9678,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.2.0"; + version = "2.1.0"; edition = "2024"; crateBin = []; - sha256 = "1n87f1lfymy00y0fnr4kjq9xn1pqp2zsv6szigsvnk9l941xx74n"; + sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -9692,7 +9692,7 @@ rec { } { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64 0.23.1"; optional = true; } { @@ -10616,6 +10616,10 @@ rec { packageId = "futures 0.3.34"; features = [ "compat" ]; } + { + name = "hmac"; + packageId = "hmac"; + } { name = "http"; packageId = "http"; @@ -10624,6 +10628,10 @@ rec { name = "json-patch"; packageId = "json-patch"; } + { + name = "rand"; + packageId = "rand 0.10.3"; + } { name = "serde"; packageId = "serde"; diff --git a/Cargo.toml b/Cargo.toml index e14d2704..99482ffa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,9 @@ chrono = "0.4" clap = "4.6" futures = { version = "0.3", features = ["compat"] } http = "1.5" +hmac = "0.13" json-patch = "4.2" +rand = "0.10" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.11" diff --git a/deploy/helm/commons-operator/templates/roles.yaml b/deploy/helm/commons-operator/templates/roles.yaml index 40cf02d3..e658d075 100644 --- a/deploy/helm/commons-operator/templates/roles.yaml +++ b/deploy/helm/commons-operator/templates/roles.yaml @@ -82,3 +82,35 @@ rules: # Required for startup condition - list - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "commons-operator.fullname" . }}-role + labels: + {{- include "commons-operator.labels" . | nindent 4 }} +rules: + # Create the Secret holding the key of the content digests on first use of the change tracking + # mechanism data-hash (annotation restarter.stackable.tech/change-tracking-mechanism). + # Reading it is covered by the ClusterRole above. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "commons-operator.fullname" . }}-rolebinding + labels: + {{- include "commons-operator.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "commons-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "commons-operator.fullname" . }}-role + apiGroup: rbac.authorization.k8s.io diff --git a/docs/modules/commons-operator/pages/restarter.adoc b/docs/modules/commons-operator/pages/restarter.adoc index d2c5f4b9..75cd9381 100644 --- a/docs/modules/commons-operator/pages/restarter.adoc +++ b/docs/modules/commons-operator/pages/restarter.adoc @@ -70,7 +70,9 @@ This annotation selects what counts as a change of a referenced ConfigMap or Sec Writes by third parties that leave the content as it is do not restart the Pods. To hash the content, the operator reads a referenced object whenever its resource version changes. It only keeps the hash, not the content. -Anyone who can read the StatefulSet or its Pods can use the hash to verify guesses of a Secret's content, so keep low-entropy Secrets such as user-chosen passwords on `resource-version`. +The hash is keyed with a random key that only the operator can read and bound to the UID of the object, so the hash in the Pod template annotations reveals nothing about the content beyond whether it changed. +On the first use of `data-hash`, the operator creates this key in the Secret `commons-operator-restarter-digest-key` in its own namespace. +Deleting that Secret, or the namespace, makes the operator create a new key on its next start, which changes every hash and therefore rolling-restarts every StatefulSet using `data-hash` once. With any other value, the operator logs an error and stops updating the Pod template annotations of the StatefulSet until the value is fixed. Switching between the mechanisms changes the values of the Pod template annotations described below and therefore rolling-restarts the StatefulSet once. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index eb416144..931db83a 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -16,7 +16,9 @@ chrono.workspace = true clap.workspace = true http.workspace = true futures.workspace = true +hmac.workspace = true json-patch.workspace = true +rand.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 0951e191..b512f5dd 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -110,7 +110,10 @@ async fn main() -> anyhow::Result<()> { ) .await?; - let (ctx, cm_store_tx, secret_store_tx) = create_context(client.clone()); + let (ctx, cm_store_tx, secret_store_tx) = create_context( + client.clone(), + operator_environment.operator_namespace.clone(), + ); let mut readiness_checks = HealthCheckRegistry::new(); let authentication_class_crd_check = readiness_checks.register(format!( diff --git a/rust/operator-binary/src/restart_controller/content_digest.rs b/rust/operator-binary/src/restart_controller/content_digest.rs index ebcc1905..5df18f82 100644 --- a/rust/operator-binary/src/restart_controller/content_digest.rs +++ b/rust/operator-binary/src/restart_controller/content_digest.rs @@ -4,14 +4,20 @@ //! The restarter only watches the metadata of ConfigMaps and Secrets. The content of an object is //! fetched on demand, only when a StatefulSet using `data-hash` references it, and only once per //! resourceVersion: the [`DigestCache`] keeps the digest, never the content. +//! +//! The digests are keyed with a [`DigestKey`] and bound to the uid of their object, so that anyone +//! who can read them in the Pod template annotations learns nothing about the content beyond +//! whether it changed. use std::{ collections::HashMap, future::Future, + marker::PhantomData, sync::{Mutex, MutexGuard}, }; -use sha2::{Digest, Sha256}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; use stackable_operator::{ k8s_openapi::{ api::core::v1::{ConfigMap, Secret}, @@ -20,49 +26,62 @@ use stackable_operator::{ kube::Resource, }; +use crate::restart_controller::digest_key::DigestKey; + /// An object whose content can be digested. pub trait ContentDigest: Resource { - /// Digest over the parts of the object that a Pod can read. - fn content_digest(&self) -> String; + /// Keyed digest over the parts of the object that a Pod can read, bound to the object's `uid`. + fn content_digest(&self, uid: &str, digest_key: &DigestKey) -> String; } impl ContentDigest for ConfigMap { - fn content_digest(&self) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"configmap/data"); + fn content_digest(&self, uid: &str, digest_key: &DigestKey) -> String { + let mut mac = object_mac(uid, digest_key); + mac.update(b"configmap/data"); for (key, value) in self.data.iter().flatten() { - update_entry(&mut hasher, key, value.as_bytes()); + update_entry(&mut mac, key, value.as_bytes()); } - hasher.update(b"configmap/binaryData"); + mac.update(b"configmap/binaryData"); for (key, value) in self.binary_data.iter().flatten() { - update_entry(&mut hasher, key, &value.0); + update_entry(&mut mac, key, &value.0); } - finish_digest(hasher) + finish_digest(mac) } } impl ContentDigest for Secret { - fn content_digest(&self) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"secret/data"); + fn content_digest(&self, uid: &str, digest_key: &DigestKey) -> String { + let mut mac = object_mac(uid, digest_key); + mac.update(b"secret/data"); for (key, value) in self.data.iter().flatten() { - update_entry(&mut hasher, key, &value.0); + update_entry(&mut mac, key, &value.0); } - finish_digest(hasher) + finish_digest(mac) } } -/// Adds one key/value pair to `hasher`, length-prefixed. -fn update_entry(hasher: &mut Sha256, key: &str, value: &[u8]) { - hasher.update((key.len() as u64).to_be_bytes()); - hasher.update(key.as_bytes()); - hasher.update((value.len() as u64).to_be_bytes()); - hasher.update(value); +/// Returns a new MAC keyed with `digest_key` and bound to the object with the given `uid`. +/// +/// Binding to the uid keeps the digests of different objects apart, even with identical content. +/// Otherwise anyone able to create an object could have the operator digest guesses of another +/// object's content and compare the results. +fn object_mac(uid: &str, digest_key: &DigestKey) -> Hmac { + let mut mac = digest_key.mac(); + update_entry(&mut mac, "uid", uid.as_bytes()); + mac +} + +/// Adds one key/value pair to `mac`, length-prefixed. +fn update_entry(mac: &mut Hmac, key: &str, value: &[u8]) { + mac.update(&(key.len() as u64).to_be_bytes()); + mac.update(key.as_bytes()); + mac.update(&(value.len() as u64).to_be_bytes()); + mac.update(value); } /// Truncates the digest to 128 bits and hex-encodes it. -fn finish_digest(hasher: Sha256) -> String { - hasher.finalize()[..16] +fn finish_digest(mac: Hmac) -> String { + mac.finalize().into_bytes()[..16] .iter() .map(|byte| format!("{byte:02x}")) .collect() @@ -81,40 +100,53 @@ pub struct CachedDigest { /// Namespace and name of a cached object. type CacheKey = (String, String); -/// Content digests of the ConfigMaps or the Secrets referenced by StatefulSets using `data-hash`, -/// keyed by namespace and name. +/// Content digests of the objects of kind `K` referenced by StatefulSets using `data-hash`, keyed +/// by namespace and name. /// /// An entry is replaced when the uid or resourceVersion changes, and removed when a fetch finds /// the object gone. Entries of objects deleted without a later fetch stay until the operator /// restarts. They cannot be served for a recreated object, which has a new uid, and are only a /// few hundred bytes each, so they are not tracked any further. -#[derive(Debug, Default)] -pub struct DigestCache { +#[derive(Debug)] +pub struct DigestCache { entries: Mutex>, + kind: PhantomData K>, } -impl DigestCache { +impl Default for DigestCache { + fn default() -> Self { + Self { + entries: Mutex::default(), + kind: PhantomData, + } + } +} + +impl DigestCache { /// Returns the digest of the object described by `metadata`, as seen by the metadata watch. /// /// The cached digest is used if it was computed for the same uid and resourceVersion, /// otherwise the object is fetched by calling `fetch` with its name and namespace, and its /// digest cached. Returns `None` if `metadata` lacks a name or namespace, or `fetch` does not /// find the object. - pub async fn get_or_fetch<'a, K, E, Fut>( + /// + /// All calls on one cache must pass the same `digest_key`, as the cached digests are not keyed + /// by it. + pub async fn get_or_fetch<'a, E, Fut>( &self, metadata: &'a ObjectMeta, + digest_key: &DigestKey, fetch: impl FnOnce(&'a str, &'a str) -> Fut, ) -> Result, E> where - K: ContentDigest, Fut: Future, E>>, { let (Some(namespace), Some(name)) = (&metadata.namespace, &metadata.name) else { return Ok(None); }; - let key = (namespace.clone(), name.clone()); + let cache_key = (namespace.clone(), name.clone()); - if let Some(cached) = self.lock().get(&key) + if let Some(cached) = self.lock().get(&cache_key) && Some(&cached.uid) == metadata.uid.as_ref() && Some(&cached.resource_version) == metadata.resource_version.as_ref() { @@ -127,7 +159,7 @@ impl DigestCache { "Fetching object to digest its content" ); let Some(object) = fetch(name, namespace).await? else { - self.lock().remove(&key); + self.lock().remove(&cache_key); return Ok(None); }; let (Some(uid), Some(resource_version)) = ( @@ -137,11 +169,11 @@ impl DigestCache { return Ok(None); }; let cached = CachedDigest { + digest: object.content_digest(&uid, digest_key), uid, resource_version, - digest: object.content_digest(), }; - self.lock().insert(key, cached.clone()); + self.lock().insert(cache_key, cached.clone()); Ok(Some(cached)) } @@ -163,6 +195,10 @@ mod tests { use super::*; + const TEST_KEY: DigestKey = DigestKey::new([42; 32]); + + const TEST_UID: &str = "f9dc0a8f-5f4b-4f52-9d0f-1a0ba1e0bd2c"; + fn config_map(data: &[(&str, &str)]) -> ConfigMap { ConfigMap { data: Some( @@ -191,18 +227,9 @@ mod tests { } } - /// Fetches `object` and counts the calls in `calls`. - async fn fetch_counted( - calls: &Cell, - object: Option, - ) -> Result, Infallible> { - calls.set(calls.get() + 1); - Ok(object) - } - + /// Changing the digest function rolling-restarts every StatefulSet using `data-hash`. #[test] fn digest_is_stable_across_releases() { - // Changing the digest function rolls every StatefulSet using data-hash let binary_config_map = ConfigMap { binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), ..ConfigMap::default() @@ -213,14 +240,17 @@ mod tests { }; assert_eq!( - config_map(&[("foo", "bar")]).content_digest(), - "401bdc692ecd7b9bb6b02f186e2976c3" + config_map(&[("foo", "bar")]).content_digest(TEST_UID, &TEST_KEY), + "aec6a1a9a372bda7f4b7a600d631186f" + ); + assert_eq!( + binary_config_map.content_digest(TEST_UID, &TEST_KEY), + "c2efb2422a076038e7e3affa23d7ffb1" ); assert_eq!( - binary_config_map.content_digest(), - "20feddae4a413582dab5c1c2698ef5b8" + secret.content_digest(TEST_UID, &TEST_KEY), + "646f7311f8401d03267d18cf5419ba32" ); - assert_eq!(secret.content_digest(), "fa14a91ea83afdfe42d96880fe813616"); } #[test] @@ -238,110 +268,76 @@ mod tests { }; assert_eq!( - config_map(&[("foo", "bar")]).content_digest(), - changed.content_digest() - ); - } - - #[test] - fn digest_distinguishes_ambiguous_entry_splits() { - // Without length prefixes these would hash identically. - assert_ne!( - config_map(&[("ab", "c")]).content_digest(), - config_map(&[("a", "bc")]).content_digest() + config_map(&[("foo", "bar")]).content_digest(TEST_UID, &TEST_KEY), + changed.content_digest(TEST_UID, &TEST_KEY) ); } + /// Each pair would collide without the key, the uid binding, the length prefixes or the field + /// prefixes. #[test] - fn digest_distinguishes_data_from_binary_data() { + fn digest_distinguishes_content() { let binary = ConfigMap { binary_data: Some([("foo".to_owned(), ByteString(b"bar".to_vec()))].into()), ..ConfigMap::default() }; + let text = config_map(&[("foo", "bar")]); - assert_ne!( - config_map(&[("foo", "bar")]).content_digest(), - binary.content_digest() - ); - } - - #[tokio::test] - async fn cache_fetches_once_per_resource_version() { - let cache = DigestCache::default(); - let calls = Cell::new(0); - let object = stored("uid-1", "100", &[("foo", "bar")]); - - for _ in 0..3 { - let cached = cache - .get_or_fetch(&metadata("uid-1", "100"), |_, _| { - fetch_counted(&calls, Some(object.clone())) - }) - .await - .expect("fetching is infallible") - .expect("the object exists"); - assert_eq!(cached.digest, object.content_digest()); + for (left, right) in [ + ( + text.content_digest(TEST_UID, &TEST_KEY), + text.content_digest(TEST_UID, &DigestKey::new([43; 32])), + ), + ( + text.content_digest(TEST_UID, &TEST_KEY), + text.content_digest("0b6f1a52-2d1e-4c43-8c55-5ad3d3a4e7f1", &TEST_KEY), + ), + ( + config_map(&[("ab", "c")]).content_digest(TEST_UID, &TEST_KEY), + config_map(&[("a", "bc")]).content_digest(TEST_UID, &TEST_KEY), + ), + ( + text.content_digest(TEST_UID, &TEST_KEY), + binary.content_digest(TEST_UID, &TEST_KEY), + ), + ] { + assert_ne!(left, right); } - - assert_eq!(calls.get(), 1); } + /// Each step looks the object up by uid and resourceVersion, with the content a fetch returns + /// (`None` if the object is gone) and whether the cache has to fetch it. #[tokio::test] - async fn cache_refetches_when_resource_version_or_uid_changes() { - let cache = DigestCache::default(); - let calls = Cell::new(0); - - for (uid, resource_version, content) in [ - ("uid-1", "100", "bar"), - ("uid-1", "101", "baz"), - ("uid-2", "101", "qux"), + async fn cache_fetches_once_per_uid_and_resource_version() { + let cache = DigestCache::::default(); + + for (uid, resource_version, content, fetches) in [ + ("uid-1", "100", Some("bar"), true), + ("uid-1", "100", Some("bar"), false), + ("uid-1", "101", Some("baz"), true), + ("uid-2", "101", Some("qux"), true), + ("uid-2", "102", None, true), + ("uid-2", "101", Some("qux"), true), ] { - let object = stored(uid, resource_version, &[("foo", content)]); + let fetched = Cell::new(false); + let object = content.map(|content| stored(uid, resource_version, &[("foo", content)])); let cached = cache - .get_or_fetch(&metadata(uid, resource_version), |_, _| { - fetch_counted(&calls, Some(object.clone())) + .get_or_fetch(&metadata(uid, resource_version), &TEST_KEY, |_, _| { + fetched.set(true); + std::future::ready(Ok::<_, Infallible>(object.clone())) }) .await - .expect("fetching is infallible") - .expect("the object exists"); + .expect("fetching should be infallible"); + assert_eq!(fetched.get(), fetches); assert_eq!( cached, - CachedDigest { + object.map(|object| CachedDigest { uid: uid.to_owned(), resource_version: resource_version.to_owned(), - digest: object.content_digest(), - } + digest: object.content_digest(uid, &TEST_KEY), + }) ); } - - assert_eq!(calls.get(), 3); - } - - #[tokio::test] - async fn cache_forgets_objects_that_are_gone() { - let cache = DigestCache::default(); - let calls = Cell::new(0); - let object = stored("uid-1", "100", &[("foo", "bar")]); - let existing = metadata("uid-1", "100"); - let fetch_existing = |_, _| fetch_counted(&calls, Some(object.clone())); - - cache - .get_or_fetch(&existing, fetch_existing) - .await - .expect("fetching is infallible"); - let missing = cache - .get_or_fetch(&metadata("uid-1", "101"), |_, _| { - fetch_counted::(&calls, None) - }) - .await - .expect("fetching is infallible"); - - assert_eq!(missing, None); - - cache - .get_or_fetch(&existing, fetch_existing) - .await - .expect("fetching is infallible"); - assert_eq!(calls.get(), 3); } } diff --git a/rust/operator-binary/src/restart_controller/digest_key.rs b/rust/operator-binary/src/restart_controller/digest_key.rs new file mode 100644 index 00000000..8b1994f5 --- /dev/null +++ b/rust/operator-binary/src/restart_controller/digest_key.rs @@ -0,0 +1,186 @@ +//! The secret key of the content digests of the `data-hash` change tracking mechanism. +//! +//! The digests end up in Pod template annotations, which anyone who can read a StatefulSet or its +//! Pods can see. Without a secret key, a digest would let them verify guesses of a Secret's +//! content offline. Each digest is also bound to the uid of its object (see +//! [`crate::restart_controller::content_digest`]), so that the operator cannot be used to verify +//! such guesses either. The key lives in a Secret in the operator namespace, so that it survives +//! operator restarts: a new key changes every digest and rolling-restarts every StatefulSet using +//! `data-hash`. + +use std::fmt; + +use hmac::{Hmac, KeyInit}; +use sha2::Sha256; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + client::Client, + k8s_openapi::{ByteString, api::core::v1::Secret}, + kvp::Label, +}; +use tokio::sync::OnceCell; + +/// Name of the Secret holding the [`DigestKey`]. +const SECRET_NAME: &str = "commons-operator-restarter-digest-key"; + +/// Entry of the Secret data holding the [`DigestKey`]. +const SECRET_DATA_KEY: &str = "key"; + +/// Length of the [`DigestKey`] in bytes. +const KEY_LENGTH: usize = 32; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to load or create the digest key Secret {secret_name:?}"))] + LoadOrCreateSecret { + source: stackable_operator::client::Error, + secret_name: String, + }, + + #[snafu(display( + "the digest key Secret {secret_name:?} does not hold a {KEY_LENGTH} byte key in {SECRET_DATA_KEY:?}, delete it to generate a new key (this rolling-restarts every StatefulSet using data-hash)" + ))] + InvalidKey { secret_name: String }, +} + +/// The secret key of the content digests. +#[derive(Clone)] +pub struct DigestKey([u8; KEY_LENGTH]); + +impl DigestKey { + pub const fn new(bytes: [u8; KEY_LENGTH]) -> Self { + Self(bytes) + } + + /// Returns a new HMAC-SHA256 keyed with this key. + pub fn mac(&self) -> Hmac { + Hmac::new_from_slice(&self.0).expect("HMAC must accept keys of any length") + } + + fn generate() -> Self { + Self::new(rand::random()) + } + + fn from_secret(secret: &Secret) -> Result { + let bytes = secret + .data + .as_ref() + .and_then(|data| data.get(SECRET_DATA_KEY)) + .and_then(|value| <[u8; KEY_LENGTH]>::try_from(value.0.as_slice()).ok()) + .context(InvalidKeySnafu { + secret_name: SECRET_NAME, + })?; + + Ok(Self::new(bytes)) + } + + fn to_secret(&self, namespace: &str) -> Secret { + Secret { + metadata: ObjectMetaBuilder::new() + .name(SECRET_NAME) + .namespace(namespace) + .with_label(Label::stackable_vendor()) + .build(), + data: Some([(SECRET_DATA_KEY.to_owned(), ByteString(self.0.to_vec()))].into()), + immutable: Some(true), + ..Secret::default() + } + } +} + +/// Redacted, so that the key never ends up in logs. +impl fmt::Debug for DigestKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("DigestKey()") + } +} + +/// Loads the [`DigestKey`] on first use, creating its Secret if it does not exist yet. +/// +/// Loading lazily keeps the operator startup independent of the Secret, so installations that do +/// not use `data-hash` never create it and do not need the permission to. +pub struct DigestKeyStore { + client: Client, + namespace: String, + key: OnceCell, +} + +impl DigestKeyStore { + pub fn new(client: Client, namespace: String) -> Self { + Self { + client, + namespace, + key: OnceCell::new(), + } + } + + /// Returns the key, loading or creating its Secret in the operator namespace on first use. + /// + /// A failed attempt is not cached, the next call tries again. + pub async fn get(&self) -> Result<&DigestKey, Error> { + self.key + .get_or_try_init(|| async { + let secret = self + .client + .create_if_missing(&DigestKey::generate().to_secret(&self.namespace)) + .await + .context(LoadOrCreateSecretSnafu { + secret_name: SECRET_NAME, + })?; + DigestKey::from_secret(&secret) + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_round_trips_through_its_secret() { + let key = DigestKey::new([7; KEY_LENGTH]); + let secret = key.to_secret("operator-namespace"); + + assert_eq!(secret.metadata.name.as_deref(), Some(SECRET_NAME)); + assert_eq!( + secret.metadata.namespace.as_deref(), + Some("operator-namespace") + ); + assert_eq!(secret.immutable, Some(true)); + assert_eq!( + DigestKey::from_secret(&secret) + .expect("a Secret built from a key should hold a valid key") + .0, + key.0 + ); + } + + #[test] + fn secret_without_valid_key_is_rejected() { + let short_key = Secret { + data: Some([(SECRET_DATA_KEY.to_owned(), ByteString(vec![7; 16]))].into()), + ..Secret::default() + }; + let wrong_entry = Secret { + data: Some([("other".to_owned(), ByteString(vec![7; KEY_LENGTH]))].into()), + ..Secret::default() + }; + + for secret in [Secret::default(), short_key, wrong_entry] { + assert!(matches!( + DigestKey::from_secret(&secret), + Err(Error::InvalidKey { .. }) + )); + } + } + + #[test] + fn debug_output_redacts_the_key() { + assert_eq!( + format!("{:?}", DigestKey::new([7; KEY_LENGTH])), + "DigestKey()" + ); + } +} diff --git a/rust/operator-binary/src/restart_controller/mod.rs b/rust/operator-binary/src/restart_controller/mod.rs index 2238597f..5f171a64 100644 --- a/rust/operator-binary/src/restart_controller/mod.rs +++ b/rust/operator-binary/src/restart_controller/mod.rs @@ -1,3 +1,4 @@ pub mod content_digest; +pub mod digest_key; pub mod pod; pub mod statefulset; diff --git a/rust/operator-binary/src/restart_controller/statefulset.rs b/rust/operator-binary/src/restart_controller/statefulset.rs index d8a6b796..87a0665f 100644 --- a/rust/operator-binary/src/restart_controller/statefulset.rs +++ b/rust/operator-binary/src/restart_controller/statefulset.rs @@ -36,10 +36,13 @@ use stackable_operator::{ logging::controller::{ReconcilerError, report_controller_reconciled}, namespace::WatchNamespace, }; -use strum::{Display, EnumDiscriminants, EnumString, IntoStaticStr}; +use strum::{EnumDiscriminants, EnumString, IntoStaticStr}; use crate::{ - restart_controller::content_digest::{ContentDigest, DigestCache}, + restart_controller::{ + content_digest::{ContentDigest, DigestCache}, + digest_key::{self, DigestKeyStore}, + }, utils::delayed_init::{DelayedInit, InitDropped, Initializer}, }; @@ -56,7 +59,7 @@ const CHANGE_TRACKING_MECHANISM_ANNOTATION: &str = "restarter.stackable.tech/change-tracking-mechanism"; /// How the restarter decides that a referenced ConfigMap or Secret has changed. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, EnumString, Display)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, EnumString)] #[strum(serialize_all = "kebab-case")] enum ChangeTrackingMechanism { /// Every write to the object counts as a change, including metadata-only writes. @@ -87,8 +90,9 @@ pub struct Ctx { client: Client, cms: DelayedInit>>, secrets: DelayedInit>>, - cm_digests: DigestCache, - secret_digests: DigestCache, + cm_digests: DigestCache, + secret_digests: DigestCache, + digest_key: DigestKeyStore, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -131,6 +135,9 @@ pub enum Error { source: stackable_operator::client::Error, obj_ref: Box>, }, + + #[snafu(display("failed to load the key for content digests"))] + LoadDigestKey { source: digest_key::Error }, } impl ReconcilerError for Error { @@ -147,6 +154,7 @@ impl ReconcilerError for Error { Error::InvalidAnnotationKey { .. } => None, Error::InvalidChangeTrackingMechanism { .. } => None, Error::FetchContentFailed { obj_ref, .. } => Some(*obj_ref.clone()), + Error::LoadDigestKey { .. } => None, } } } @@ -154,6 +162,7 @@ impl ReconcilerError for Error { #[allow(clippy::type_complexity)] pub fn create_context( client: Client, + operator_namespace: String, ) -> ( Arc, Initializer>>, @@ -162,11 +171,12 @@ pub fn create_context( let (cm_store_tx, cm_store_delayed) = DelayedInit::new(); let (secret_store_tx, secret_store_delayed) = DelayedInit::new(); let ctx = Arc::new(Ctx { - client, + client: client.clone(), cms: cm_store_delayed, secrets: secret_store_delayed, cm_digests: DigestCache::default(), secret_digests: DigestCache::default(), + digest_key: DigestKeyStore::new(client, operator_namespace), }); (ctx, cm_store_tx, secret_store_tx) @@ -350,8 +360,8 @@ fn resource_version_value(metadata: &ObjectMeta) -> Option { /// For [`ChangeTrackingMechanism::DataHash`] this fetches the object, unless `digests` already /// holds the digest for its current resourceVersion. async fn tracking_value( - client: &Client, - digests: &DigestCache, + ctx: &Ctx, + digests: &DigestCache, object: &PartialObjectMeta, mechanism: ChangeTrackingMechanism, ) -> Result, Error> @@ -362,9 +372,10 @@ where match mechanism { ChangeTrackingMechanism::ResourceVersion => Ok(resource_version_value(metadata)), ChangeTrackingMechanism::DataHash => { + let digest_key = ctx.digest_key.get().await.context(LoadDigestKeySnafu)?; let cached = digests - .get_or_fetch(metadata, |name, namespace| { - client.get_opt::(name, namespace) + .get_or_fetch(metadata, digest_key, |name, namespace| { + ctx.client.get_opt::(name, namespace) }) .await .with_context(|_| FetchContentFailedSnafu { @@ -444,7 +455,7 @@ pub async fn get_updated_restarter_annotations( // StatefulSet. let value = match cm { Some(cm) if !ignored_cms.contains(cm_name) => { - tracking_value(&ctx.client, &ctx.cm_digests, &cm, mechanism).await? + tracking_value(&ctx, &ctx.cm_digests, &cm, mechanism).await? } _ => None, } @@ -495,7 +506,7 @@ pub async fn get_updated_restarter_annotations( // Same as for ConfigMaps above, "changes-ignored" is intended for Secrets not in the store. let value = match secret { Some(secret) if !ignored_secrets.contains(secret_name) => { - tracking_value(&ctx.client, &ctx.secret_digests, &secret, mechanism).await? + tracking_value(&ctx, &ctx.secret_digests, &secret, mechanism).await? } _ => None, } @@ -624,25 +635,11 @@ mod tests { (Some("data-hash"), Some(ChangeTrackingMechanism::DataHash)), (Some("DataHash"), None), ] { - let parsed = - ChangeTrackingMechanism::from_statefulset(&statefulset_with_mechanism(annotation)); - - match expected { - Some(expected) => { - assert_eq!( - parsed.expect("a missing or known mechanism must be accepted"), - expected - ); - if let Some(annotation) = annotation { - assert_eq!(expected.to_string(), annotation); - } - } - None => assert!(matches!( - parsed, - Err(Error::InvalidChangeTrackingMechanism { ref value, .. }) - if Some(value.as_str()) == annotation - )), - } + assert_eq!( + ChangeTrackingMechanism::from_statefulset(&statefulset_with_mechanism(annotation)) + .ok(), + expected + ); } } diff --git a/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs b/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs index 6aa310c9..19f3aeb6 100644 --- a/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs +++ b/rust/operator-binary/src/webhooks/restarter_mutate_sts.rs @@ -96,9 +96,12 @@ fn get_sts_restarter_mutating_webhook_configuration() -> MutatingWebhookConfigur // // We read in the state of the world using the ConfigMap and Secret store and, for the // `data-hash` change tracking mechanism, by fetching the referenced ConfigMaps and - // Secrets. So, technically our outcome depends on external factors, *but* this webhook - // only reads and is not creating or changing any external objects, so from our - // understanding it's side-effect free. + // Secrets. So, technically our outcome depends on external factors. + // The only object this webhook may create is the Secret holding the digest key of + // `data-hash`, on its first use (see `DigestKeyStore`). Creating it is idempotent and + // happens on the first reconcile of such a StatefulSet anyway, so even when triggered + // by a dry-run request it has no effect beyond what the real request would have. We + // therefore consider this webhook side-effect free. side_effects: "None".to_owned(), ..Default::default() }]), @@ -138,11 +141,13 @@ async fn add_sts_restarter_annotations_handler( let annotations = match get_updated_restarter_annotations(sts, ctx).await { Ok(annotations) => annotations, // Rejecting the StatefulSet would block its creation over a typo in the change tracking - // mechanism or a transient API error while fetching a ConfigMap or Secret. Admitting it - // without the annotations instead has the same outcome as `failure_policy: Ignore`: the - // controller adds them later, which restarts the Pods once. + // mechanism or an API error while fetching a ConfigMap or Secret or loading the digest key. + // Admitting it without the annotations instead has the same outcome as + // `failure_policy: Ignore`: the controller adds them later, which restarts the Pods once. Err( - err @ (Error::InvalidChangeTrackingMechanism { .. } | Error::FetchContentFailed { .. }), + err @ (Error::InvalidChangeTrackingMechanism { .. } + | Error::FetchContentFailed { .. } + | Error::LoadDigestKey { .. }), ) => { tracing::warn!( error = &err as &dyn std::error::Error, From feb81ec808e305ce2ce8644915245019cf3507da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20M=C3=BCller?= Date: Fri, 20 Nov 2026 16:54:53 +0100 Subject: [PATCH 5/5] cargo deny changes --- Cargo.lock | 14 +++++++------- Cargo.nix | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b195dd3b..2ba0864d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1107,9 +1107,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" +checksum = "9bd04d0d9c7079b416b458d2317aea4aa964c81938cb7c20f559758c4f422dc8" dependencies = [ "arraydeque", "smallvec", @@ -1866,9 +1866,9 @@ checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" [[package]] name = "libz-sys" -version = "1.1.31" +version = "1.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" +checksum = "fee3f59f4321dab0c78d828942e7aa09d0bd8ad6e78a501977e6bf3d02938045" dependencies = [ "cc", "libc", @@ -2814,12 +2814,12 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "969cde0349344dbbf58b5f9badbfb8f806db13969364eb8007c057ef687007d9" dependencies = [ "annotate-snippets", - "base64 0.23.1", + "base64 0.22.1", "encoding_rs_io", "granit-parser", "nohash-hasher", diff --git a/Cargo.nix b/Cargo.nix index 79c87ff6..e3dad5db 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -3612,10 +3612,10 @@ rec { }; "granit-parser" = rec { crateName = "granit-parser"; - version = "2.1.0"; + version = "2.2.0"; edition = "2021"; crateBin = []; - sha256 = "1pq1ljzfc7n0sfx0f6la8f5p4lmj81n841rj293kvpylsj35bw05"; + sha256 = "1j1d897qqxarylh7rjrq37469aaax9x33ljqnhbb8ybhkh6lvl4v"; libName = "granit_parser"; license = "MIT OR Apache-2.0"; authors = [ @@ -6336,10 +6336,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.31"; + version = "1.1.33"; edition = "2018"; links = "z"; - sha256 = "1cjw8nisxy5iv8yms7dsxwhclb26djl38n6xq4nmlqxxra73087z"; + sha256 = "0ic0jc13vgz6fwcm12p7ss5bvl09mbkl52c2ip3v1ni18fgzbqzy"; libName = "libz_sys"; license = "MIT OR Apache-2.0"; authors = [ @@ -9678,10 +9678,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.2.0"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1n87f1lfymy00y0fnr4kjq9xn1pqp2zsv6szigsvnk9l941xx74n"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -9692,7 +9692,7 @@ rec { } { name = "base64"; - packageId = "base64 0.23.1"; + packageId = "base64 0.22.1"; optional = true; } {