From 1ddd65412080d52166f9e36eccdc8181ee49229d Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 31 Aug 2026 12:29:38 -0700 Subject: [PATCH 1/4] X-Smart-Branch-Parent: main From 973848ce6bed495df6d5bf9218c446aeac6bcf79 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 31 Aug 2026 13:28:35 -0700 Subject: [PATCH 2/4] docs: snapshot FACT operator SigNoz deployment --- deploy/fact-signoz/README.md | 79 ++++++++++++++ deploy/fact-signoz/fact.yaml | 22 ++++ deploy/fact-signoz/operator.yaml | 157 +++++++++++++++++++++++++++ deploy/fact-signoz/signoz-route.yaml | 16 +++ deploy/fact-signoz/signoz-scc.yaml | 35 ++++++ 5 files changed, 309 insertions(+) create mode 100644 deploy/fact-signoz/README.md create mode 100644 deploy/fact-signoz/fact.yaml create mode 100644 deploy/fact-signoz/operator.yaml create mode 100644 deploy/fact-signoz/signoz-route.yaml create mode 100644 deploy/fact-signoz/signoz-scc.yaml diff --git a/deploy/fact-signoz/README.md b/deploy/fact-signoz/README.md new file mode 100644 index 00000000..f7e088ce --- /dev/null +++ b/deploy/fact-signoz/README.md @@ -0,0 +1,79 @@ +# FACT operator + SigNoz experiment + +This directory records the deployment validated on the OpenShift cluster +`rc-dev-cluster` on 2026-08-31. It is intentionally a short runbook for a +future agent, not a supported installer. + +## Tested inputs + +- FACT source: `1ddd654120` +- StackRox submodule source: `8a9c85b426` +- Mauro's operator source: `origin/mauro/feat/fact-operator` at `651a9e7383` +- OTel FACT image: + `quay.io/rcochran/scratch@sha256:846110c1455070e0d28594b0567b60488427a0c82e8495e549373b0c12657be9` +- Operator image: + `quay.io/rcochran/scratch@sha256:a6730235378f7b150c7c827f4c9c44f8a4e94c292e851f865241bd073fb8988b` +- SigNoz chart `signoz/signoz` version `0.139.0` + +The images were built as `linux/amd64`. When rebuilding, construct a clean +source archive from the recorded gitlink revisions. Do not copy the currently +checked-out `third_party/stackrox`: a different StackRox revision failed to +compile because the ACL/xattr protobuf definitions did not match FACT. + +Build FACT with `CARGO_ARGS=--features otel`. Use descriptive, deep Quay tags, +for example: + +```text +quay.io/rcochran/scratch:fact-otel--src--linux-amd64- +quay.io/rcochran/scratch:fact-operator-mauro--linux-amd64- +``` + +Deploy by digest after pushing. Copy a Quay pull secret into `fact-system` and +`fact-operator-test`, then link it to the `fact-operator` and `default` service +accounts respectively. Never commit registry credentials. + +## Deploy + +Set `KUBECONFIG` to the infractl-downloaded kubeconfig, then: + +```sh +oc apply -f deploy/fact-signoz/signoz-scc.yaml + +helm repo add signoz https://charts.signoz.io +helm repo update signoz +helm upgrade --install signoz signoz/signoz \ + --namespace observability --create-namespace \ + --version 0.139.0 \ + --set global.storageClass=ssd-csi \ + --set clickhouse.installCustomStorageClass=true \ + --wait --timeout 30m + +oc apply -f deploy/fact-signoz/signoz-route.yaml +oc apply -f deploy/fact-signoz/operator.yaml +oc apply -f deploy/fact-signoz/fact.yaml +oc adm policy add-scc-to-user privileged -z default -n fact-operator-test +``` + +The SigNoz chart initially let OpAMP replace the collector pipelines with +`nop` pipelines before first-user onboarding. Force the tested collector to use +the chart's static configuration: + +```sh +oc -n observability patch deployment signoz-otel-collector --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args","value":["--config=/conf/otel-collector-config.yaml"]}]' +oc -n observability rollout status deployment/signoz-otel-collector +``` + +This patch is outside Helm state and must be checked after a Helm upgrade. +Confirm ports 4317/4318 are listening before testing FACT. + +The operator-generated DaemonSet uses the `default` service account and needs +the privileged SCC. It performs an initial host scan; files created later are +not necessarily tracked without a configured periodic scan. For deterministic +testing, create seed files under `/tmp/fact-operator-validation` before the +FACT pod starts, then modify/chmod/rename/unlink them. + +Open , +create the first account, and use Logs Explorer. Filter on +`service.name = fact`; the useful fields are `body`, `hostname`, `file`, and +`process`. The validated marker was `fact-live-20260831T201728Z`. diff --git a/deploy/fact-signoz/fact.yaml b/deploy/fact-signoz/fact.yaml new file mode 100644 index 00000000..97f622dc --- /dev/null +++ b/deploy/fact-signoz/fact.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: fact-operator-test +--- +apiVersion: fact.stackrox.io/v1alpha1 +kind: Fact +metadata: + name: fact + namespace: fact-operator-test +spec: + image: quay.io/rcochran/scratch@sha256:846110c1455070e0d28594b0567b60488427a0c82e8495e549373b0c12657be9 + logLevel: info + config: + json: true + paths: + - /etc + - /etc/**/* + - /tmp/fact-operator-validation + - /tmp/fact-operator-validation/**/* + otel: + endpoint: http://signoz-otel-collector.observability.svc.cluster.local:4318/v1/logs diff --git a/deploy/fact-signoz/operator.yaml b/deploy/fact-signoz/operator.yaml new file mode 100644 index 00000000..6c78d7d5 --- /dev/null +++ b/deploy/fact-signoz/operator.yaml @@ -0,0 +1,157 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: facts.fact.stackrox.io +spec: + group: fact.stackrox.io + names: + kind: Fact + plural: facts + singular: fact + shortNames: [ft] + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: [image] + properties: + image: + type: string + logLevel: + type: string + default: info + config: + type: object + properties: + paths: + type: array + items: + type: string + grpc: + type: object + properties: + url: + type: string + certs: + type: string + otel: + type: object + properties: + endpoint: + type: string + endpoint: + type: object + properties: + address: + type: string + exposeMetrics: + type: boolean + healthCheck: + type: boolean + bpf: + type: object + properties: + ringbufSize: + type: integer + inodesMax: + type: integer + skipPreFlight: + type: boolean + json: + type: boolean + hotreload: + type: boolean + scanInterval: + type: integer + rateLimit: + type: integer + default: 0 + status: + type: object + properties: + readyNodes: + type: integer + desiredNodes: + type: integer + subresources: + status: {} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: fact-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fact-operator + namespace: fact-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fact-operator +rules: + - apiGroups: [fact.stackrox.io] + resources: [facts, facts/status] + verbs: [get, list, watch, patch, update] + - apiGroups: [""] + resources: [configmaps] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [apps] + resources: [daemonsets] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [""] + resources: [events] + verbs: [create, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fact-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fact-operator +subjects: + - kind: ServiceAccount + name: fact-operator + namespace: fact-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fact-operator + namespace: fact-system +spec: + replicas: 1 + selector: + matchLabels: + app: fact-operator + template: + metadata: + labels: + app: fact-operator + spec: + serviceAccountName: fact-operator + containers: + - name: fact-operator + image: quay.io/rcochran/scratch@sha256:a6730235378f7b150c7c827f4c9c44f8a4e94c292e851f865241bd073fb8988b + imagePullPolicy: IfNotPresent + env: + - name: RUST_LOG + value: debug + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi diff --git a/deploy/fact-signoz/signoz-route.yaml b/deploy/fact-signoz/signoz-route.yaml new file mode 100644 index 00000000..d177ec8f --- /dev/null +++ b/deploy/fact-signoz/signoz-route.yaml @@ -0,0 +1,16 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: signoz + namespace: observability +spec: + host: signoz-observability.apps.rc-dev-cluster.ocp.infra.rox.systems + port: + targetPort: 8080 + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: signoz + weight: 100 diff --git a/deploy/fact-signoz/signoz-scc.yaml b/deploy/fact-signoz/signoz-scc.yaml new file mode 100644 index 00000000..3163fd85 --- /dev/null +++ b/deploy/fact-signoz/signoz-scc.yaml @@ -0,0 +1,35 @@ +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: signoz-scc +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: [] +defaultAddCapabilities: [] +fsGroup: + type: RunAsAny +groups: [] +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: [] +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +supplementalGroups: + type: RunAsAny +users: + - system:serviceaccount:observability:signoz-clickhouse + - system:serviceaccount:observability:default +volumes: + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - projected + - secret From 96190289b3f4d0233ae62f231c4614fbcde0f2b3 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 1 Sep 2026 08:57:49 -0700 Subject: [PATCH 3/4] feat: add opt-in OCI runtime diagnostics --- Containerfile | 1 + docs/references.md | 22 ++ fact/build.rs | 8 +- fact/src/config/mod.rs | 20 ++ fact/src/config/tests.rs | 32 ++ fact/src/event/mod.rs | 275 ++++++++++++++++- fact/src/event/process.rs | 99 +++++++ fact/src/lib.rs | 3 + fact/src/oci.rs | 609 ++++++++++++++++++++++++++++++++++++++ fact/src/output/mod.rs | 6 + fact/src/output/otel.rs | 20 +- 11 files changed, 1084 insertions(+), 11 deletions(-) create mode 100644 fact/src/oci.rs diff --git a/Containerfile b/Containerfile index e296dc64..728e8d9f 100644 --- a/Containerfile +++ b/Containerfile @@ -40,6 +40,7 @@ COPY . . FROM builder AS build ARG FACT_VERSION +ARG FACT_BUILD_SHA=unknown ARG CARGO_ARGS="" RUN --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/app/target \ diff --git a/docs/references.md b/docs/references.md index 94109070..221e935c 100644 --- a/docs/references.md +++ b/docs/references.md @@ -8,6 +8,25 @@ * `FACT_LOGLEVEL`: At which level produce log messages. +* `FACT_OCI_RUNTIME_SPEC_DEBUG`: Development-only OCI runtime-spec + diagnostics. When `true`, Fact adds a curated subset of `config.json`, + including the root, configured process, capabilities, namespaces, and + matching mount, to debug logs and `container.oci.*` OpenTelemetry + attributes. OCI configuration is not read when this option is `false`. + Diagnostics never filter events or change the Sensor gRPC message. The + equivalent top-level YAML setting is `oci_runtime_spec_debug: true`. + + The supported diagnostic modes are: + + - default: Sensor gRPC output with no OCI diagnostics; + - OCI debug: unchanged Sensor output plus debug-log diagnostics; + - OCI debug with `FACT_OTEL_ENDPOINT`: unchanged Sensor output plus the same + diagnostics in debug logs and OpenTelemetry. + + Debug-log records include the Fact version and build SHA. OpenTelemetry + resources include `service.version` and, when the image was built with + `FACT_BUILD_SHA`, `fact.build.sha`, so records can be tied to an exact build. + ### Commandline options * `--skip-pre-flight`: Do not perform pre-flight checks. Before starting up @@ -17,3 +36,6 @@ * `-p, --paths`: List of file paths to monitor. This option could be used multiple times, instructing Fact to monitor multiple files. + +* `--oci-runtime-spec-debug`: Equivalent to `FACT_OCI_RUNTIME_SPEC_DEBUG`; + accepts `true` or `false`. diff --git a/fact/build.rs b/fact/build.rs index 131bcee7..95fa46e0 100644 --- a/fact/build.rs +++ b/fact/build.rs @@ -4,6 +4,7 @@ use anyhow::{Context, bail}; fn main() -> anyhow::Result<()> { println!("cargo::rerun-if-changed=../.git/HEAD"); + println!("cargo::rerun-if-env-changed=FACT_BUILD_SHA"); let out_dir: PathBuf = std::env::var("OUT_DIR") .context("Failed to interpret OUT_DIR environment variable")? .into(); @@ -18,10 +19,15 @@ fn main() -> anyhow::Result<()> { } let version = String::from_utf8(cmd.stdout)?; + let build_sha = std::env::var("FACT_BUILD_SHA").unwrap_or_else(|_| "unknown".to_owned()); let out_path = out_dir.join("version.rs"); std::fs::write( &out_path, - format!(r#"pub const FACT_VERSION: &str = "{}";"#, version.trim()), + format!( + "pub const FACT_VERSION: &str = {:?};\npub const FACT_BUILD_SHA: &str = {:?};", + version.trim(), + build_sha.trim() + ), )?; Ok(()) } diff --git a/fact/src/config/mod.rs b/fact/src/config/mod.rs index 1542f86a..b3851d71 100644 --- a/fact/src/config/mod.rs +++ b/fact/src/config/mod.rs @@ -38,6 +38,7 @@ pub struct FactConfig { pub otel: OTelConfig, pub endpoint: EndpointConfig, pub bpf: BpfConfig, + oci_runtime_spec_debug: Option, skip_pre_flight: Option, json: Option, hotreload: Option, @@ -90,6 +91,9 @@ impl FactConfig { self.otel.update(&from.otel); self.endpoint.update(&from.endpoint); self.bpf.update(&from.bpf); + if let Some(oci_runtime_spec_debug) = from.oci_runtime_spec_debug { + self.oci_runtime_spec_debug = Some(oci_runtime_spec_debug); + } if let Some(skip_pre_flight) = from.skip_pre_flight { self.skip_pre_flight = Some(skip_pre_flight); @@ -124,6 +128,11 @@ impl FactConfig { self.skip_pre_flight.unwrap_or(false) } + /// Whether development-only OCI configuration is logged and exported. + pub fn oci_runtime_spec_debug(&self) -> bool { + self.oci_runtime_spec_debug.unwrap_or(false) + } + pub fn json(&self) -> bool { self.json.unwrap_or(false) } @@ -234,6 +243,12 @@ impl TryFrom> for FactConfig { }; config.bpf = BpfConfig::try_from(bpf)?; } + "oci_runtime_spec_debug" => { + let Some(oci_runtime_spec_debug) = v.as_bool() else { + bail!("oci_runtime_spec_debug field has incorrect type: {v:?}"); + }; + config.oci_runtime_spec_debug = Some(oci_runtime_spec_debug); + } "hotreload" => { let Some(hotreload) = v.as_bool() else { bail!("hotreload field has incorrect type: {v:?}"); @@ -795,6 +810,10 @@ pub struct FactCli { #[arg(long, env = "FACT_OTEL_ENDPOINT")] otel_endpoint: Option, + /// Add curated OCI runtime configuration to debug logs and OpenTelemetry + #[arg(long, env = "FACT_OCI_RUNTIME_SPEC_DEBUG")] + oci_runtime_spec_debug: Option, + /// The port to bind for all exposed endpoints #[arg(long, short, env = "FACT_ENDPOINT_ADDRESS")] address: Option, @@ -942,6 +961,7 @@ impl FactCli { d_instantiate_ctx_size: self.d_instantiate_ctx_size, programs: HashMap::new(), }, + oci_runtime_spec_debug: self.oci_runtime_spec_debug, skip_pre_flight: resolve_bool_arg(self.skip_pre_flight, self.no_skip_pre_flight), json: resolve_bool_arg(self.json, self.no_json), hotreload: resolve_bool_arg(self.hotreload, self.no_hotreload), diff --git a/fact/src/config/tests.rs b/fact/src/config/tests.rs index 596ca711..b93bb84d 100644 --- a/fact/src/config/tests.rs +++ b/fact/src/config/tests.rs @@ -9,6 +9,13 @@ use super::*; fn parsing() { let tests = [ ("", FactConfig::default()), + ( + "oci_runtime_spec_debug: true", + FactConfig { + oci_runtime_spec_debug: Some(true), + ..Default::default() + }, + ), ( "paths:", FactConfig { @@ -510,6 +517,7 @@ fn parsing() { "#, FactConfig { paths: Some(vec![PathBuf::from("/etc")]), + oci_runtime_spec_debug: None, grpc: GrpcConfig { url: Some(String::from("https://svc.sensor.stackrox:9090")), certs: Some(PathBuf::from("/etc/stackrox/certs")), @@ -968,6 +976,10 @@ paths: "replay field has incorrect type: Boolean(true)", ), ("unknown:", "Invalid field 'unknown' with value: Null"), + ( + "oci_runtime_spec_debug: definitely", + "oci_runtime_spec_debug field has incorrect type: String(\"definitely\")", + ), ]; for (input, expected) in tests { let Err(err) = FactConfig::try_from(input) else { @@ -981,6 +993,14 @@ paths: fn update() { let tests = [ ("", FactConfig::default(), FactConfig::default()), + ( + "oci_runtime_spec_debug: true", + FactConfig::default(), + FactConfig { + oci_runtime_spec_debug: Some(true), + ..Default::default() + }, + ), ( "paths:", FactConfig::default(), @@ -1937,6 +1957,7 @@ fn update() { "#, FactConfig { paths: Some(vec![PathBuf::from("/etc"), PathBuf::from("/bin")]), + oci_runtime_spec_debug: None, grpc: GrpcConfig { url: Some(String::from("http://localhost")), certs: Some(PathBuf::from("/etc/certs")), @@ -1977,6 +1998,7 @@ fn update() { }, FactConfig { paths: Some(vec![PathBuf::from("/etc")]), + oci_runtime_spec_debug: None, grpc: GrpcConfig { url: Some(String::from("https://svc.sensor.stackrox:9090")), certs: Some(PathBuf::from("/etc/stackrox/certs")), @@ -2385,6 +2407,16 @@ fn env_vars() { ..Default::default() }, ), + ( + EnvVar { + name: "FACT_OCI_RUNTIME_SPEC_DEBUG", + value: "true", + }, + FactConfig { + oci_runtime_spec_debug: Some(true), + ..Default::default() + }, + ), ( EnvVar { name: "FACT_ENDPOINT_ADDRESS", diff --git a/fact/src/event/mod.rs b/fact/src/event/mod.rs index 3bed7a04..8d9666d3 100644 --- a/fact/src/event/mod.rs +++ b/fact/src/event/mod.rs @@ -374,7 +374,74 @@ impl Event { self.get_monitored() == monitored_t::MONITORED_BY_PARENT } - #[cfg(feature = "otel")] + pub(crate) fn log_oci_debug(&self) { + let Some(short_id) = self.process.container_id() else { + return; + }; + let Some(metadata) = crate::oci::resolve(short_id) else { + log::debug!( + "OCI config event: fact_version={} fact_build_sha={} status=unavailable reason=config_unavailable container_id={short_id} event_path={}", + crate::version::FACT_VERSION, + crate::version::FACT_BUILD_SHA, + self.get_filename().display(), + ); + return; + }; + let oci = metadata.oci_debug(); + let info = metadata.match_mount(self.get_filename()); + log::debug!( + "OCI config event: fact_version={} fact_build_sha={} status=parsed container_id={} oci_version={} root_path={} root_read_only={} configured_executable={} configured_args={:?} configured_cwd={} effective_capabilities={:?} bounding_capabilities={:?} namespaces={:?} event_path={} mount_status={} mount_destination={} mount_source={} mount_type={} mount_options={:?} resolved_source_path={}", + crate::version::FACT_VERSION, + crate::version::FACT_BUILD_SHA, + oci.container_id, + oci.version, + oci.root_path, + oci.root_read_only, + oci.process_args + .first() + .map(String::as_str) + .unwrap_or_default(), + oci.process_args, + oci.process_cwd, + oci.effective_capabilities, + oci.bounding_capabilities, + oci.namespaces, + self.get_filename().display(), + info.status, + info.destination + .as_deref() + .unwrap_or(Path::new("")) + .display(), + info.source.as_deref().unwrap_or(Path::new("")).display(), + info.mount_type.as_deref().unwrap_or_default(), + info.options, + info.resolved_source_path + .as_deref() + .unwrap_or(Path::new("")) + .display(), + ); + if let Some(path) = self.get_old_filename() { + let info = metadata.match_mount(path); + log::debug!( + "OCI config old path: container_id={} event_path={} mount_status={} mount_destination={} mount_source={} mount_type={} mount_options={:?} resolved_source_path={}", + oci.container_id, + path.display(), + info.status, + info.destination + .as_deref() + .unwrap_or(Path::new("")) + .display(), + info.source.as_deref().unwrap_or(Path::new("")).display(), + info.mount_type.as_deref().unwrap_or_default(), + info.options, + info.resolved_source_path + .as_deref() + .unwrap_or(Path::new("")) + .display(), + ); + } + } + pub(crate) fn event_type(&self) -> &'static str { self.file.event_type() } @@ -419,15 +486,162 @@ impl From for fact_api::FileActivity { } } +#[cfg(feature = "otel")] +impl Event { + pub(crate) fn into_otel(self, oci_debug: bool) -> AnyValue { + let mut map = HashMap::from([ + ("file".into(), self.file.clone().into()), + ("timestamp".into(), AnyValue::Int(self.timestamp as i64)), + ("process".into(), self.process.clone().into()), + ("hostname".into(), self.hostname.to_string().into()), + ]); + if oci_debug { + map.insert("event.name".into(), self.event_type().into()); + map.insert( + "file.path".into(), + self.get_filename().to_string_lossy().to_string().into(), + ); + map.insert( + "file.host_path".into(), + self.get_host_path().to_string_lossy().to_string().into(), + ); + map.insert("host.name".into(), self.hostname.to_string().into()); + if let Some(path) = self.get_old_filename() { + map.insert( + "file.old.path".into(), + path.to_string_lossy().to_string().into(), + ); + } + if let Some(path) = self.get_old_host_path() { + map.insert( + "file.old.host_path".into(), + path.to_string_lossy().to_string().into(), + ); + } + add_oci_debug_attributes(&mut map, &self); + self.process.add_debug_otel_attributes(&mut map); + } + AnyValue::Map(Box::new(map)) + } +} + +#[cfg(feature = "otel")] +fn add_oci_debug_attributes(map: &mut HashMap, event: &Event) { + let Some(container_id) = event.process.container_id() else { + return; + }; + let Some(metadata) = crate::oci::resolve(container_id) else { + map.insert("container.oci.config.status".into(), "unavailable".into()); + map.insert( + "container.oci.config.reason".into(), + "config_unavailable".into(), + ); + map.insert( + "container.oci.container_id".into(), + container_id.to_owned().into(), + ); + return; + }; + let oci = metadata.oci_debug(); + map.insert("container.oci.config.status".into(), "parsed".into()); + map.insert( + "container.oci.container_id".into(), + oci.container_id.clone().into(), + ); + map.insert("container.oci.version".into(), oci.version.clone().into()); + map.insert( + "container.oci.root.path".into(), + oci.root_path.clone().into(), + ); + map.insert( + "container.oci.root.read_only".into(), + oci.root_read_only.into(), + ); + if let Some(executable) = oci.process_args.first() { + map.insert( + "container.oci.process.executable".into(), + executable.clone().into(), + ); + } + map.insert( + "container.oci.process.args".into(), + serde_json::to_string(&oci.process_args) + .unwrap_or_default() + .into(), + ); + map.insert( + "container.oci.process.cwd".into(), + oci.process_cwd.clone().into(), + ); + map.insert( + "container.oci.capabilities.effective".into(), + serde_json::to_string(&oci.effective_capabilities) + .unwrap_or_default() + .into(), + ); + map.insert( + "container.oci.capabilities.bounding".into(), + serde_json::to_string(&oci.bounding_capabilities) + .unwrap_or_default() + .into(), + ); + map.insert( + "container.oci.linux.namespaces".into(), + serde_json::to_string(&oci.namespaces) + .unwrap_or_default() + .into(), + ); + + add_oci_path_attributes( + map, + "container.oci.mount", + metadata.match_mount(event.get_filename()), + ); + if let Some(path) = event.get_old_filename() { + add_oci_path_attributes(map, "container.oci.mount.old", metadata.match_mount(path)); + } +} + +#[cfg(feature = "otel")] +fn add_oci_path_attributes( + map: &mut HashMap, + prefix: &str, + info: crate::oci::OciPathDebugInfo, +) { + map.insert(format!("{prefix}.status").into(), info.status.into()); + if let Some(destination) = info.destination { + map.insert( + format!("{prefix}.destination").into(), + destination.to_string_lossy().to_string().into(), + ); + } + if let Some(source) = info.source { + map.insert( + format!("{prefix}.source").into(), + source.to_string_lossy().to_string().into(), + ); + } + if let Some(mount_type) = info.mount_type { + map.insert(format!("{prefix}.type").into(), mount_type.into()); + } + map.insert( + format!("{prefix}.options").into(), + serde_json::to_string(&info.options) + .unwrap_or_default() + .into(), + ); + if let Some(resolved_path) = info.resolved_source_path { + map.insert( + format!("{prefix}.resolved_source_path").into(), + resolved_path.to_string_lossy().to_string().into(), + ); + } +} + #[cfg(feature = "otel")] impl From for opentelemetry::logs::AnyValue { fn from(value: Event) -> Self { - AnyValue::Map(Box::new(HashMap::from([ - ("file".into(), value.file.into()), - ("timestamp".into(), AnyValue::Int(value.timestamp as i64)), - ("process".into(), value.process.into()), - ("hostname".into(), value.hostname.into()), - ]))) + value.into_otel(false) } } @@ -556,7 +770,6 @@ impl FileData { Ok(file) } - #[cfg(feature = "otel")] fn event_type(&self) -> &'static str { match self { FileData::Open(_) => "open", @@ -1103,6 +1316,52 @@ mod tests { use super::test_utils::*; use super::*; + #[cfg(feature = "otel")] + #[test] + fn oci_diagnostics_only_add_debug_attributes_when_enabled() { + let event = Event { + timestamp: 1, + hostname: "test-host".into(), + process: Process::default(), + file: FileData::Open(BaseFileData { + filename: PathBuf::from("/etc/example"), + ..Default::default() + }), + }; + + let AnyValue::Map(default) = event.clone().into_otel(false) else { + panic!("event did not serialize to a map"); + }; + assert_eq!(default.len(), 4); + assert!( + default + .keys() + .all(|key| matches!(key.as_str(), "file" | "timestamp" | "process" | "hostname")) + ); + + let AnyValue::Map(debug) = event.into_otel(true) else { + panic!("event did not serialize to a map"); + }; + for key in [ + "event.name", + "file.path", + "file.host_path", + "host.name", + "process.command", + "process.executable.path", + ] { + assert!( + debug.keys().any(|candidate| candidate.as_str() == key), + "missing debug attribute {key}" + ); + } + assert!( + !debug + .keys() + .any(|key| key.as_str().starts_with("container.oci.")) + ); + } + #[test] fn slice_to_string_valid_utf8() { let tests = [ diff --git a/fact/src/event/process.rs b/fact/src/event/process.rs index 6f0733e0..57fc7f71 100644 --- a/fact/src/event/process.rs +++ b/fact/src/event/process.rs @@ -4,6 +4,8 @@ use std::{ffi::CStr, path::PathBuf}; use fact_ebpf::{lineage_t, process_t}; #[cfg(feature = "otel")] +use opentelemetry::Key; +#[cfg(feature = "otel")] use opentelemetry::logs::AnyValue; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -130,6 +132,103 @@ impl Process { None } } + + pub(crate) fn container_id(&self) -> Option<&str> { + self.container_id.as_deref() + } +} + +#[cfg(feature = "otel")] +impl Process { + pub(super) fn add_debug_otel_attributes(&self, map: &mut HashMap) { + map.insert("process.command".into(), self.comm.clone().into()); + map.insert( + "process.command_line".into(), + shlex::try_join(self.args.iter().map(String::as_str)) + .unwrap_or_else(|_| self.args.join(" ")) + .into(), + ); + map.insert( + "process.executable.path".into(), + self.exe_path.to_string_lossy().to_string().into(), + ); + map.insert("process.pid".into(), (self.pid as i64).into()); + map.insert("process.user.id".into(), (self.uid as i64).into()); + map.insert("process.user.name".into(), self.username.into()); + map.insert("process.group.id".into(), (self.gid as i64).into()); + map.insert("process.login_uid".into(), (self.login_uid as i64).into()); + map.insert( + "process.in_root_mount_ns".into(), + self.in_root_mount_ns.into(), + ); + + let Some(container_id) = &self.container_id else { + return; + }; + map.insert("container.id".into(), container_id.clone().into()); + let Some(container) = crate::oci::resolve(container_id) else { + return; + }; + + insert_string(map, "k8s.namespace.name", &container.namespace); + insert_string(map, "k8s.pod.uid", &container.pod_uid); + insert_string(map, "k8s.pod.name", &container.pod_name); + insert_string(map, "k8s.container.name", &container.container_name); + insert_string(map, "container.image.name", &container.image_name); + insert_string(map, "container.image.id", &container.image_ref); + insert_string(map, "container.runtime.type", &container.container_type); + insert_string(map, "container.created_at", &container.created); + insert_string(map, "openshift.scc", &container.openshift_scc); + map.insert("openshift.debug".into(), container.oc_debug.into()); + map.insert( + "container.security_context.privileged".into(), + container.privileged.into(), + ); + map.insert("container.host_pid".into(), container.host_pid.into()); + map.insert( + "container.host_network".into(), + container.host_network.into(), + ); + map.insert( + "container.host_root_mount".into(), + container.host_root_mount.into(), + ); + insert_json_map(map, "k8s.container.labels", &container.labels); + insert_json_map(map, "k8s.container.annotations", &container.annotations); + + if let Some(sandbox) = &container.sandbox { + insert_string(map, "container.sandbox.id", &sandbox.id); + insert_string(map, "container.sandbox.oci.version", &sandbox.oci_version); + insert_string(map, "container.sandbox.image.name", &sandbox.image_name); + insert_string(map, "container.sandbox.image.id", &sandbox.image_ref); + insert_json_map(map, "k8s.pod.labels", &sandbox.labels); + insert_json_map(map, "k8s.pod.annotations", &sandbox.annotations); + } else { + insert_json_map(map, "k8s.pod.labels", &container.labels); + insert_json_map(map, "k8s.pod.annotations", &container.annotations); + } + } +} + +#[cfg(feature = "otel")] +fn insert_string(map: &mut HashMap, key: &'static str, value: &str) { + if !value.is_empty() { + map.insert(key.into(), value.to_owned().into()); + } +} + +#[cfg(feature = "otel")] +fn insert_json_map( + map: &mut HashMap, + key: &'static str, + value: &HashMap, +) { + if !value.is_empty() { + map.insert( + key.into(), + serde_json::to_string(value).unwrap_or_default().into(), + ); + } } #[cfg(test)] diff --git a/fact/src/lib.rs b/fact/src/lib.rs index 2c0d335a..f57b71de 100644 --- a/fact/src/lib.rs +++ b/fact/src/lib.rs @@ -21,6 +21,7 @@ mod event; mod host_info; mod host_scanner; mod metrics; +mod oci; mod output; mod pre_flight; mod rate_limiter; @@ -114,6 +115,7 @@ pub async fn run(config: FactConfig) -> anyhow::Result<()> { let (host_scanner_intro_tx, host_scanner_intro_rx) = mpsc::channel(10); let stdout_enabled = config.json(); + let oci_debug = config.oci_runtime_spec_debug(); let skip_pre_flight = config.skip_pre_flight(); let replay = config.replay().map(PathBuf::from); let bpf_config = config.bpf.clone(); @@ -148,6 +150,7 @@ pub async fn run(config: FactConfig) -> anyhow::Result<()> { reloader.grpc(), reloader.otel(), stdout_enabled, + oci_debug, ); rate_limiter.start(&mut task_set); diff --git a/fact/src/oci.rs b/fact/src/oci.rs new file mode 100644 index 00000000..7b9f2b84 --- /dev/null +++ b/fact/src/oci.rs @@ -0,0 +1,609 @@ +use std::{ + collections::HashMap, + fs, + path::{Component, Path, PathBuf}, + sync::{LazyLock, RwLock}, +}; + +use anyhow::{Context, bail}; +use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::host_info; + +const RUNTIME_ROOTS: [&str; 2] = [ + "run/containers/storage/overlay-containers", + "var/lib/containers/storage/overlay-containers", +]; + +static CACHE: LazyLock>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContainerMetadata { + pub namespace: String, + pub pod_uid: String, + pub pod_name: String, + pub container_name: String, + pub image_name: String, + pub image_ref: String, + pub container_type: String, + pub openshift_scc: String, + pub created: String, + pub oc_debug: bool, + pub privileged: bool, + pub host_pid: bool, + pub host_network: bool, + pub host_root_mount: bool, + pub labels: HashMap, + pub annotations: HashMap, + pub sandbox: Option, + #[serde(skip)] + oci: OciDebugMetadata, +} + +/// Curated OCI runtime configuration retained for development diagnostics. +/// It is intentionally excluded from the Sensor protobuf and normal JSON. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct OciDebugMetadata { + pub container_id: String, + pub version: String, + pub root_path: String, + pub root_read_only: bool, + pub process_args: Vec, + pub process_cwd: String, + pub effective_capabilities: Vec, + pub bounding_capabilities: Vec, + pub namespaces: Vec, + mounts: Vec, +} + +/// Relationship between one event path and the OCI mount table. This exposes +/// runtime facts only; callers must not treat it as a filtering decision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OciPathDebugInfo { + pub status: &'static str, + pub destination: Option, + pub source: Option, + pub mount_type: Option, + pub options: Vec, + pub resolved_source_path: Option, +} + +impl OciPathDebugInfo { + pub(crate) fn unavailable(status: &'static str) -> Self { + Self { + status, + destination: None, + source: None, + mount_type: None, + options: Vec::new(), + resolved_source_path: None, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxMetadata { + pub id: String, + pub oci_version: String, + pub image_name: String, + pub image_ref: String, + pub labels: HashMap, + pub annotations: HashMap, +} + +impl ContainerMetadata { + pub(crate) fn oci_debug(&self) -> &OciDebugMetadata { + &self.oci + } + + pub(crate) fn match_mount(&self, path: &Path) -> OciPathDebugInfo { + if !is_normal_absolute_path(path) { + return OciPathDebugInfo::unavailable("invalid_event_path"); + } + + let matched = self + .oci + .mounts + .iter() + .filter(|mount| { + let destination = Path::new(&mount.destination); + is_normal_absolute_path(destination) && path.starts_with(destination) + }) + .max_by_key(|mount| Path::new(&mount.destination).components().count()); + + let Some(mount) = matched else { + return OciPathDebugInfo::unavailable("no_mount_match"); + }; + + let destination = PathBuf::from(&mount.destination); + let source = PathBuf::from(&mount.source); + let suffix = path.strip_prefix(&destination).ok(); + OciPathDebugInfo { + status: "matched", + destination: Some(destination), + source: Some(source.clone()), + mount_type: Some(mount.mount_type.clone()), + options: mount.options.clone(), + resolved_source_path: suffix.map(|suffix| source.join(suffix)), + } + } +} + +fn is_normal_absolute_path(path: &Path) -> bool { + path.is_absolute() + && path + .components() + .all(|component| matches!(component, Component::RootDir | Component::Normal(_))) +} + +pub fn resolve(short_id: &str) -> Option { + if let Some(metadata) = CACHE.read().ok()?.get(short_id) { + return Some(metadata.clone()); + } + + match resolve_from_root(host_info::get_host_mount(), short_id) { + Ok(Some(metadata)) => { + if let Ok(mut cache) = CACHE.write() { + cache.insert(short_id.to_owned(), metadata.clone()); + } + Some(metadata) + } + Ok(None) => None, + Err(error) => { + debug!("Failed to resolve OCI metadata for container {short_id}: {error:#}"); + None + } + } +} + +fn resolve_from_root( + host_root: &Path, + short_id: &str, +) -> anyhow::Result> { + if short_id.is_empty() || !short_id.chars().all(|c| c.is_ascii_hexdigit()) { + return Ok(None); + } + + for runtime_root in RUNTIME_ROOTS { + let root = host_root.join(runtime_root); + let Some(config) = find_config(&root, short_id)? else { + continue; + }; + let spec = read_spec(&config)?; + let sandbox_id = annotation(&spec.annotations, "io.kubernetes.cri-o.SandboxID"); + let mut metadata = ContainerMetadata::from(spec); + metadata.oci.container_id = config + .parent() + .and_then(Path::parent) + .and_then(Path::file_name) + .and_then(|id| id.to_str()) + .unwrap_or_default() + .to_owned(); + metadata.sandbox = match resolve_sandbox(host_root, &sandbox_id, &metadata) { + Ok(sandbox) => sandbox, + Err(error) => { + debug!( + "Failed to resolve OCI sandbox metadata for container {short_id}: {error:#}" + ); + None + } + }; + return Ok(Some(metadata)); + } + + Ok(None) +} + +fn read_spec(config: &Path) -> anyhow::Result { + let contents = + fs::read_to_string(config).with_context(|| format!("reading {}", config.display()))?; + serde_json::from_str(&contents).with_context(|| format!("parsing {}", config.display())) +} + +fn resolve_sandbox( + host_root: &Path, + sandbox_id: &str, + container: &ContainerMetadata, +) -> anyhow::Result> { + if sandbox_id.len() != 64 || !sandbox_id.chars().all(|c| c.is_ascii_hexdigit()) { + return Ok(None); + } + + for runtime_root in RUNTIME_ROOTS { + let config = host_root + .join(runtime_root) + .join(sandbox_id) + .join("userdata/config.json"); + if !config.is_file() { + continue; + } + + let spec = read_spec(&config)?; + if annotation(&spec.annotations, "io.kubernetes.cri-o.ContainerType") != "sandbox" { + bail!("{} is not a sandbox OCI config", config.display()); + } + if annotation(&spec.annotations, "io.kubernetes.pod.uid") != container.pod_uid { + bail!("sandbox pod UID does not match its container"); + } + + return Ok(Some(SandboxMetadata { + id: sandbox_id.to_owned(), + oci_version: spec.oci_version, + image_name: annotation(&spec.annotations, "io.kubernetes.cri-o.ImageName"), + image_ref: annotation(&spec.annotations, "io.kubernetes.cri-o.ImageRef"), + labels: parse_map(spec.annotations.get("io.kubernetes.cri-o.Labels")), + annotations: parse_map(spec.annotations.get("io.kubernetes.cri-o.Annotations")), + })); + } + + Ok(None) +} + +fn find_config(root: &Path, short_id: &str) -> anyhow::Result> { + let entries = match fs::read_dir(root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error).with_context(|| format!("reading {}", root.display())), + }; + + let mut matches = Vec::new(); + for entry in entries { + let entry = entry.with_context(|| format!("reading an entry in {}", root.display()))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if name.len() == 64 + && name.starts_with(short_id) + && name.chars().all(|c| c.is_ascii_hexdigit()) + { + let config = entry.path().join("userdata/config.json"); + if config.is_file() { + matches.push(config); + } + } + if matches.len() > 1 { + bail!( + "container ID prefix {short_id} is ambiguous under {}", + root.display() + ); + } + } + + Ok(matches.pop()) +} + +#[derive(Debug, Default, Deserialize)] +struct OciSpec { + #[serde(default, rename = "ociVersion")] + oci_version: String, + #[serde(default)] + annotations: HashMap, + #[serde(default)] + mounts: Vec, + #[serde(default)] + root: OciRoot, + #[serde(default)] + process: OciProcess, + #[serde(default)] + linux: OciLinux, +} + +#[derive(Debug, Default, Deserialize)] +struct OciRoot { + #[serde(default)] + path: String, + #[serde(default, rename = "readonly")] + read_only: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +struct OciMount { + #[serde(default)] + destination: String, + #[serde(default, rename = "type")] + mount_type: String, + #[serde(default)] + source: String, + #[serde(default)] + options: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct OciProcess { + #[serde(default)] + args: Vec, + #[serde(default)] + cwd: String, + #[serde(default)] + capabilities: OciCapabilities, +} + +#[derive(Debug, Default, Deserialize)] +struct OciCapabilities { + #[serde(default)] + effective: Vec, + #[serde(default)] + bounding: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct OciLinux { + #[serde(default)] + namespaces: Vec, + #[serde(default, rename = "maskedPaths")] + masked_paths: Vec, + #[serde(default, rename = "readonlyPaths")] + readonly_paths: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct OciNamespace { + #[serde(default, rename = "type")] + kind: String, +} + +impl From for ContainerMetadata { + fn from(spec: OciSpec) -> Self { + let labels = parse_map(spec.annotations.get("io.kubernetes.cri-o.Labels")); + let annotations = parse_map(spec.annotations.get("io.kubernetes.cri-o.Annotations")); + let oc_debug = spec + .annotations + .keys() + .chain(labels.keys()) + .chain(annotations.keys()) + .any(|key| key.starts_with("debug.openshift.io/")); + let has_namespace = |kind: &str| spec.linux.namespaces.iter().any(|ns| ns.kind == kind); + let privileged = spec + .process + .capabilities + .effective + .iter() + .any(|capability| capability == "CAP_SYS_ADMIN") + && spec.linux.masked_paths.is_empty() + && spec.linux.readonly_paths.is_empty(); + let host_root_mount = spec.mounts.iter().any(|mount| { + mount.source == "/" + && mount.destination != "/" + && mount.options.iter().any(|option| option == "rw") + }); + let oci = OciDebugMetadata { + version: spec.oci_version, + root_path: spec.root.path, + root_read_only: spec.root.read_only, + process_args: spec.process.args, + process_cwd: spec.process.cwd, + effective_capabilities: spec.process.capabilities.effective, + bounding_capabilities: spec.process.capabilities.bounding, + namespaces: spec + .linux + .namespaces + .iter() + .map(|namespace| namespace.kind.clone()) + .collect(), + mounts: spec.mounts, + ..Default::default() + }; + + Self { + namespace: annotation(&spec.annotations, "io.kubernetes.pod.namespace"), + pod_uid: annotation(&spec.annotations, "io.kubernetes.pod.uid"), + pod_name: annotation(&spec.annotations, "io.kubernetes.pod.name"), + container_name: annotation(&spec.annotations, "io.kubernetes.container.name"), + image_name: annotation(&spec.annotations, "io.kubernetes.cri-o.ImageName"), + image_ref: annotation(&spec.annotations, "io.kubernetes.cri-o.ImageRef"), + container_type: annotation(&spec.annotations, "io.kubernetes.cri-o.ContainerType"), + openshift_scc: annotation(&spec.annotations, "openshift.io/scc"), + created: annotation(&spec.annotations, "io.kubernetes.cri-o.Created"), + oc_debug, + privileged, + host_pid: !has_namespace("pid"), + host_network: !has_namespace("network"), + host_root_mount, + labels, + annotations, + sandbox: None, + oci, + } + } +} + +fn annotation(annotations: &HashMap, key: &str) -> String { + annotations.get(key).cloned().unwrap_or_default() +} + +fn parse_map(value: Option<&String>) -> HashMap { + value + .and_then(|value| serde_json::from_str(value).ok()) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + const FULL_ID: &str = "e72fce5766ead840d848faae8843678ff84e779443c6776e941421a6617acdbd"; + const SANDBOX_ID: &str = "dbb94bac0914953156552635dbc8e2125913c8c38bbab78930020f6cb5b57427"; + + fn write_config(root: &Path, id: &str) { + let directory = root.join(RUNTIME_ROOTS[0]).join(id).join("userdata"); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("config.json"), + r#"{ + "ociVersion": "1.2.0", + "root": {"path":"/var/lib/containers/storage/overlay/rootfs","readonly":true}, + "annotations": { + "io.kubernetes.pod.namespace": "test-ns", + "io.kubernetes.pod.uid": "pod-uid", + "io.kubernetes.pod.name": "pod-name", + "io.kubernetes.container.name": "container-name", + "io.kubernetes.cri-o.ImageName": "example/image:tag", + "io.kubernetes.cri-o.ImageRef": "sha256:image", + "io.kubernetes.cri-o.ContainerType": "container", + "io.kubernetes.cri-o.ContainerID": "e72fce5766ead840d848faae8843678ff84e779443c6776e941421a6617acdbd", + "io.kubernetes.cri-o.Name": "k8s_container-name_pod-name_test-ns_pod-uid_0", + "io.kubernetes.cri-o.LogPath": "/var/log/pods/test-ns_pod-name_pod-uid/container-name/0.log", + "io.kubernetes.cri-o.SandboxID": "dbb94bac0914953156552635dbc8e2125913c8c38bbab78930020f6cb5b57427", + "io.kubernetes.cri-o.Created": "2026-08-31T20:18:23.652874525Z", + "io.kubernetes.cri-o.Labels": "{\"app\":\"test\",\"debug.openshift.io/managed-by\":\"oc-debug\"}", + "io.kubernetes.cri-o.Annotations": "{\"example\":\"value\"}", + "openshift.io/scc": "privileged", + "io.container.manager": "cri-o" + }, + "mounts": [{"destination":"/host","source":"/","options":["rbind","rw"]}], + "process": { + "args":["/usr/bin/example","--serve"], + "cwd":"/work", + "capabilities":{ + "effective":["CAP_SYS_ADMIN"], + "bounding":["CAP_SYS_ADMIN","CAP_CHOWN"] + } + }, + "linux": {"namespaces":[{"type":"ipc"},{"type":"mount"}]} + }"#, + ) + .unwrap(); + + let sandbox_directory = root + .join(RUNTIME_ROOTS[0]) + .join(SANDBOX_ID) + .join("userdata"); + fs::create_dir_all(&sandbox_directory).unwrap(); + fs::write( + sandbox_directory.join("config.json"), + r#"{ + "ociVersion": "1.3.0", + "annotations": { + "io.kubernetes.cri-o.ContainerType": "sandbox", + "io.kubernetes.cri-o.ImageName": "example/pause@sha256:digest", + "io.kubernetes.cri-o.ImageRef": "sandbox-image-id", + "io.kubernetes.pod.uid": "pod-uid", + "io.kubernetes.cri-o.Labels": "{\"app\":\"sandbox-label\",\"pod-template-hash\":\"abc123\"}", + "io.kubernetes.cri-o.Annotations": "{\"pod.example/annotation\":\"value\"}" + } + }"#, + ) + .unwrap(); + } + + #[test] + fn resolves_unique_prefix_and_extracts_metadata() { + let root = tempfile::tempdir().unwrap(); + write_config(root.path(), FULL_ID); + + let metadata = resolve_from_root(root.path(), &FULL_ID[..12]) + .unwrap() + .unwrap(); + + assert_eq!(metadata.namespace, "test-ns"); + assert_eq!(metadata.pod_uid, "pod-uid"); + assert_eq!(metadata.image_name, "example/image:tag"); + assert_eq!(metadata.labels.get("app").unwrap(), "test"); + assert!(metadata.oc_debug); + assert!(metadata.privileged); + assert!(metadata.host_pid); + assert!(metadata.host_network); + assert!(metadata.host_root_mount); + assert_eq!(metadata.oci.container_id, FULL_ID); + assert_eq!(metadata.oci.version, "1.2.0"); + assert!(metadata.oci.root_read_only); + assert_eq!(metadata.oci.process_args[0], "/usr/bin/example"); + assert_eq!(metadata.oci.process_cwd, "/work"); + assert_eq!(metadata.oci.namespaces, ["ipc", "mount"]); + let sandbox = metadata.sandbox.unwrap(); + assert_eq!(sandbox.id, SANDBOX_ID); + assert_eq!(sandbox.oci_version, "1.3.0"); + assert_eq!(sandbox.image_name, "example/pause@sha256:digest"); + assert_eq!(sandbox.labels.get("app").unwrap(), "sandbox-label"); + assert_eq!(sandbox.labels.get("pod-template-hash").unwrap(), "abc123"); + } + + #[test] + fn ambiguous_prefix_fails_open() { + let root = tempfile::tempdir().unwrap(); + write_config(root.path(), FULL_ID); + write_config( + root.path(), + "e72fce5766eaffffffffffffffffffffffffffffffffffffffffffffffffffff", + ); + + assert!(resolve_from_root(root.path(), &FULL_ID[..12]).is_err()); + } + + #[test] + fn invalid_or_missing_prefix_is_unresolved() { + let root = tempfile::tempdir().unwrap(); + assert!(resolve_from_root(root.path(), "not-hex").unwrap().is_none()); + assert!( + resolve_from_root(root.path(), "0123456789ab") + .unwrap() + .is_none() + ); + } + + #[test] + fn matches_longest_mount_destination() { + let metadata = ContainerMetadata { + oci: OciDebugMetadata { + mounts: vec![ + OciMount { + destination: "/etc".into(), + mount_type: "bind".into(), + source: "/host/etc".into(), + options: vec!["ro".into(), "bind".into()], + }, + OciMount { + destination: "/etc/prometheus/config_out".into(), + mount_type: "bind".into(), + source: "/var/lib/kubelet/pods/pod-uid/volumes/kubernetes.io~empty-dir/config-out".into(), + options: vec!["rw".into(), "rbind".into()], + }, + ], + ..Default::default() + }, + ..Default::default() + }; + + let info = + metadata.match_mount(Path::new("/etc/prometheus/config_out/prometheus.env.yaml")); + + assert_eq!(info.status, "matched"); + assert_eq!( + info.destination.as_deref(), + Some(Path::new("/etc/prometheus/config_out")) + ); + assert_eq!( + info.resolved_source_path.as_deref(), + Some(Path::new( + "/var/lib/kubelet/pods/pod-uid/volumes/kubernetes.io~empty-dir/config-out/prometheus.env.yaml" + )) + ); + assert_eq!(info.options, ["rw", "rbind"]); + } + + #[test] + fn mount_match_is_component_aware() { + let metadata = ContainerMetadata { + oci: OciDebugMetadata { + mounts: vec![OciMount { + destination: "/data".into(), + mount_type: "bind".into(), + source: "/source".into(), + options: vec!["rw".into()], + }], + ..Default::default() + }, + ..Default::default() + }; + + assert_eq!( + metadata.match_mount(Path::new("/database/file")).status, + "no_mount_match" + ); + let invalid = metadata.match_mount(Path::new("/data/../host/file")); + assert_eq!(invalid.status, "invalid_event_path"); + } +} diff --git a/fact/src/output/mod.rs b/fact/src/output/mod.rs index 049f6e9c..531afca5 100644 --- a/fact/src/output/mod.rs +++ b/fact/src/output/mod.rs @@ -31,6 +31,7 @@ pub fn start( grpc_config: watch::Receiver, #[allow(unused)] otel_config: watch::Receiver, stdout_enabled: bool, + oci_debug: bool, ) { let (broad_tx, _) = broadcast::channel(100); let (subs_req, mut subs_rx) = mpsc::channel(10); @@ -54,6 +55,7 @@ pub fn start( running.subscribe(), metrics.otel.clone(), otel_config, + oci_debug, ); non_stdout_enabled = non_stdout_enabled || otel_client.is_enabled(); otel_client.start(&mut handles); @@ -81,6 +83,10 @@ pub fn start( break Ok(()); }; + if oci_debug { + event.log_oci_debug(); + } + if let Err(e) = broad_tx.send(Arc::new(event)) { warn!("Failed to forward output event: {e}"); } diff --git a/fact/src/output/otel.rs b/fact/src/output/otel.rs index 3af202df..2847646e 100644 --- a/fact/src/output/otel.rs +++ b/fact/src/output/otel.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use anyhow::bail; use log::{debug, info, warn}; +use opentelemetry::KeyValue; use opentelemetry::logs::{AnyValue, LogRecord, Logger, LoggerProvider, Severity}; use opentelemetry_otlp::{LogExporter, WithExportConfig}; use opentelemetry_sdk::Resource; @@ -18,6 +19,7 @@ pub(super) struct Client { running: watch::Receiver, config: watch::Receiver, metrics: EventCounter, + oci_debug: bool, } impl Client { @@ -26,12 +28,14 @@ impl Client { running: watch::Receiver, metrics: EventCounter, config: watch::Receiver, + oci_debug: bool, ) -> Self { Client { subscriber, running, config, metrics, + oci_debug, } } @@ -68,9 +72,21 @@ impl Client { .with_endpoint(endpoint) .build()?; + let mut resource = Resource::builder() + .with_service_name("fact") + .with_attribute(KeyValue::new( + "service.version", + crate::version::FACT_VERSION, + )); + if crate::version::FACT_BUILD_SHA != "unknown" { + resource = resource.with_attribute(KeyValue::new( + "fact.build.sha", + crate::version::FACT_BUILD_SHA, + )); + } let logger_provider = SdkLoggerProvider::builder() .with_batch_exporter(exporter_otlp) - .with_resource(Resource::builder().with_service_name("fact").build()) + .with_resource(resource.build()) .build(); let logger = logger_provider.logger("fact"); @@ -92,7 +108,7 @@ impl Client { event.event_type(), event.get_filename().display(), event.get_host_path().display()).into()); - if let AnyValue::Map(map) = event.into() { + if let AnyValue::Map(map) = event.into_otel(self.oci_debug) { for (k, v) in *map { record.add_attribute(k, v); } From 98803b85c6de43d6f56e2266c566712c591a6830 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 1 Sep 2026 10:18:01 -0700 Subject: [PATCH 4/4] docs: add ACS-controlled FACT validation lab --- deploy/acs-fact-lab/README.md | 120 ++++++++++++++++ deploy/acs-fact-lab/RESULTS-2026-09-01.md | 52 +++++++ deploy/acs-fact-lab/configure-lab.sh | 59 ++++++++ deploy/acs-fact-lab/lib.sh | 82 +++++++++++ deploy/acs-fact-lab/policies.json | 168 ++++++++++++++++++++++ deploy/acs-fact-lab/query-alerts.sh | 36 +++++ deploy/acs-fact-lab/query-otel.sh | 40 ++++++ deploy/acs-fact-lab/roxie.yaml | 43 ++++++ deploy/acs-fact-lab/run-experiments.sh | 51 +++++++ deploy/acs-fact-lab/setup-acs.sh | 57 ++++++++ deploy/acs-fact-lab/teardown.sh | 52 +++++++ deploy/acs-fact-lab/workloads.yaml | 70 +++++++++ 12 files changed, 830 insertions(+) create mode 100644 deploy/acs-fact-lab/README.md create mode 100644 deploy/acs-fact-lab/RESULTS-2026-09-01.md create mode 100755 deploy/acs-fact-lab/configure-lab.sh create mode 100755 deploy/acs-fact-lab/lib.sh create mode 100644 deploy/acs-fact-lab/policies.json create mode 100755 deploy/acs-fact-lab/query-alerts.sh create mode 100755 deploy/acs-fact-lab/query-otel.sh create mode 100644 deploy/acs-fact-lab/roxie.yaml create mode 100755 deploy/acs-fact-lab/run-experiments.sh create mode 100755 deploy/acs-fact-lab/setup-acs.sh create mode 100755 deploy/acs-fact-lab/teardown.sh create mode 100644 deploy/acs-fact-lab/workloads.yaml diff --git a/deploy/acs-fact-lab/README.md b/deploy/acs-fact-lab/README.md new file mode 100644 index 00000000..e5c4aef6 --- /dev/null +++ b/deploy/acs-fact-lab/README.md @@ -0,0 +1,120 @@ +# ACS-controlled FACT validation lab + +This is a development lab for comparing raw FACT events with RHACS file +activity policy results. RHACS policies are the only source of monitored paths; +do not edit the generated `stackrox/fact-config` ConfigMap. + +See [RESULTS-2026-09-01.md](RESULTS-2026-09-01.md) for the captured validation +counts and conclusions. + +The deployment was validated on `rc-dev-cluster` on 2026-09-01 with: + +- StackRox source `d5255a30c33` and Roxie `v0.4.9` +- development main image `quay.io/rcochran/main:4.12.x-529-gc619d5385e` +- FACT source `96190289b3f4d0233ae62f231c4614fbcde0f2b3` +- FACT image `quay.io/rcochran/scratch@sha256:4e6968b475595baf96425bbe04f45e005a398ff7babb762cdbc6a9c0d0f95655` +- SigNoz chart `0.139.0` from the earlier `deploy/fact-signoz` lab + +The Roxie overlay selects FACT's third operating mode: + +1. Sensor output only: normal ACS configuration. +2. Sensor plus OCI diagnostics: add `FACT_OCI_RUNTIME_SPEC_DEBUG=true`. +3. Sensor plus OCI diagnostics plus OTLP: also add `FACT_OTEL_ENDPOINT`. + +OCI diagnostics and OTLP are additive. FACT remains connected to Sensor and the +diagnostic feature does not change event selection or filtering. + +## Deploy + +Prerequisites are `oc`, `jq`, `curl`, the adjacent `stackrox/stackrox` checkout, +an existing SigNoz deployment in `observability`, and registry credentials for +the private development images. Do not commit the registry config or Roxie +environment file. + +```sh +export KUBECONFIG=/path/to/infractl-cluster-artifacts/kubeconfig +export DOCKER_CONFIG_JSON=/path/to/registry/config.json +export ROXIE_ENVRC=/tmp/roxie-fact-lab.envrc + +deploy/acs-fact-lab/setup-acs.sh +deploy/acs-fact-lab/configure-lab.sh +deploy/acs-fact-lab/run-experiments.sh +``` + +If SigNoz needs to be rebuilt, follow `deploy/fact-signoz/README.md` through the +SigNoz installation and static collector configuration, but do not deploy the +old standalone FACT operator or `Fact` custom resource. + +The setup script creates the `quay-rcochran` pull secret from the supplied +registry config before Roxie deploys Central. The environment file contains the +generated ACS password and CA path; keep it private. + +## What the experiment proves + +The controlled container sequence emits create, writable open, permission +change, two renames, and unlink. The exact same sequence runs against container +rootfs and an EmptyDir mounted at `/tmp`. + +| Case | Raw FACT identity | ACS result | +|---|---|---| +| container rootfs | empty `host_path`; OCI `no_mount_match` | deployment alert; empty `actualPath` | +| EmptyDir | empty `host_path`; OCI matched bind mount from `kubernetes.io~empty-dir` | deployment alert unless excluded | +| direct `oc debug node` host write | populated `host_path`; `openshift.debug=true`; container ID present | deployment alert attributed to the transient debug pod | +| host `systemd-run` write | populated `host_path`; no container ID | node alert attributed to the node | + +The process policy (`Process Name=chmod`) and operation policy (`File +Operation=open`) each select only their matching event while the raw OTLP stream +retains the complete sequence. The exclusion policy suppresses the EmptyDir +deployment only; it does not alter FACT collection or OTLP. + +Node versus deployment evaluation is currently determined by Sensor enrichment: +a nonempty deployment ID is a deployment event. It is not determined by whether +`host_path` is populated. This is why `oc debug node` is visible as a deployment +event while a detached host systemd unit is a node event. + +The OCI mount match for a chrooted debug process is currently `no_mount_match`: +the observed path is `/var/tmp/...` after chroot while the OCI mount destination +is `/host`. The independent inode mapping still supplies the correct host path. + +## Inspect + +Open the SigNoz Logs Explorer and filter on `service.name = fact`. Useful fields +are: + +```text +fact.build.sha +file.path +file.host_path +process.executable.path +k8s.namespace.name +k8s.pod.name +openshift.debug +container.oci.config.status +container.oci.mount.status +container.oci.mount.destination +container.oci.mount.source +``` + +For terminal output: + +```sh +SINCE_UTC=2026-09-01T16:00:00Z deploy/acs-fact-lab/query-otel.sh +deploy/acs-fact-lab/query-alerts.sh +oc -n stackrox get configmap fact-config -o jsonpath='{.data.fact\.yml}' +``` + +The ACS UI shows the same results under Violations. Central and Sensor logs are +also available with `oc -n stackrox logs deployment/central` and +`oc -n stackrox logs deployment/sensor`. + +## Tear down + +The default removes only the six lab policies, workloads, and isolated host +marker. Add `--acs` to remove Roxie-deployed ACS. Add `--signoz` only when its +persistent telemetry data should also be deleted. + +```sh +deploy/acs-fact-lab/teardown.sh +deploy/acs-fact-lab/teardown.sh --acs +deploy/acs-fact-lab/teardown.sh --acs --signoz +``` diff --git a/deploy/acs-fact-lab/RESULTS-2026-09-01.md b/deploy/acs-fact-lab/RESULTS-2026-09-01.md new file mode 100644 index 00000000..16a1de97 --- /dev/null +++ b/deploy/acs-fact-lab/RESULTS-2026-09-01.md @@ -0,0 +1,52 @@ +# Validated results — 2026-09-01 + +The repeatable matrix ran at `2026-09-01T17:13:49Z` on +`rc-dev-cluster`. All six collector pods were ready. Every FACT process was +simultaneously connected to Sensor and exporting OTLP to SigNoz. + +Sensor compiled the six RHACS policies to this FACT configuration: + +```yaml +paths: + - /tmp/acs-fact-lab-marker + - /var/tmp/acs-fact-host-marker +``` + +No FACT ConfigMap was edited by the experiment. + +## Raw FACT stream + +The run produced 16 marker events in SigNoz: + +| Source | Count | Operations | Host path | OCI result | +|---|---:|---|---|---| +| container rootfs | 6 | create, open, chmod, rename, rename, unlink | empty | parsed, `no_mount_match` | +| EmptyDir at `/tmp` | 6 | create, open, chmod, rename, rename, unlink | empty | parsed, `matched` | +| direct `oc debug node` | 2 | open, chmod | `/var/tmp/acs-fact-host-marker` | parsed, `no_mount_match`, `openshift.debug=true` | +| detached host systemd unit | 2 | open, chmod | `/var/tmp/acs-fact-host-marker` | no container or OCI attributes | + +The EmptyDir match identified destination `/tmp` and source +`/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/scratch`. This is +the positive evidence that distinguishes it from container rootfs even though +both have an empty protobuf `host_path`. + +## ACS results + +| Policy | Alerts | Violations | +|---|---:|---| +| Container marker activity | 2 | six for rootfs and six for EmptyDir | +| Marker changed by chmod | 2 | one chmod for each container deployment | +| Marker writable open only | 2 | one writable open for each container deployment | +| Exclude EmptyDir writer deployment | 1 | six for rootfs; none for EmptyDir | +| Node host marker activity | 1 | open and chmod from the host systemd unit | +| Containerized host marker activity | 1 | open and chmod attributed to the transient debug pod | + +For both rootfs and EmptyDir, ACS `actualPath` was empty. For both host-write +cases it was `/var/tmp/acs-fact-host-marker`. The direct debug process retained +pod, namespace, image, container, executable, and node attribution after the +transient pod was removed. + +These results establish that RHACS criteria and exclusions reduce violations at +policy evaluation time while raw OTLP remains complete. They also establish +that ACS node/deployment routing follows Sensor's deployment enrichment, not +the presence of `host_path`. diff --git a/deploy/acs-fact-lab/configure-lab.sh b/deploy/acs-fact-lab/configure-lab.sh new file mode 100755 index 00000000..04a1acd3 --- /dev/null +++ b/deploy/acs-fact-lab/configure-lab.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command oc +require_command jq +require_command curl +require_kubeconfig +load_roxie_env + +oc apply -f "${SCRIPT_DIR}/workloads.yaml" +oc -n "${LAB_NAMESPACE}" rollout status deployment/rootfs-writer --timeout=5m +oc -n "${LAB_NAMESPACE}" rollout status deployment/emptydir-writer --timeout=5m + +node=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}') +oc -n "${LAB_NAMESPACE}" create configmap fact-lab-state \ + --from-literal=node="${node}" \ + --dry-run=client -o yaml | oc apply -f - + +# Seed the exact host inode before the policy reaches FACT. This makes host +# scanner behavior deterministic and limits all host writes to /var/tmp. +oc -n "${LAB_NAMESPACE}" debug "node/${node}" -- \ + chroot /host sh -c 'touch /var/tmp/acs-fact-host-marker && chmod 600 /var/tmp/acs-fact-host-marker' + +while IFS= read -r policy_name; do + while IFS= read -r policy_id; do + [[ -z "${policy_id}" ]] || rox_api DELETE "/v1/policies/${policy_id}" >/dev/null + done < <(policy_ids_by_name "${policy_name}") +done < <(jq -r '.[].name' "${SCRIPT_DIR}/policies.json") + +policy_dir=$(mktemp -d) +trap 'rm -rf "${policy_dir}"' EXIT +policy_count=$(jq 'length' "${SCRIPT_DIR}/policies.json") +for index in $(seq 0 $((policy_count - 1))); do + policy_file="${policy_dir}/policy-${index}.json" + jq ".[${index}]" "${SCRIPT_DIR}/policies.json" >"${policy_file}" + rox_api POST /v1/policies "${policy_file}" | jq -r '"created policy: \(.name) [\(.id)]"' +done + +wait_for_fact_paths +oc -n stackrox get configmap fact-config -o jsonpath='{.data.fact\.yml}' + +for deployment in rootfs-writer emptydir-writer; do + query=$(jq -rn --arg value "Deployment:${deployment}" '$value|@uri') + for _ in $(seq 1 60); do + count=$(rox_api GET "/v1/deployments?query=${query}" | jq --arg name "${deployment}" '[.deployments[]? | select(.name == $name)] | length') + [[ "${count}" -gt 0 ]] && break + sleep 2 + done + [[ "${count}" -gt 0 ]] || { + echo "ACS did not inventory deployment ${deployment}" >&2 + exit 1 + } +done + +echo "lab policies, workloads, host marker, and FACT paths are ready" diff --git a/deploy/acs-fact-lab/lib.sh b/deploy/acs-fact-lab/lib.sh new file mode 100755 index 00000000..bd681268 --- /dev/null +++ b/deploy/acs-fact-lab/lib.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +set -euo pipefail + +LAB_NAMESPACE=acs-file-activity-lab +ROXIE_ENVRC=${ROXIE_ENVRC:-/tmp/roxie-fact-lab.envrc} + +require_command() { + command -v "$1" >/dev/null 2>&1 || { + echo "required command not found: $1" >&2 + exit 1 + } +} + +require_kubeconfig() { + if [[ -z "${KUBECONFIG:-}" || ! -f "${KUBECONFIG}" ]]; then + echo "set KUBECONFIG to the downloaded cluster kubeconfig" >&2 + exit 1 + fi +} + +load_roxie_env() { + if [[ ! -f "${ROXIE_ENVRC}" ]]; then + echo "Roxie environment file not found: ${ROXIE_ENVRC}" >&2 + exit 1 + fi + set -a + # shellcheck disable=SC1090 + source "${ROXIE_ENVRC}" + set +a + : "${ROX_ENDPOINT:?ROX_ENDPOINT is missing from the Roxie environment}" + : "${ROX_USERNAME:?ROX_USERNAME is missing from the Roxie environment}" + : "${ROX_ADMIN_PASSWORD:?ROX_ADMIN_PASSWORD is missing from the Roxie environment}" + : "${ROX_CA_CERT_FILE:?ROX_CA_CERT_FILE is missing from the Roxie environment}" +} + +rox_api() { + local method=$1 + local api_path=$2 + local data_file=${3:-} + local endpoint_host=${ROX_ENDPOINT%:*} + local args=( + --silent --show-error --fail-with-body --noproxy '*' + --resolve "central.stackrox:443:${endpoint_host}" + --user "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" + --cacert "${ROX_CA_CERT_FILE}" + --request "${method}" + ) + if [[ -n "${data_file}" ]]; then + args+=(--header 'Content-Type: application/json' --data-binary "@${data_file}") + fi + curl "${args[@]}" "https://central.stackrox${api_path}" +} + +policy_ids_by_name() { + local policy_name=$1 + local query + query=$(jq -rn --arg value "Policy:${policy_name}" '$value|@uri') + rox_api GET "/v1/policies?query=${query}" | jq -r --arg name "${policy_name}" '.policies[]? | select(.name == $name) | .id' +} + +wait_for_fact_paths() { + local config + for _ in $(seq 1 90); do + config=$(oc -n stackrox get configmap fact-config -o jsonpath='{.data.fact\.yml}' 2>/dev/null || true) + if grep -q '/tmp/acs-fact-lab-marker' <<<"${config}" && grep -q '/var/tmp/acs-fact-host-marker' <<<"${config}"; then + return 0 + fi + sleep 2 + done + echo "timed out waiting for Sensor to compile the lab paths into fact-config" >&2 + return 1 +} + +lab_node() { + local node + node=$(oc -n "${LAB_NAMESPACE}" get configmap fact-lab-state -o jsonpath='{.data.node}' 2>/dev/null || true) + if [[ -z "${node}" ]]; then + node=$(oc get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}') + fi + printf '%s\n' "${node}" +} diff --git a/deploy/acs-fact-lab/policies.json b/deploy/acs-fact-lab/policies.json new file mode 100644 index 00000000..98f04595 --- /dev/null +++ b/deploy/acs-fact-lab/policies.json @@ -0,0 +1,168 @@ +[ + { + "name": "FACT Lab - Container marker activity", + "description": "Validation policy for controlled FACT and ACS file-activity experiments.", + "rationale": "Correlate a unique container path across FACT OTLP, Sensor, and ACS violations.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "DEPLOYMENT_EVENT", + "exclusions": [], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "marker path", + "policyGroups": [{ + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/tmp/acs-fact-lab-marker"}] + }] + }] + }, + { + "name": "FACT Lab - Marker changed by chmod", + "description": "Validation policy combining a file path with Process Name.", + "rationale": "Compare all raw events with the process-scoped ACS result.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "DEPLOYMENT_EVENT", + "exclusions": [], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "marker path and chmod", + "policyGroups": [ + { + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/tmp/acs-fact-lab-marker"}] + }, + { + "fieldName": "Process Name", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "chmod"}] + } + ] + }] + }, + { + "name": "FACT Lab - Marker writable open only", + "description": "Validation policy combining a file path with File Operation open.", + "rationale": "Compare all raw events with the operation-scoped ACS result.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "DEPLOYMENT_EVENT", + "exclusions": [], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "marker path and open", + "policyGroups": [ + { + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/tmp/acs-fact-lab-marker"}] + }, + { + "fieldName": "File Operation", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "open"}] + } + ] + }] + }, + { + "name": "FACT Lab - Exclude EmptyDir writer deployment", + "description": "Validation policy proving deployment and namespace exclusion at ACS evaluation time.", + "rationale": "Compare excluded ACS results with the unchanged raw FACT stream.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "DEPLOYMENT_EVENT", + "exclusions": [{ + "name": "controlled EmptyDir workload", + "deployment": { + "name": "emptydir-writer", + "scope": {"namespace": "acs-file-activity-lab"} + } + }], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "marker path", + "policyGroups": [{ + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/tmp/acs-fact-lab-marker"}] + }] + }] + }, + { + "name": "FACT Lab - Node host marker activity", + "description": "Validation policy for an isolated host path modified by a non-containerized process.", + "rationale": "Correlate a true node file event across FACT OTLP, Sensor, and ACS.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "NODE_EVENT", + "exclusions": [], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "isolated host marker", + "policyGroups": [{ + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/var/tmp/acs-fact-host-marker"}] + }] + }] + }, + { + "name": "FACT Lab - Containerized host marker activity", + "description": "Validation policy for containerized host writes, including oc debug node.", + "rationale": "Determine ACS classification when a containerized process modifies a host inode.", + "remediation": "This is an intentionally generated lab event.", + "disabled": false, + "categories": ["System Modification"], + "lifecycleStages": ["RUNTIME"], + "eventSource": "DEPLOYMENT_EVENT", + "exclusions": [], + "scope": [], + "severity": "LOW_SEVERITY", + "enforcementActions": [], + "notifiers": [], + "policySections": [{ + "sectionName": "isolated host marker", + "policyGroups": [{ + "fieldName": "File Path", + "booleanOperator": "OR", + "negate": false, + "values": [{"value": "/var/tmp/acs-fact-host-marker"}] + }] + }] + } +] diff --git a/deploy/acs-fact-lab/query-alerts.sh b/deploy/acs-fact-lab/query-alerts.sh new file mode 100755 index 00000000..3b45fda5 --- /dev/null +++ b/deploy/acs-fact-lab/query-alerts.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command jq +require_command curl +load_roxie_env + +while IFS= read -r policy_name; do + query=$(jq -rn --arg value "Policy:${policy_name}" '$value|@uri') + response=$(rox_api GET "/v1/alerts?query=${query}") + count=$(jq --arg name "${policy_name}" '[.alerts[]? | select(.policy.name == $name)] | length' <<<"${response}") + printf '\n%s: %s alert(s)\n' "${policy_name}" "${count}" + while IFS= read -r alert_id; do + rox_api GET "/v1/alerts/${alert_id}" | jq '{ + id, + policy: .policy.name, + deployment: (.deployment.name // null), + namespace: (.deployment.namespace // null), + node: (.node.name // null), + violations: [.violations[] | { + operation: .fileAccess.operation, + effectivePath: .fileAccess.file.effectivePath, + actualPath: .fileAccess.file.actualPath, + process: .fileAccess.process.signal.name, + executable: .fileAccess.process.signal.execFilePath, + containerId: .fileAccess.process.signal.containerId, + pod: .fileAccess.process.podId, + hostname: .fileAccess.hostname + }] + }' + done < <(jq -r --arg name "${policy_name}" '.alerts[]? | select(.policy.name == $name) | .id' <<<"${response}") +done < <(jq -r '.[].name' "${SCRIPT_DIR}/policies.json") diff --git a/deploy/acs-fact-lab/query-otel.sh b/deploy/acs-fact-lab/query-otel.sh new file mode 100755 index 00000000..143d37fd --- /dev/null +++ b/deploy/acs-fact-lab/query-otel.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command oc +require_kubeconfig + +SINCE_UTC=${SINCE_UTC:-1970-01-01T00:00:00Z} +clickhouse_pod=$(oc -n observability get pod -l clickhouse.altinity.com/chi=signoz-clickhouse -o jsonpath='{.items[0].metadata.name}') +since_sql=${SINCE_UTC/T/ } +since_sql=${since_sql%Z} + +query="SELECT + formatDateTime(fromUnixTimestamp64Nano(timestamp), '%FT%T.%fZ') AS time, + attributes_string['event.name'] AS operation, + attributes_string['file.path'] AS effective_path, + attributes_string['file.host_path'] AS host_path, + attributes_string['process.command'] AS process, + attributes_string['process.executable.path'] AS executable, + attributes_string['container.id'] AS container_id, + attributes_string['k8s.namespace.name'] AS namespace, + attributes_string['k8s.pod.name'] AS pod, + attributes_bool['openshift.debug'] AS openshift_debug, + attributes_string['container.oci.config.status'] AS oci_status, + attributes_string['container.oci.mount.status'] AS mount_status, + attributes_string['container.oci.mount.destination'] AS mount_destination, + attributes_string['container.oci.mount.source'] AS mount_source, + resources_string['fact.build.sha'] AS fact_build_sha +FROM signoz_logs.distributed_logs_v2 +WHERE timestamp >= toUnixTimestamp64Nano(toDateTime64('${since_sql}', 9, 'UTC')) + AND resources_string['service.name'] = 'fact' + AND (attributes_string['file.path'] LIKE '%acs-fact-%' + OR attributes_string['file.host_path'] LIKE '%acs-fact-%') +ORDER BY timestamp +FORMAT PrettyCompactMonoBlock" + +oc -n observability exec "${clickhouse_pod}" -- clickhouse-client --query "${query}" diff --git a/deploy/acs-fact-lab/roxie.yaml b/deploy/acs-fact-lab/roxie.yaml new file mode 100644 index 00000000..2ec83ba0 --- /dev/null +++ b/deploy/acs-fact-lab/roxie.yaml @@ -0,0 +1,43 @@ +roxie: + version: 4.12.x-nightly-20260715 + featureFlags: + ROX_SENSITIVE_FILE_ACTIVITY: true + +operator: + envVars: + RELATED_IMAGE_MAIN: quay.io/rcochran/main:4.12.x-529-gc619d5385e + RELATED_IMAGE_FACT: quay.io/rcochran/scratch@sha256:4e6968b475595baf96425bbe04f45e005a398ff7babb762cdbc6a9c0d0f95655 + +central: + namespace: stackrox + resourceProfile: auto + earlyReadiness: false + deployTimeout: 30m + spec: + imagePullSecrets: + - name: quay-rcochran + +securedCluster: + namespace: stackrox + resourceProfile: auto + earlyReadiness: false + deployTimeout: 30m + spec: + imagePullSecrets: + - name: quay-rcochran + perNode: + fileActivityMonitoring: + mode: Enabled + overlays: + - apiVersion: apps/v1 + kind: DaemonSet + name: collector + patches: + - path: spec.template.spec.containers[name:fact].env[-1] + value: | + name: FACT_OCI_RUNTIME_SPEC_DEBUG + value: "true" + - path: spec.template.spec.containers[name:fact].env[-1] + value: | + name: FACT_OTEL_ENDPOINT + value: http://signoz-otel-collector.observability.svc.cluster.local:4318/v1/logs diff --git a/deploy/acs-fact-lab/run-experiments.sh b/deploy/acs-fact-lab/run-experiments.sh new file mode 100755 index 00000000..eb872a84 --- /dev/null +++ b/deploy/acs-fact-lab/run-experiments.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command oc +require_kubeconfig +load_roxie_env + +start_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ) +node=$(lab_node) + +exercise_container_path() { + local deployment=$1 + oc -n "${LAB_NAMESPACE}" exec "deployment/${deployment}" -- sh -c ' + printf create > /tmp/acs-fact-lab-marker + printf append >> /tmp/acs-fact-lab-marker + chmod 640 /tmp/acs-fact-lab-marker + mv /tmp/acs-fact-lab-marker /tmp/acs-fact-lab-marker.moved + mv /tmp/acs-fact-lab-marker.moved /tmp/acs-fact-lab-marker + rm /tmp/acs-fact-lab-marker + ' +} + +echo "experiment start: ${start_utc}" +echo "rootfs: six operations" +exercise_container_path rootfs-writer +echo "EmptyDir: same six operations" +exercise_container_path emptydir-writer + +echo "oc debug: containerized writes to a monitored host inode" +oc -n "${LAB_NAMESPACE}" debug "node/${node}" -- \ + chroot /host sh -c 'printf debug-write >> /var/tmp/acs-fact-host-marker; chmod 600 /var/tmp/acs-fact-host-marker' + +echo "host systemd unit: non-containerized writes to the same host inode" +oc -n "${LAB_NAMESPACE}" debug "node/${node}" -- \ + chroot /host systemd-run --wait --collect --unit=acs-fact-lab-host-write \ + /bin/sh -c 'printf host-write >> /var/tmp/acs-fact-host-marker; chmod 640 /var/tmp/acs-fact-host-marker' + +echo "waiting for Sensor and Central to persist the results" +sleep 10 + +echo +echo "Raw FACT OTEL events since ${start_utc}" +SINCE_UTC=${start_utc} "${SCRIPT_DIR}/query-otel.sh" + +echo +echo "ACS alerts" +"${SCRIPT_DIR}/query-alerts.sh" diff --git a/deploy/acs-fact-lab/setup-acs.sh b/deploy/acs-fact-lab/setup-acs.sh new file mode 100755 index 00000000..537a99e3 --- /dev/null +++ b/deploy/acs-fact-lab/setup-acs.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +FACT_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) +STACKROX_DIR=${STACKROX_DIR:-"${FACT_ROOT}/../stackrox"} +ROXIE_ENVRC=${ROXIE_ENVRC:-/tmp/roxie-fact-lab.envrc} +DOCKER_CONFIG_JSON=${DOCKER_CONFIG_JSON:-} + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command oc +require_command jq +require_command curl +require_kubeconfig + +if [[ ! -x "${STACKROX_DIR}/scripts/roxie.sh" ]]; then + echo "StackRox Roxie wrapper not found at ${STACKROX_DIR}/scripts/roxie.sh" >&2 + exit 1 +fi +if [[ -z "${DOCKER_CONFIG_JSON}" || ! -f "${DOCKER_CONFIG_JSON}" ]]; then + echo "set DOCKER_CONFIG_JSON to a registry config authorized for quay.io/rcochran" >&2 + exit 1 +fi +if ! oc -n observability get service signoz-otel-collector >/dev/null 2>&1; then + echo "SigNoz OTLP service observability/signoz-otel-collector is not available" >&2 + echo "deploy it with deploy/fact-signoz before enabling FACT_OTEL_ENDPOINT" >&2 + exit 1 +fi + +oc create namespace stackrox --dry-run=client -o yaml | oc apply -f - +oc -n stackrox create secret generic quay-rcochran \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson="${DOCKER_CONFIG_JSON}" \ + --dry-run=client -o yaml | oc apply -f - + +( + cd "${STACKROX_DIR}" + ./scripts/roxie.sh \ + --skip-user-config \ + --config "${SCRIPT_DIR}/roxie.yaml" \ + deploy both \ + --deploy-operator \ + --envrc "${ROXIE_ENVRC}" +) + +oc -n stackrox rollout status deployment/central --timeout=10m +oc -n stackrox rollout status deployment/sensor --timeout=10m +oc -n stackrox rollout status daemonset/collector --timeout=10m + +fact_image=$(oc -n stackrox get daemonset collector -o json | jq -r '.spec.template.spec.containers[] | select(.name == "fact") | .image') +fact_env=$(oc -n stackrox get daemonset collector -o json | jq -r '.spec.template.spec.containers[] | select(.name == "fact") | .env[] | select(.name == "FACT_OCI_RUNTIME_SPEC_DEBUG" or .name == "FACT_OTEL_ENDPOINT") | "\(.name)=\(.value)"') + +printf 'Roxie environment: %s\n' "${ROXIE_ENVRC}" +printf 'FACT image: %s\n' "${fact_image}" +printf '%s\n' "${fact_env}" diff --git a/deploy/acs-fact-lab/teardown.sh b/deploy/acs-fact-lab/teardown.sh new file mode 100755 index 00000000..e1d6afc0 --- /dev/null +++ b/deploy/acs-fact-lab/teardown.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +FACT_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) +STACKROX_DIR=${STACKROX_DIR:-"${FACT_ROOT}/../stackrox"} +TEARDOWN_ACS=false +TEARDOWN_SIGNOZ=false + +for arg in "$@"; do + case "${arg}" in + --acs) TEARDOWN_ACS=true ;; + --signoz) TEARDOWN_SIGNOZ=true ;; + *) echo "usage: $0 [--acs] [--signoz]" >&2; exit 2 ;; + esac +done + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib.sh" +require_command oc +require_command jq +require_command curl +require_kubeconfig + +if [[ -f "${ROXIE_ENVRC}" ]]; then + load_roxie_env + while IFS= read -r policy_name; do + while IFS= read -r policy_id; do + [[ -z "${policy_id}" ]] || rox_api DELETE "/v1/policies/${policy_id}" >/dev/null + done < <(policy_ids_by_name "${policy_name}") + done < <(jq -r '.[].name' "${SCRIPT_DIR}/policies.json") +fi + +if oc get namespace "${LAB_NAMESPACE}" >/dev/null 2>&1; then + node=$(lab_node) + oc -n "${LAB_NAMESPACE}" debug "node/${node}" -- chroot /host rm -f /var/tmp/acs-fact-host-marker || true + oc delete namespace "${LAB_NAMESPACE}" --wait=true +fi + +if [[ "${TEARDOWN_ACS}" == true ]]; then + ( + cd "${STACKROX_DIR}" + ./scripts/roxie.sh --skip-user-config --config "${SCRIPT_DIR}/roxie.yaml" teardown both + ./scripts/roxie.sh --skip-user-config --config "${SCRIPT_DIR}/roxie.yaml" teardown operator + ) +fi + +if [[ "${TEARDOWN_SIGNOZ}" == true ]]; then + oc delete namespace observability --wait=true + oc delete securitycontextconstraints signoz-scc --ignore-not-found +fi diff --git a/deploy/acs-fact-lab/workloads.yaml b/deploy/acs-fact-lab/workloads.yaml new file mode 100644 index 00000000..da430181 --- /dev/null +++ b/deploy/acs-fact-lab/workloads.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: acs-file-activity-lab + labels: + fact.stackrox.io/experiment: acs-policy +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rootfs-writer + namespace: acs-file-activity-lab +spec: + replicas: 1 + selector: + matchLabels: + app: rootfs-writer + template: + metadata: + labels: + app: rootfs-writer + fact.stackrox.io/scope: rootfs + spec: + automountServiceAccountToken: false + containers: + - name: writer + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + command: [/bin/sh, -c, sleep infinity] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emptydir-writer + namespace: acs-file-activity-lab +spec: + replicas: 1 + selector: + matchLabels: + app: emptydir-writer + template: + metadata: + labels: + app: emptydir-writer + fact.stackrox.io/scope: emptydir + spec: + automountServiceAccountToken: false + containers: + - name: writer + image: registry.access.redhat.com/ubi9/ubi-minimal:latest + command: [/bin/sh, -c, sleep infinity] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + - name: scratch + emptyDir: {}