I’m migrating from the Splunk Connect for Kubernetes Helm Chart to the SignalFX Splunk Otel Collector Helm Chart. I’m having trouble translating the custom filters that we have in the SCK chart to add metadata and send logs to indexes based on container name.
The custom filters we have on the fluentd
section of the values files for SCK are below
customFilters:
<truncated>
# Write each log into the index which matches the namespace's name
# If a container is marked as containing audit logs, we push it to a dedicated audit index
IndexNameFilter:
tag: tail.containers.**
type: jq_transformer
body: |
jq 'if .record.container_name|startswith("audit") then .record.splunk_index = ("audit-" + .record.namespace) | .record else .record.splunk_index = .record.namespace | .record end'
# Write k8s audit logs to its own index
K8sAuditIndexNameFilter:
tag: tail.file.kube:apiserver-audit
type: jq_transformer
body: |
jq '.record.splunk_index = "audit-kube" | .record'
Looking at the example values file and the logs only example I’ve translated that to the Otel chart with the transform processor
agent:
enabled: true
config:
processors:
<truncated>
transform/namespace_to_index:
error_mode: ignore
log_statements:
- context: log
statements:
- set(resource.attributes["com.splunk.index"], Concat(["audit", resource.attributes["k8s.namespace.name"]], "-")) where IsMatch(resource.attributes["k8s.container.name"], "audit-.*")
- set(resource.attributes["com.splunk.index"], resource.attributes["k8s.namespace.name"])
The transform/namespace_to_index
is included in service.pipeline.logs.processors
And the logsCollection.extraFileLogs
are set up to watch the audit files which I have triple checked exist in the expected places
extraFileLogs:
filelog/kube-audit: # sck logs go to audit-kube index, but got it in otel index for now.
include:
- /var/log/kubernetes/audit/kube-apiserver-audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-kube
com.splunk.sourcetype: kube:apiserver-audit
com.splunk.source: /var/log/kubernetes/audit/kube-apiserver-audit.log
filelog/linux-audit:
include:
- /var/log/audit/audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-linux
com.splunk.sourcetype: linux:audit
com.splunk.source: /var/log/audit/audit.log
However the pod logs show that the collector cannot find the audit files
2024-05-16T14:23:44.206Z warn fileconsumer/file.go:132 finding files: no files match the configured criteria {"kind": "receiver", "name": "filelog/kube-audit", "data_type": "logs", "component": "fileconsumer"}
Here is the entire values file for the Splunk Otel Collector
---
splunk-otel-collector:
clusterName: ${env:CLUSTER_NAME}
priorityClassName: "system-cluster-critical"
splunkPlatform:
# sets Splunk Platform as a destination. Use the /services/collector/event
# endpoint for proper extraction of fields.
endpoint: "https://our-endpoint.splunkcloud.com:443/services/collector/event"
token: "fake-placeholder-token"
index: "k8s" # should be able to replace with "" to dynamically set index as was done with SCK
logsEnabled: true
secret:
create: false
name: splunk-otel-credentials
validateSecret: false
logsCollection:
containers:
enabled: true
excludePaths:
- /var/log/containers/*fluent-bit*
- /var/log/containers/*speaker*
- /var/log/containers/*datadog*
- /var/log/containers/*collectd*
- /var/log/containers/*rook-ceph*
- /var/log/containers/*bird*
- /var/log/containers/*logdna*
- /var/log/containers/*6c6f616462616c2d*
- /var/log/containers/*lb-6c6f616462616c2d*
# extraOperators:
# - type: copy
# # Copy the name of the namespace associated with the log record.
# from: resource["k8s.namespace.name"]
# # Copy to the index key, so the record will be ingested under the index named after the k8s namespace.
# to: resource["com.splunk.index"]
extraFileLogs:
filelog/kube-audit: # sck logs go to audit-kube index, but got it in otel index for now.
include:
- /var/log/kubernetes/audit/kube-apiserver-audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-kube
com.splunk.sourcetype: kube:apiserver-audit
com.splunk.source: /var/log/kubernetes/audit/kube-apiserver-audit.log
filelog/linux-audit:
include:
- /var/log/audit/audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-linux
com.splunk.sourcetype: linux:audit
com.splunk.source: /var/log/audit/audit.log # can't find these results for SCK yet
extraAttributes:
fromLabels:
- key: k8s.pod.labels.cluster.name
tag_name: cluster_name
from: pod
- key: k8s.namespace.labels.cluster.class
tag_name: cluster_class
from: namespace
- key: k8s.namespace.labels.cluster.env
from: namespace
- key: k8s.node.name
tag_name: host
from: node
agent:
enabled: true
config:
processors:
# add cluster metadata to each logged event
# these are pulled in as environment variables due to a limitation
# as helm is unable to use templating when specifying values.
attributes/cluster_name_filter:
actions:
- key: cluster_name
action: upsert
value: ${env:CLUSTER_NAME}
attributes/cluster_class_filter:
actions:
- key: cluster_class
action: upsert
value: ${env:CLUSTER_CLASS}
attributes/cluster_env_filter:
actions:
- key: cluster_env
action: upsert
value: ${env:CLUSTER_ENV}
transform/namespace_to_index:
error_mode: ignore
log_statements:
- context: log
statements:
- set(resource.attributes["com.splunk.index"], Concat(["audit", resource.attributes["k8s.namespace.name"]], "-")) where IsMatch(resource.attributes["k8s.container.name"], "audit-.*")
- set(resource.attributes["com.splunk.index"], resource.attributes["k8s.namespace.name"])
# attributes/namespace_filter:
# actions:
# - key: com.splunk.index
# action: upsert
# value: k8s.namespace.name
# - key: logindex
# action: delete
exporters:
debug:
verbosity: detailed
service:
pipelines:
logs:
processors:
- memory_limiter
- k8sattributes
- filter/logs
- batch
- resourcedetection
- resource
- resource/logs
- attributes/cluster_name_filter
- attributes/cluster_class_filter
- attributes/cluster_env_filter
- transform/namespace_to_index
# - attributes/namespace_filter
receivers:
kubeletstats:
metric_groups:
- node
- pod
- container
filelog:
include:
- /var/log/pods/*/*/*.log
- /var/log/kubernetes/audit/*.log
- /var/log/audit/audit*.log
start_at: beginning
include_file_name: false
include_file_path: true
operators:
# parse cri-o format
- type: regex_parser
id: parser-crio
regex:
'^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)
?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout_type: gotime
layout: '2006-01-02T15:04:05.999999999Z07:00'
# Parse CRI-Containerd format
- type: regex_parser
id: parser-containerd
regex:
'^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)
?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
- type: copy
from: resource["k8s.namespace.name"]
to: resource["com.splunk.index"]
# Set Environment Variables to be set on every Pod in the DaemonSet
# Many of these are used as a work-around to include additional log metadata
# from what is available in `.Values` but inaccessible due to limitations of
# Helm.
extraEnvs:
- name: CLUSTER_NAME
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_NAME
- name: CLUSTER_CLASS
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_CLASS
- name: CLUSTER_ENV
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_ENV
# The container logs may actually be a series of symlinks. In order to read
# them, all directories need to be accessible by the logging pods. We use
# volumes and volume mounts to achieve that.
extraVolumes:
- name: containerdlogs
hostPath:
path: /var/lib/containerd/pod-logs
- name: podlogs
hostPath:
path: /var/log/pods
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: kubeauditlogs
hostPath:
path: /var/log/kubernetes/audit
- name: linuxauditlogs
hostPath:
path: /var/log/audit
extraVolumeMounts:
- name: containerdlogs
mountPath: /var/lib/containerd/pod-logs
readOnly: true
- name: podlogs
mountPath: /var/log/pods
readOnly: true
- name: varlogcontainers
mountPath: /var/log/containers
readOnly: true
- name: kubeauditlogs
mountPath: /var/log/kubernetes/audit
readOnly: true
- name: linuxauditlogs
mountPath: /var/log/audit
readOnly: true
resources:
limits:
cpu: 1
memory: 4Gi
requests:
cpu: 1
memory: 1Gi
1