360 lines
13 KiB
YAML
360 lines
13 KiB
YAML
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
|
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role
|
|
rules:
|
|
- nonResourceURLs:
|
|
- /metrics
|
|
verbs:
|
|
- get
|
|
- apiGroups: [""]
|
|
resources:
|
|
- namespaces
|
|
- pods
|
|
- pods/logs
|
|
verbs: ["get", "list", "watch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role-binding
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
|
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: fluent-bit-cluster-info
|
|
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
|
data:
|
|
cluster.name: {{ .Values.statefulset.clusterName }}
|
|
http.server: "On"
|
|
http.port: "2020"
|
|
read.head: "Off"
|
|
read.tail: "On"
|
|
logs.region: {{ .Values.statefulset.logsRegion }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-config
|
|
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
|
labels:
|
|
k8s-app: fluent-bit
|
|
data:
|
|
fluent-bit.conf: |
|
|
[SERVICE]
|
|
Flush 5
|
|
Log_Level info
|
|
Daemon off
|
|
Parsers_File parsers.conf
|
|
HTTP_Server ${HTTP_SERVER}
|
|
HTTP_Listen 0.0.0.0
|
|
HTTP_Port ${HTTP_PORT}
|
|
storage.path /var/fluent-bit/state/flb-storage/
|
|
storage.sync normal
|
|
storage.checksum off
|
|
storage.backlog.mem_limit 5M
|
|
|
|
@INCLUDE nexus-log.conf
|
|
@INCLUDE nexus-request-log.conf
|
|
@INCLUDE nexus-audit-log.conf
|
|
@INCLUDE nexus-tasks-log.conf
|
|
|
|
nexus-log.conf: |
|
|
[INPUT]
|
|
Name tail
|
|
Tag nexus.nexus-log
|
|
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_nxrm-app-*.log
|
|
Parser docker
|
|
DB /var/fluent-bit/state/flb_container.db
|
|
Mem_Buf_Limit 5MB
|
|
Skip_Long_Lines Off
|
|
Refresh_Interval 10
|
|
Rotate_Wait 30
|
|
storage.type filesystem
|
|
Read_from_Head ${READ_FROM_HEAD}
|
|
|
|
[FILTER]
|
|
Name kubernetes
|
|
Match nexus.nexus-log
|
|
Kube_URL https://kubernetes.default.svc:443
|
|
Kube_Tag_Prefix application.var.log.containers.
|
|
Merge_Log On
|
|
Merge_Log_Key log_processed
|
|
K8S-Logging.Parser On
|
|
K8S-Logging.Exclude Off
|
|
Labels Off
|
|
Annotations Off
|
|
|
|
[OUTPUT]
|
|
Name cloudwatch_logs
|
|
Match nexus.nexus-log
|
|
region ${AWS_REGION}
|
|
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
|
log_stream_prefix ${HOST_NAME}-nexus.log-
|
|
auto_create_group true
|
|
extra_user_agent container-insights
|
|
|
|
nexus-request-log.conf: |
|
|
[INPUT]
|
|
Name tail
|
|
Tag nexus.request-log
|
|
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_request-log-*.log
|
|
Parser docker
|
|
DB /var/fluent-bit/state/flb_container.db
|
|
Mem_Buf_Limit 5MB
|
|
Skip_Long_Lines Off
|
|
Refresh_Interval 10
|
|
Rotate_Wait 30
|
|
storage.type filesystem
|
|
Read_from_Head ${READ_FROM_HEAD}
|
|
|
|
[FILTER]
|
|
Name kubernetes
|
|
Match nexus.request-log
|
|
Kube_URL https://kubernetes.default.svc:443
|
|
Kube_Tag_Prefix application.var.log.containers.
|
|
Merge_Log On
|
|
Merge_Log_Key log_processed
|
|
K8S-Logging.Parser On
|
|
K8S-Logging.Exclude Off
|
|
Labels Off
|
|
Annotations Off
|
|
|
|
[OUTPUT]
|
|
Name cloudwatch_logs
|
|
Match nexus.request-log
|
|
region ${AWS_REGION}
|
|
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
|
log_stream_prefix ${HOST_NAME}-request.log-
|
|
auto_create_group true
|
|
extra_user_agent container-insights
|
|
|
|
nexus-audit-log.conf: |
|
|
[INPUT]
|
|
Name tail
|
|
Tag nexus.audit-log
|
|
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_audit-log-*.log
|
|
Parser docker
|
|
DB /var/fluent-bit/state/flb_container.db
|
|
Mem_Buf_Limit 5MB
|
|
Skip_Long_Lines Off
|
|
Refresh_Interval 10
|
|
Rotate_Wait 30
|
|
storage.type filesystem
|
|
Read_from_Head ${READ_FROM_HEAD}
|
|
|
|
[FILTER]
|
|
Name kubernetes
|
|
Match nexus.audit-log
|
|
Kube_URL https://kubernetes.default.svc:443
|
|
Kube_Tag_Prefix application.var.log.containers.
|
|
Merge_Log On
|
|
Merge_Log_Key log_processed
|
|
K8S-Logging.Parser On
|
|
K8S-Logging.Exclude Off
|
|
Labels Off
|
|
Annotations Off
|
|
|
|
[OUTPUT]
|
|
Name cloudwatch_logs
|
|
Match nexus.audit-log
|
|
region ${AWS_REGION}
|
|
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
|
log_stream_prefix ${HOST_NAME}-audit.log-
|
|
auto_create_group true
|
|
extra_user_agent container-insights
|
|
|
|
nexus-tasks-log.conf: |
|
|
[INPUT]
|
|
Name tail
|
|
Tag nexus.tasks-log
|
|
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version | replace "." "-" }}-{{ .Release.Name }}-{{ .Values.statefulset.name }}*{{ .Values.namespaces.nexusNs }}_tasks-log-*.log
|
|
Parser docker
|
|
DB /var/fluent-bit/state/flb_container.db
|
|
Mem_Buf_Limit 5MB
|
|
Skip_Long_Lines Off
|
|
Refresh_Interval 10
|
|
Rotate_Wait 30
|
|
storage.type filesystem
|
|
Read_from_Head ${READ_FROM_HEAD}
|
|
|
|
[FILTER]
|
|
Name kubernetes
|
|
Match nexus.tasks-log
|
|
Kube_URL https://kubernetes.default.svc:443
|
|
Kube_Tag_Prefix application.var.log.containers.
|
|
Merge_Log On
|
|
Merge_Log_Key log_processed
|
|
K8S-Logging.Parser On
|
|
K8S-Logging.Exclude Off
|
|
Labels Off
|
|
Annotations Off
|
|
|
|
[OUTPUT]
|
|
Name cloudwatch_logs
|
|
Match nexus.tasks-log
|
|
region ${AWS_REGION}
|
|
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
|
log_stream_prefix ${HOST_NAME}-tasks.log-
|
|
auto_create_group true
|
|
extra_user_agent container-insights
|
|
|
|
parsers.conf: |
|
|
[PARSER]
|
|
Name docker
|
|
Format json
|
|
Time_Key time
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
|
|
|
[PARSER]
|
|
Name syslog
|
|
Format regex
|
|
Regex ^(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
|
|
Time_Key time
|
|
Time_Format %b %d %H:%M:%S
|
|
|
|
[PARSER]
|
|
Name container_firstline
|
|
Format regex
|
|
Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
|
|
Time_Key time
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
|
|
|
[PARSER]
|
|
Name cwagent_firstline
|
|
Format regex
|
|
Regex (?<log>(?<="log":")\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
|
|
Time_Key time
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
|
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
|
labels:
|
|
k8s-app: fluent-bit
|
|
version: v1
|
|
kubernetes.io/cluster-service: "true"
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
k8s-app: fluent-bit
|
|
template:
|
|
metadata:
|
|
labels:
|
|
k8s-app: fluent-bit
|
|
version: v1
|
|
kubernetes.io/cluster-service: "true"
|
|
spec:
|
|
containers:
|
|
- name: fluent-bit
|
|
image: amazon/aws-for-fluent-bit:{{ .Values.statefulset.fluentBitVersion }}
|
|
imagePullPolicy: Always
|
|
env:
|
|
- name: AWS_REGION
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: logs.region
|
|
- name: CLUSTER_NAME
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: cluster.name
|
|
- name: HTTP_SERVER
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: http.server
|
|
- name: HTTP_PORT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: http.port
|
|
- name: READ_FROM_HEAD
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: read.head
|
|
- name: READ_FROM_TAIL
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: fluent-bit-cluster-info
|
|
key: read.tail
|
|
- name: HOST_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: CI_VERSION
|
|
value: "k8s/1.3.7"
|
|
# the below var is just to force DaemonSet restarts when changing configuration stored in ConfigMap above
|
|
- name: FOO_VERSION
|
|
value: "16"
|
|
resources:
|
|
limits:
|
|
memory: 200Mi
|
|
requests:
|
|
cpu: 500m
|
|
memory: 100Mi
|
|
volumeMounts:
|
|
# Please don't change below read-only permissions
|
|
- name: fluentbitstate
|
|
mountPath: /var/fluent-bit/state
|
|
- name: varlog
|
|
mountPath: /var/log
|
|
readOnly: true
|
|
- name: varlibdockercontainers
|
|
mountPath: /var/lib/docker/containers
|
|
readOnly: true
|
|
- name: fluent-bit-config
|
|
mountPath: /fluent-bit/etc/
|
|
- name: runlogjournal
|
|
mountPath: /run/log/journal
|
|
readOnly: true
|
|
- name: dmesg
|
|
mountPath: /var/log/dmesg
|
|
readOnly: true
|
|
terminationGracePeriodSeconds: 120
|
|
volumes:
|
|
- name: fluentbitstate
|
|
hostPath:
|
|
path: /var/fluent-bit/state
|
|
- name: varlog
|
|
hostPath:
|
|
path: /var/log
|
|
- name: varlibdockercontainers
|
|
hostPath:
|
|
path: /var/lib/docker/containers
|
|
- name: fluent-bit-config
|
|
configMap:
|
|
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-config
|
|
- name: runlogjournal
|
|
hostPath:
|
|
path: /run/log/journal
|
|
- name: dmesg
|
|
hostPath:
|
|
path: /var/log/dmesg
|
|
serviceAccountName: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/master
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- operator: "Exists"
|
|
effect: "NoExecute"
|
|
- operator: "Exists"
|
|
effect: "NoSchedule" |