Creating public repository
This commit is contained in:
Mike Oliverio
360
aws-single-instance-resiliency/templates/fluent-bit.yaml
Normal file
360
aws-single-instance-resiliency/templates/fluent-bit.yaml
Normal file
@@ -0,0 +1,360 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
||||
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role
|
||||
rules:
|
||||
- nonResourceURLs:
|
||||
- /metrics
|
||||
verbs:
|
||||
- get
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- namespaces
|
||||
- pods
|
||||
- pods/logs
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role-binding
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
||||
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: fluent-bit-cluster-info
|
||||
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
||||
data:
|
||||
cluster.name: {{ .Values.deployment.clusterName }}
|
||||
http.server: "On"
|
||||
http.port: "2020"
|
||||
read.head: "Off"
|
||||
read.tail: "On"
|
||||
logs.region: {{ .Values.deployment.logsRegion }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-config
|
||||
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
||||
labels:
|
||||
k8s-app: fluent-bit
|
||||
data:
|
||||
fluent-bit.conf: |
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Daemon off
|
||||
Parsers_File parsers.conf
|
||||
HTTP_Server ${HTTP_SERVER}
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port ${HTTP_PORT}
|
||||
storage.path /var/fluent-bit/state/flb-storage/
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 5M
|
||||
|
||||
@INCLUDE nexus-log.conf
|
||||
@INCLUDE nexus-request-log.conf
|
||||
@INCLUDE nexus-audit-log.conf
|
||||
@INCLUDE nexus-tasks-log.conf
|
||||
|
||||
nexus-log.conf: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Tag nexus.nexus-log
|
||||
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment-*-*_{{ .Values.namespaces.nexusNs }}_nxrm-app-*.log
|
||||
Parser docker
|
||||
DB /var/fluent-bit/state/flb_container.db
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines Off
|
||||
Refresh_Interval 10
|
||||
Rotate_Wait 30
|
||||
storage.type filesystem
|
||||
Read_from_Head ${READ_FROM_HEAD}
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match nexus.nexus-log
|
||||
Kube_URL https://kubernetes.default.svc:443
|
||||
Kube_Tag_Prefix application.var.log.containers.
|
||||
Merge_Log On
|
||||
Merge_Log_Key log_processed
|
||||
K8S-Logging.Parser On
|
||||
K8S-Logging.Exclude Off
|
||||
Labels Off
|
||||
Annotations Off
|
||||
|
||||
[OUTPUT]
|
||||
Name cloudwatch_logs
|
||||
Match nexus.nexus-log
|
||||
region ${AWS_REGION}
|
||||
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
||||
log_stream_prefix ${HOST_NAME}-nexus.log-
|
||||
auto_create_group true
|
||||
extra_user_agent container-insights
|
||||
|
||||
nexus-request-log.conf: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Tag nexus.request-log
|
||||
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment-*-*_{{ .Values.namespaces.nexusNs }}_request-log-*.log
|
||||
Parser docker
|
||||
DB /var/fluent-bit/state/flb_container.db
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines Off
|
||||
Refresh_Interval 10
|
||||
Rotate_Wait 30
|
||||
storage.type filesystem
|
||||
Read_from_Head ${READ_FROM_HEAD}
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match nexus.request-log
|
||||
Kube_URL https://kubernetes.default.svc:443
|
||||
Kube_Tag_Prefix application.var.log.containers.
|
||||
Merge_Log On
|
||||
Merge_Log_Key log_processed
|
||||
K8S-Logging.Parser On
|
||||
K8S-Logging.Exclude Off
|
||||
Labels Off
|
||||
Annotations Off
|
||||
|
||||
[OUTPUT]
|
||||
Name cloudwatch_logs
|
||||
Match nexus.request-log
|
||||
region ${AWS_REGION}
|
||||
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
||||
log_stream_prefix ${HOST_NAME}-request.log-
|
||||
auto_create_group true
|
||||
extra_user_agent container-insights
|
||||
|
||||
nexus-audit-log.conf: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Tag nexus.audit-log
|
||||
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment-*-*_{{ .Values.namespaces.nexusNs }}_audit-log-*.log
|
||||
Parser docker
|
||||
DB /var/fluent-bit/state/flb_container.db
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines Off
|
||||
Refresh_Interval 10
|
||||
Rotate_Wait 30
|
||||
storage.type filesystem
|
||||
Read_from_Head ${READ_FROM_HEAD}
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match nexus.audit-log
|
||||
Kube_URL https://kubernetes.default.svc:443
|
||||
Kube_Tag_Prefix application.var.log.containers.
|
||||
Merge_Log On
|
||||
Merge_Log_Key log_processed
|
||||
K8S-Logging.Parser On
|
||||
K8S-Logging.Exclude Off
|
||||
Labels Off
|
||||
Annotations Off
|
||||
|
||||
[OUTPUT]
|
||||
Name cloudwatch_logs
|
||||
Match nexus.audit-log
|
||||
region ${AWS_REGION}
|
||||
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
||||
log_stream_prefix ${HOST_NAME}-audit.log-
|
||||
auto_create_group true
|
||||
extra_user_agent container-insights
|
||||
|
||||
nexus-tasks-log.conf: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Tag nexus.tasks-log
|
||||
Path /var/log/containers/{{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-nxrm.deployment-*-*_{{ .Values.namespaces.nexusNs }}_tasks-log-*.log
|
||||
Parser docker
|
||||
DB /var/fluent-bit/state/flb_container.db
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines Off
|
||||
Refresh_Interval 10
|
||||
Rotate_Wait 30
|
||||
storage.type filesystem
|
||||
Read_from_Head ${READ_FROM_HEAD}
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match nexus.tasks-log
|
||||
Kube_URL https://kubernetes.default.svc:443
|
||||
Kube_Tag_Prefix application.var.log.containers.
|
||||
Merge_Log On
|
||||
Merge_Log_Key log_processed
|
||||
K8S-Logging.Parser On
|
||||
K8S-Logging.Exclude Off
|
||||
Labels Off
|
||||
Annotations Off
|
||||
|
||||
[OUTPUT]
|
||||
Name cloudwatch_logs
|
||||
Match nexus.tasks-log
|
||||
region ${AWS_REGION}
|
||||
log_group_name /aws/containerinsights/${CLUSTER_NAME}/nexus-logs
|
||||
log_stream_prefix ${HOST_NAME}-tasks.log-
|
||||
auto_create_group true
|
||||
extra_user_agent container-insights
|
||||
|
||||
parsers.conf: |
|
||||
[PARSER]
|
||||
Name docker
|
||||
Format json
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
||||
|
||||
[PARSER]
|
||||
Name syslog
|
||||
Format regex
|
||||
Regex ^(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
|
||||
Time_Key time
|
||||
Time_Format %b %d %H:%M:%S
|
||||
|
||||
[PARSER]
|
||||
Name container_firstline
|
||||
Format regex
|
||||
Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
||||
|
||||
[PARSER]
|
||||
Name cwagent_firstline
|
||||
Format regex
|
||||
Regex (?<log>(?<="log":")\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
||||
namespace: {{ .Values.namespaces.cloudwatchNs }}
|
||||
labels:
|
||||
k8s-app: fluent-bit
|
||||
version: v1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: fluent-bit
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: fluent-bit
|
||||
version: v1
|
||||
kubernetes.io/cluster-service: "true"
|
||||
spec:
|
||||
containers:
|
||||
- name: fluent-bit
|
||||
image: amazon/aws-for-fluent-bit:2.10.0
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: AWS_REGION
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: logs.region
|
||||
- name: CLUSTER_NAME
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: cluster.name
|
||||
- name: HTTP_SERVER
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: http.server
|
||||
- name: HTTP_PORT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: http.port
|
||||
- name: READ_FROM_HEAD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: read.head
|
||||
- name: READ_FROM_TAIL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: fluent-bit-cluster-info
|
||||
key: read.tail
|
||||
- name: HOST_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: CI_VERSION
|
||||
value: "k8s/1.3.7"
|
||||
# the below var is just to force DaemonSet restarts when changing configuration stored in ConfigMap above
|
||||
- name: FOO_VERSION
|
||||
value: "16"
|
||||
resources:
|
||||
limits:
|
||||
memory: 200Mi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 100Mi
|
||||
volumeMounts:
|
||||
# Please don't change below read-only permissions
|
||||
- name: fluentbitstate
|
||||
mountPath: /var/fluent-bit/state
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
readOnly: true
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
- name: fluent-bit-config
|
||||
mountPath: /fluent-bit/etc/
|
||||
- name: runlogjournal
|
||||
mountPath: /run/log/journal
|
||||
readOnly: true
|
||||
- name: dmesg
|
||||
mountPath: /var/log/dmesg
|
||||
readOnly: true
|
||||
terminationGracePeriodSeconds: 120
|
||||
volumes:
|
||||
- name: fluentbitstate
|
||||
hostPath:
|
||||
path: /var/fluent-bit/state
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
- name: fluent-bit-config
|
||||
configMap:
|
||||
name: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit-config
|
||||
- name: runlogjournal
|
||||
hostPath:
|
||||
path: /run/log/journal
|
||||
- name: dmesg
|
||||
hostPath:
|
||||
path: /var/log/dmesg
|
||||
serviceAccountName: {{ .Chart.Name }}-{{ .Chart.Version }}.{{ .Release.Name }}-fluent-bit
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/master
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- operator: "Exists"
|
||||
effect: "NoExecute"
|
||||
- operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
Reference in New Issue
Block a user