This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This setup works seemlessly if you deploy microservices in any of the big 3 cloud providers using terraform and helm as I described in the following github projects: | |
# https://github.com/nestoru/gcp-microservices-poc | |
# https://github.com/nestoru/azure-microservices-poc | |
# https://github.com/nestoru/aws-microservices-poc | |
#1. Set in your protected environment specific IaC project the helm vars: | |
# api-helm/values.yaml | |
fluentBit: | |
enabled: true | |
s3: | |
accessKey: *** | |
secretKey: *** | |
bucket: *** | |
region: *** | |
#2. Define a deployment descriptor for a fluent-bit pod to be deployed by helm | |
# api-helm/templates/fluent-bit-deployment.yaml | |
{{- if .Values.fluentBit.enabled }} | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: fluent-bit | |
labels: | |
app: fluent-bit | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: fluent-bit | |
template: | |
metadata: | |
labels: | |
app: fluent-bit | |
spec: | |
serviceAccountName: fluent-bit | |
containers: | |
- name: fluent-bit | |
image: fluent/fluent-bit:latest | |
volumeMounts: | |
- name: fluent-bit-config | |
mountPath: /fluent-bit/etc/ | |
- name: varlog | |
mountPath: /var/log | |
readOnly: false # Ensure this is false to allow writing | |
env: | |
- name: AWS_ACCESS_KEY_ID | |
valueFrom: | |
secretKeyRef: | |
name: fluent-bit-aws-credentials | |
key: AWS_ACCESS_KEY_ID | |
- name: AWS_SECRET_ACCESS_KEY | |
valueFrom: | |
secretKeyRef: | |
name: fluent-bit-aws-credentials | |
key: AWS_SECRET_ACCESS_KEY | |
volumes: | |
- name: fluent-bit-config | |
configMap: | |
name: fluent-bit-config | |
- name: varlog | |
hostPath: | |
path: /var/log | |
type: DirectoryOrCreate # This ensures the directory is created if it doesn't exist | |
{{- end }} | |
#3. Define a configmap kubernetes descriptor containing your settings for log collection for helm to deploy | |
# api-helm/templates/fluent-bit-configmap.yaml | |
{{- if .Values.fluentBit.enabled }} | |
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: fluent-bit-config | |
data: | |
fluent-bit.conf: |- | |
[SERVICE] | |
Flush 5 | |
Log_Level info | |
Parsers_File parsers.conf | |
HTTP_Server On | |
HTTP_Listen 0.0.0.0 | |
HTTP_Port 2020 | |
[INPUT] | |
Name tail | |
Path /var/log/containers/{{ .Values.appServiceName }}-{{ .Values.majorVersion }}-test*.log | |
Tag {{ .Values.appServiceName }}.{{ .Values.majorVersion }} | |
Parser docker | |
DB /var/log/flb_kube.db | |
Mem_Buf_Limit 5MB | |
Skip_Long_Lines On | |
[OUTPUT] | |
Name s3 | |
Match {{ .Values.appServiceName }}.{{ .Values.majorVersion }} | |
bucket {{ .Values.fluentBit.s3.bucket }} | |
region {{ .Values.fluentBit.s3.region }} | |
s3_key_format /%Y/%m/%d/{{ .Values.appServiceName }}-{{ .Values.majorVersion }}.log | |
upload_timeout 1m | |
use_put_object On | |
storage.total_limit_size 1G | |
parsers.conf: |- | |
[PARSER] | |
Name docker | |
Format json | |
Time_Key time | |
Time_Format %Y-%m-%dT%H:%M:%S.%L | |
Time_Keep On | |
{{- end }} | |
#4. Define a kubernetes secret descriptor for the S3 bucket to be deployed by helm | |
# api-helm/templates/fluent-bit-secret.yaml | |
apiVersion: v1 | |
kind: Secret | |
metadata: | |
name: fluent-bit-aws-credentials | |
type: Opaque | |
stringData: # Note: Using `stringData` for convenience with plain text values | |
AWS_ACCESS_KEY_ID: {{ .Values.fluentBit.s3.accessKey }} | |
AWS_SECRET_ACCESS_KEY: {{ .Values.fluentBit.s3.secretKey }} | |
#5 Define a kubernetes serviceaccount descriptor to be deployed by helm | |
# api-helm/templates/fluent-bit-service-account.yaml | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: fluent-bit | |
namespace: {{ .Release.Namespace | default "default" }} |
No comments:
Post a Comment