Build Centos
yum -y install cmake make flex bison
yum -y install gcc gcc-c++ openssl-devel
wget https://github.com/fluent/fluent-bit/archive/refs/tags/v1.9.8.tar.gz
tar xzvf v1.9.8.tar.gz
cd fluent-bit-1.9.8/
cd build
cmake ../ -DFLB_CONFIG_YAML=Off
make
make DESTDIR=/opt/fluent-bit install
echo 'export PATH="/opt/fluent-bit/usr/local/bin:$PATH"' >> ~/.bashrc
. ~/.bashrc
https://docs.fluentbit.io/manual/pipeline/outputs/standard-output
https://docs.fluentbit.io/manual/pipeline/filters/modify
build.sh
#!/bin/bash
docker rm -f test
docker build -t test .
docker run -dit --name test test
Dockerfile
FROM python:3.7.4
RUN apt update; apt install less -y
RUN pip3 install flask flask_cors supervisor
COPY supervisord.conf /etc/supervisord.conf
COPY file /file
COPY test-flask.py /test-flask.py
COPY test-write.py /test-write.py
COPY fluent-bit /fluent-bit
COPY fluent.conf /fluent.conf
CMD ["/usr/local/bin/supervisord"]
fluent.conf
[SERVICE]
tag log.fluentbit_svc
log_level info
flush 5
[INPUT]
name tail
tag log.supervisord
read_from_head true
path /tmp/supervisord.log
#[INPUT]
# name tail
# tag log.fluentbit
# read_from_head true
# path /tmp/fluentbit.log
[INPUT]
name tail
tag log.p1
read_from_head true
path /tmp/p1.log
[INPUT]
name tail
tag log.p2
read_from_head true
path /tmp/p2.log
[FILTER]
name modify
match log.supervisord
add source log.supervisord
[FILTER]
name modify
match log.p1
add source log.p1
[FILTER]
name modify
match log.p2
add source log.p2
[OUTPUT]
name stdout
format json_lines
match *
supervisord.conf
[supervisord]
nodaemon=true
logfile=/tmp/supervisord.log
logfile_maxbytes=5MB
[program:fluent-bit]
command=/fluent-bit -c fluent.conf
stdout_logfile=/tmp/fluentbit.log
directory=/
logfile_maxbytes=1MB
redirect_stderr=true
[program:p-write-to-file]
command=/usr/local/bin/python3 /test-write.py
stdout_logfile=/tmp/p1.log
logfile_maxbytes=1MB
#stdout_logfile=/proc/1/fd/1
#stdout_logfile_maxbytes = 0
redirect_stderr=true
environment=PYTHONUNBUFFERED=1
[program:p-flask]
command=/usr/local/bin/python3 /test-flask.py
stdout_logfile=/tmp/p2.log
logfile_maxbytes=1MB
#stdout_logfile=/proc/1/fd/1
#stdout_logfile_maxbytes = 0
redirect_stderr=true
environment=PYTHONUNBUFFERED=1
test-flask.py
from flask import Flask, request, jsonify, json, abort, redirect, url_for, render_template
from flask_cors import CORS, cross_origin
import os
import re
import subprocess
import traceback
app = Flask(__name__, template_folder='template')
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'
@app.route('/', methods=['GET', 'POST'])
@cross_origin()
def main():
txt = open("/file").read()
return txt
return "hello world"
# gunicorn --workers=2 'app:create_app()' --bind=0.0.0.0:<port>
def create_app():
return app
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8080)
#test
#with app.test_client() as c:
# rs = c.get("/")
# print(rs.data)
test-write.py
import time
idx = 0
while True:
print("write")
with open("/file", "a+") as f:
f.write(f"{idx}\n")
idx+=1
time.sleep(1)