Carlos Aguni

Highly motivated self-taught IT analyst. Always learning and ready to explore new skills. An eternal apprentice.

Fluent bit from stdin or programs examples

17 Sep 2022 »

program + jq + fluent-bit stdin

PYTHONUNBUFFERED=1 python3 app.py 2>&1 | \
    jq --unbuffered --stream -R -c '{line: . , app: "lala"}' | \
    fluent-bit -i stdin -o 


[FILTER]
    name modify
    match *
    add source lala


[OUTPUT]
    name stdout
    format json_lines 
    match *

inside docker

touch /logoutput

fluent-bit -i tail -p path=/logoutput -p read_from_head=true -o stdout -m '*'

python3 -u app000.py > /test 2>&1

with named pipe

PYTHONUNBUFFERED=1 python3 app000.py 2>&1 | tee testfile

fluent-bit -i tail -p path=testfile -p 'read_from_head=true' -o stdout -m '*'

while [ 1 ]; do du -sh testfile; > testfile; sleep 5; done

multilog

python3 app000.py 2>&1 | tee >(mutilog s1000000 n10 ./mtest)

[root@lab002 ~]# ls mtest
@40000000632541e90e595cdc.s  @40000000632541e90e6668b4.s  @40000000632541e90e731ab4.s  current
@40000000632541e90e5d0274.s  @40000000632541e90e69f2f4.s  @40000000632541e90e76b0ac.s  lock
@40000000632541e90e6065a4.s  @40000000632541e90e6d523c.s  @40000000632541e90e7a4e74.s  state

cat mtest/current