Specific log file

Sending logs from a specific folder¶

Using fluentd enables logs to transfer to the log collecting server by reading the log from a local JSON file as shown picture below.

Example¶

Example for the log in local JSON file

{
  "dateTime": "2022-04-15 13:02:45",
  "timezone": "GMT+09:00",
  "channel": "C2S",
  "source": "Web",
  "serverId": "1",
  "osVer": "4.3.1",
  "appId": "com.gcp2.hivesdk.android.google.global.normal",
  "company": "C2S",
  "model": "SHW-M250S",
  "lang": "ko",
  "class": "sample value of class",
  "os": "A",
  "appVer": "1.0",
  "cpu": "armeabi-v7a",
  "market": "GO",
  "clientIp": "127.0.0.1",
  "guid": "4314nbh31211234ld",
  "sdkVer": "2.0.1",
  "serverIp": "127.0.0.2",
  "category": "hive_login_log",
  "newUser": "N",
  "hiveSnsType": "FB",
  "deviceName": "SM-G935",
  "osVersionFixed": "9.2.1",
  "serverUid": "QA-2",
  "level": "7",
  "isEmulator": "1"
}

How to stack logs in the log file
Configuring td-agent.conf as shown below, you can implement stacking logs in the log file.

<source>
@type tail
path /game/sample/log/com2us.log
pos_file /var/log/td-agent/com2us_filelog.pos
tag ha2union.game.sample.com2us_log
format json
</source>

<source>
@type tail
path /game/sample/log/com2usHoldings.log
pos_file /var/log/td-agent/com2usHoldings_filelog.pos
tag ha2union.game.sample.com2usHoldings_log
format json
</source>

<source>
@type tail
path /game/sample/log/com2usPlatform.log
pos_file /var/log/td-agent/com2usPlatform_filelog.pos
tag ha2union.game.sample.com2usPlatform_log
format json
</source>


<match ha2union.** >
@type copy
@include td-agent-forest.conf  # fluentd-plugin-forest
    <store>
        @type forward
        send_timeout 60s
        recover_wait 10s
        heartbeat_interval 1s
        #heartbeat_type tcp
        transport tcp
        phi_threshold 16
        hard_timeout 60s

        require_ack_response true # at-least-once

        buffer_type file
        buffer_chunk_limit 8m                # 허용되는 최대 청크의 크기
        buffer_queue_limit 10240             # 출력 큐의 최대길이
        buffer_path /data/log/td-agent/buffer/forward.buffer
        flush_interval 1s                    # 다음 버퍼를 호출하기 전에 대기하는 간격

        <server>
            name hiveanalyticslog-server-03
            host analytics-hivelog-03.withhive.com
            port 24224
            weight 50
        </server>

        <server>
            name hiveanalyticslog-server-04
            host analytics-hivelog-04.withhive.com
            port 24224
            weight 50
        </server>

        <secondary>
            @type file
            path /var/log/td-agent/forward-failed # 각자 환경에 따라 변경
        </secondary>

    </store>
</match>

<source>
    @type forward
</source>

secondary: Backup option. You can use secondary when it is unavailable to use the server because an error occurs, and back up the logs to path as a file format.
Buffer Plugin : Buffer is a set of chunks. Chunks are flushed to the output queue periodically and transferred to the designated target. The following image shows the operating sequence.