Example commonlog-kb

config

let conf = {};
conf.projectName = 'PROJECT_NAME';  //project name

//Stream log server
conf.logStream = {};
conf.logStream.host = 'xx.xx.xx.xx';
conf.logStream.port = 80;
conf.logStream.maxQueueSize = 10000; //maxQueueSize

//Kafka 
conf.kafka = {};
conf.kafka.logger = true; //default false
conf.kafka.partition = 0; //default 0
conf.kafka.kafkaHost = [{},{}];
conf.kafka.kafkaHost[0].host = 'xx.xx.xx.xx';
conf.kafka.kafkaHost[0].port = 9092;
conf.kafka.kafkaHost[1].host = 'xx.xx.xx.xx';
conf.kafka.kafkaHost[1].port = 9092;

//Enable appLog
conf.log = {};
conf.log.time = 15;  //Minute
conf.log.size = null;   //maxsize per file, K
conf.log.path = './appLogPath/';  //path file
conf.log.level = 'debug'; //debug,info,warn,error
conf.log.autoAddResBody = true; //default  true
conf.log.format = 'json'; //json, pipe, json_cauldron //default json
conf.log.console = false; //output
conf.log.file = true;     //output
conf.log.stream = false;  //output //Stream to tcp server //needs conf.logStream
conf.log.kafka = false;    //Send appLog to kafka server //needs conf.kafka

//Enable custom log
conf.clog =  [{}];
conf.clog[0].name = 'refname'   //clog("refname", "log log log");
conf.clog[0].fileName = 'customFileName'     //file name
conf.clog[0].fileExt = 'customExtension'     //file extension default txt
conf.clog[0].time = 15;     //Minute
conf.clog[0].size = null;   //maxsize per file, K
conf.clog[0].path = './cPath/';  //path file
conf.clog[0].console = false; //output
conf.clog[0].file = true;     //output
conf.clog[0].stream = false;  //output //Stream to tcp server //needs conf.logStream
conf.clog[0].kafka = false;   //Send appLog to kafka server //needs conf.kafka

//Enable summaryLog
conf.summary = {};
conf.summary.time = 15;
conf.summary.size = null;   //maxsize per file, K
conf.summary.path = './summaryPath/';
conf.summary.format = 'json'; //write summaryLog in format "json" OR "pipe" OR "json_cauldron"  //default json
conf.summary.console = false; //output
conf.summary.file = true;     //output
conf.summary.stream = false;  //output //Stream to tcp server //needs conf.logStream
conf.summary.kafka = false;    //Send summaryLog to kafka server //needs conf.kafka

//Enable detail
conf.detail = {};
conf.detail.time = 15;
conf.detail.size = null;   //maxsize per file, K
conf.detail.path = './detailPath/';
conf.detail.rawData = true; //true == show raw data
conf.detail.format = 'json'; //write detailLog in format "json" OR "json_cauldron" OR "json_cauldron_analytic"//default json
conf.detail.console = false;  //output
conf.detail.file = true;      //output
conf.detail.stream = false;   //output //Stream to tcp server //needs conf.logStream
conf.detail.kafka = false;    //Send detailLog to kafka server //needs conf.kafka
conf.detail.analyticData = {  //valid with conf.detail.format == json_cauldron_analytic
  input:{
    default:{
      "Result": "path.to.result", //path to get value from input[].data
      "Desc": "path.to.desc"
    },node_cmd1:{
      "Result": "path.to.result",
      "Desc": "path.to.desc"
    },node_cmd2:{
      "Result": "path.to.result",
      "Desc": "path.to.desc"
    }
  },
  output:{
    default:{
      "Result": "path.to.result", //path to get value from output[].data
      "Desc": "path.to.desc"
    },node_cmd2:{
      "Result": "path.to.result",
      "Desc": "path.to.desc"
    }
  }
}

//Enable stat
conf.stat={};
conf.stat.time = 15;
conf.stat.size = null;   //maxsize per file, K
conf.stat.path = './statPath/';
conf.stat.pathDB = undefined; //optional, "Folder" path DB store
conf.stat.statInterval = 15;
conf.stat.console = false;    //output
conf.stat.file = true;        //output
conf.stat.stream = false;     //output //Stream to tcp server //needs conf.logStream
conf.stat.kafka = false;      //Send stat to kafka server //needs conf.kafka
conf.stat.prometheus = false; //output//support prometheus
conf.stat.prometheusMode = 'single'; //single, multi default single
conf.stat.prometheusCustomLabels = null; //custom label ["label1","label2"]
conf.stat.flush = false; //true == enable flush stat
conf.stat.format = 'json'; //json, pipe, json_cauldron //default json (json_cauldron will replace whitespce to "__")
//conf.stat.process  => String | Array
conf.stat.process = '/path/that/contain/data'
//OR
conf.stat.process = [{
  "name":"stat_name_a",
  "threshold": 3,
  "severity":"critical",  //critical, major
  "promLabel":{
      "label1":"value",
      "code":"200",
      "method":"get"
  }
},{
  "name":"stat_name_b",
  "threshold": 1
}]; 

//Enable alarm
conf.alarm = {};
conf.alarm.time = 15;
conf.alarm.size = null;
conf.alarm.path = './alarmPath/';
conf.alarm.console = false;
conf.alarm.file = true;
//conf.alarm.process  => String | Array
conf.alarm.process = '/path/that/contain/data'
//OR
conf.alarm.process = [{
  "id": 1,
  "name":"stat_name_a"
}]; 

init log with config (call once)

Please calli init() after lasted middleware

//simple
let logg = require('commonlog-kb').init(conf);

//integrate with express
let app = express();
let logg = require('commonlog-kb').init(conf, app);

logg.sessionID = (req,res) =>{
  return 'how to find session'
};

Check log already init

let alreadyInit = logg.ready();

Close log

logg.close();
//OR
logg.close((result)=>{
  //...
});

Example appLog

logg.debug('without session');
logg.debug('session', 'text to log');
logg.debug('session', {foo:'bar'},['foo','bar']);

logg.info('without session');
logg.info('session', 'text to log');
logg.info('session', {foo:'bar'},['foo','bar']);

logg.warn('without session');
logg.warn('session', 'text to log');
logg.warn('session', {foo:'bar'},['foo','bar']);

logg.error('without session');
logg.error('session', 'text to log');
logg.error('session', {foo:'bar'},['foo','bar']);

• example data (conf.log.format="pipe")

20190410 16:09:38.289|DESKTOP-E0NGPUA|PROJECT_NAME|0||debug|without session
20190410 16:09:38.290|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|debug|text to log
20190410 16:09:38.290|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|debug|{"foo":"bar"} ["foo","bar"]
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0||info|without session
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|info|text to log
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|info|{"foo":"bar"} ["foo","bar"]
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0||warn|without session
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|warn|text to log
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|warn|{"foo":"bar"} ["foo","bar"]
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0||error|without session
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|error|text to log
20190410 16:09:38.291|DESKTOP-E0NGPUA|PROJECT_NAME|0|session|error|{"foo":"bar"} ["foo","bar"]

• example data (conf.log.format="json")

{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"","InputTimeStamp":"20190826 16:01:21.715","Level":"debug","Message":"without session"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.721","Level":"debug","Message":"text to log"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.722","Level":"debug","Message":[{"foo":"bar"},["foo","bar"]]}

{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"","InputTimeStamp":"20190826 16:01:21.733","Level":"info","Message":"without session"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.735","Level":"info","Message":"text to log"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.736","Level":"info","Message":[{"foo":"bar"},["foo","bar"]]}

{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"","InputTimeStamp":"20190826 16:01:21.747","Level":"warn","Message":"without session"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.749","Level":"warn","Message":"text to log"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.750","Level":"warn","Message":[{"foo":"bar"},["foo","bar"]]}

{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"","InputTimeStamp":"20190826 16:01:21.751","Level":"error","Message":"without session"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.752","Level":"error","Message":"text to log"}
{"LogType":"App","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","Session":"session","InputTimeStamp":"20190826 16:01:21.753","Level":"error","Message":[{"foo":"bar"},["foo","bar"]]}

Example stat

logg.stat('recv command a');
logg.stat('recv command a');

• example app log (conf.log.format="pipe")

TIMESTAMP|HOSTNAME|STATNAME|COUNTER
2019-04-10 11:10:04.000|DESKTOP-E0NGPUA|recv command a|2
END

• example app log (conf.log.format="json")

{"LogType":"Stat","Host":"DESKTOP-E0NGPUA","AppName":"PROJECT_NAME","Instance":"0","TimeStamp":"2019-08-26 16:18:00.000","StatName":"recv command a","StatCount":2}

Example alarm

• example alarm

2019-04-10 12:20:34.000|DESKTOP-E0NGPUA|major|id|stat_name_b[1:2]

Force flush stat

http://IP:PORT/flushStat

Example summary

//CREATE summaryLog
var s1 = logg.summary('session1', 'initInvoke', 'cmd', 'identity');
//CREATE summaryLog without initInvoke parameter
var s1 = logg.summary('session1', undefined, 'cmd', 'identity');

//OPTIONAL Change identity after init summaryLog
s1.setIdentity("new identity");

//OPTIONAL add field
s1.addField('Field_1','value');
s1.addField('Field_1',{'example':'value'});

//OPTIONAL add sequence 
s1.addSuccessBlock('node', 'a', '20000', 'resultDesc');
s1.addSuccessBlock('node', 'b', 'resultCode', 'resultDesc');
s1.addSuccessBlock('node1', 'c', 'resultCode', 'resultDesc');
s1.addSuccessBlock('node1', 'cmd', 'resultCode', 'resultDesc');
s1.addErrorBlock('node1', 'cmd', 'resultCode', 'resultDesc');
//WITE summaryLog (sync)
s1.end('responseResult','responseDesc');
//WITE summaryLog (async)
s1.endASync('responseResult', 'responseDesc','transactionResult ','transactionDesc');

example summary log (pipe format)

20190724 09:31:33.344|DESKTOP-E0NGPUA|PROJECT_NAME|0|20190724 09:31:33.343|session1|initInvoke|cmd|identity|20000|sucesss|[node; a(1); [20000; resultDesc(1)], node; b(1); [resultCode; resultDesc(1)], node1; c(1); [resultCode; resultDesc(1)], node1; cmd(2); [resultCode; resultDesc(2)]]|20190724 09:31:33.344|1 ms

example summary log (json format)

{
  "InputTimeStamp": "20190724 09:32:59.870",
  "Host": "DESKTOP-E0NGPUA",
  "AppName": "PROJECT_NAME",
  "Instance": "0",
  "Session": "session1",
  "InitInvoke": "initInvoke",
  "Scenario": "cmd",
  "Identity": "identity",
  "ResponseResult": "20000",
  "ResponseDesc": "sucesss",
  "Sequences": [
    {
      "Node": "node",
      "Command": "a",
      "result": [
        {
          "Result": "20000",
          "Desc": "resultDesc"
        }
      ]
    },
    {
      "Node": "node",
      "Command": "b",
      "result": [
        {
          "Result": "resultCode",
          "Desc": "resultDesc"
        }
      ]
    },
    {
      "Node": "node1",
      "Command": "c",
      "result": [
        {
          "Result": "resultCode",
          "Desc": "resultDesc"
        }
      ]
    },
    {
      "Node": "node1",
      "Command": "cmd",
      "result": [
        {
          "Result": "resultCode",
          "Desc": "resultDesc"
        }
      ]
    }
  ],
  "EndProcessTimeStamp": "20190724 09:32:59.871",
  "ProcessTime": "1 ms"
}

Example detail

//Type of detailLog
- Input
    req, res, res_timeout, res_error
- Output
    req, res, req_retry_$COUNT/$MAX_COUNT(EX. req_retry_1/2)

//CREATE detailLog
var ddd = logg.detail('session1', 'initInvoke','cmd', 'identity');
//CREATE detailLog without initInvoke parameter
var ddd = logg.detail('session1', undefined, 'cmd', 'identity');


//OPTIONAL Change identity after init detailLog
ddd.setIdentity("new identity");

//Input without protocol
ddd.addInputRequest( 'node', 'cmd', 'invoke', 'rawData', {} );
ddd.addInputRequestimeout( 'node', 'cmd', 'invoke');
let resTimeInMilliSec = 1007;
ddd.addInputResponse( 'node', 'cmd', 'invoke', 'rawData', {}, resTimeInMilliSec );
ddd.addInputResponse( 'node', 'cmd', 'invoke', 'rawData', {} );
ddd.addInputResponseTimeout( 'node', 'cmd', 'invoke');
ddd.addInputResponseError( 'node', 'cmd', 'invoke');

//Output without protocol
ddd.addOutputRequest( 'node', 'cmd', 'invoke', 'rawData', {});
ddd.addOutputResponse( 'node', 'cmd', 'invoke', 'rawData', {});
let totalCount_ex1 = 1;
let maxRetry_ex1 = 2;
ddd.addOutputRequestRetry( 'node', 'cmd', 'invoke', 'rawData', {}, totalCount_ex1, maxRetry_ex1 );
ddd.end();

//Input/Output with protocol
let protocol="http";
let protocolMethod="get";
ddd.addInputRequest( 'node', 'cmd', 'invoke', 'rawData', {}, protocol, protocolMethod );
ddd.addOutputRequest( 'node', 'cmd', 'invoke', 'rawData', {}, protocol, protocolMethod);
ddd.end();

• example detail log

{
  "Host": "DESKTOP-E0NGPUA",
  "AppName": "PROJECT_NAME",
  "Instance": "0",
  "Session": "session1",
  "InitInvoke": "initInvoke",
  "Scenario": "cmd",
  "Identity": "identity",
  "InputTimeStamp": "20190724 09:37:49.200",
  "Input": [
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "req",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Protocol": "http.get",
      "Type": "req",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "res",
      "Data": {
        
      },
      "ResTime": "1007 ms"
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "res",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "res_timeout"
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "res_error"
    }
  ],
  "OutputTimeStamp": "20190724 09:37:49.200",
  "Output": [
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "req",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Protocol": "http.get",
      "Type": "req",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "res",
      "Data": {
        
      }
    },
    {
      "Invoke": "invoke",
      "Event": "node.cmd",
      "Type": "req_retry_1/2",
      "Data": {
        
      }
    }
  ],
  "ProcessingTime": "1 ms"
}

Version History

version 1.5.1

Add function addField(key, value) to summary log.

version 1.6.0

Add function stream log to logStash.
Fix calulate detaillog ProcessingTime.
Add config conf.stat.format pipe or json (default json).
Add config conf.log.format pipe or json (default json).

version 1.6.1

Add field "ProcessApp" in app log.

version 1.6.2, 1.6.3

Fix error init log without conf.stat.

version 1.6.4

Fix bug call add stat without conf

version 1.6.5

Change summaryLog.Sequences.result to summaryLog.Sequences.Result

version 1.6.6

Fixed appLog write Instances of Error objects with {}

version 1.6.7

Fixed detailLog case only have input

version 1.6.8

DetailLog add ResTime to input responseType res,res_timeout,res_error

version 1.6.9

DetailLog add function isRawDataEnabled()

version 1.6.10, 1.6.11, 1.6.12

SummaryLog add function isEnd()
SummaryLog check end,endASync twice

version 1.6.13

SummaryLog add function setIdentity(identity)
DetailLog add function setIdentity(identity)

version 1.6.14

fixed bug SummaryLog generate log type json by sequential

version 1.6.15

add function close(), to close I/O

version 2.0.0/2.0.1

change status management (avoid memory leak)

version 2.0.2

handle case delete current log file

version 2.1.0

add new format (EDR) for detailLog
add new format (CDR) for summaryLog
support Prometheus - Monitoring system

version 2.1.1

Do not write app log case get prometheus metrics

version 2.1.2

init metric from template file(conf.stat.process)

version 2.1.3, 2.1.4, 2.1.5

Fixed case A metric with the name $name has already been registered.

version 2.1.6

Fixed CDR log format EndProcessTimeStamp to dd/mm/yyyy HH:MM:ss.l

version 2.1.7

add app log type json_cauldron

version 2.1.8, 2.1.9

support multi attribute of metric (conf.stat.prometheusMode=single OR multi)

version 2.1.10

support multi attribute of metric from stat (ignore case)

version 2.1.11

change custom fields from summaryLog.addField() to summaryLog.CustomDes.$fieldName

version 2.1.12

add writeApplogOutgoing for force call onFinished

version 2.1.13

add isDebug() for check level log

version 2.1.14

fix bug flushstat not write file

version 2.1.15

support feature hide credential log

version 2.1.16

change formart date to YYYY-MM-DDThh:mm:ss.sssTZD (eg 1997-07-16T19:20:30.450+01:00)

[2.2.0, 2.2.1] - 2021-02-17

support push log to kafka

Added

conf.kafka conf.log.kafka conf.summary.kafka conf.detail.kafka conf.stat.kafka

Changed

Fixed

[2.2.2] - 2021-02-19

Added

Changed

change Kafka debug log to console log

Fixed

[2.3.0] - 2021-03-11

Added

add customlog clog("$refName","log log log");

Changed

Fixed

[2.3.1, 2.3.2] - 2021-06-28

Added

add detailLog.addInputRequestTimeout(node, cmd, invoke)

Changed

Fixed

[2.4.0] - 2021-10-20

Added

add detailLog new format="json_cauldron_analytic" add new config(conf.detail.analyticData) for detailLog

Changed

Fixed

[2.4.1] - 2022-02-01

Added

Changed

change detailLog field name custom1 to custom

Fixed

[2.4.2] - 2022-08-26

Added

conf.stat.pathDB = undefined; //optional, "Folder" path stat DB store

Changed

Fixed

[2.5.0] - 2022-11.07

Added

conf.stat.prometheusCustomLabels = null; //custom label ["label1","label2"] conf.stat.process = [{ "name":"stat_name_a", "promLabel":{ //custom label and value "label1":"value", "code":"200", "method":"get" } },{

Changed

Fixed

[2.5.1] - 2023-11.27

Added

add bypassExitHandler() to disable exit signal

Changed

Fixed