THIS IS A TEST INSTANCE. ALL YOUR CHANGES WILL BE LOST!!!!
Install syslog service
1) set yum repo for rsyslog
...
rsyslog-7.6.7-1.el6.x86_64
Add eagle log4j config for appending logs to syslog server
log4j.rootLogger=INFO
log4j.logger.org.apache.eagle.executor.AlertExecutor=DEBUG,SYSLOG
# Syslog Appender
log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
log4j.appender.SYSLOG.syslogHost=<syslog_server_hostname>
log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
log4j.appender.SYSLOG.layout.conversionPattern=%-4r [%t] %-5p %c %x - %m%n
log4j.appender.SYSLOG.Facility=LOCAL0
Add syslog config to filter & store eagle alert log
:msg, !contains, "A new alert is triggered: " ~
local0.* /home/eagle/eagle.alert.log
After the following configuration, the following eagle log will be persisted in file located at /home/eagle/eagle.alert.log
if(LOG.isDebugEnabled()) LOG.debug("A new alert is triggered: "+alertExecutorId + ", partition " + partitionSeq + ", Got an alert with output context: " + entity.getAlertContext() + ", for policy " + evaluator);
Convert Eagle alert log to splunk log format
When forwarding Eagle alert info to syslog server, we need convert it to splunk key value log format like following:
[Timestamp] Hostname key1=value1 key2=value2 key3=value3...
| Code Block | ||
|---|---|---|
| ||
{
"timestamp": 1452222222991,
"tags": {
"site": "sandbox",
"alertSource": "pid@hostname",
"dataSource": "NNGCLog",
"sourceStreams": "NNGCLogStream",
"policyId": "NamenodeGCAlert",
"alertExecutorId": "NNGCAlert"
},
"alertContext": {
"properties": {
"tenuredAreaGCed": "false",
"youngAreaGCed": "true",
"eventType": "YoungGC",
"youngTotalHeapK": "9437184",
"totalHeapUsageAvailable": "true",
"permUsedHeapK": "0",
"permTotalHeapK": "0",
"tenuredUsedHeapK": "0",
"pausedGCTimeSec": "0.118064",
"totalHeapK": "124780544",
"severity": "WARNING",
"logLine": "2016-01-07T06:25:50.223-0700: 6327495.031: [GC2016-01-07T06:25:50.224-0700: 6327495.031: [ParNew: 8767575K->338334K(9437184K), 0.1177600 secs] 87972843K->79573655K(124780544K), 0.1180640 secs] [Times: user=3.05 sys=0.00, real=0.12 secs] ",
"permAreaGCed": "false",
"tenuredTotalHeapK": "0",
"youngUsedHeapK": "8767575",
"usedTotalHeapK": "87972843",
...
}
}
} |
| Code Block | ||
|---|---|---|
| ||
<timestamp> <hostname> site=localhost dataSource=NNGCLog sourceStreams=NNGCLogStream policyId=NamenodeGCAlert alertExecutorId=NNGCAlert alertContext.trnuredAreaGCed=false alertContext.youngTotalHeapK=9437184... |