You are viewing an old version of this page. View the current version.

Compare with Current View Page History

« Previous Version 6 Next »

Support Cassandra query log as example:

host:/192.168.6.233|source:/192.168.6.227|user:cassandra|timestamp:1454965365762|category:QUERY|type:CQL_SELECT|ks:dg_keyspace|cf:customer_details|operation:Select  id,name,email,contact from customer_details|masked_columns:[email,contact]|other_columns:[id,name]

1. Flow data stream into kafka

{
    "host": "/192.168.6.227",
    "source": "/192.168.6.227",
    "user": "jaspa",
    "timestamp": 1455574202864,
    "category": "QUERY",
    "type": "CQL_SELECT",
    "ks": "dg_keyspace",
    "cf": "customer_details",
    "operation": "CQL_SELECT",
    "masked_columns": "bank|ccno|email|ip|name|sal|ssn|tel|url",
    "other_columns": "id|npi"
}

 

2. Define new application metadata

curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
 "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService" \
  -d '
  [
     {
        "tags":{
           "site":"sandbox",
           "application":"cassandraQueryLogMonitor"
        },
        "enabled": true,
        "config": "{}"
     }
  ]
  '

curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
 "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=ApplicationDescService" \
  -d '
  [
     {
        "tags":{
           "application":"cassandraQueryLogMonitor"
        },
        "desc":"cassandra Query Log Monitoring",
        "alias":"QueryLogMonitor",
        "group":"Cassandra",
        "config":"{}",
        "features":["common","metadata"]
     }
  ]
  '

## AlertStreamService: alert streams generated from data source
echo ""
echo "Importing AlertStreamService for HDFS... "
curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
 "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamService" \
 -d '
 [
    {
       "prefix":"alertStream",
       "tags":{
          "dataSource":"hadoopJmxMetricDataSource",
          "streamName":"hadoopJmxMetricEventStream"
       },
       "desc":"hadoop"
    }
 ]
 '

## AlertExecutorService: what alert streams are consumed by alert executor
echo ""
echo "Importing AlertExecutorService for HDFS... "
curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
 "http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertExecutorService" \
 -d '
 [
    {
       "prefix":"alertExecutor",
       "tags":{
          "dataSource":"hadoopJmxMetricDataSource",
          "alertExecutorId":"hadoopJmxMetricAlertExecutor",
          "streamName":"hadoopJmxMetricEventStream"
       },
       "desc":"aggregate executor for hadoop jmx metric event stream"
    }
 ]
 '

## AlertStreamSchemaService: schema for event from alert stream
echo ""
echo "Importing AlertStreamSchemaService for HDFS... "
curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json' \
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService" \
 -d '
 [
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "host"
       },
       "attrDescription": "the host that current metric comes form",
       "attrType": "string",
       "category": "",
       "attrValueResolver": ""
    },
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "timestamp"
       },
       "attrDescription": "the metric timestamp",
       "attrType": "long",
       "category": "",
       "attrValueResolver": ""
    },
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "metric"
       },
       "attrDescription": "the metric name",
       "attrType": "string",
       "category": "",
       "attrValueResolver": ""
    },
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "component"
       },
       "attrDescription": "the component that the metric comes from",
       "attrType": "string",
       "category": "",
       "attrValueResolver": ""
    },
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "site"
       },
       "attrDescription": "the site that the metric belongs to",
       "attrType": "string",
       "category": "",
       "attrValueResolver": ""
    },
    {
       "prefix": "alertStreamSchema",
       "tags": {
          "dataSource": "hadoopJmxMetricDataSource",
          "streamName": "hadoopJmxMetricEventStream",
          "attrName": "value"
       },
       "attrDescription": "the metric value in string presentation",
       "attrType": "double",
       "category": "",
       "attrValueResolver": ""
    }
 ]
 '

## Finished
echo ""
echo "Finished initialization for eagle topology"

 

3. Start Monitoring Topology with Template

  • No labels