Page History

...

    tail -F /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list [kafka_broker]:[kafka_port]$KAFKA_BROKER_URL --topic squid

4. Ensure that the parser topology for Squid continues to run based on the steps outlined in the previous tutorials.

...

1. Run the following command to create an index template for Squid.

 curl -XPOST ec2-52-40-44-64.us-west-2.compute.amazonaws.com:9200$ES_HOST:$ES_PORT/_template/squid_index -d '
 {
 "template": "squid_index*",
 "mappings": {
 "bro_doc": {
 "_timestamp": {
 "enabled": true
 },
 "properties": {
 "timestamp": {
 "type": "date",
 "format": "epoch_millis"
 },
 "source:type": {
 "type": "string",
 "index": "not_analyzed"
 },
 "action": {
 "type": "string",
 "index": "not_analyzed"
 },
 "bytes": {
 "type": "integer"
 },
 "code": {
 "type": "string",
 "index": "not_analyzed"
 },
 "domain_without_subdomains": {
 "type": "string",
 "index": "not_analyzed"
 },
 "full_hostname": {
 "type": "string",
 "index": "not_analyzed"
 },
 "elapsed": {
 "type": "integer"
 },
 "method": {
 "type": "string",
 "index": "not_analyzed"
 },
 "ip_dst_addr": {
 "type": "string",
 "index": "not_analyzed"
 }
 }
 }
 }
 }'

...

3. An index template will only apply for indices that are created after the template is created. Delete the existing Squid indices so that new ones can be generated with the index template.

    curl -XDELETE node1:9200/squid*

4. Wait for the Squid index to be re-created. This may take a minute or two based on how fast the Squid data is being consumed in your environment.

    curl -XGET node1:9200/squid*

Configure the Squid Index in Kibana

...