InfluxDB Installation/Collectors
From r00tedvw.com wiki
(Difference between revisions)
m (→vmstat collector) |
(→Script) |
||
| (15 intermediate revisions by one user not shown) | |||
| Line 1: | Line 1: | ||
[[InfluxDB_Installation|InfluxDB Installation]] | [[InfluxDB_Installation/Collectors|Collector Scripts]] | [[InfluxDB_Installation|InfluxDB Installation]] | [[InfluxDB_Installation/Collectors|Collector Scripts]] | ||
==vmstat collector== | ==vmstat collector== | ||
| + | ===script=== | ||
| + | place in <code>/etc/statcollector/statcollector.sh</code> | ||
<nowiki> | <nowiki> | ||
| − | #!/bin/bash | + | #!/bin/bash -x |
| − | # | + | #user defined variables |
| − | + | dbhostname=('dbhost fqdn') | |
| − | + | ||
| − | dbhostname=(' | + | |
dbport=('8086') | dbport=('8086') | ||
| − | dbtable=(' | + | dbtable=('influxdb') |
| − | dbuser=(' | + | dbuser=('influx') |
| − | dbpassword=(' | + | dbpassword=('influxdb') |
| + | logdir=("/var/log/statcollector/") | ||
| − | # | + | #function that checks for connectivity to db host and continues if found. |
| − | + | function inet_test { | |
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | + | ||
| − | # | + | #define variables |
| − | + | HOSTNAME=$(hostname) | |
| + | DOMAIN=$(hostname | awk -F "." '/1/ { printf $2"."$3"."$4};') | ||
| + | log=("statcollector-$(date +%F).txt") | ||
| + | DATE=$(date +%F"|"%R:%S) | ||
| − | # | + | #function that extracts values and names from vmstat |
| − | + | function vmstatvar { | |
| + | #set word boundary as a space, read the line and insert each variable into an array. | ||
| + | #this reads the value names from vmstat | ||
| + | string=$(vmstat 1 1 -S k | awk 'NR==2') | ||
| + | IFS=' ' read -r -a vmstatdef <<<$string | ||
| − | #loop that brings everything together and outputs to db via http post | + | #same as above, but this reads the 10 second average values from vmstat and saves them into an array. |
| − | for ((i=0;i<=(${#vmstatvalues[@]}-1);i++)) | + | string2=$(vmstat 10 2 -S k | awk 'NR==4') |
| − | do | + | IFS=' ' read -r -a vmstatvalues <<<$string2 |
| − | + | } | |
| − | + | #function that checks for access to log dir and creates log file either in log directory or in /var/tmp/ | |
| − | + | function log_dir { | |
| − | + | if ls "$logdir" >/dev/null 2>&1; then | |
| + | if [ -e "$logdir""$log" ] >/dev/null 2>&1; then | ||
| + | printf "$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "$logdir""$log" | ||
| + | else | ||
| + | printf "Start New Log\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "$logdir""$log" | ||
| + | fi | ||
| + | elif [ -e "/var/tmp/""$log" ] >/dev/null 2>&1; then | ||
| + | printf "$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "/var/tmp/""$log" | ||
| + | else | ||
| + | printf "Start New Log\n""$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "/var/tmp/""$log" | ||
| + | fi | ||
| + | } | ||
| + | #loop that brings everything together and outputs to db via http post | ||
| + | function influx_post { | ||
| + | for ((i=0;i<=(${#vmstatvalues[@]}-1);i++)) | ||
| + | do | ||
| + | curl -i --max-time 10\ | ||
| + | -H "Accept: application/json" \ | ||
| + | -H "Content-Type:application/json" \ | ||
| + | -X POST "http://""$dbhostname"":""$dbport""/write?db=""$dbtable" -u "$dbuser":"$dbpassword" --data-binary "${vmstatdef[i]}"",host=""$HOSTNAME"",domain=""$DOMAIN"" value=""${vmstatvalues[i]}" | ||
| + | done | ||
| + | } | ||
| + | #portion of code that checks db connectivity and then calls nested functions | ||
| + | if ping -q -c 1 -W 1 "$dbhostname" >/dev/null 2>&1; then | ||
| + | is=("is") \ | ||
| + | && log_dir \ | ||
| + | && vmstatvar \ | ||
| + | && influx_post | ||
| + | else | ||
| + | is=('is NOT') \ | ||
| + | && log_dir | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | while true; do | ||
| + | inet_test | ||
| + | sleep 5 | ||
done | done | ||
| + | </nowiki> | ||
| + | |||
| + | ===Service=== | ||
| + | place in <code>/etc/init.d/statcollector</code> | ||
| + | <nowiki> | ||
| + | # chkconfig: 345 90 60 | ||
| + | |||
| + | ## BEGIN INIT INFO | ||
| + | # Provides: statcollector | ||
| + | # Required-Start: $remote_fs $syslog | ||
| + | # Required-Stop: $remote_fs $syslog | ||
| + | # Default-Start: 3 4 5 | ||
| + | # Default-Stop: 90 | ||
| + | # Short-Description: Start statcollector at boot time | ||
| + | # Description: Load statcollector at startup to report vmstat values to InfluxDB | ||
| + | ### END INIT INFO | ||
| + | |||
| + | #startup function | ||
| + | start_stat_collector() { | ||
| + | printf "Starting Stat Collector ...\n" | ||
| + | /etc/statcollector/statcollector.sh > /dev/null 2>&1 & | ||
| + | printf "... started.\n" | ||
| + | } | ||
| + | |||
| + | #stopping function | ||
| + | stop_stat_collector () { | ||
| + | printf "Stopping Stat Collector ...\n" | ||
| + | killall -9 statcollector.sh > /dev/null 2>&1 & | ||
| + | printf "... stopped.\n" | ||
| + | } | ||
| + | |||
| + | #restarting function that stops and starts stat collector | ||
| + | restart_stat_collector () { | ||
| + | printf "Restarting Stat Collector ..." | ||
| + | stop_stat_collector | ||
| + | start_stat_collector | ||
| + | } | ||
| + | |||
| + | #status determination | ||
| + | status_stat_collector () { | ||
| + | if [ $(ps -ef | grep -v grep | grep 'statcollector.sh' | wc -l) -ge 1 ]; then | ||
| + | printf "Stat Collector is running.\n" | ||
| + | else | ||
| + | printf "Stat Collector is not running.\n" | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | case "$1" in | ||
| + | start) | ||
| + | start_stat_collector | ||
| + | ;; | ||
| + | stop) | ||
| + | stop_stat_collector | ||
| + | ;; | ||
| + | restart) | ||
| + | restart_stat_collector | ||
| + | ;; | ||
| + | status) | ||
| + | status_stat_collector | ||
| + | ;; | ||
| + | *) | ||
| + | printf "Usage: /etc/init.d/statcollector {start|stop|restart|status}\n" | ||
| + | exit 1 | ||
| + | ;; | ||
| + | |||
| + | esac | ||
| + | exit 0 | ||
| + | </nowiki> | ||
| + | |||
| + | <nowiki> | ||
| + | # sudo chmod +x /etc/init.d/statcollector | ||
| + | # sudo chkconfig --add /etc/init.d/statcollector | ||
| + | </nowiki> | ||
| + | |||
| + | |||
| + | ==File Count Collector== | ||
| + | This was for a service that deposits files into a cache directory and then works on them. It would display the "queue". | ||
| + | ===Script=== | ||
| + | place in <code>/etc/mail_queue/mail_queue.sh</code> | ||
| + | <nowiki> | ||
| + | #!/bin/bash -x | ||
| + | |||
| + | #user defined variables | ||
| + | dbhostname=('dbhost fqdn') | ||
| + | dbport=('8086') | ||
| + | dbtable=('influxdb') | ||
| + | dbuser=('influx') | ||
| + | dbpassword=('influxdb') | ||
| + | logdir=("/var/log/mail_queue/") | ||
| + | |||
| + | #function that checks for connectivity to db host and continues if found. | ||
| + | function inet_test { | ||
| + | |||
| + | #define variables | ||
| + | HOSTNAME=$(hostname) | ||
| + | DOMAIN=$(hostname | awk -F "." '/1/ { printf $2"."$3"."$4};') | ||
| + | log=("mail_queue-$(date +%F).txt") | ||
| + | DATE=$(date +%F"|"%R:%S) | ||
| + | |||
| + | #function to extract the mail queue | ||
| + | function mail_queue { | ||
| + | #determine the total number of folders for mail-accepted clients | ||
| + | #string0=$(find /var/tmp/smtp/ -maxdepth 1 -type d | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done | awk 'NR==1' | awk '{ printf$2}') | ||
| + | #determine the total number of messages and folders for each client | ||
| + | #string1=$(find /var/tmp/smtp/ -maxdepth 1 -type d | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done | awk 'NR==2' | awk '{ printf$2}') | ||
| + | #subtract the total folder count from total message count | ||
| + | #string2="$((string1 - string0))" | ||
| + | |||
| + | #determine the number of files in directory and children, excluding the directories themselves and only looking for txt files. | ||
| + | string2=$(find /var/tmp/smtp/feed/ -type f -name "*.txt" | wc -l) | ||
| + | } | ||
| + | |||
| + | #function that checks for access to log dir and creates log file either in log directory or in /var/tmp/ | ||
| + | function log_dir { | ||
| + | if ls "$logdir" >/dev/null 2>&1; then | ||
| + | if [ -e "$logdir""$log" ] >/dev/null 2>&1; then | ||
| + | printf "$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "$logdir""$log" | ||
| + | else | ||
| + | printf "Start New Log\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "$logdir""$log" | ||
| + | fi | ||
| + | elif [ -e "/var/tmp/""$log" ] >/dev/null 2>&1; then | ||
| + | printf "$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "/var/tmp/""$log" else | ||
| + | printf "Start New Log\n""$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > /var/tmp/""$log" | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | function print_message_count { | ||
| + | string2="0" | ||
| + | printf "$string2""\n" | ||
| + | } | ||
| + | |||
| + | #function that outputs to db via http post | ||
| + | function influx_post { | ||
| + | curl -i --max-time 10\ | ||
| + | -H "Accept: application/json" \ | ||
| + | -H "Content-Type:application/json" \ | ||
| + | -X POST "http://""$dbhostname"":""$dbport""/write?db=""$dbtable" -u "$dbuser":"$dbpassword" --data-binary "mail_queue"",host=""$HOSTNAME"",domain=""$DOMAIN"" value=""$string2" | ||
| + | } | ||
| + | |||
| + | #portion of code that checks db connectivity and then calls nested functions | ||
| + | if ping -q -c 1 -W 1 "$dbhostname" >/dev/null 2>&1; then | ||
| + | is=("is") \ | ||
| + | && log_dir \ | ||
| + | && mail_queue \ | ||
| + | && influx_post | ||
| + | else | ||
| + | is=('is NOT') \ | ||
| + | && log_dir | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | while true; do | ||
| + | inet_test | ||
| + | sleep 30 | ||
| + | done | ||
| + | </nowiki> | ||
| + | |||
| + | ===Service=== | ||
| + | place in <code>/etc/init.d/mail_queue</code> | ||
| + | <nowiki> | ||
| + | # chkconfig: 345 90 60 | ||
| + | |||
| + | ## BEGIN INIT INFO | ||
| + | # Provides: mail_queue | ||
| + | # Required-Start: $remote_fs $syslog | ||
| + | # Required-Stop: $remote_fs $syslog | ||
| + | # Default-Start: 3 4 5 | ||
| + | # Default-Stop: 90 | ||
| + | # Short-Description: Start mail_queue at boot time | ||
| + | # Description: Load mail_queue at startup to report queue values to InfluxDB | ||
| + | ### END INIT INFO | ||
| + | |||
| + | #startup function | ||
| + | start_mail_queue() { | ||
| + | printf "Starting Mail Queue ...\n" | ||
| + | /etc/mail_queue/mail_queue.sh > /dev/null 2>&1 & | ||
| + | printf "... started.\n" | ||
| + | } | ||
| + | |||
| + | #stopping function | ||
| + | stop_mail_queue () { | ||
| + | printf "Stopping Mail Queue ...\n" | ||
| + | killall -9 mail_queue.sh > /dev/null 2>&1 & | ||
| + | printf "... stopped.\n" | ||
| + | } | ||
| + | |||
| + | #restarting function that stops and starts stat collector | ||
| + | restart_mail_queue () { | ||
| + | printf "Restarting Mail Queue ..." | ||
| + | stop_mail_queue | ||
| + | start_mail_queue | ||
| + | } | ||
| + | |||
| + | #status determination | ||
| + | status_mail_queue () { | ||
| + | if [ $(ps -ef | grep -v grep | grep 'mail_queue.sh' | wc -l) -ge 1 ]; then | ||
| + | printf "Mail Queue is running.\n" | ||
| + | else | ||
| + | printf "Mail Queue is not running.\n" | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | case "$1" in | ||
| + | start) | ||
| + | start_mail_queue | ||
| + | ;; | ||
| + | stop) | ||
| + | stop_mail_queue | ||
| + | ;; | ||
| + | restart) | ||
| + | restart_mail_queue | ||
| + | ;; | ||
| + | status) | ||
| + | status_mail_queue | ||
| + | ;; | ||
| + | *) | ||
| + | printf "Usage: /etc/init.d/mail_queue {start|stop|restart|status}\n" | ||
| + | exit 1 | ||
| + | ;; | ||
| + | |||
| + | esac | ||
| + | exit 0 | ||
| + | </nowiki> | ||
| + | |||
| + | <nowiki> | ||
| + | # sudo chmod +x /etc/init.d/mail_queue | ||
| + | # sudo chkconfig --add /etc/init.d/mail_queue | ||
</nowiki> | </nowiki> | ||
Latest revision as of 11:19, 19 October 2016
InfluxDB Installation | Collector Scripts
Contents |
[edit] vmstat collector
[edit] script
place in /etc/statcollector/statcollector.sh
#!/bin/bash -x
#user defined variables
dbhostname=('dbhost fqdn')
dbport=('8086')
dbtable=('influxdb')
dbuser=('influx')
dbpassword=('influxdb')
logdir=("/var/log/statcollector/")
#function that checks for connectivity to db host and continues if found.
function inet_test {
#define variables
HOSTNAME=$(hostname)
DOMAIN=$(hostname | awk -F "." '/1/ { printf $2"."$3"."$4};')
log=("statcollector-$(date +%F).txt")
DATE=$(date +%F"|"%R:%S)
#function that extracts values and names from vmstat
function vmstatvar {
#set word boundary as a space, read the line and insert each variable into an array.
#this reads the value names from vmstat
string=$(vmstat 1 1 -S k | awk 'NR==2')
IFS=' ' read -r -a vmstatdef <<<$string
#same as above, but this reads the 10 second average values from vmstat and saves them into an array.
string2=$(vmstat 10 2 -S k | awk 'NR==4')
IFS=' ' read -r -a vmstatvalues <<<$string2
}
#function that checks for access to log dir and creates log file either in log directory or in /var/tmp/
function log_dir {
if ls "$logdir" >/dev/null 2>&1; then
if [ -e "$logdir""$log" ] >/dev/null 2>&1; then
printf "$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "$logdir""$log"
else
printf "Start New Log\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "$logdir""$log"
fi
elif [ -e "/var/tmp/""$log" ] >/dev/null 2>&1; then
printf "$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "/var/tmp/""$log"
else
printf "Start New Log\n""$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "/var/tmp/""$log"
fi
}
#loop that brings everything together and outputs to db via http post
function influx_post {
for ((i=0;i<=(${#vmstatvalues[@]}-1);i++))
do
curl -i --max-time 10\
-H "Accept: application/json" \
-H "Content-Type:application/json" \
-X POST "http://""$dbhostname"":""$dbport""/write?db=""$dbtable" -u "$dbuser":"$dbpassword" --data-binary "${vmstatdef[i]}"",host=""$HOSTNAME"",domain=""$DOMAIN"" value=""${vmstatvalues[i]}"
done
}
#portion of code that checks db connectivity and then calls nested functions
if ping -q -c 1 -W 1 "$dbhostname" >/dev/null 2>&1; then
is=("is") \
&& log_dir \
&& vmstatvar \
&& influx_post
else
is=('is NOT') \
&& log_dir
fi
}
while true; do
inet_test
sleep 5
done
[edit] Service
place in /etc/init.d/statcollector
# chkconfig: 345 90 60
## BEGIN INIT INFO
# Provides: statcollector
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 3 4 5
# Default-Stop: 90
# Short-Description: Start statcollector at boot time
# Description: Load statcollector at startup to report vmstat values to InfluxDB
### END INIT INFO
#startup function
start_stat_collector() {
printf "Starting Stat Collector ...\n"
/etc/statcollector/statcollector.sh > /dev/null 2>&1 &
printf "... started.\n"
}
#stopping function
stop_stat_collector () {
printf "Stopping Stat Collector ...\n"
killall -9 statcollector.sh > /dev/null 2>&1 &
printf "... stopped.\n"
}
#restarting function that stops and starts stat collector
restart_stat_collector () {
printf "Restarting Stat Collector ..."
stop_stat_collector
start_stat_collector
}
#status determination
status_stat_collector () {
if [ $(ps -ef | grep -v grep | grep 'statcollector.sh' | wc -l) -ge 1 ]; then
printf "Stat Collector is running.\n"
else
printf "Stat Collector is not running.\n"
fi
}
case "$1" in
start)
start_stat_collector
;;
stop)
stop_stat_collector
;;
restart)
restart_stat_collector
;;
status)
status_stat_collector
;;
*)
printf "Usage: /etc/init.d/statcollector {start|stop|restart|status}\n"
exit 1
;;
esac
exit 0
# sudo chmod +x /etc/init.d/statcollector # sudo chkconfig --add /etc/init.d/statcollector
[edit] File Count Collector
This was for a service that deposits files into a cache directory and then works on them. It would display the "queue".
[edit] Script
place in /etc/mail_queue/mail_queue.sh
#!/bin/bash -x
#user defined variables
dbhostname=('dbhost fqdn')
dbport=('8086')
dbtable=('influxdb')
dbuser=('influx')
dbpassword=('influxdb')
logdir=("/var/log/mail_queue/")
#function that checks for connectivity to db host and continues if found.
function inet_test {
#define variables
HOSTNAME=$(hostname)
DOMAIN=$(hostname | awk -F "." '/1/ { printf $2"."$3"."$4};')
log=("mail_queue-$(date +%F).txt")
DATE=$(date +%F"|"%R:%S)
#function to extract the mail queue
function mail_queue {
#determine the total number of folders for mail-accepted clients
#string0=$(find /var/tmp/smtp/ -maxdepth 1 -type d | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done | awk 'NR==1' | awk '{ printf$2}')
#determine the total number of messages and folders for each client
#string1=$(find /var/tmp/smtp/ -maxdepth 1 -type d | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done | awk 'NR==2' | awk '{ printf$2}')
#subtract the total folder count from total message count
#string2="$((string1 - string0))"
#determine the number of files in directory and children, excluding the directories themselves and only looking for txt files.
string2=$(find /var/tmp/smtp/feed/ -type f -name "*.txt" | wc -l)
}
#function that checks for access to log dir and creates log file either in log directory or in /var/tmp/
function log_dir {
if ls "$logdir" >/dev/null 2>&1; then
if [ -e "$logdir""$log" ] >/dev/null 2>&1; then
printf "$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "$logdir""$log"
else
printf "Start New Log\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > "$logdir""$log"
fi
elif [ -e "/var/tmp/""$log" ] >/dev/null 2>&1; then
printf "$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" >> "/var/tmp/""$log" else
printf "Start New Log\n""$DATE"" -- ""$logdir"" is NOT accessible\n""$DATE"" -- ""$dbhostname"" ""$is"" accessible\n" > /var/tmp/""$log"
fi
}
function print_message_count {
string2="0"
printf "$string2""\n"
}
#function that outputs to db via http post
function influx_post {
curl -i --max-time 10\
-H "Accept: application/json" \
-H "Content-Type:application/json" \
-X POST "http://""$dbhostname"":""$dbport""/write?db=""$dbtable" -u "$dbuser":"$dbpassword" --data-binary "mail_queue"",host=""$HOSTNAME"",domain=""$DOMAIN"" value=""$string2"
}
#portion of code that checks db connectivity and then calls nested functions
if ping -q -c 1 -W 1 "$dbhostname" >/dev/null 2>&1; then
is=("is") \
&& log_dir \
&& mail_queue \
&& influx_post
else
is=('is NOT') \
&& log_dir
fi
}
while true; do
inet_test
sleep 30
done
[edit] Service
place in /etc/init.d/mail_queue
# chkconfig: 345 90 60
## BEGIN INIT INFO
# Provides: mail_queue
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 3 4 5
# Default-Stop: 90
# Short-Description: Start mail_queue at boot time
# Description: Load mail_queue at startup to report queue values to InfluxDB
### END INIT INFO
#startup function
start_mail_queue() {
printf "Starting Mail Queue ...\n"
/etc/mail_queue/mail_queue.sh > /dev/null 2>&1 &
printf "... started.\n"
}
#stopping function
stop_mail_queue () {
printf "Stopping Mail Queue ...\n"
killall -9 mail_queue.sh > /dev/null 2>&1 &
printf "... stopped.\n"
}
#restarting function that stops and starts stat collector
restart_mail_queue () {
printf "Restarting Mail Queue ..."
stop_mail_queue
start_mail_queue
}
#status determination
status_mail_queue () {
if [ $(ps -ef | grep -v grep | grep 'mail_queue.sh' | wc -l) -ge 1 ]; then
printf "Mail Queue is running.\n"
else
printf "Mail Queue is not running.\n"
fi
}
case "$1" in
start)
start_mail_queue
;;
stop)
stop_mail_queue
;;
restart)
restart_mail_queue
;;
status)
status_mail_queue
;;
*)
printf "Usage: /etc/init.d/mail_queue {start|stop|restart|status}\n"
exit 1
;;
esac
exit 0
# sudo chmod +x /etc/init.d/mail_queue # sudo chkconfig --add /etc/init.d/mail_queue