high resolution resource monitoring

• collect information about used resources, several sources at a time
• compile these into an xml structure
• deliver for further processing
• postprocess
• plot graphs

• overall:
  • the container element per data set is <s>
  • the container element carries an attribute “t” which contains a unix timestamp

• data tags:
  • <df> disk usage, output of “df -kP”
  • <l> load, /proc/loadavg
  • network: /proc/net/dev
    • <rb> received bytes
    • <rp> received packets
    • <rbpc> received avg. bytes per packet
    • <tb> transmitted bytes
    • <tp> transmitted packets
    • <tbpc> transmitted avg. bytes per packet
    • <sb> total bytes
    • <sp> total packets
  • disk i/o
    • <ds> disk stats, /proc/diskstats, see https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
  • application:
    • <aq> “asbman -i asbroker1 -q”

• directory: /var/log/resource-mon
• file names:
  • YYYY-MM
  • YYYY-MM-DD

mkdir /var/log/resource-mon

• cron job

/etc/cron.d/resource-mon

# logrotate & startup for /usr/local/bin/resource-mon
#off# @reboot      root /usr/local/bin/resource-mon
*/5 *  * * * root /usr/local/bin/resource-mon
35 00 01 * * root find /var/log/resource-mon -name "????-??"       -mmin +30 -print0 | xargs -0r xz
35 00  * * * root find /var/log/resource-mon -name "????-??-??"    -mmin +30 -print0 | xargs -0r xz
35  *  * * * root find /var/log/resource-mon -name "????-??-??T??" -mmin +30 -print0 | xargs -0r xz

• simple filesystem monitoring

/usr/local/bin/resource-mon

#!/bin/bash
 
mkdir -p /var/log/resource-mon
 
(
echo "<s t=\"$(date +%s)\">"
echo "<df>"
df -kP
echo "</df>"
echo "</s>"
) >> /var/log/resource-mon/$( date +%Y-%m ) 

• extensive load / network & disk i/o / application performance mapping

/usr/local/bin/resource-mon

#!/bin/bash 
 
mkdir -p /var/log/resource-mon
 
OC1=0
OC2=0
OC3=0
OC4=0
 
while true ; do 
    DATE=$( date +%s )
    LOAD=$( cat /proc/loadavg )
	DISKSTATS=$( cat /proc/diskstats )
    eval $( awk '$1 == "br0:" { print "C1="$2,"C2="$3,"C3="$10,"C4="$11 }' /proc/net/dev )
    AQ=$( a q )
 
    DC1=$(( $C1 - $OC1 ))
    DC2=$(( $C2 - $OC2 ))
    DC3=$(( $C3 - $OC3 ))
    DC4=$(( $C4 - $OC4 ))
    # catch div by zero:
    if [ "$DC2" == 0 ]; then 
            RDC12="?"
    else 
            RDC12=$(( $DC1 / $DC2 ))
    fi
    if [ "$DC4" == 0 ]; then 
            RDC34="?"
    else 
            RDC34=$(( $DC3 / $DC4 ))
    fi
 
    (
        echo "<s t=\"$DATE\">"
        echo "<rb>$DC1</rb><rp>$DC2</rp><rbpc>$RDC12</rbpc>"
        echo "<tb>$DC3</tb><tp>$DC4</tp><tbpc>$RDC34</tbpc>"
        echo "<sb>$(( $DC1 + $DC3 ))</sb><sp>$(( $DC2 + $DC4 ))</sp><l>$LOAD</l>"
        echo "<ds>"
        echo "$DISKSTATS"
        echo "</ds>"
        echo "<aq>"
        echo "$AQ"
        echo "</aq>"
        echo "</s>"
    ) >> /var/log/resource-mon/$( date +%FT%H )
 
    #echo $DATE R: $DC1 $DC2 $RDC12 T: $DC3 $DC4 $RDC34 S: $(( $DC1 + $DC3 )) $(( $DC2 + $DC4 )) L: $LOAD
    OC1=$C1
    OC2=$C2
    OC3=$C3
    OC4=$C4
    sleep 1
done

chmod 755 /usr/local/bin/resource-mon
bash -n 
/usr/local/bin/resource-mon &