====== resourcemon ======

high resolution resource monitoring

===== objective =====

  * collect information about used resources, several sources at a time
  * compile these into an xml structure
  * deliver for further processing
  * postprocess
  * plot graphs

===== xml =====

  * overall:
    * the container element per data set is <s>
    * the container element carries an attribute "t" which contains a unix timestamp 

  * data tags:
    * <df> disk usage, output of "df -kP"
    * <l>  load, /proc/loadavg
    * network: /proc/net/dev
      * <rb> received bytes
      * <rp> received packets
      * <rbpc> received avg. bytes per packet
      * <tb> transmitted bytes
      * <tp> transmitted packets
      * <tbpc> transmitted avg. bytes per packet
      * <sb> total bytes
      * <sp> total packets
    * disk i/o
      * <ds> disk stats, /proc/diskstats, see https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
    * application:
      * <aq> "asbman -i asbroker1 -q"

===== output =====

  * directory: /var/log/resource-mon
  * file names:
    * YYYY-MM
    * YYYY-MM-DD

===== deployment =====

<code>
mkdir /var/log/resource-mon
</code>

  * cron job

<file txt /etc/cron.d/resource-mon>
# logrotate & startup for /usr/local/bin/resource-mon
#off# @reboot      root /usr/local/bin/resource-mon
*/5 *  * * * root /usr/local/bin/resource-mon
35 00 01 * * root find /var/log/resource-mon -name "????-??"       -mmin +30 -print0 | xargs -0r xz
35 00  * * * root find /var/log/resource-mon -name "????-??-??"    -mmin +30 -print0 | xargs -0r xz
35  *  * * * root find /var/log/resource-mon -name "????-??-??T??" -mmin +30 -print0 | xargs -0r xz
</file>

  * simple filesystem monitoring

<file txt /usr/local/bin/resource-mon>
#!/bin/bash

mkdir -p /var/log/resource-mon

(
echo "<s t=\"$(date +%s)\">"
echo "<df>"
df -kP
echo "</df>"
echo "</s>"
) >> /var/log/resource-mon/$( date +%Y-%m ) 
</file>

  * extensive load / network & disk i/o / application performance mapping

<file txt /usr/local/bin/resource-mon>
#!/bin/bash 

mkdir -p /var/log/resource-mon

OC1=0
OC2=0
OC3=0
OC4=0

while true ; do 
    DATE=$( date +%s )
    LOAD=$( cat /proc/loadavg )
	DISKSTATS=$( cat /proc/diskstats )
    eval $( awk '$1 == "br0:" { print "C1="$2,"C2="$3,"C3="$10,"C4="$11 }' /proc/net/dev )
    AQ=$( a q )
    
    DC1=$(( $C1 - $OC1 ))
    DC2=$(( $C2 - $OC2 ))
    DC3=$(( $C3 - $OC3 ))
    DC4=$(( $C4 - $OC4 ))
    # catch div by zero:
    if [ "$DC2" == 0 ]; then 
            RDC12="?"
    else 
            RDC12=$(( $DC1 / $DC2 ))
    fi
    if [ "$DC4" == 0 ]; then 
            RDC34="?"
    else 
            RDC34=$(( $DC3 / $DC4 ))
    fi

    (
        echo "<s t=\"$DATE\">"
        echo "<rb>$DC1</rb><rp>$DC2</rp><rbpc>$RDC12</rbpc>"
        echo "<tb>$DC3</tb><tp>$DC4</tp><tbpc>$RDC34</tbpc>"
        echo "<sb>$(( $DC1 + $DC3 ))</sb><sp>$(( $DC2 + $DC4 ))</sp><l>$LOAD</l>"
        echo "<ds>"
        echo "$DISKSTATS"
        echo "</ds>"
        echo "<aq>"
        echo "$AQ"
        echo "</aq>"
        echo "</s>"
    ) >> /var/log/resource-mon/$( date +%FT%H )

    #echo $DATE R: $DC1 $DC2 $RDC12 T: $DC3 $DC4 $RDC34 S: $(( $DC1 + $DC3 )) $(( $DC2 + $DC4 )) L: $LOAD
    OC1=$C1
    OC2=$C2
    OC3=$C3
    OC4=$C4
    sleep 1
done
</file>

<code>
chmod 755 /usr/local/bin/resource-mon
bash -n 
/usr/local/bin/resource-mon &
</code>