summaryrefslogtreecommitdiffstats
path: root/perf-framework
diff options
context:
space:
mode:
Diffstat (limited to 'perf-framework')
-rw-r--r--perf-framework/.runfile1
-rw-r--r--perf-framework/README85
-rwxr-xr-xperf-framework/acl42
-rwxr-xr-xperf-framework/analyze_perf_data466
-rwxr-xr-xperf-framework/automate_pwl_ssh43
-rwxr-xr-xperf-framework/batchrun50
-rwxr-xr-xperf-framework/buildit6
-rwxr-xr-xperf-framework/calc_all49
-rwxr-xr-xperf-framework/calc_avg54
-rwxr-xr-xperf-framework/calc_best31
-rwxr-xr-xperf-framework/calc_worst31
-rwxr-xr-xperf-framework/check_install13
-rwxr-xr-xperf-framework/check_install.new33
-rwxr-xr-xperf-framework/create_gluster_vol122
-rwxr-xr-xperf-framework/deploy_gluster53
-rwxr-xr-xperf-framework/diff_perfrun17
-rwxr-xr-xperf-framework/file_open_analyse430
-rwxr-xr-xperf-framework/file_open_test159
-rwxr-xr-xperf-framework/gen_perf_data191
-rwxr-xr-xperf-framework/get_date5
-rw-r--r--perf-framework/gf_perf_config34
-rwxr-xr-xperf-framework/glusterfs-precreate.sh32
-rw-r--r--perf-framework/motd25
-rwxr-xr-xperf-framework/newtest.sh331
-rwxr-xr-xperf-framework/nfs-reread.sh331
-rwxr-xr-xperf-framework/nfs_delete.sh331
-rw-r--r--perf-framework/ops17
-rwxr-xr-xperf-framework/parallel_create71
-rwxr-xr-xperf-framework/perf-test.sh337
-rwxr-xr-xperf-framework/perf.sh23
-rwxr-xr-xperf-framework/pwl_ssh50
-rwxr-xr-xperf-framework/quota_gsync_run42
-rwxr-xr-xperf-framework/remove_export_dirs10
-rwxr-xr-xperf-framework/run9
-rwxr-xr-xperf-framework/send_mail8
-rwxr-xr-xperf-framework/setbatch25
-rwxr-xr-xperf-framework/setrun14
-rwxr-xr-xperf-framework/start_perf_measure166
-rwxr-xr-xperf-framework/stat_collect36
39 files changed, 3773 insertions, 0 deletions
diff --git a/perf-framework/.runfile b/perf-framework/.runfile
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/perf-framework/.runfile
@@ -0,0 +1 @@
+0 \ No newline at end of file
diff --git a/perf-framework/README b/perf-framework/README
new file mode 100644
index 0000000..7733c35
--- /dev/null
+++ b/perf-framework/README
@@ -0,0 +1,85 @@
+Brief desciption of individual scripts:
+----------------------------------------
+acl - Enable/Disable acl on the backend filesystem.
+analyze_perf_data - Create plots using the data generated during perf runs.
+batchrun - Used for running multiple tarballs one after the other.
+buildit - Build scripts used by deploy_gluster for parallel builds.
+calc_all - Print results of all 3 runs performed in perf.sh
+calc_avg - Print average of 3 runs performed in perf.sh
+calc_best - Print best of the 3 runs performed in perf.sh
+calc_worst - Print worst of the 3 runs performed in perf.sh
+check_install - Print checksums of built and installed binaries
+check_install.new - Check if installation succeeded on all machines.
+create_gluster_vol - Create a gluster volume using parameters in gf_perf_conig
+deploy_gluster - Install gluster from source on multiple machines
+gf_perf_config - Example config file. Most scripts depend on this config file.
+perf.sh - Perf test wrapper that runs perf-test.sh thrice.
+perf-test.sh - Perf test
+quota_gsync_run - Used to run different Quota/Gsync combinations
+run - Used to run any command on multiple systems
+setbatch - Set the tarballs for batch run
+setrun - Set the tarball for a single run
+start_perf_measure - Launch statistics collection script on all machines
+involved in a perf run, start the perf run and save statistics post run.
+stat_collect - Helper script to collect system statistics during perf runs.
+
+diff_perfrun
+------------
+<File Begin>
+DATA FOR BASELINE - Output of calc_avg for the baseline
+followed by
+DATA FOR THE NEW RUN - Output of calc_avg for the new run
+<FILE END>
+
+It also assumes that there are 4 columns - one each for Quota/Gsync combination.
+You can then put in a criteria for percentage deviation from the baseline which should send out an alert.
+
+
+Setting up the environment:
+------------------------------------
+* Initialize the required variables by modifying the example gf_perf_config
+present in the git repo.
+
+Building and deploying GlusterFS
+------------------------------------
+* Populate the BRICKS variable in gf_perf_config
+* If you want to deploy a tarball, update the tarball in gf_perf_config
+and then run "deploy_gluster". This will copy the tarball to all the bricks and
+will build it there. This uses a script called buildit which is part of the
+repo.
+
+Running the tests
+------------------------------------
+* Use start_perf_measure to run the tests. The test run details can be changed
+in gf_perf_config before the run. You can specify whether the configuration is
+a distribute, replicate or distributed-replicate (distrep).
+* This will generate statistics in the log_repo for the current run under the
+director - "$log_repo/<current run>"
+* The run "id" is maintained by a ".runfile" which gets generated on the first
+run and is later updated as required during subsequent runs.
+* NOTE: You can use "batchrun" to start tests for all combinations. Downloading
+the tarball to the cwd is the responsibility of the user.
+
+Analyzing the output
+------------------------------------
+* Use analyze_perf_measure to generate plots of resource utilization for all the
+bricks and the client.
+
+Running Quota and Gsync combinations:
+------------------------------------
+* After an initial run (as described above) is complete, you can run different
+combinations of Quota/Gsync by running quota_gsync_run.
+* NOTE: Before running this script, it is important to comment out the line -
+./create_gluster_vol in the script start_perf_measure.
+Otherwise, a new volume will be created for every combination, which will wipe
+out the Quota/Gsync settings. Look at the usage in batchrun.
+TODO: Need to use the gluster command line to enable/disable quota/gsync instead
+of directly modifying the volume file.
+
+Setting up automated-email setup:
+--------------------------------
+* Setup automated emails to send results as soon as the runs are done. The whole
+email setup is done using Lakshmipathi's inotify program available at
+http://github.com/gluster/qa/blob/master/c_pgms/inotify.c & when a file is scpd
+to the particular destination, inotify sends across the mail from a shell.gluster.com
+account. \ No newline at end of file
diff --git a/perf-framework/acl b/perf-framework/acl
new file mode 100755
index 0000000..c971fee
--- /dev/null
+++ b/perf-framework/acl
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+source gf_perf_config
+
+function usage()
+{
+ echo "Usage: $0 <on|off>"
+ exit 1
+}
+
+function get_mount()
+{
+ brick=$1
+ export_dir=$2
+ ssh -l root $brick ls -d $export_dir > /dev/null
+ if [ $? -ne 0 ]
+ then
+ echo "FATAL : Export directory does not exist. Exiting..."
+ exit 1
+ fi
+ echo `ssh -l root $brick df -h $export_dir | grep -v ^Filesystem | awk '{print $NF}'`
+}
+
+
+if [ $# -ne 1 ]
+then
+ usage
+fi
+
+for brick in $BRICK_IP_ADDRS
+do
+ mount_point=`get_mount $brick $SERVER_EXPORT_DIR`
+ if [ "$1" == "on" ]
+ then
+ ssh -l root $brick mount -o remount,acl $mount_point
+ elif [ "$1" == "off" ]
+ then
+ ssh -l root $brick mount -o remount,noacl $mount_point
+ else
+ usage
+ fi
+done
diff --git a/perf-framework/analyze_perf_data b/perf-framework/analyze_perf_data
new file mode 100755
index 0000000..129a1e9
--- /dev/null
+++ b/perf-framework/analyze_perf_data
@@ -0,0 +1,466 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+if [ $# -le 0 -o $# -gt 2 ]
+then
+ echo "Usage : $0 <run directory>"
+ exit 1
+fi
+
+rundir=$1
+cmpdir=""
+
+if [ $# -eq 2 ]
+then
+ cmpdir=$2
+fi
+
+function gen_cpu_data()
+{
+ time=0
+ sum=0
+
+ if [ -f times ]
+ then
+ rm times
+ fi
+
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ sum=$((sum + $time))
+ echo $sum >> times
+ done
+ sed -i 's/$/ 100/g' times
+
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+ idle_col_tweak=$?
+ echo `grep CPU $MPSTAT_LOG | head -1 | awk '{print $3}'` | grep -o CPU > /dev/null
+ cpu_col_tweak=$?
+ for i in "all" 0 `seq $((num_procs-1))`
+ do
+ egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$cpu_col_tweak -v v2=$idle_col_tweak '{print $(3-v1) " " $(NF-v2)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+ cat -n cpu-$i > cpu-$i.new
+ done
+}
+
+function plot_cpu_usage()
+{
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+ mpstat_interval=5
+ plot_info=pinfo.$$
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ ltype=1
+ identity=$1
+
+ for i in "all" 0 `seq $((num_procs-1))`
+ do
+ echo -ne "\"cpu-$i.new\" using (\$1*$mpstat_interval):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+ ltype=$((ltype+1))
+ done
+ echo "\"times\" using 1:2 title '' with impulse lt 2 lw 1" >> $plot_info
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "CPU utilization : All CPUs ($identity)"
+ set xlabel "Time"
+ set ylabel "% CPU utilization"
+ set xr [0:$xrange]
+ set yr [0:100]
+ set terminal png nocrop size 1024,768
+ set output "$CPU_PLOT_OUTPUT"
+ plot `cat $plot_info`
+EOF
+ rm $plot_info
+}
+
+function gen_vm_data()
+{
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $4}' > vm_datafile
+ totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+ cat -n memfile > memfile.new
+}
+
+function plot_vm_usage()
+{
+ vmstat_interval=5
+ total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+ identity=$1
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "Memory utilization ($identity)"
+ set xlabel "Time"
+ set ylabel "Memory utilization in bytes"
+ set xr [0:$xrange]
+ set yr [0:$total_mem]
+ set terminal png nocrop size 1024,768
+ set output "$VM_PLOT_OUTPUT"
+ plot "memfile.new" using (\$1*$vmstat_interval):2 title 'memory-usage' with lines lt 2 lw 2,\
+ "times" using 1:(\$2*$total_mem/100) title '' with impulse lt 2 lw 1
+EOF
+}
+
+function gen_iostats()
+{
+ brick=$1
+ dev=`ssh $brick "df -h $SERVER_EXPORT_DIR" | tail -1 | awk '{print $1}' | cut -f3 -d'/'`
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $10}' | cut -f1 -d'.' > io_await
+ cat -n io_await > io_await.new
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $6}' | cut -f1 -d'.' > read_tput
+ cat -n read_tput > read_tput.new
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $7}' | cut -f1 -d'.' > write_tput
+ cat -n write_tput > write_tput.new
+}
+
+function plot_iostats()
+{
+ iostat_interval=5
+ max_wait=$((`sort -n io_await | tail -1` + 50))
+ max_read=$(((`sort -n read_tput | tail -1`) / 2 + 100))
+ max_write=$(((`sort -n write_tput | tail -1`) / 2 + 100))
+ max_io=$max_write
+ if [ $max_read -gt $max_write ]
+ then
+ max_io=$max_read;
+ fi
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+ identity=$1
+
+ gnuplot <<EOF1
+ set autoscale
+ set grid
+ set title "IO Wait times ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "IO Wait times in milliseconds"
+ set xr [0:$xrange]
+ set yr [0:$max_wait]
+ set terminal png nocrop size 1024,768
+ set output "$IO_TIMES_PLOT_OUTPUT"
+ plot "io_await.new" using (\$1*$iostat_interval):2 title 'IO wait times' with lines lt 3 lw 2,\
+ "times" using 1:(\$2*$max_wait/100) title '' with impulse lt 2 lw 1
+EOF1
+ gnuplot <<EOF2
+ set autoscale
+ set grid
+ set title "Disk Read-Write throughput ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Throughput in KB/sec"
+ set xr [0:$xrange]
+ set yr [0:$max_io]
+ set terminal png nocrop size 1024,768
+ set output "$IO_TPUT_PLOT_OUTPUT"
+ plot "read_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Read throughput' with lines lt 4 lw 2,\
+ "write_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Write throughput' with lines lt 5 lw 2,\
+ "times" using 1:(\$2*$max_io/100) title '' with impulse lt 2 lw 1
+EOF2
+}
+
+function gen_cmp_data()
+{
+ perflog_baseline=$1
+ perflog_current=$2
+
+ time=0
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> btimes.$$
+ done
+
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> ctimes.$$
+ done
+
+ paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+ rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+ a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+ yrange=`echo $a + $a/5 | bc`
+ b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+ xrange=`echo $b - 0.5 | bc`
+
+ gnuplot <<EOF
+ reset
+ set key at graph 0.15, 0.85 horizontal samplen 0.1
+ set style data histogram
+ set style histogram cluster gap 1
+ set style fill solid border -1
+ set boxwidth 0.8
+ set xtic rotate by 90 scale 0
+ unset ytics
+ set y2tics rotate by 90
+ set terminal png nocrop size 1024,768
+ set xlabel ' '
+ set size 0.6, 1
+ set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+ set y2label '$XLABEL' offset -2
+ set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+ set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+ set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+ set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+ set output "tmp.$$.png"
+ p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+ convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+ rm tmp.$$.png
+}
+
+function gen_intr_csw_stats()
+{
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $11}' > intrstat
+ cat -n intrstat > intrstat.new
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $12}' > cswstat
+ cat -n cswstat > cswstat.new
+}
+
+function plot_intr_csw_stats()
+{
+ vmstat_interval=5
+ xrange=$((`tail -1 times | awk '{print $1}'` + 50))
+ max_intr=$((`sort -n intrstat | tail -1` + 100))
+ max_csw=$((`sort -n cswstat | tail -1` + 100))
+ max_val=$max_csw
+ if [ $max_intr -gt $max_csw ]
+ then
+ max_val=$max_intr;
+ fi
+ identity=$1
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "Interrupts and context switches ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Interrupts/Context Switches"
+ set xr [0:$xrange]
+ set yr [0:$max_val]
+ set terminal png nocrop size 1024,768
+ set output "$INTR_CSW_PLOT_OUTPUT"
+ plot "intrstat.new" using (\$1*$vmstat_interval):2 title 'Interrupts' with lines lt 4 lw 2,\
+ "cswstat.new" using (\$1*$vmstat_interval):2 title 'Context Switches' with lines lt 5 lw 2,\
+ "times" using 1:(\$2*$max_val/100) title '' with impulse lt 2 lw 1
+EOF
+}
+
+function gen_netstats()
+{
+ ip_addr=$1
+ dev=`ssh $ip_addr ifconfig | grep -B1 $ip_addr | head -1 | cut -f1 -d' '`
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $3}' | cut -f1 -d'.' > rpkts
+ cat -n rpkts > rpkts.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $4}' | cut -f1 -d'.' > wpkts
+ cat -n wpkts > wpkts.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $5}' | cut -f1 -d'.' > rkbytes
+ cat -n rkbytes > rkbytes.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $6}' | cut -f1 -d'.' > wkbytes
+ cat -n wkbytes > wkbytes.new
+}
+
+function plot_netstats()
+{
+ sar_netstat_interval=5
+ max_read_pkts=$((`sort -n rpkts | tail -1` + 50))
+ max_write_pkts=$((`sort -n wpkts | tail -1` + 50))
+ max_read_kbytes=$(((`sort -n rkbytes | tail -1`)/1024 + 100))
+ max_write_kbytes=$(((`sort -n wkbytes | tail -1`)/1024 + 100))
+
+ max_pkts=$max_write_pkts
+ if [ $max_read_pkts -gt $max_write_pkts ]
+ then
+ max_pkts=$max_read_pkts;
+ fi
+
+ max_kbytes=$max_write_kbytes
+ if [ $max_read_kbytes -gt $max_write_kbytes ]
+ then
+ max_kbytes=$max_read_kbytes;
+ fi
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+ identity=$1
+
+ gnuplot <<EOF1
+ set autoscale
+ set grid
+ set title "Network statistics - Packet Read/Write ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Number of Packets"
+ set xr [0:$xrange]
+ set yr [0:$max_pkts]
+ set terminal png nocrop size 1024,768
+ set output "$NET_PKTS_PLOT_OUTPUT"
+ plot "rpkts.new" using (\$1*$sar_netstat_interval):2 title 'Read Packets' with lines lt 3 lw 2,\
+ "wpkts.new" using (\$1*$sar_netstat_interval):2 title 'Write Packets' with lines lt 4 lw 2,\
+ "times" using 1:(\$2*$max_pkts/100) title '' with impulse lt 2 lw 1
+EOF1
+ gnuplot <<EOF2
+ set autoscale
+ set grid
+ set title "Network Read-Write throughput ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Throughput in KB/sec"
+ set xr [0:$xrange]
+ set yr [0:$max_kbytes]
+ set terminal png nocrop size 1024,768
+ set output "$NET_TPUT_PLOT_OUTPUT"
+ plot "rkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Read throughput' with lines lt 3 lw 2,\
+ "wkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Write throughput' with lines lt 4 lw 2,\
+ "times" using 1:(\$2*$max_kbytes/100) title '' with impulse lt 2 lw 1
+EOF2
+}
+
+function analyse_plot_data()
+{
+ identity=$1
+
+ if [ $identity != "client" ]
+ then
+ brick=$2
+ fi
+ # Generate CPU data
+ gen_cpu_data
+ # plot CPU data
+ plot_cpu_usage $identity
+
+ # Generate VM data
+ gen_vm_data
+ # plot VM data
+ plot_vm_usage $identity
+
+ if [ $identity != "client" ]
+ then
+ # Generate io-times and io-throughput data
+ # This makes sense only for the bricks since the client is not involved in disk IO
+ gen_iostats $brick
+ # plot io-times and io-throughput data
+ plot_iostats $identity
+ fi
+
+ # Generate interrupt and context switch data
+ gen_intr_csw_stats
+ # plot interrupt and context switch data
+ plot_intr_csw_stats $identity
+
+ # Generate network packet statistics and throughput data
+ if [ $identity != "client" ]
+ then
+ gen_netstats $brick
+ else
+ gen_netstats $CLIENT_IP_ADDR
+ fi
+ # Generate network packet statistics and throughput data
+ plot_netstats $identity
+
+ # cleanup tmp files
+ rm vm_datafile memfile*
+ rm cpu-* times
+ if [ $identity != "client" ]
+ then
+ rm io_await* read_tput* write_tput*
+ fi
+ rm cswstat* intrstat*
+ rm rpkts* wpkts* rkbytes* wkbytes*
+}
+
+function analyse_client_data()
+{
+ MPSTAT_LOG=$rundir/client/mpstat_log
+ VMSTAT_LOG=$rundir/client/vmstat_log
+ IOSTAT_LOG=$rundir/client/iostat_log
+ SAR_NETSTAT_LOG=$rundir/client/sar_netstat_log
+ SYSINFO=$rundir/client/sysinfo
+ PERFLOG=$rundir/client/perf-test.log
+ CPU_PLOT_OUTPUT=$rundir/client/cpu.png
+ VM_PLOT_OUTPUT=$rundir/client/vm.png
+ IO_TIMES_PLOT_OUTPUT=$rundir/client/io_times.png
+ IO_TPUT_PLOT_OUTPUT=$rundir/client/io_tput.png
+ INTR_CSW_PLOT_OUTPUT=$rundir/client/intr_csw.png
+ NET_PKTS_PLOT_OUTPUT=$rundir/client/net-pkts.png
+ NET_TPUT_PLOT_OUTPUT=$rundir/client/net-tput.png
+ identity="client"
+ analyse_plot_data $identity
+}
+
+function analyse_brick_data()
+{
+ ind=0
+ for b in $BRICK_IP_ADDRS
+ do
+ ind=$((ind+1))
+ MPSTAT_LOG=$rundir/brick$ind*mpstat-log
+ VMSTAT_LOG=$rundir/brick$ind*vmstat-log
+ IOSTAT_LOG=$rundir/brick$ind*iostat-log
+ SAR_NETSTAT_LOG=$rundir/brick$ind-*sar_netstat-log
+ SYSINFO=$rundir/brick$ind*sysinfo-log
+ PERFLOG=$rundir/client/perf-test.log
+ CPU_PLOT_OUTPUT=$rundir/brick$ind-cpu.png
+ VM_PLOT_OUTPUT=$rundir/brick$ind-vm.png
+ IO_TIMES_PLOT_OUTPUT=$rundir/brick$ind-io_times.png
+ IO_TPUT_PLOT_OUTPUT=$rundir/brick$ind-io_tput.png
+ INTR_CSW_PLOT_OUTPUT=$rundir/brick$ind-intr_csw.png
+ NET_PKTS_PLOT_OUTPUT=$rundir/brick$ind-net-pkts.png
+ NET_TPUT_PLOT_OUTPUT=$rundir/brick$ind-net-tput.png
+
+ identity="brick$ind"
+ analyse_plot_data $identity $b
+ done
+}
+
+function do_comparison()
+{
+ # Generate comparison data
+ gen_cmp_data $1 $2
+
+ # plot perf comparison
+ plot_comparison
+}
+
+cp ops $LOCAL_LOG_REPO
+cd $LOCAL_LOG_REPO
+analyse_client_data
+analyse_brick_data
+rm ops
+
+if [ "$cmpdir" != "" ]
+then
+ CMP_PLOT_OUTPUT=$cmpdir-$rundir-cmp.png
+ CMP_DATAFILE=$rundir/client/cmp-with-$cmpdir-data.dat
+
+ XLABEL="Time in seconds"
+ YLABEL="Operations"
+ LEGEND_A="$cmpdir"
+ LEGEND_B="$rundir"
+ PLOT_TITLE="Performance comparison - $cmpdir vs $rundir"
+ do_comparison $cmpdir/client/perf-test.log $rundir/client/perf-test.log
+fi
+
+# Create tarball of the plots
+echo "Creating plots.tar..."
+cd $rundir
+tar cf plots.tar *.png client/*.png
diff --git a/perf-framework/automate_pwl_ssh b/perf-framework/automate_pwl_ssh
new file mode 100755
index 0000000..78e4c47
--- /dev/null
+++ b/perf-framework/automate_pwl_ssh
@@ -0,0 +1,43 @@
+#!/bin/bash -u
+
+expect_log=./expectlog
+
+if [ ! -f /root/.ssh/id_rsa.pub ]
+then
+ ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa
+fi
+
+if [ ! -f /usr/bin/expect ]
+then
+ echo "expect not found. Attempting to install..."
+ if grep -i centos /etc/issue > /dev/null 2>&1
+ then
+ echo "CentOS. Using yum"
+ yum install -y `yum whatprovides expect | grep -o "expect.*x86_64"` > /dev/null 2>&1
+ if [ $? -ne 0 ]
+ then
+ echo "Failed installing expect. Exiting..."
+ exit 1
+ fi
+ elif grep -i ubuntu /etc/issue > /dev/null 2>&1
+ then
+ echo "Ubuntu. Using apt-get"
+ apt-get install -y expect > /dev/null 2>&1
+ if [ $? -ne 0 ]
+ then
+ echo "Failed installing expect. Exiting..."
+ exit 1
+ fi
+ else
+ echo "Unknown distribution"
+ echo "Install expect and try again."
+ exit 1
+ fi
+fi
+
+echo "Attempting passwordless ssh setup on multiple hosts."
+echo "log can be found in $expect_log"
+
+export ROOT_PW1
+export ROOT_PW2
+./pwl_ssh $expect_log $SSH_HOSTS
diff --git a/perf-framework/batchrun b/perf-framework/batchrun
new file mode 100755
index 0000000..d45455e
--- /dev/null
+++ b/perf-framework/batchrun
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+RUNLIST="glusterfs-3git.tar.gz"
+
+function do_run()
+{
+ FIRST=`cat .runfile`
+ ./start_perf_measure
+# sed -i 's/^\.\/create_gluster_vol/#&/' start_perf_measure
+# sleep 10
+# if [ $MINOR -gt 1 ]; then
+# LAST=$(($FIRST+3))
+# ./quota_gsync_run $run
+# sed -i 's/^#\.\/create_gluster_vol/\.\/create_gluster_vol/' start_perf_measure
+# fi
+}
+
+function extract_release()
+{
+ MAJOR=`echo $runlabel | cut -f1 -d'.' | grep -o "^[0-9]"`
+ MINOR=`echo $runlabel | cut -f2 -d'.' | grep -o "^[0-9]"`
+}
+
+for run in $RUNLIST
+do
+ cp tarballs/$run .
+ ./setrun $run
+ ./deploy_gluster
+ ./check_install.new
+ if [ $? -ne 0 ]
+ then
+ echo "Installation of run $run failed. Continuing with next run"
+ continue
+ fi
+ echo "Sleeping for 10 seconds.."
+ runlabel=`echo $run|sed -e 's/^glusterfs-//' -e 's/\.tar\.gz//'`
+ echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota off, gsync off)" >> $RUNLOG
+ sleep 10
+ extract_release
+ do_run
+ rm $run
+ sed -i 's/ENABLE_ACL=no/ENABLE_ACL=yes/' gf_perf_config
+ if [ $MINOR -gt 1 ]; then
+ ./send_mail `seq $FIRST $LAST`
+ else
+ ./send_mail $FIRST
+ fi
+done
diff --git a/perf-framework/buildit b/perf-framework/buildit
new file mode 100755
index 0000000..df3f176
--- /dev/null
+++ b/perf-framework/buildit
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+./autogen.sh > autogen.out.remote 2> autogen.err.remote
+./configure CFLAGS="-g -O0 -DDEBUG" > configure.out.remote 2> configure.err.remote
+make > make.out.remote 2> make.err.remote
+make install > make_install.out.remote 2> make_install.err.remote
diff --git a/perf-framework/calc_all b/perf-framework/calc_all
new file mode 100755
index 0000000..01e4c91
--- /dev/null
+++ b/perf-framework/calc_all
@@ -0,0 +1,49 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+num=""
+for run in "$@"
+do
+ egrep_patt+="^`echo run$run\|`"
+ PERFLOG="log_repo/run$run/client/perf-test.log"
+ if [ $run -lt 9 ]
+ then
+ run="0$run"
+ fi
+ echo "RUN$run " > /tmp/run$run.$$
+ echo "----------------" >> /tmp/run$run.$$
+ for op in `cat ops`
+ do
+ printf "%-6s" `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | tr '\n' '\t'|sed 's/ $//'` >> /tmp/run$run.$$
+ printf "\n" >> /tmp/run$run.$$
+ done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+if [ $# -lt 2 ]; then
+ paste /tmp/tmp_ops.$$ /tmp/run*.$$;
+else
+ for list in "$@"; do
+ if [ $list -lt 9 ]; then
+ list="0$list"
+ fi
+ if [ "$1" -eq $list ]; then
+ num="{$list"
+ else
+ num="$num,$list";
+ fi
+
+ if [ "${@: -1}" -eq $list ]; then
+ num="$num}"
+ fi
+ done
+ eval "paste /tmp/tmp_ops.$$ /tmp/run$num.$$";
+fi
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_avg b/perf-framework/calc_avg
new file mode 100755
index 0000000..596554f
--- /dev/null
+++ b/perf-framework/calc_avg
@@ -0,0 +1,54 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+num=""
+for run in "$@"
+do
+ egrep_patt+="^`echo run$run\|`"
+ PERFLOG="log_repo/run$run/client/perf-test.log"
+ if [ $run -lt 9 ]
+ then
+ run="0$run"
+ fi
+ echo "RUN$run" > /tmp/run$run.$$
+ echo "-------" >> /tmp/run$run.$$
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> /tmp/run$run.$$
+ done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+if [ $# -lt 2 ]; then
+ paste /tmp/tmp_ops.$$ /tmp/run*.$$;
+else
+ for list in "$@"; do
+ if [ $list -lt 9 ]; then
+ list="0$list"
+ fi
+ if [ "$1" -eq $list ]; then
+ num="{$list"
+ else
+ num="$num,$list";
+ fi
+
+ if [ "${@: -1}" -eq $list ]; then
+ num="$num}"
+ fi
+ done
+ eval "paste /tmp/tmp_ops.$$ /tmp/run$num.$$";
+fi
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_best b/perf-framework/calc_best
new file mode 100755
index 0000000..fed9fda
--- /dev/null
+++ b/perf-framework/calc_best
@@ -0,0 +1,31 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+
+for run in "$@"
+do
+ egrep_patt+="^`echo run$run\|`"
+ PERFLOG="log_repo/run$run/client/perf-test.log"
+ if [ $run -lt 9 ]
+ then
+ run="0$run"
+ fi
+ echo "RUN$run " > /tmp/run$run.$$
+ echo "------------" >> /tmp/run$run.$$
+ for op in `cat ops`
+ do
+ printf "%-6s" `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | cat -n | sort -n -k2 | head -1` >> /tmp/run$run.$$
+ printf "\n" >> /tmp/run$run.$$
+ done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+paste /tmp/tmp_ops.$$ /tmp/run*.$$
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_worst b/perf-framework/calc_worst
new file mode 100755
index 0000000..363ac58
--- /dev/null
+++ b/perf-framework/calc_worst
@@ -0,0 +1,31 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+
+for run in "$@"
+do
+ egrep_patt+="^`echo run$run\|`"
+ PERFLOG="log_repo/run$run/client/perf-test.log"
+ if [ $run -lt 9 ]
+ then
+ run="0$run"
+ fi
+ echo "RUN$run " > /tmp/run$run.$$
+ echo "------------" >> /tmp/run$run.$$
+ for op in `cat ops`
+ do
+ printf "%-6s" `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | cat -n | sort -n -k2 | tail -1` >> /tmp/run$run.$$
+ printf "\n" >> /tmp/run$run.$$
+ done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+paste /tmp/tmp_ops.$$ /tmp/run*.$$
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/check_install b/perf-framework/check_install
new file mode 100755
index 0000000..aa72447
--- /dev/null
+++ b/perf-framework/check_install
@@ -0,0 +1,13 @@
+#!/bin/bash -ue
+
+source gf_perf_config
+
+gluster_build_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+def_gfd_loc=/usr/local/sbin/glusterfsd
+
+echo "Checking server installation"
+./run cksum $SERVER_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd $def_gfd_loc
+
+echo ""
+echo "Checking client installation"
+cksum $CLIENT_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd $def_gfd_loc
diff --git a/perf-framework/check_install.new b/perf-framework/check_install.new
new file mode 100755
index 0000000..d571ae1
--- /dev/null
+++ b/perf-framework/check_install.new
@@ -0,0 +1,33 @@
+#!/bin/bash -ue
+
+source gf_perf_config
+
+gluster_build_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+def_gfd_loc=/usr/local/sbin/glusterfsd
+
+echo "Checking server installation"
+for brick in $BRICK_IP_ADDRS
+do
+ build_file_cksum=`ssh -l root $brick "cksum $SERVER_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd" | cut -f1 -d' '`
+ current_installed_file_cksum=`ssh -l root $brick cksum $def_gfd_loc | cut -f1 -d' '`
+ if [ "$build_file_cksum" != "$current_installed_file_cksum" ]
+ then
+ echo "Checksum on $brick do not match. Aborting..."
+ exit 1
+ else
+ echo "Checksum on $brick OK"
+ fi
+done
+
+echo ""
+echo "Checking client installation"
+build_file_cksum=`cksum $CLIENT_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd | cut -f1 -d' '`
+current_installed_file_cksum=`cksum $def_gfd_loc | cut -f1 -d' '`
+
+if [ "$build_file_cksum" != "$current_installed_file_cksum" ]
+then
+ echo "Checksum on client do not match. Aborting..."
+ exit 1
+else
+ echo "Checksum on client OK"
+fi
diff --git a/perf-framework/create_gluster_vol b/perf-framework/create_gluster_vol
new file mode 100755
index 0000000..fcb91ce
--- /dev/null
+++ b/perf-framework/create_gluster_vol
@@ -0,0 +1,122 @@
+#!/bin/bash -ue
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+GF_BIN=/usr/local/sbin
+
+function mgmt_vol_create()
+{
+ if [ $ENABLE_ACL == "yes" ]
+ then
+ ./acl on
+ else
+ ./acl off
+ fi
+ for brick in $BRICK_IP_ADDRS
+ do
+ ssh -l root $MGMT_NODE "$GF_BIN/gluster peer probe $brick"
+ done
+ ssh -l root $MGMT_NODE "$@"
+ ssh -l root $MGMT_NODE "$GF_BIN/gluster volume start $VOLNAME"
+ if [ $GF_CONFIG == "stripe" ]; then
+ echo "Setting cache-size to 128MB for stripe tests..."
+ ssh -l root $MGMT_NODE "$GF_BIN/gluster volume set $VOLNAME cache-size 128MB"
+ fi
+ #mem-factor is set for mount type of nfs to increase performance.
+ if [ $MOUNT_TYPE == "nfs" ]; then
+ echo "Setting mem-factor to 20 since mount type is nfs..."
+ ssh -l root $MGMT_NODE "$GF_BIN/gluster volume set $VOLNAME nfs.mem-factor 20"
+ fi
+}
+
+function cleanup_brick()
+{
+ ssh -l root $1 "killall glusterd glusterfs glusterfsd > /dev/null 2>&1; rm -rf /etc/glusterd"
+
+ if [ "$SERVER_EXPORT_DIR" != "" ]
+ then
+ echo "Deleting export dir $SERVER_EXPORT_DIR on $brick..."
+ ssh -l root $1 "rm -rf $SERVER_EXPORT_DIR"
+ else
+ echo "************* Empty export dir in config *************"
+ exit
+ fi
+}
+
+echo "Cleaning bricks..."
+for brick in $BRICK_IP_ADDRS
+do
+ cleanup_brick $brick &
+done
+
+wait
+
+echo "Creating export directory..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "mkdir -p $SERVER_EXPORT_DIR"
+done
+
+# Start glusterd on all the bricks
+echo "Starting glusterd..."
+for brick in $BRICK_IP_ADDRS
+do
+ if [ $ENABLE_MEM_ACCT == "yes" ]
+ then
+ ssh -l root $brick "GLUSTERFS_DISABLE_MEM_ACCT=0 $GF_BIN/glusterd"
+ else
+ ssh -l root $brick "$GF_BIN/glusterd"
+ fi
+done
+
+# Create the gluster volume according to the config
+
+bricklist=""
+count=0
+for brick in $BRICK_IP_ADDRS
+do
+ bricklist+="$brick:$SERVER_EXPORT_DIR "
+ count=$((count+1))
+done
+
+if [ "$GF_CONFIG" == "replicate" -a "$count" -gt "2" ]
+then
+ echo "NOTE:"
+ echo "Replicate volume with more than 2 bricks created."
+ echo "This will result in a distributed replicate config."
+fi
+
+echo "Creating $GF_CONFIG volume..."
+if [ "$GF_CONFIG" == "distribute" ]
+then
+ mgmt_vol_create "gluster volume create $VOLNAME $bricklist"
+elif [ "$GF_CONFIG" == "replicate" ]
+then
+ mgmt_vol_create "gluster volume create $VOLNAME replica 2 $bricklist"
+elif [ "$GF_CONFIG" == "distrep" ]
+then
+ mgmt_vol_create "gluster volume create $VOLNAME replica 2 $bricklist"
+elif [ "$GF_CONFIG" == "stripe" ]
+then
+ mgmt_vol_create "gluster volume create $VOLNAME stripe 4 $bricklist"
+else
+ echo "Unknown configuration. Exiting..."
+ exit 1
+fi
+
+sleep 5
+
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+ for brick in $BRICK_IP_ADDRS
+ do
+ echo "Memory accounting status on : $brick -"
+ ssh -l root $brick "\
+ echo \"x/x &gf_mem_acct_enable\" > gf_gdb_commands;\
+ echo \"quit\" >> gf_gdb_commands;\
+ gdb -q --command=gf_gdb_commands -p \`pidof glusterfsd\` 2> /dev/null | \
+ grep gf_mem_acct_enable | awk '{print \$(NF-1) \$NF}';\
+ rm gf_gdb_commands > /dev/null 2>&1"
+ echo ""
+ done
+fi
diff --git a/perf-framework/deploy_gluster b/perf-framework/deploy_gluster
new file mode 100755
index 0000000..49543f7
--- /dev/null
+++ b/perf-framework/deploy_gluster
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+function prepare_server()
+{
+ ip_addr=$1
+ ssh -l root $ip_addr mkdir -p $SERVER_BUILD_DIR
+ scp -p $RELEASE_TARBALL $BUILD_SCRIPT root@$ip_addr:$SERVER_BUILD_DIR
+ ssh -l root $ip_addr "cd $SERVER_BUILD_DIR && tar xf $RELEASE_TARBALL"
+ ssh -l root $ip_addr "cd $SERVER_BUILD_DIR/$release_dir && ../$BUILD_SCRIPT"
+}
+
+function prepare_client()
+{
+ mkdir -p $CLIENT_BUILD_DIR
+ cp $RELEASE_TARBALL $BUILD_SCRIPT $CLIENT_BUILD_DIR
+ cd $CLIENT_BUILD_DIR && tar xf $RELEASE_TARBALL
+ cd $CLIENT_BUILD_DIR/$release_dir && ../$BUILD_SCRIPT
+}
+
+
+CONFIG_FILE=gf_perf_config
+BUILD_SCRIPT=buildit
+source $CONFIG_FILE
+
+if [ "$RELEASE_TARBALL" == "" ]
+then
+ echo "ERROR: Empty release tarball"
+ exit 1
+fi
+
+if [ ! -f $RELEASE_TARBALL ]
+then
+ echo "ERROR: Unable to find source tarball $RELEASE_TARBALL"
+ exit 1
+fi
+
+release_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+
+# Build gluster on the bricks
+for brick in `echo $BRICK_IP_ADDRS`
+do
+ echo "Starting build on server $brick..."
+ prepare_server $brick &
+done
+
+echo "Waiting for jobs to finish..."
+sleep 5
+
+# Build gluster on the client
+echo "Starting build on client..."
+prepare_client &
+
+wait
diff --git a/perf-framework/diff_perfrun b/perf-framework/diff_perfrun
new file mode 100755
index 0000000..65ea316
--- /dev/null
+++ b/perf-framework/diff_perfrun
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+echo ""
+echo "Q - Quota ON, q - Quota OFF, G - Gsync ON, g - Gsync OFF"
+echo ""
+echo "======================================================================"
+echo "OPERATIONS q-g q-G Q-G Q-g"
+echo "======================================================================"
+for i in `cat /tmp/ops`
+do
+ printf "%-25s" $i
+ for j in 2 3 4 5
+ do
+ printf "%10.1f" `echo \`grep -w ^$i /tmp/$1 | awk -v col=$j '{print $col}'\` | awk '{print ($2 - $1)*100/$1}' | tr '\n' '\t'`
+ done
+ echo ""
+done
diff --git a/perf-framework/file_open_analyse b/perf-framework/file_open_analyse
new file mode 100755
index 0000000..f7c98f5
--- /dev/null
+++ b/perf-framework/file_open_analyse
@@ -0,0 +1,430 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+if [ $# -le 0 -o $# -gt 2 ]
+then
+ echo "Usage : $0 <run directory>"
+ exit 1
+fi
+
+rundir=$1
+cmpdir=""
+
+if [ $# -eq 2 ]
+then
+ cmpdir=$2
+fi
+
+function gen_cpu_data()
+{
+ time=0
+ sum=0
+
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+ idle_col_tweak=$?
+ echo `grep CPU $MPSTAT_LOG | head -1 | awk '{print $3}'` | grep -o CPU > /dev/null
+ cpu_col_tweak=$?
+ for i in "all" 0 `seq $((num_procs-1))`
+ do
+ egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$cpu_col_tweak -v v2=$idle_col_tweak '{print $(3-v1) " " $(NF-v2)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+ cat -n cpu-$i > cpu-$i.new
+ done
+}
+
+function plot_cpu_usage()
+{
+ mpstat_interval=5
+ plot_info=pinfo.$$
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ ltype=2
+ identity=$1
+
+ for i in 0 `seq $((num_procs-1))`
+ do
+ echo -ne "\"cpu-$i.new\" using (\$1*$mpstat_interval):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+ ltype=$((ltype+1))
+ done
+ echo -ne "\"cpu-all.new\" using (\$1*$mpstat_interval):2 title 'cpu-all' with lines lt 1 lw 2" >> $plot_info
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "CPU utilization : All CPUs ($identity)"
+ set xlabel "Time"
+ set ylabel "% CPU utilization"
+ set yr [0:100]
+ set terminal png nocrop size 1024,768
+ set output "$CPU_PLOT_OUTPUT"
+ plot `cat $plot_info`
+EOF
+ rm $plot_info
+}
+
+function gen_vm_data()
+{
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $4}' > vm_datafile
+ totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+ cat -n memfile > memfile.new
+}
+
+function plot_vm_usage()
+{
+ vmstat_interval=5
+ total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ identity=$1
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "Memory utilization ($identity)"
+ set xlabel "Time"
+ set ylabel "Memory utilization in bytes"
+ set yr [0:$total_mem]
+ set terminal png nocrop size 1024,768
+ set output "$VM_PLOT_OUTPUT"
+ plot "memfile.new" using (\$1*$vmstat_interval):2 title 'memory-usage' with lines lt 2 lw 2
+EOF
+}
+
+function gen_iostats()
+{
+ brick=$1
+ dev=`ssh $brick "df -h $SERVER_EXPORT_DIR" | tail -1 | awk '{print $1}' | cut -f3 -d'/'`
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $10}' | cut -f1 -d'.' > io_await
+ cat -n io_await > io_await.new
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $6}' | cut -f1 -d'.' > read_tput
+ cat -n read_tput > read_tput.new
+ egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $7}' | cut -f1 -d'.' > write_tput
+ cat -n write_tput > write_tput.new
+}
+
+function plot_iostats()
+{
+ iostat_interval=5
+ max_wait=$((`sort -n io_await | tail -1` + 50))
+ max_read=$(((`sort -n read_tput | tail -1`) / 2 + 100))
+ max_write=$(((`sort -n write_tput | tail -1`) / 2 + 100))
+ max_io=$max_write
+ if [ $max_read -gt $max_write ]
+ then
+ max_io=$max_read;
+ fi
+ identity=$1
+
+ gnuplot <<EOF1
+ set autoscale
+ set grid
+ set title "IO Wait times ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "IO Wait times in milliseconds"
+ set yr [0:$max_wait]
+ set terminal png nocrop size 1024,768
+ set output "$IO_TIMES_PLOT_OUTPUT"
+ plot "io_await.new" using (\$1*$iostat_interval):2 title 'IO wait times' with lines lt 3 lw 2
+EOF1
+ gnuplot <<EOF2
+ set autoscale
+ set grid
+ set title "Disk Read-Write throughput ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Throughput in KB/sec"
+ set yr [0:$max_io]
+ set terminal png nocrop size 1024,768
+ set output "$IO_TPUT_PLOT_OUTPUT"
+ plot "read_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Read throughput' with lines lt 4 lw 2,\
+ "write_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Write throughput' with lines lt 5 lw 2
+EOF2
+}
+
+function gen_cmp_data()
+{
+ perflog_baseline=$1
+ perflog_current=$2
+
+ time=0
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> btimes.$$
+ done
+
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> ctimes.$$
+ done
+
+ paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+ rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+ a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+ yrange=`echo $a + $a/5 | bc`
+ b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+ xrange=`echo $b - 0.5 | bc`
+
+ gnuplot <<EOF
+ reset
+ set key at graph 0.15, 0.85 horizontal samplen 0.1
+ set style data histogram
+ set style histogram cluster gap 1
+ set style fill solid border -1
+ set boxwidth 0.8
+ set xtic rotate by 90 scale 0
+ unset ytics
+ set y2tics rotate by 90
+ set terminal png nocrop size 1024,768
+ set xlabel ' '
+ set size 0.6, 1
+ set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+ set y2label '$XLABEL' offset -2
+ set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+ set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+ set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+ set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+ set output "tmp.$$.png"
+ p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+ convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+ rm tmp.$$.png
+}
+
+function gen_intr_csw_stats()
+{
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $11}' > intrstat
+ cat -n intrstat > intrstat.new
+ egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $12}' > cswstat
+ cat -n cswstat > cswstat.new
+}
+
+function plot_intr_csw_stats()
+{
+ vmstat_interval=5
+ max_intr=$((`sort -n intrstat | tail -1` + 100))
+ max_csw=$((`sort -n cswstat | tail -1` + 100))
+ max_val=$max_csw
+ if [ $max_intr -gt $max_csw ]
+ then
+ max_val=$max_intr;
+ fi
+ identity=$1
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "Interrupts and context switches ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Interrupts/Context Switches"
+ set yr [0:$max_val]
+ set terminal png nocrop size 1024,768
+ set output "$INTR_CSW_PLOT_OUTPUT"
+ plot "intrstat.new" using (\$1*$vmstat_interval):2 title 'Interrupts' with lines lt 4 lw 2,\
+ "cswstat.new" using (\$1*$vmstat_interval):2 title 'Context Switches' with lines lt 5 lw 2
+EOF
+}
+
+function gen_netstats()
+{
+ ip_addr=$1
+ dev=`ssh $ip_addr ifconfig | grep -B1 $ip_addr | head -1 | cut -f1 -d' '`
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $3}' | cut -f1 -d'.' > rpkts
+ cat -n rpkts > rpkts.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $4}' | cut -f1 -d'.' > wpkts
+ cat -n wpkts > wpkts.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $5}' | cut -f1 -d'.' > rkbytes
+ cat -n rkbytes > rkbytes.new
+ egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $6}' | cut -f1 -d'.' > wkbytes
+ cat -n wkbytes > wkbytes.new
+}
+
+function plot_netstats()
+{
+ sar_netstat_interval=5
+ max_read_pkts=$((`sort -n rpkts | tail -1` + 50))
+ max_write_pkts=$((`sort -n wpkts | tail -1` + 50))
+ max_read_kbytes=$(((`sort -n rkbytes | tail -1`)/1024 + 100))
+ max_write_kbytes=$(((`sort -n wkbytes | tail -1`)/1024 + 100))
+
+ max_pkts=$max_write_pkts
+ if [ $max_read_pkts -gt $max_write_pkts ]
+ then
+ max_pkts=$max_read_pkts;
+ fi
+
+ max_kbytes=$max_write_kbytes
+ if [ $max_read_kbytes -gt $max_write_kbytes ]
+ then
+ max_kbytes=$max_read_kbytes;
+ fi
+ identity=$1
+
+ gnuplot <<EOF1
+ set autoscale
+ set grid
+ set title "Network statistics - Packet Read/Write ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Number of Packets"
+ set yr [0:$max_pkts]
+ set terminal png nocrop size 1024,768
+ set output "$NET_PKTS_PLOT_OUTPUT"
+ plot "rpkts.new" using (\$1*$sar_netstat_interval):2 title 'Read Packets' with lines lt 3 lw 2,\
+ "wpkts.new" using (\$1*$sar_netstat_interval):2 title 'Write Packets' with lines lt 4 lw 2
+EOF1
+ gnuplot <<EOF2
+ set autoscale
+ set grid
+ set title "Network Read-Write throughput ($identity)"
+ set xlabel "Time in seconds"
+ set ylabel "Throughput in KB/sec"
+ set yr [0:$max_kbytes]
+ set terminal png nocrop size 1024,768
+ set output "$NET_TPUT_PLOT_OUTPUT"
+ plot "rkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Read throughput' with lines lt 3 lw 2,\
+ "wkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Write throughput' with lines lt 4 lw 2
+EOF2
+}
+
+function analyse_plot_data()
+{
+ identity=$1
+
+ if [ $identity != "client" ]
+ then
+ brick=$2
+ fi
+ # Generate CPU data
+ gen_cpu_data
+ # plot CPU data
+ plot_cpu_usage $identity
+
+ # Generate VM data
+ gen_vm_data
+ # plot VM data
+ plot_vm_usage $identity
+
+ if [ $identity != "client" ]
+ then
+ # Generate io-times and io-throughput data
+ # This makes sense only for the bricks since the client is not involved in disk IO
+ gen_iostats $brick
+ # plot io-times and io-throughput data
+ plot_iostats $identity
+ fi
+
+ # Generate interrupt and context switch data
+ gen_intr_csw_stats
+ # plot interrupt and context switch data
+ plot_intr_csw_stats $identity
+
+ # Generate network packet statistics and throughput data
+ if [ $identity != "client" ]
+ then
+ gen_netstats $brick
+ else
+ gen_netstats $CLIENT_IP_ADDR
+ fi
+ # Generate network packet statistics and throughput data
+ plot_netstats $identity
+
+ # cleanup tmp files
+ rm vm_datafile memfile*
+ rm cpu-*
+ if [ $identity != "client" ]
+ then
+ rm io_await* read_tput* write_tput*
+ fi
+ rm cswstat* intrstat*
+ rm rpkts* wpkts* rkbytes* wkbytes*
+}
+
+function analyse_client_data()
+{
+ MPSTAT_LOG=$rundir/client/mpstat_log
+ VMSTAT_LOG=$rundir/client/vmstat_log
+ IOSTAT_LOG=$rundir/client/iostat_log
+ SAR_NETSTAT_LOG=$rundir/client/sar_netstat_log
+ SYSINFO=$rundir/client/sysinfo
+ PERFLOG=$rundir/client/perf-test.log
+ CPU_PLOT_OUTPUT=$rundir/client/cpu.png
+ VM_PLOT_OUTPUT=$rundir/client/vm.png
+ IO_TIMES_PLOT_OUTPUT=$rundir/client/io_times.png
+ IO_TPUT_PLOT_OUTPUT=$rundir/client/io_tput.png
+ INTR_CSW_PLOT_OUTPUT=$rundir/client/intr_csw.png
+ NET_PKTS_PLOT_OUTPUT=$rundir/client/net-pkts.png
+ NET_TPUT_PLOT_OUTPUT=$rundir/client/net-tput.png
+ identity="client"
+ analyse_plot_data $identity
+}
+
+function analyse_brick_data()
+{
+ ind=0
+ for b in $BRICK_IP_ADDRS
+ do
+ ind=$((ind+1))
+ MPSTAT_LOG=$rundir/brick$ind*mpstat-log
+ VMSTAT_LOG=$rundir/brick$ind*vmstat-log
+ IOSTAT_LOG=$rundir/brick$ind*iostat-log
+ SAR_NETSTAT_LOG=$rundir/brick$ind-*sar_netstat-log
+ SYSINFO=$rundir/brick$ind*sysinfo-log
+ PERFLOG=$rundir/client/perf-test.log
+ CPU_PLOT_OUTPUT=$rundir/brick$ind-cpu.png
+ VM_PLOT_OUTPUT=$rundir/brick$ind-vm.png
+ IO_TIMES_PLOT_OUTPUT=$rundir/brick$ind-io_times.png
+ IO_TPUT_PLOT_OUTPUT=$rundir/brick$ind-io_tput.png
+ INTR_CSW_PLOT_OUTPUT=$rundir/brick$ind-intr_csw.png
+ NET_PKTS_PLOT_OUTPUT=$rundir/brick$ind-net-pkts.png
+ NET_TPUT_PLOT_OUTPUT=$rundir/brick$ind-net-tput.png
+
+ identity="brick$ind"
+ analyse_plot_data $identity $b
+ done
+}
+
+function do_comparison()
+{
+ # Generate comparison data
+ gen_cmp_data $1 $2
+
+ # plot perf comparison
+ plot_comparison
+}
+
+cp ops $LOCAL_LOG_REPO
+cd $LOCAL_LOG_REPO
+analyse_client_data
+analyse_brick_data
+rm ops
+
+if [ "$cmpdir" != "" ]
+then
+ CMP_PLOT_OUTPUT=$cmpdir-$rundir-cmp.png
+ CMP_DATAFILE=$rundir/client/cmp-with-$cmpdir-data.dat
+
+ XLABEL="Time in seconds"
+ YLABEL="Operations"
+ LEGEND_A="$cmpdir"
+ LEGEND_B="$rundir"
+ PLOT_TITLE="Performance comparison - $cmpdir vs $rundir"
+ do_comparison $cmpdir/client/perf-test.log $rundir/client/perf-test.log
+fi
+
+# Create tarball of the plots
+echo "Creating plots.tar..."
+cd $rundir
+tar cf plots.tar *.png client/*.png
diff --git a/perf-framework/file_open_test b/perf-framework/file_open_test
new file mode 100755
index 0000000..83cbe97
--- /dev/null
+++ b/perf-framework/file_open_test
@@ -0,0 +1,159 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+SETTLE_TIME=10
+RUNFILE=.runfile
+STAT_COLLECTOR=stat_collect
+
+# Generate current run, update runfile
+if [ ! -f $RUNFILE ]
+then
+ run=1
+else
+ run=`cat $RUNFILE`
+fi
+echo $((run+1)) > $RUNFILE
+
+# Drop vm caches on all the bricks before starting the runs
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "echo 3 > /proc/sys/vm/drop_caches"
+done
+
+# Create the gluster volume
+./create_gluster_vol
+
+mount | grep $MOUNT_POINT > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+ umount $MOUNT_POINT
+fi
+
+if [ ! -d $MOUNT_POINT ]
+then
+ mkdir -p $MOUNT_POINT
+fi
+
+# Make sure that the fuse kernel module is loaded
+/sbin/lsmod | grep -w fuse > /dev/null 2>&1
+if [ $? -ne 0 ]
+then
+ /sbin/modprobe fuse > /dev/null 2>&1
+fi
+
+ps -eaf | egrep -w 'glusterfs|glusterfsd|glusterd' > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+ killall glusterfsd glusterd glusterfs > /dev/null 2>&1
+fi
+
+# Mount the client
+# Sleep for a while. Sometimes, NFS mounts fail if attempted soon after creating the volume
+sleep $SETTLE_TIME
+
+if [ $ENABLE_ACL == "yes" ]
+then
+ acl_opts="-o acl"
+else
+ acl_opts=""
+fi
+
+echo "Mounting volume..."
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+ echo "Memory accounting status on client -"
+ echo "x/x &gf_mem_acct_enable" > commands.$$
+ echo "quit" >> commands.$$
+ GLUSTERFS_DISABLE_MEM_ACCT=0 mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+ mount_status=$?
+ gdb -q --command=commands.$$ -p `pidof glusterfs` | grep gf_mem_acct_enable | awk '{print $(NF-1) $NF}'
+ rm commands.$$ > /dev/null 2>&1
+else
+ mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+ mount_status=$?
+fi
+
+if [ $mount_status -ne 0 ]
+then
+ echo "mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT failed..."
+ echo "Exiting..."
+ exit 1
+fi
+
+# Copy statistics collection scripts to the server
+
+echo ""
+echo "Copying stat collection script to bricks..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "mkdir -p $SERVER_SCRIPTS_DIR"
+ scp -p $STAT_COLLECTOR root@$brick:$SERVER_SCRIPTS_DIR > /dev/null 2>&1
+done
+
+# Run statistics collection scripts on the server
+
+echo ""
+echo "Starting server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "mkdir -p $SERVER_LOG_DIR"
+ ssh -l root $brick "$SERVER_SCRIPTS_DIR/$STAT_COLLECTOR $SERVER_LOG_DIR" &
+done
+
+# Run statistics collection on client
+
+mkdir -p $LOCAL_LOG_REPO
+./$STAT_COLLECTOR $LOCAL_LOG_REPO/run$run/client &
+
+# Start perf test
+
+echo ""
+echo "Starting run $run..."
+
+sleep $SETTLE_TIME
+./parallel_create 50 10000
+sleep $SETTLE_TIME
+
+# Stop statistics collection scripts on the client
+
+killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+
+# Stop statistics collection scripts on the server
+
+echo ""
+echo "Stopping server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+done
+
+# Since the ssh processes were backgrounded, they will be hanging around.
+# Kill them
+kill `jobs -l | awk '{print $2}'` > /dev/null 2>&1
+
+# Copy statistics from the server
+
+echo ""
+echo "Copying server logfiles for run $run..."
+cur_log_dump_dir=$LOCAL_LOG_REPO/run$run
+mkdir -p $cur_log_dump_dir
+count=1
+for brick in $BRICK_IP_ADDRS
+do
+ for statf in mpstat vmstat iostat sysinfo sar_netstat
+ do
+ scp root@$brick:$SERVER_LOG_DIR/*$statf* $cur_log_dump_dir/brick$count-$brick-$statf-log > /dev/null 2>&1
+ done
+ count=$((count + 1))
+done
+
+# Cleanup statistics collected for this run on the server
+
+echo ""
+echo "Cleaning server logfiles..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "cd $SERVER_LOG_DIR; rm mpstat_log vmstat_log iostat_log sysinfo sar_netstat_log"
+done
diff --git a/perf-framework/gen_perf_data b/perf-framework/gen_perf_data
new file mode 100755
index 0000000..31110f6
--- /dev/null
+++ b/perf-framework/gen_perf_data
@@ -0,0 +1,191 @@
+#!/bin/bash -u
+
+MPSTAT_LOG=mpstat_log1908
+VMSTAT_LOG=vmstat_log1908
+SYSINFO=sysinfo1908
+PERFLOG=perf-fuse-ami.log
+
+source gnuplotrc
+
+function gen_cpu_data()
+{
+ time=0
+ sum=0
+
+ if [ -f times ]
+ then
+ rm times
+ fi
+
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ sum=$((sum + $time))
+ echo $sum >> times
+ done
+ sed -i 's/$/ 200/g' times
+
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+ col_tweak=$?
+ for i in "all" 0 `seq $((num_procs-1))`
+ do
+ egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$col_tweak '{print $3 " " $(NF-v1)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+ cat -n cpu-$i > cpu-$i.new
+ done
+}
+
+function plot_cpu_usage()
+{
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+ MPSTAT_INTERVAL=5
+ plot_info=pinfo.$$
+ num_procs=`grep -w ^processor $SYSINFO | wc -l`
+ ltype=1
+
+ for i in "all" 0 `seq $((num_procs-1))`
+ do
+ echo -ne "\"cpu-$i.new\" using (\$1*$MPSTAT_INTERVAL):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+ ltype=$((ltype+1))
+ done
+ echo "\"times\" using 1:2 title '' with impulse lt 2 lw 1 lc rgb \"#FF0000\"" >> $plot_info
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "CPU utilization : All CPUs"
+ set xlabel "Time"
+ set ylabel "% CPU utilization"
+ set xr [0:$xrange]
+ set yr [0:100]
+ set terminal png nocrop size 1024,768
+ set output "$CPU_PLOT_OUTPUT"
+ plot `cat $plot_info`
+EOF
+ rm $plot_info
+}
+
+function gen_vm_data()
+{
+ egrep -v 'memory|free' $VMSTAT_LOG | awk '{print $4}' > vm_datafile
+ totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+}
+
+function plot_vm_usage()
+{
+ vmstat_interval=5
+ total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+ xrange=$((`tail -1 times | awk '{print $1}'`+50))
+
+ gnuplot <<EOF
+ set autoscale
+ set grid
+ set title "Memory utilization"
+ set xlabel "Time"
+ set ylabel "Memory utilization in bytes"
+ set xr [0:$xrange]
+ set yr [0:$total_mem]
+ set terminal png nocrop size 1024,768
+ set output "$VM_PLOT_OUTPUT"
+ plot "vm_datafile" using 1 title 'free-memory' with lines lt 1 lw 2, \
+ "memfile" using 1 title 'memory-usage' with lines lt 2 lw 2,\
+ "times" using 1:(\$2*$total_mem/100) title '' with impulse lt 2 lw 1 lc rgb "#FF0000"
+EOF
+}
+
+function gen_cmp_data()
+{
+ perflog_baseline=$1
+ perflog_current=$2
+
+ time=0
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> btimes.$$
+ done
+
+ for op in `cat ops`
+ do
+ time=0;
+ for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+ do
+ time=$((time+$i))
+ done;
+ time=$((time/3)) # Average over three runs
+ echo $time >> ctimes.$$
+ done
+
+ paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+ rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+ a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+ yrange=`echo $a + $a/5 | bc`
+ b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+ xrange=`echo $b - 0.5 | bc`
+
+ gnuplot <<EOF
+ reset
+ set key at graph 0.15, 0.85 horizontal samplen 0.1
+ set style data histogram
+ set style histogram cluster gap 1
+ set style fill solid border -1
+ set boxwidth 0.8
+ set xtic rotate by 90 scale 0
+ unset ytics
+ set y2tics rotate by 90
+ set terminal png nocrop size 1024,768
+ set xlabel ' '
+ set size 0.6, 1
+ set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+ set y2label '$XLABEL' offset -2
+ set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+ set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+ set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+ set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+ set output "tmp.$$.png"
+ p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+ convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+ rm tmp.$$.png
+}
+
+# Generate CPU data
+gen_cpu_data
+
+# plot CPU data
+plot_cpu_usage
+
+# Generate VM data
+gen_vm_data
+
+# plot VM data
+plot_vm_usage
+
+# Generate comparison data - baseline vs current run
+gen_cmp_data perf-fuse-ami.log perf-fuse-aws.log
+
+# plot perf comparison - baseline vs current run
+plot_comparison
+
+# cleanup tmp files
+rm vm_datafile memfile
+rm cpu-* times
+
+display cpu.png &
+display vm.png &
+display cmp.png &
diff --git a/perf-framework/get_date b/perf-framework/get_date
new file mode 100755
index 0000000..335359d
--- /dev/null
+++ b/perf-framework/get_date
@@ -0,0 +1,5 @@
+date +%s
+ssh -l root 10.1.10.30 "date +%s"
+ssh -l root 10.1.10.31 "date +%s"
+ssh -l root 10.1.10.35 "date +%s"
+ssh -l root 10.1.10.36 "date +%s"
diff --git a/perf-framework/gf_perf_config b/perf-framework/gf_perf_config
new file mode 100644
index 0000000..017feb8
--- /dev/null
+++ b/perf-framework/gf_perf_config
@@ -0,0 +1,34 @@
+# Common config variables
+RELEASE_TARBALL=glusterfs-3git.tar.gz
+BRICK_IP_ADDRS="10.1.10.30 10.1.10.31 10.1.10.35 10.1.10.36"
+GF_CONFIG="stripe"
+VOLNAME="testvol"
+MGMT_NODE="10.1.10.30"
+
+# Server specific config variables
+SERVER_BUILD_DIR=/data/rahul/gluster_builds
+SERVER_LOG_DIR=/data/rahul/PERF/server_logs
+SERVER_SCRIPTS_DIR=/data/rahul/PERF/scripts
+SERVER_EXPORT_DIR=/data/rahul/datastore
+
+# Client specific config variables
+MOUNT_TYPE="glusterfs"
+#MOUNT_TYPE="nfs"
+MOUNT_POINT=/mnt/perfmount
+LOCAL_LOG_REPO=/root/rahul/test_deploy/log_repo
+CLIENT_BUILD_DIR=/root/rahul/test_deploy/gluster_builds
+CLIENT_IP_ADDR=10.1.10.29
+RUNLOG=$LOCAL_LOG_REPO/runlog
+
+# Special runs
+ENABLE_ACL=yes
+ENABLE_MEM_ACCT=no
+
+# For passwordless ssh setup
+ROOT_PW1=abc123
+ROOT_PW2=abc321
+# SSH_HOSTS is a space separated list of hosts
+SSH_HOSTS="0"
+
+# Setup automated emails
+EMAIL="rahul@shell.gluster.com:/home/rahul/perf_logs" \ No newline at end of file
diff --git a/perf-framework/glusterfs-precreate.sh b/perf-framework/glusterfs-precreate.sh
new file mode 100755
index 0000000..d1c92a3
--- /dev/null
+++ b/perf-framework/glusterfs-precreate.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e;
+
+if [ $# -eq 0 ]; then
+ echo "Usage: $0 <dir1> [dir2 [dir3 [dir4 ...]]]";
+ exit 1
+fi
+
+for dir in "$@"; do
+ if [ ! -d "$dir" ]; then
+ echo "$0: $dir not a directory"
+ exit 1;
+ fi
+done
+
+subdirs="{00"
+for i in {1..255}; do
+ n=$(printf "%02x" $i);
+ subdirs="$subdirs,$n";
+done
+subdirs="$subdirs}"
+
+mkdir -v "$dir/.glusterfs";
+
+for dir in $@; do
+ for i in {0..255}; do
+ n=$(printf "%02x" $i);
+ mkdir -v "$dir/.glusterfs/$n"
+ eval "mkdir -v $dir/.glusterfs/$n/$subdirs"
+ done
+done \ No newline at end of file
diff --git a/perf-framework/motd b/perf-framework/motd
new file mode 100644
index 0000000..dd934b4
--- /dev/null
+++ b/perf-framework/motd
@@ -0,0 +1,25 @@
+/*
+ * System being used by Rahul for performance runs.
+ * It is part of the setup described below.
+ * Please contact tcp@gluster.com before running anything on this system.
+ *
+ * DO NOT DISTURB THE SETUP SINCE IT MIGHT AFFECT THE PERFORMANCE RUNS.
+ *
+ * /etc/motd last updated:
+ * Mon Jul 4 04:04:29 PDT 2011
+ */
+
++-------------+ +-------------+ +-------------+ +-------------+
+| 10.1.10.30 |<---->| 10.1.10.31 |<---->| 10.1.10.35 |<---->| 10.1.10.36 |
+| (client-10) | | (client-11) | | (client-15) | | (client-16) |
+| (Brick-1) | | (Brick-2) | | (Brick-3) | | (Brick-4) |
++------+------+ +------+------+ +------+------+ +------+------+
+ | | | |
+ +--------------------+---------+----------+--------------------+
+ |
+ v
+ +--------------+
+ | 10.1.10.29 |
+ | (client-9) |
+ | (client) |
+ +--------------+
diff --git a/perf-framework/newtest.sh b/perf-framework/newtest.sh
new file mode 100755
index 0000000..789c784
--- /dev/null
+++ b/perf-framework/newtest.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+ mkdir -p $PG/emptyfiles;
+ for i in $(seq 1 $smallfilecount); do
+ : > $PG/emptyfiles/file.$i;
+ done
+}
+
+
+function emptyfiles_delete()
+{
+ rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+ mkdir -p $PG/emptydirs;
+
+ eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+ for top in $(seq 1 $emptytops); do
+ eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+ done
+}
+
+
+function emptydirs_delete()
+{
+ rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+ mkdir -p $PG/smallfiles;
+ for i in $(seq 1 $smallfilecount); do
+ echo -n $smallblob >$PG/smallfiles/file.$i
+ done
+}
+
+
+function smallfiles_rewrite()
+{
+ smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+ for i in $(seq 1 $smallfilecount); do
+ cat $PG/smallfiles/file.$i > /dev/null
+ done
+}
+
+
+function smallfiles_reread()
+{
+ smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+ rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+ mkdir -p $PG/largefile;
+ dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+ largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+ dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+ largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+ rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+ local subpath;
+ local depth;
+
+ subpath="$1";
+ depth="$2";
+
+ if [ $depth -eq 0 ]; then
+ for i in $(seq 1 $leafcount); do
+ : > $subpath/file.$i;
+ done
+ return
+ fi
+
+ depth=$(($depth - 1));
+
+ eval "echo $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+ for i in $(seq 1 $crawlwidth); do
+ crawl_create_recurse "$subpath/dir.$i" $depth;
+ done
+}
+
+
+function directory_crawl_create ()
+{
+ crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+ ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+ directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+ rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+ chmod -R 777 "$PG/crawl";
+ chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+ #run emptyfiles_create;
+ #run emptyfiles_delete;
+
+
+ # run emptydirs_create;
+ # run emptydirs_delete;
+
+
+ #run smallfiles_create;
+ #run smallfiles_rewrite;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run smallfiles_read;
+ #run smallfiles_reread;
+ #run smallfiles_delete;
+
+
+ #run largefile_create;
+ #run largefile_rewrite;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run largefile_read;
+ #run largefile_reread;
+ #run largefile_delete;
+
+
+ run directory_crawl_create;
+ echo 3 > /proc/sys/vm/drop_caches;
+ sleep 10;
+ run directory_crawl;
+ run directory_recrawl;
+ run metadata_modify;
+ run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+ rm -rvf $PG;
+ mkdir -p $PG;
+}
+
+
+function params_init()
+{
+ emptytops=10;
+ emptydirs=10000;
+
+ smallfilecount=100000;
+ smallblock=4096;
+
+ largeblock=64K;
+ largecount=16K;
+
+ crawlwidth=10;
+ crawldepth=3;
+ leafcount=100;
+
+ smallblob=;
+ for i in $(seq 1 $smallblock); do
+ smallblob=a$smallblob
+ done
+}
+
+
+function _init()
+{
+ params_init;
+
+ TSTDOUT=251
+ TSTDERR=252
+ LOGFD=253
+ LOGFILE=/tmp/perf$$
+
+ eval "exec $TSTDOUT>&1"
+ eval "exec $TSTDERR>&2"
+ eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+ MOUNT=;
+
+ if [ "x$1" == "x" ] ; then
+ echo "Usage: $0 /gluster/mount"
+ exit 1
+ fi
+
+ MOUNT="$1";
+ PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+ "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+ set -o pipefail;
+ (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+ local t;
+ local rest;
+
+ t=$1;
+ shift;
+ rest="$@";
+
+ echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+ local t;
+
+ echo -n "running $@ ... "
+ t=$(measure "$@");
+
+ if [ $? -eq 0 ]; then
+ echo "done ($t secs)";
+ log "$t" "$@";
+ else
+ echo "FAILED!!!"
+ fi
+}
+
+
+function verify_mount()
+{
+ if [ ! -d "$MOUNT" ] ; then
+ echo "Can't access '$MOUNT'"
+ exit 1
+ fi
+}
+
+
+function show_report()
+{
+ (echo "Testname Time"; cat $LOGFILE) | column -t;
+ rm -f $LOGFILE;
+}
+
+
+function main()
+{
+ parse_cmdline "$@";
+
+ verify_mount;
+
+ cleanup_playground;
+
+ run_tests;
+
+ show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/nfs-reread.sh b/perf-framework/nfs-reread.sh
new file mode 100755
index 0000000..1e2f51b
--- /dev/null
+++ b/perf-framework/nfs-reread.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+ mkdir -p $PG/emptyfiles;
+ for i in $(seq 1 $smallfilecount); do
+ : > $PG/emptyfiles/file.$i;
+ done
+}
+
+
+function emptyfiles_delete()
+{
+ rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+ mkdir -p $PG/emptydirs;
+
+ eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+ for top in $(seq 1 $emptytops); do
+ eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+ done
+}
+
+
+function emptydirs_delete()
+{
+ rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+ mkdir -p $PG/smallfiles;
+ for i in $(seq 1 $smallfilecount); do
+ echo -n $smallblob >$PG/smallfiles/file.$i
+ done
+}
+
+
+function smallfiles_rewrite()
+{
+ smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+ for i in $(seq 1 $smallfilecount); do
+ cat $PG/smallfiles/file.$i > /dev/null
+ done
+}
+
+
+function smallfiles_reread()
+{
+ smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+ rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+ mkdir -p $PG/largefile;
+ dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+ largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+ dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+ largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+ rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+ local subpath;
+ local depth;
+
+ subpath="$1";
+ depth="$2";
+
+ if [ $depth -eq 0 ]; then
+ for i in $(seq 1 $leafcount); do
+ : > $subpath/file.$i;
+ done
+ return
+ fi
+
+ depth=$(($depth - 1));
+
+ eval "echo $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+ for i in $(seq 1 $crawlwidth); do
+ crawl_create_recurse "$subpath/dir.$i" $depth;
+ done
+}
+
+
+function directory_crawl_create ()
+{
+ crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+ ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+ directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+ rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+ chmod -R 777 "$PG/crawl";
+ chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+ #run emptyfiles_create;
+ #run emptyfiles_delete;
+
+
+ # run emptydirs_create;
+ # run emptydirs_delete;
+
+
+ run smallfiles_create;
+ run smallfiles_rewrite;
+ echo 3 > /proc/sys/vm/drop_caches;
+ sleep 10;
+ run smallfiles_read;
+ run smallfiles_reread;
+ run smallfiles_delete;
+
+
+ #run largefile_create;
+ #run largefile_rewrite;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run largefile_read;
+ #run largefile_reread;
+ #run largefile_delete;
+
+
+ #run directory_crawl_create;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run directory_crawl;
+ #run directory_recrawl;
+ #run metadata_modify;
+ #run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+ rm -rvf $PG;
+ mkdir -p $PG;
+}
+
+
+function params_init()
+{
+ emptytops=10;
+ emptydirs=10000;
+
+ smallfilecount=100000;
+ smallblock=4096;
+
+ largeblock=64K;
+ largecount=16K;
+
+ crawlwidth=10;
+ crawldepth=3;
+ leafcount=100;
+
+ smallblob=;
+ for i in $(seq 1 $smallblock); do
+ smallblob=a$smallblob
+ done
+}
+
+
+function _init()
+{
+ params_init;
+
+ TSTDOUT=251
+ TSTDERR=252
+ LOGFD=253
+ LOGFILE=/tmp/perf$$
+
+ eval "exec $TSTDOUT>&1"
+ eval "exec $TSTDERR>&2"
+ eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+ MOUNT=;
+
+ if [ "x$1" == "x" ] ; then
+ echo "Usage: $0 /gluster/mount"
+ exit 1
+ fi
+
+ MOUNT="$1";
+ PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+ "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+ set -o pipefail;
+ (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+ local t;
+ local rest;
+
+ t=$1;
+ shift;
+ rest="$@";
+
+ echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+ local t;
+
+ echo -n "running $@ ... "
+ t=$(measure "$@");
+
+ if [ $? -eq 0 ]; then
+ echo "done ($t secs)";
+ log "$t" "$@";
+ else
+ echo "FAILED!!!"
+ fi
+}
+
+
+function verify_mount()
+{
+ if [ ! -d "$MOUNT" ] ; then
+ echo "Can't access '$MOUNT'"
+ exit 1
+ fi
+}
+
+
+function show_report()
+{
+ (echo "Testname Time"; cat $LOGFILE) | column -t;
+ rm -f $LOGFILE;
+}
+
+
+function main()
+{
+ parse_cmdline "$@";
+
+ verify_mount;
+
+ cleanup_playground;
+
+ run_tests;
+
+ show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/nfs_delete.sh b/perf-framework/nfs_delete.sh
new file mode 100755
index 0000000..a6cb5bc
--- /dev/null
+++ b/perf-framework/nfs_delete.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+ mkdir -p $PG/emptyfiles;
+ for i in $(seq 1 $smallfilecount); do
+ : > $PG/emptyfiles/file.$i;
+ done
+}
+
+
+function emptyfiles_delete()
+{
+ rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+ mkdir -p $PG/emptydirs;
+
+ eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+ for top in $(seq 1 $emptytops); do
+ eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+ done
+}
+
+
+function emptydirs_delete()
+{
+ rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+ mkdir -p $PG/smallfiles;
+ for i in $(seq 1 $smallfilecount); do
+ echo -n $smallblob >$PG/smallfiles/file.$i
+ done
+}
+
+
+function smallfiles_rewrite()
+{
+ smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+ for i in $(seq 1 $smallfilecount); do
+ cat $PG/smallfiles/file.$i > /dev/null
+ done
+}
+
+
+function smallfiles_reread()
+{
+ smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+ rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+ mkdir -p $PG/largefile;
+ dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+ largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+ dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+ largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+ rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+ local subpath;
+ local depth;
+
+ subpath="$1";
+ depth="$2";
+
+ if [ $depth -eq 0 ]; then
+ for i in $(seq 1 $leafcount); do
+ : > $subpath/file.$i;
+ done
+ return
+ fi
+
+ depth=$(($depth - 1));
+
+ eval "echo $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+ for i in $(seq 1 $crawlwidth); do
+ crawl_create_recurse "$subpath/dir.$i" $depth;
+ done
+}
+
+
+function directory_crawl_create ()
+{
+ crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+ ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+ directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+ rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+ chmod -R 777 "$PG/crawl";
+ chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+ run emptyfiles_create;
+ #run emptyfiles_delete;
+
+
+ # run emptydirs_create;
+ # run emptydirs_delete;
+
+
+ #run smallfiles_create;
+ #run smallfiles_rewrite;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run smallfiles_read;
+ #run smallfiles_reread;
+ #run smallfiles_delete;
+
+
+ #run largefile_create;
+ #run largefile_rewrite;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run largefile_read;
+ #run largefile_reread;
+ #run largefile_delete;
+
+
+ #run directory_crawl_create;
+ #echo 3 > /proc/sys/vm/drop_caches;
+ #sleep 10;
+ #run directory_crawl;
+ #run directory_recrawl;
+ #run metadata_modify;
+ #run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+ rm -rvf $PG;
+ mkdir -p $PG;
+}
+
+
+function params_init()
+{
+ emptytops=10;
+ emptydirs=10000;
+
+ smallfilecount=80000;
+ smallblock=4096;
+
+ largeblock=64K;
+ largecount=16K;
+
+ crawlwidth=10;
+ crawldepth=3;
+ leafcount=100;
+
+ smallblob=;
+ for i in $(seq 1 $smallblock); do
+ smallblob=a$smallblob
+ done
+}
+
+
+function _init()
+{
+ params_init;
+
+ TSTDOUT=251
+ TSTDERR=252
+ LOGFD=253
+ LOGFILE=/tmp/perf$$
+
+ eval "exec $TSTDOUT>&1"
+ eval "exec $TSTDERR>&2"
+ eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+ MOUNT=;
+
+ if [ "x$1" == "x" ] ; then
+ echo "Usage: $0 /gluster/mount"
+ exit 1
+ fi
+
+ MOUNT="$1";
+ PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+ "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+ set -o pipefail;
+ (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+ local t;
+ local rest;
+
+ t=$1;
+ shift;
+ rest="$@";
+
+ echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+ local t;
+
+ echo -n "running $@ ... "
+ t=$(measure "$@");
+
+ if [ $? -eq 0 ]; then
+ echo "done ($t secs)";
+ log "$t" "$@";
+ else
+ echo "FAILED!!!"
+ fi
+}
+
+
+function verify_mount()
+{
+ if [ ! -d "$MOUNT" ] ; then
+ echo "Can't access '$MOUNT'"
+ exit 1
+ fi
+}
+
+
+function show_report()
+{
+ (echo "Testname Time"; cat $LOGFILE) | column -t;
+ rm -f $LOGFILE;
+}
+
+
+function main()
+{
+ parse_cmdline "$@";
+
+ verify_mount;
+
+ cleanup_playground;
+
+ run_tests;
+
+ show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/ops b/perf-framework/ops
new file mode 100644
index 0000000..74e1939
--- /dev/null
+++ b/perf-framework/ops
@@ -0,0 +1,17 @@
+emptyfiles_create
+emptyfiles_delete
+smallfiles_create
+smallfiles_rewrite
+smallfiles_read
+smallfiles_reread
+smallfiles_delete
+largefile_create
+largefile_rewrite
+largefile_read
+largefile_reread
+largefile_delete
+directory_crawl_create
+directory_crawl
+directory_recrawl
+metadata_modify
+directory_crawl_delete
diff --git a/perf-framework/parallel_create b/perf-framework/parallel_create
new file mode 100755
index 0000000..0bd9406
--- /dev/null
+++ b/perf-framework/parallel_create
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+FILE_REPO="/mnt/perfmount/file_repo"
+TEST=0
+DD_BS=512
+RAND_SCALE=20
+
+function usage()
+{
+ echo "usage: $1 <thread_count> <file_count>"
+ exit 1
+}
+
+function create_files()
+{
+ tid=$1
+ count=$2
+ start=$((tid*count))
+
+ if [ $TEST -eq "1" ]; then
+ for ((i=0; i<${count}; i++)); do
+ outfile=$FILE_REPO/tid${tid}_file$((start+i))
+ echo "dd if=/dev/urandom of=$outfile bs=1b count=$((RANDOM*20))"
+ done
+ else
+ for ((i=0; i<${count}; i++)); do
+ filesize=$((RANDOM*RAND_SCALE))
+ dd_count=$((filesize/DD_BS))
+ outfile=$FILE_REPO/tid${tid}_file$((start+i))
+ dd if=/dev/zero of=$outfile bs=1b count=$dd_count > /dev/null 2>&1
+ echo "$tid $i"
+ done
+ fi
+}
+
+function Not_A_Number()
+{
+ echo $1$2 | grep ^[0-9]*$ > /dev/null
+ return $?
+}
+
+function main()
+{
+ if [ $# -ne 2 ]; then
+ usage $0
+ fi
+
+ if ! Not_A_Number $1 $2; then
+ echo "Invalid input. Enter only numbers."
+ usage
+ fi
+
+ if [ $2 -lt $1 ]; then
+ echo "Number of files cannot be less than number of threads"
+ usage $0
+ fi
+
+ mkdir -p $FILE_REPO
+
+ nthr=$1
+ fc=$2
+ files_per_thr=$((fc/nthr))
+
+ for i in `seq 0 $((nthr-1))`; do
+ create_files $i $files_per_thr &
+ done
+
+ wait
+}
+
+main $@
diff --git a/perf-framework/perf-test.sh b/perf-framework/perf-test.sh
new file mode 100755
index 0000000..6ce5b55
--- /dev/null
+++ b/perf-framework/perf-test.sh
@@ -0,0 +1,337 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+function emptyfiles_create()
+{
+ mkdir -p $PG/emptyfiles;
+ for i in $(seq 1 $smallfilecount); do
+ : > $PG/emptyfiles/file.$i;
+ done
+}
+
+
+function emptyfiles_delete()
+{
+ rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+ mkdir -p $PG/emptydirs;
+
+ eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+ for top in $(seq 1 $emptytops); do
+ eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+ done
+}
+
+
+function emptydirs_delete()
+{
+ rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+ mkdir -p $PG/smallfiles;
+ for i in $(seq 1 $smallfilecount); do
+ echo -n $smallblob >$PG/smallfiles/file.$i
+ done
+}
+
+
+function smallfiles_rewrite()
+{
+ smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+ for i in $(seq 1 $smallfilecount); do
+ cat $PG/smallfiles/file.$i > /dev/null
+ done
+}
+
+
+function smallfiles_reread()
+{
+ smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+ rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+ mkdir -p $PG/largefile;
+ dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+ largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+ dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+ largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+ rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+ local subpath;
+ local depth;
+
+ subpath="$1";
+ depth="$2";
+
+ if [ $depth -eq 0 ]; then
+ for i in $(seq 1 $leafcount); do
+ : > $subpath/file.$i;
+ done
+ return
+ fi
+
+ depth=$(($depth - 1));
+
+ eval "echo $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+ for i in $(seq 1 $crawlwidth); do
+ crawl_create_recurse "$subpath/dir.$i" $depth;
+ done
+}
+
+
+function directory_crawl_create ()
+{
+ crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+ ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+ directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+ rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+ chmod -R 777 "$PG/crawl";
+ chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+ run emptyfiles_create;
+ run emptyfiles_delete;
+
+
+ # run emptydirs_create;
+ # run emptydirs_delete;
+
+
+ run smallfiles_create;
+ run smallfiles_rewrite;
+ echo 3 > /proc/sys/vm/drop_caches;
+ sleep 10;
+ run smallfiles_read;
+ run smallfiles_reread;
+ run smallfiles_delete;
+
+
+ run largefile_create;
+ run largefile_rewrite;
+ echo 3 > /proc/sys/vm/drop_caches;
+ sleep 10;
+ run largefile_read;
+ run largefile_reread;
+ run largefile_delete;
+
+
+ run directory_crawl_create;
+ echo 3 > /proc/sys/vm/drop_caches;
+ sleep 10;
+ run directory_crawl;
+ run directory_recrawl;
+ run metadata_modify;
+ run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+ rm -rvf $PG;
+ mkdir -p $PG;
+}
+
+
+function params_init()
+{
+ emptytops=10;
+ emptydirs=10000;
+
+ if [ $MOUNT_TYPE == "nfs" ]; then
+ smallfilecount=85000;
+ else
+ smallfilecount=100000;
+ fi
+ smallblock=4096;
+
+ largeblock=64K;
+ largecount=16K;
+
+ crawlwidth=10;
+ crawldepth=3;
+ leafcount=100;
+
+ smallblob=;
+ for i in $(seq 1 $smallblock); do
+ smallblob=a$smallblob
+ done
+}
+
+
+function _init()
+{
+ params_init;
+
+ TSTDOUT=251
+ TSTDERR=252
+ LOGFD=253
+ LOGFILE=/tmp/perf$$
+
+ eval "exec $TSTDOUT>&1"
+ eval "exec $TSTDERR>&2"
+ eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+ MOUNT=;
+
+ if [ "x$1" == "x" ] ; then
+ echo "Usage: $0 /gluster/mount"
+ exit 1
+ fi
+
+ MOUNT="$1";
+ PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+ "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+ set -o pipefail;
+ (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+ local t;
+ local rest;
+
+ t=$1;
+ shift;
+ rest="$@";
+
+ echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+ local t;
+
+ echo -n "running $@ ... "
+ t=$(measure "$@");
+
+ if [ $? -eq 0 ]; then
+ echo "done ($t secs)";
+ log "$t" "$@";
+ else
+ echo "FAILED!!!"
+ fi
+}
+
+
+function verify_mount()
+{
+ if [ ! -d "$MOUNT" ] ; then
+ echo "Can't access '$MOUNT'"
+ exit 1
+ fi
+}
+
+
+function show_report()
+{
+ (echo "Testname Time"; cat $LOGFILE) | column -t;
+ rm -f $LOGFILE;
+}
+
+
+function main()
+{
+ parse_cmdline "$@";
+
+ verify_mount;
+
+ cleanup_playground;
+
+ run_tests;
+
+ show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/perf.sh b/perf-framework/perf.sh
new file mode 100755
index 0000000..1a4bdc6
--- /dev/null
+++ b/perf-framework/perf.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+perf_test ()
+{
+ ./perf-test.sh $mount_point >> $logfile
+}
+
+_init ()
+{
+ mount_point=$1;
+ logfile=$2
+}
+
+main ()
+{
+ echo "" > $logfile
+ perf_test;
+ perf_test;
+ perf_test;
+}
+
+
+_init "$@" && main "$@";
diff --git a/perf-framework/pwl_ssh b/perf-framework/pwl_ssh
new file mode 100755
index 0000000..fc837f1
--- /dev/null
+++ b/perf-framework/pwl_ssh
@@ -0,0 +1,50 @@
+#!/usr/bin/expect -f
+
+proc setup_pwl_ssh { host pw1 pw2 } {
+ spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$host
+ set timeout 50
+ expect {
+ "yes/no" {
+ send "yes\r"
+ exp_continue
+ } "password: " {
+ send "$pw1\r"
+ expect {
+ "#" {
+ # Success. Nothing to do
+ } "password: " {
+ send "$pw2\r";
+ expect {
+ "#" {
+ # Success. Nothing to do
+ } (.*) {
+ catch {close -i $spawn_id}
+ wait -nowait -i $spawn_id
+ }
+ }
+ }
+ }
+ } timeout {
+ catch {close -i $spawn_id}
+ wait -nowait -i $spawn_id
+ } eof {
+ }
+ }
+}
+
+if { $argc < 2 } {
+ puts "usage $argv0 <logfile> <list-of-hosts>"
+ exit
+}
+
+log_user 0
+set logfile [ lindex $argv 0 ]
+exp_log_file -a -noappend $logfile
+
+set pw1 $env(ROOT_PW1)
+set pw2 $env(ROOT_PW2)
+
+for {set i 1} {$i<[llength $argv]} {incr i} {
+ set host [lindex $argv $i]
+ setup_pwl_ssh $host $pw1 $pw2
+}
diff --git a/perf-framework/quota_gsync_run b/perf-framework/quota_gsync_run
new file mode 100755
index 0000000..a090b66
--- /dev/null
+++ b/perf-framework/quota_gsync_run
@@ -0,0 +1,42 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+run=$1
+runlabel=`echo $run|sed -e 's/^glusterfs-//' -e 's/\.tar\.gz//'`
+
+a=`cat .runfile`
+echo "run$a - Quota off; gsync on..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume set $VOLNAME geo-replication.indexing on"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota off, gsync on)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+echo "run$((a+1)) - Quota on; gsync on..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume quota $VOLNAME enable"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume set $VOLNAME geo-replication.indexing on"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota on, gsync on)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+echo "run$((a+2)) - Quota on; gsync off..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume quota $VOLNAME enable"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota on, gsync off)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+# Restore
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+
diff --git a/perf-framework/remove_export_dirs b/perf-framework/remove_export_dirs
new file mode 100755
index 0000000..c84d881
--- /dev/null
+++ b/perf-framework/remove_export_dirs
@@ -0,0 +1,10 @@
+#!/bin/bash -ue
+
+SERVER_EXPORT_DIR=$1
+
+if [ "$SERVER_EXPORT_DIR" != "" -a "$SERVER_EXPORT_DIR" != "/" ]
+then
+ echo "$SERVER_EXPORT_DIR"
+else
+ echo "Empty export dir"
+fi
diff --git a/perf-framework/run b/perf-framework/run
new file mode 100755
index 0000000..965d7bb
--- /dev/null
+++ b/perf-framework/run
@@ -0,0 +1,9 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+for brick in $BRICK_IP_ADDRS
+do
+ echo $brick
+ ssh -l root $brick "$@"
+done
diff --git a/perf-framework/send_mail b/perf-framework/send_mail
new file mode 100755
index 0000000..e44f29b
--- /dev/null
+++ b/perf-framework/send_mail
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+./calc_avg "$@" > /tmp/run_all
+
+scp /tmp/run_all $EMAIL/run_all;
diff --git a/perf-framework/setbatch b/perf-framework/setbatch
new file mode 100755
index 0000000..fb18585
--- /dev/null
+++ b/perf-framework/setbatch
@@ -0,0 +1,25 @@
+#!/bin/bash -u
+
+TARBALL_DIR=./tarballs
+function usage()
+{
+ echo "Usage : $0 <gluster_release_tarball_list>"
+ exit 1
+}
+
+if [ $# -lt 1 ]
+then
+ usage
+fi
+
+list=`echo $@`
+for i in $list
+do
+ if [ ! -f $TARBALL_DIR/$i ]
+ then
+ echo "$i not found in directory $TARBALL_DIR. Provide correct list."
+ exit
+ fi
+done
+
+sed -i "s/^RUNLIST=.*$/RUNLIST=\"$list\"/" batchrun
diff --git a/perf-framework/setrun b/perf-framework/setrun
new file mode 100755
index 0000000..7992051
--- /dev/null
+++ b/perf-framework/setrun
@@ -0,0 +1,14 @@
+#!/bin/bash -u
+
+function usage()
+{
+ echo "Usage : $0 <gluster_release_tarball>"
+ exit 1
+}
+
+if [ $# -ne 1 ]
+then
+ usage
+fi
+
+sed -i "s/^RELEASE_TARBALL=.*$/RELEASE_TARBALL=$1/" gf_perf_config
diff --git a/perf-framework/start_perf_measure b/perf-framework/start_perf_measure
new file mode 100755
index 0000000..2e06db5
--- /dev/null
+++ b/perf-framework/start_perf_measure
@@ -0,0 +1,166 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+SETTLE_TIME=10
+RUNFILE=.runfile
+STAT_COLLECTOR=stat_collect
+
+# Generate current run, update runfile
+if [ ! -f $RUNFILE ]
+then
+ run=1
+else
+ run=`cat $RUNFILE`
+fi
+echo $((run+1)) > $RUNFILE
+
+# Drop vm caches on all the bricks before starting the runs
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "echo 3 > /proc/sys/vm/drop_caches"
+done
+
+# Create the gluster volume
+./create_gluster_vol
+
+mount | grep $MOUNT_POINT > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+ umount $MOUNT_POINT
+fi
+
+if [ ! -d $MOUNT_POINT ]
+then
+ mkdir -p $MOUNT_POINT
+fi
+
+# Make sure that the fuse kernel module is loaded
+/sbin/lsmod | grep -w fuse > /dev/null 2>&1
+if [ $? -ne 0 ]
+then
+ /sbin/modprobe fuse > /dev/null 2>&1
+fi
+
+ps -eaf | egrep -w 'glusterfs|glusterfsd|glusterd' > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+ killall glusterfsd glusterd glusterfs > /dev/null 2>&1
+fi
+
+# Mount the client
+# Sleep for a while. Sometimes, NFS mounts fail if attempted soon after creating the volume
+sleep $SETTLE_TIME
+
+if [ $ENABLE_ACL == "yes" ]
+then
+ acl_opts="-o acl"
+else
+ acl_opts=""
+fi
+
+if [ $MOUNT_TYPE == "nfs" ]
+then
+ nfs_opts="-o vers=3"
+else
+ nfs_opts=""
+fi
+
+echo "Mounting volume..."
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+ echo "Memory accounting status on client -"
+ echo "x/x &gf_mem_acct_enable" > commands.$$
+ echo "quit" >> commands.$$
+ GLUSTERFS_DISABLE_MEM_ACCT=0 mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+ mount_status=$?
+ gdb -q --command=commands.$$ -p `pidof glusterfs` | grep gf_mem_acct_enable | awk '{print $(NF-1) $NF}'
+ rm commands.$$ > /dev/null 2>&1
+else
+ mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+ mount_status=$?
+fi
+
+if [ $mount_status -ne 0 ]
+then
+ echo "mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT failed..."
+ echo "Exiting..."
+ exit 1
+fi
+
+# Copy statistics collection scripts to the server
+
+echo ""
+echo "Copying stat collection script to bricks..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "mkdir -p $SERVER_SCRIPTS_DIR"
+ scp -p $STAT_COLLECTOR root@$brick:$SERVER_SCRIPTS_DIR > /dev/null 2>&1
+done
+
+# Run statistics collection scripts on the server
+
+echo ""
+echo "Starting server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "mkdir -p $SERVER_LOG_DIR"
+ ssh -l root $brick "$SERVER_SCRIPTS_DIR/$STAT_COLLECTOR $SERVER_LOG_DIR" &
+done
+
+# Run statistics collection on client
+
+mkdir -p $LOCAL_LOG_REPO
+./$STAT_COLLECTOR $LOCAL_LOG_REPO/run$run/client &
+
+# Start perf test
+
+echo ""
+echo "Starting run $run..."
+
+sleep $SETTLE_TIME
+./perf.sh $MOUNT_POINT $LOCAL_LOG_REPO/run$run/client/perf-test.log
+sleep $SETTLE_TIME
+
+# Stop statistics collection scripts on the client
+
+killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+
+# Stop statistics collection scripts on the server
+
+echo ""
+echo "Stopping server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+done
+
+# Since the ssh processes were backgrounded, they will be hanging around.
+# Kill them
+kill `jobs -l | awk '{print $2}'` > /dev/null 2>&1
+
+# Copy statistics from the server
+
+echo ""
+echo "Copying server logfiles for run $run..."
+cur_log_dump_dir=$LOCAL_LOG_REPO/run$run
+mkdir -p $cur_log_dump_dir
+count=1
+for brick in $BRICK_IP_ADDRS
+do
+ for statf in mpstat vmstat iostat sysinfo sar_netstat
+ do
+ scp root@$brick:$SERVER_LOG_DIR/*$statf* $cur_log_dump_dir/brick$count-$brick-$statf-log > /dev/null 2>&1
+ done
+ count=$((count + 1))
+done
+
+# Cleanup statistics collected for this run on the server
+
+echo ""
+echo "Cleaning server logfiles..."
+for brick in $BRICK_IP_ADDRS
+do
+ ssh -l root $brick "cd $SERVER_LOG_DIR; rm mpstat_log vmstat_log iostat_log sysinfo sar_netstat_log"
+done
diff --git a/perf-framework/stat_collect b/perf-framework/stat_collect
new file mode 100755
index 0000000..8b63203
--- /dev/null
+++ b/perf-framework/stat_collect
@@ -0,0 +1,36 @@
+#!/bin/bash -ue
+
+function start_stat_collect()
+{
+ echo "Interrupts details:" >> $SYSINFO
+ cat /proc/interrupts >> $SYSINFO
+ echo "CPU details:" >> $SYSINFO
+ cat /proc/cpuinfo >> $SYSINFO
+ echo "Memory details:" >> $SYSINFO
+ cat /proc/meminfo >> $SYSINFO
+
+ iostat -xcdh 5 >> $IOSTAT_LOG &
+ mpstat -P ALL 5 >> $MPSTAT_LOG &
+ vmstat 5 >> $VMSTAT_LOG &
+ LC_TIME="POSIX" sar -n DEV 5 0 >> $SAR_NETSTAT_LOG &
+}
+
+function init()
+{
+ mkdir -p $LOGDIR
+ IOSTAT_LOG=$LOGDIR/iostat_log
+ VMSTAT_LOG=$LOGDIR/vmstat_log
+ MPSTAT_LOG=$LOGDIR/mpstat_log
+ SAR_NETSTAT_LOG=$LOGDIR/sar_netstat_log
+ SYSINFO=$LOGDIR/sysinfo
+ echo "" > $IOSTAT_LOG
+ echo "" > $VMSTAT_LOG
+ echo "" > $MPSTAT_LOG
+ echo "" > $SAR_NETSTAT_LOG
+ echo "" > $SYSINFO
+}
+
+LOGDIR=$1
+init $LOGDIR;
+start_stat_collect;
+wait