From db468693ef5faa294d9bc3cd3c5d70c0d99d488b Mon Sep 17 00:00:00 2001
From: Rahul C S <rahulcs@redhat.com>
Date: Fri, 17 Feb 2012 17:23:28 +0530
Subject: Adding the performance framework to the qa repo

Change-Id: Ia7dbd82e9bb2e5e65e9345234ce34f8518a091ad
Signed-off-by: Rahul C S <rahulcs@redhat.com>
---
 perf-framework/.runfile               |   1 +
 perf-framework/README                 |  85 +++++++
 perf-framework/acl                    |  42 +++
 perf-framework/analyze_perf_data      | 466 ++++++++++++++++++++++++++++++++++
 perf-framework/automate_pwl_ssh       |  43 ++++
 perf-framework/batchrun               |  50 ++++
 perf-framework/buildit                |   6 +
 perf-framework/calc_all               |  49 ++++
 perf-framework/calc_avg               |  54 ++++
 perf-framework/calc_best              |  31 +++
 perf-framework/calc_worst             |  31 +++
 perf-framework/check_install          |  13 +
 perf-framework/check_install.new      |  33 +++
 perf-framework/create_gluster_vol     | 122 +++++++++
 perf-framework/deploy_gluster         |  53 ++++
 perf-framework/diff_perfrun           |  17 ++
 perf-framework/file_open_analyse      | 430 +++++++++++++++++++++++++++++++
 perf-framework/file_open_test         | 159 ++++++++++++
 perf-framework/gen_perf_data          | 191 ++++++++++++++
 perf-framework/get_date               |   5 +
 perf-framework/gf_perf_config         |  34 +++
 perf-framework/glusterfs-precreate.sh |  32 +++
 perf-framework/motd                   |  25 ++
 perf-framework/newtest.sh             | 331 ++++++++++++++++++++++++
 perf-framework/nfs-reread.sh          | 331 ++++++++++++++++++++++++
 perf-framework/nfs_delete.sh          | 331 ++++++++++++++++++++++++
 perf-framework/ops                    |  17 ++
 perf-framework/parallel_create        |  71 ++++++
 perf-framework/perf-test.sh           | 337 ++++++++++++++++++++++++
 perf-framework/perf.sh                |  23 ++
 perf-framework/pwl_ssh                |  50 ++++
 perf-framework/quota_gsync_run        |  42 +++
 perf-framework/remove_export_dirs     |  10 +
 perf-framework/run                    |   9 +
 perf-framework/send_mail              |   8 +
 perf-framework/setbatch               |  25 ++
 perf-framework/setrun                 |  14 +
 perf-framework/start_perf_measure     | 166 ++++++++++++
 perf-framework/stat_collect           |  36 +++
 39 files changed, 3773 insertions(+)
 create mode 100644 perf-framework/.runfile
 create mode 100644 perf-framework/README
 create mode 100755 perf-framework/acl
 create mode 100755 perf-framework/analyze_perf_data
 create mode 100755 perf-framework/automate_pwl_ssh
 create mode 100755 perf-framework/batchrun
 create mode 100755 perf-framework/buildit
 create mode 100755 perf-framework/calc_all
 create mode 100755 perf-framework/calc_avg
 create mode 100755 perf-framework/calc_best
 create mode 100755 perf-framework/calc_worst
 create mode 100755 perf-framework/check_install
 create mode 100755 perf-framework/check_install.new
 create mode 100755 perf-framework/create_gluster_vol
 create mode 100755 perf-framework/deploy_gluster
 create mode 100755 perf-framework/diff_perfrun
 create mode 100755 perf-framework/file_open_analyse
 create mode 100755 perf-framework/file_open_test
 create mode 100755 perf-framework/gen_perf_data
 create mode 100755 perf-framework/get_date
 create mode 100644 perf-framework/gf_perf_config
 create mode 100755 perf-framework/glusterfs-precreate.sh
 create mode 100644 perf-framework/motd
 create mode 100755 perf-framework/newtest.sh
 create mode 100755 perf-framework/nfs-reread.sh
 create mode 100755 perf-framework/nfs_delete.sh
 create mode 100644 perf-framework/ops
 create mode 100755 perf-framework/parallel_create
 create mode 100755 perf-framework/perf-test.sh
 create mode 100755 perf-framework/perf.sh
 create mode 100755 perf-framework/pwl_ssh
 create mode 100755 perf-framework/quota_gsync_run
 create mode 100755 perf-framework/remove_export_dirs
 create mode 100755 perf-framework/run
 create mode 100755 perf-framework/send_mail
 create mode 100755 perf-framework/setbatch
 create mode 100755 perf-framework/setrun
 create mode 100755 perf-framework/start_perf_measure
 create mode 100755 perf-framework/stat_collect

diff --git a/perf-framework/.runfile b/perf-framework/.runfile
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/perf-framework/.runfile
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/perf-framework/README b/perf-framework/README
new file mode 100644
index 0000000..7733c35
--- /dev/null
+++ b/perf-framework/README
@@ -0,0 +1,85 @@
+Brief desciption of individual scripts:
+----------------------------------------
+acl - Enable/Disable acl on the backend filesystem.
+analyze_perf_data - Create plots using the data generated during perf runs.
+batchrun - Used for running multiple tarballs one after the other.
+buildit - Build scripts used by deploy_gluster for parallel builds.
+calc_all - Print results of all 3 runs performed in perf.sh
+calc_avg - Print average of 3 runs performed in perf.sh
+calc_best - Print best of the 3 runs performed in perf.sh
+calc_worst - Print worst of the 3 runs performed in perf.sh
+check_install - Print checksums of built and installed binaries
+check_install.new - Check if installation succeeded on all machines.
+create_gluster_vol - Create a gluster volume using parameters in gf_perf_conig
+deploy_gluster - Install gluster from source on multiple machines
+gf_perf_config - Example config file. Most scripts depend on this config file.
+perf.sh - Perf test wrapper that runs perf-test.sh thrice.
+perf-test.sh - Perf test
+quota_gsync_run - Used to run different Quota/Gsync combinations
+run - Used to run any command on multiple systems
+setbatch - Set the tarballs for batch run
+setrun - Set the tarball for a single run
+start_perf_measure - Launch statistics collection script on all machines
+involved in a perf run, start the perf run and save statistics post run.
+stat_collect - Helper script to collect system statistics during perf runs.
+
+diff_perfrun
+------------
+<File Begin>
+DATA FOR BASELINE - Output of calc_avg for the baseline
+followed by
+DATA FOR THE NEW RUN - Output of calc_avg for the new run
+<FILE END>
+
+It also assumes that there are 4 columns - one each for Quota/Gsync combination.
+You can then put in a criteria for percentage deviation from the baseline which should send out an alert.
+
+
+Setting up the environment:
+------------------------------------
+* Initialize the required variables by modifying the example gf_perf_config
+present in the git repo.
+
+Building and deploying GlusterFS
+------------------------------------
+* Populate the BRICKS variable in gf_perf_config
+* If you want to deploy a tarball, update the tarball in gf_perf_config
+and then run "deploy_gluster". This will copy the tarball to all the bricks and
+will build it there. This uses a script called buildit which is part of the
+repo.
+
+Running the tests
+------------------------------------
+* Use start_perf_measure to run the tests. The test run details can be changed
+in gf_perf_config before the run. You can specify whether the configuration is
+a distribute, replicate or distributed-replicate (distrep).
+* This will generate statistics in the log_repo for the current run under the
+director - "$log_repo/<current run>"
+* The run "id" is maintained by a ".runfile" which gets generated on the first
+run and is later updated as required during subsequent runs.
+* NOTE: You can use "batchrun" to start tests for all combinations. Downloading
+the tarball to the cwd is the responsibility of the user.
+
+Analyzing the output
+------------------------------------
+* Use analyze_perf_measure to generate plots of resource utilization for all the
+bricks and the client.
+
+Running Quota and Gsync combinations:
+------------------------------------
+* After an initial run (as described above) is complete, you can run different
+combinations of Quota/Gsync by running quota_gsync_run.
+* NOTE: Before running this script, it is important to comment out the line -
+./create_gluster_vol in the script start_perf_measure.
+Otherwise, a new volume will be created for every combination, which will wipe
+out the Quota/Gsync settings. Look at the usage in batchrun.
+TODO: Need to use the gluster command line to enable/disable quota/gsync instead
+of directly modifying the volume file.
+
+Setting up automated-email setup:
+--------------------------------
+* Setup automated emails to send results as soon as the runs are done. The whole
+email setup is done using Lakshmipathi's inotify program available at
+http://github.com/gluster/qa/blob/master/c_pgms/inotify.c & when a file is scpd
+to the particular destination, inotify sends across the mail from a shell.gluster.com
+account.
\ No newline at end of file
diff --git a/perf-framework/acl b/perf-framework/acl
new file mode 100755
index 0000000..c971fee
--- /dev/null
+++ b/perf-framework/acl
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+source gf_perf_config
+
+function usage()
+{
+	echo "Usage: $0 <on|off>"
+	exit 1
+}
+
+function get_mount()
+{
+        brick=$1
+	export_dir=$2
+        ssh -l root $brick ls -d $export_dir > /dev/null
+	if [ $? -ne 0 ]
+	then
+		echo "FATAL : Export directory does not exist. Exiting..."
+		exit 1
+	fi
+        echo `ssh -l root $brick df -h $export_dir | grep -v ^Filesystem | awk '{print $NF}'`
+}
+	
+	
+if [ $# -ne 1 ]
+then
+	usage
+fi
+
+for brick in $BRICK_IP_ADDRS
+do
+	mount_point=`get_mount $brick $SERVER_EXPORT_DIR`
+	if [ "$1" == "on" ]
+	then
+		ssh -l root $brick mount -o remount,acl $mount_point
+	elif [ "$1" == "off" ]
+	then
+		ssh -l root $brick  mount -o remount,noacl $mount_point
+	else
+		usage
+	fi
+done
diff --git a/perf-framework/analyze_perf_data b/perf-framework/analyze_perf_data
new file mode 100755
index 0000000..129a1e9
--- /dev/null
+++ b/perf-framework/analyze_perf_data
@@ -0,0 +1,466 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+if [ $# -le 0 -o $# -gt 2 ]
+then
+	echo "Usage : $0 <run directory>"
+	exit 1
+fi
+
+rundir=$1
+cmpdir=""
+
+if [ $# -eq 2 ]
+then
+	cmpdir=$2
+fi
+
+function gen_cpu_data()
+{
+	time=0
+	sum=0
+	
+	if [ -f times ]
+	then
+	        rm times
+	fi
+	
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        sum=$((sum + $time))
+	        echo $sum >> times
+	done
+	sed -i 's/$/  100/g' times
+	
+	num_procs=`grep -w ^processor $SYSINFO | wc -l`
+	echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+	idle_col_tweak=$?
+	echo `grep CPU $MPSTAT_LOG | head -1 | awk '{print $3}'` | grep -o CPU > /dev/null
+	cpu_col_tweak=$?
+	for i in "all" 0 `seq $((num_procs-1))`
+	do
+	        egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$cpu_col_tweak -v v2=$idle_col_tweak '{print $(3-v1) " " $(NF-v2)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+	        cat -n cpu-$i > cpu-$i.new
+	done
+}
+
+function plot_cpu_usage()
+{
+	xrange=$((`tail -1 times | awk '{print $1}'`+50))
+	mpstat_interval=5
+        plot_info=pinfo.$$
+        num_procs=`grep -w ^processor $SYSINFO | wc -l`
+        ltype=1
+	identity=$1
+	
+        for i in "all" 0 `seq $((num_procs-1))`
+        do
+                echo -ne "\"cpu-$i.new\" using (\$1*$mpstat_interval):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+                ltype=$((ltype+1))
+        done
+        echo "\"times\" using 1:2 title '' with impulse lt 2 lw 1" >> $plot_info
+
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "CPU utilization : All CPUs ($identity)"
+	set xlabel "Time"
+	set ylabel "% CPU utilization"
+	set xr [0:$xrange]
+	set yr [0:100]
+	set terminal png nocrop size 1024,768
+	set output "$CPU_PLOT_OUTPUT"
+        plot `cat $plot_info`
+EOF
+        rm $plot_info
+}
+
+function gen_vm_data()
+{
+	egrep -v 'memory|free|^$' $VMSTAT_LOG  | awk '{print $4}' > vm_datafile
+	totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+	cat -n memfile > memfile.new
+}
+
+function plot_vm_usage()
+{
+	vmstat_interval=5
+	total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	xrange=$((`tail -1 times | awk '{print $1}'`+50))
+	identity=$1
+	
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "Memory utilization ($identity)"
+	set xlabel "Time"
+	set ylabel "Memory utilization in bytes"
+	set xr [0:$xrange]
+	set yr [0:$total_mem]
+	set terminal png nocrop size 1024,768
+	set output "$VM_PLOT_OUTPUT"
+	plot  "memfile.new" using (\$1*$vmstat_interval):2 title 'memory-usage' with lines lt 2 lw 2,\
+	      "times" using 1:(\$2*$total_mem/100) title '' with impulse lt 2 lw 1
+EOF
+}
+
+function gen_iostats()
+{
+        brick=$1
+        dev=`ssh $brick "df -h $SERVER_EXPORT_DIR" | tail -1 | awk '{print $1}' | cut -f3 -d'/'`
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $10}' | cut -f1 -d'.' > io_await
+	cat -n io_await > io_await.new
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $6}' | cut -f1 -d'.' > read_tput
+	cat -n read_tput > read_tput.new
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $7}' | cut -f1 -d'.' > write_tput
+	cat -n write_tput > write_tput.new
+}
+
+function plot_iostats()
+{
+	iostat_interval=5
+	max_wait=$((`sort -n io_await | tail -1` + 50))
+	max_read=$(((`sort -n read_tput | tail -1`) / 2 + 100))
+	max_write=$(((`sort -n write_tput | tail -1`) / 2 + 100))
+        max_io=$max_write
+        if [ $max_read -gt $max_write ]
+        then
+                max_io=$max_read;
+        fi
+	xrange=$((`tail -1 times | awk '{print $1}'`+50))
+	identity=$1
+	
+	gnuplot <<EOF1
+	set autoscale 
+	set grid
+	set title "IO Wait times ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "IO Wait times in milliseconds"
+	set xr [0:$xrange]
+	set yr [0:$max_wait]
+	set terminal png nocrop size 1024,768
+	set output "$IO_TIMES_PLOT_OUTPUT"
+	plot  "io_await.new" using (\$1*$iostat_interval):2 title 'IO wait times' with lines lt 3 lw 2,\
+	      "times" using 1:(\$2*$max_wait/100) title '' with impulse lt 2 lw 1
+EOF1
+	gnuplot <<EOF2
+	set autoscale 
+	set grid
+	set title "Disk Read-Write throughput ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "Throughput in KB/sec"
+	set xr [0:$xrange]
+	set yr [0:$max_io]
+	set terminal png nocrop size 1024,768
+	set output "$IO_TPUT_PLOT_OUTPUT"
+	plot  "read_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Read throughput' with lines lt 4 lw 2,\
+	      "write_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Write throughput' with lines lt 5 lw 2,\
+	      "times" using 1:(\$2*$max_io/100) title '' with impulse lt 2 lw 1
+EOF2
+}
+
+function gen_cmp_data()
+{
+	perflog_baseline=$1
+	perflog_current=$2
+	
+	time=0
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> btimes.$$
+	done
+	
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> ctimes.$$
+	done
+	
+	paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+	rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+	a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+	yrange=`echo $a + $a/5 | bc`
+	b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+	xrange=`echo $b - 0.5 | bc` 
+	
+	gnuplot <<EOF
+	reset
+	set key at graph 0.15, 0.85 horizontal samplen 0.1
+	set style data histogram
+	set style histogram cluster gap 1
+	set style fill solid border -1
+	set boxwidth 0.8
+	set xtic rotate by 90 scale 0
+	unset ytics
+	set y2tics rotate by 90
+	set terminal png nocrop size 1024,768
+	set xlabel ' '
+	set size 0.6, 1
+	set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+	set y2label '$XLABEL' offset -2
+	set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+	set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+	set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+	set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+	set output "tmp.$$.png"
+	p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+        convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+        rm tmp.$$.png
+}
+
+function gen_intr_csw_stats()
+{
+	egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $11}' > intrstat
+	cat -n intrstat > intrstat.new
+	egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $12}' > cswstat
+	cat -n cswstat > cswstat.new
+}
+
+function plot_intr_csw_stats()
+{
+	vmstat_interval=5
+	xrange=$((`tail -1 times | awk '{print $1}'` + 50))
+	max_intr=$((`sort -n intrstat | tail -1` + 100))
+	max_csw=$((`sort -n cswstat | tail -1` + 100))
+        max_val=$max_csw
+        if [ $max_intr -gt $max_csw ]
+        then
+                max_val=$max_intr;
+        fi
+	identity=$1
+	
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "Interrupts and context switches ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "Interrupts/Context Switches"
+	set xr [0:$xrange]
+	set yr [0:$max_val]
+	set terminal png nocrop size 1024,768
+	set output "$INTR_CSW_PLOT_OUTPUT"
+	plot  "intrstat.new" using (\$1*$vmstat_interval):2 title 'Interrupts' with lines lt 4 lw 2,\
+	      "cswstat.new" using (\$1*$vmstat_interval):2 title 'Context Switches' with lines lt 5 lw 2,\
+	      "times" using 1:(\$2*$max_val/100) title '' with impulse lt 2 lw 1
+EOF
+}
+
+function gen_netstats()
+{
+	ip_addr=$1
+        dev=`ssh $ip_addr ifconfig | grep -B1 $ip_addr | head -1 | cut -f1 -d' '`
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $3}' | cut -f1 -d'.' > rpkts
+        cat -n rpkts > rpkts.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $4}' | cut -f1 -d'.' > wpkts
+        cat -n wpkts > wpkts.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $5}' | cut -f1 -d'.' > rkbytes
+        cat -n rkbytes > rkbytes.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $6}' | cut -f1 -d'.' > wkbytes
+        cat -n wkbytes > wkbytes.new
+}
+
+function plot_netstats()
+{
+        sar_netstat_interval=5
+        max_read_pkts=$((`sort -n rpkts | tail -1` + 50))
+        max_write_pkts=$((`sort -n wpkts | tail -1` + 50))
+        max_read_kbytes=$(((`sort -n rkbytes | tail -1`)/1024 + 100))
+        max_write_kbytes=$(((`sort -n wkbytes | tail -1`)/1024 + 100))
+
+        max_pkts=$max_write_pkts
+        if [ $max_read_pkts -gt $max_write_pkts ]
+        then   
+                max_pkts=$max_read_pkts;
+        fi
+
+        max_kbytes=$max_write_kbytes
+        if [ $max_read_kbytes -gt $max_write_kbytes ]
+        then   
+                max_kbytes=$max_read_kbytes;
+        fi
+        xrange=$((`tail -1 times | awk '{print $1}'`+50))
+        identity=$1
+
+        gnuplot <<EOF1
+        set autoscale
+        set grid
+        set title "Network statistics - Packet Read/Write ($identity)"
+        set xlabel "Time in seconds"
+        set ylabel "Number of Packets"
+        set xr [0:$xrange]
+        set yr [0:$max_pkts]
+        set terminal png nocrop size 1024,768
+        set output "$NET_PKTS_PLOT_OUTPUT"
+        plot  "rpkts.new" using (\$1*$sar_netstat_interval):2 title 'Read Packets' with lines lt 3 lw 2,\
+              "wpkts.new" using (\$1*$sar_netstat_interval):2 title 'Write Packets' with lines lt 4 lw 2,\
+              "times" using 1:(\$2*$max_pkts/100) title '' with impulse lt 2 lw 1
+EOF1
+        gnuplot <<EOF2
+        set autoscale
+        set grid
+        set title "Network Read-Write throughput ($identity)"
+        set xlabel "Time in seconds"
+        set ylabel "Throughput in KB/sec"
+        set xr [0:$xrange]
+        set yr [0:$max_kbytes]
+        set terminal png nocrop size 1024,768
+        set output "$NET_TPUT_PLOT_OUTPUT"
+        plot  "rkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Read throughput' with lines lt 3 lw 2,\
+              "wkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Write throughput' with lines lt 4 lw 2,\
+              "times" using 1:(\$2*$max_kbytes/100) title '' with impulse lt 2 lw 1
+EOF2
+}
+
+function analyse_plot_data()
+{
+	identity=$1
+
+	if [ $identity != "client" ]
+	then
+        	brick=$2
+	fi
+	# Generate CPU data
+	gen_cpu_data
+	# plot CPU data
+	plot_cpu_usage $identity
+
+	# Generate VM data
+	gen_vm_data
+	# plot VM data
+	plot_vm_usage $identity
+
+	if [ $identity != "client" ]
+	then
+		# Generate io-times and io-throughput data
+		# This makes sense only for the bricks since the client is not involved in disk IO
+		gen_iostats $brick
+		# plot io-times and io-throughput data
+		plot_iostats $identity
+	fi
+
+        # Generate interrupt and context switch data
+        gen_intr_csw_stats
+        # plot interrupt and context switch data
+        plot_intr_csw_stats $identity
+
+	# Generate network packet statistics and throughput data
+	if [ $identity != "client" ]
+	then
+		gen_netstats $brick
+	else
+		gen_netstats $CLIENT_IP_ADDR
+	fi
+	# Generate network packet statistics and throughput data
+	plot_netstats $identity
+
+	# cleanup tmp files
+	rm vm_datafile memfile*
+	rm cpu-* times
+	if [ $identity != "client" ]
+	then
+		rm io_await* read_tput* write_tput*
+	fi
+	rm cswstat* intrstat*
+	rm rpkts* wpkts* rkbytes* wkbytes*
+}
+
+function analyse_client_data()
+{
+	MPSTAT_LOG=$rundir/client/mpstat_log
+	VMSTAT_LOG=$rundir/client/vmstat_log
+	IOSTAT_LOG=$rundir/client/iostat_log
+	SAR_NETSTAT_LOG=$rundir/client/sar_netstat_log
+	SYSINFO=$rundir/client/sysinfo
+	PERFLOG=$rundir/client/perf-test.log
+	CPU_PLOT_OUTPUT=$rundir/client/cpu.png
+	VM_PLOT_OUTPUT=$rundir/client/vm.png
+        IO_TIMES_PLOT_OUTPUT=$rundir/client/io_times.png
+        IO_TPUT_PLOT_OUTPUT=$rundir/client/io_tput.png
+        INTR_CSW_PLOT_OUTPUT=$rundir/client/intr_csw.png
+	NET_PKTS_PLOT_OUTPUT=$rundir/client/net-pkts.png
+	NET_TPUT_PLOT_OUTPUT=$rundir/client/net-tput.png
+	identity="client"
+	analyse_plot_data $identity
+}
+
+function analyse_brick_data()
+{
+	ind=0
+	for b in $BRICK_IP_ADDRS
+	do
+		ind=$((ind+1))
+		MPSTAT_LOG=$rundir/brick$ind*mpstat-log
+		VMSTAT_LOG=$rundir/brick$ind*vmstat-log
+		IOSTAT_LOG=$rundir/brick$ind*iostat-log
+		SAR_NETSTAT_LOG=$rundir/brick$ind-*sar_netstat-log
+		SYSINFO=$rundir/brick$ind*sysinfo-log
+		PERFLOG=$rundir/client/perf-test.log
+		CPU_PLOT_OUTPUT=$rundir/brick$ind-cpu.png
+		VM_PLOT_OUTPUT=$rundir/brick$ind-vm.png
+                IO_TIMES_PLOT_OUTPUT=$rundir/brick$ind-io_times.png
+                IO_TPUT_PLOT_OUTPUT=$rundir/brick$ind-io_tput.png
+                INTR_CSW_PLOT_OUTPUT=$rundir/brick$ind-intr_csw.png
+		NET_PKTS_PLOT_OUTPUT=$rundir/brick$ind-net-pkts.png
+		NET_TPUT_PLOT_OUTPUT=$rundir/brick$ind-net-tput.png
+
+		identity="brick$ind"
+		analyse_plot_data $identity $b
+	done
+}
+
+function do_comparison()
+{
+	# Generate comparison data
+	gen_cmp_data $1 $2
+
+	# plot perf comparison
+	plot_comparison
+}
+
+cp ops $LOCAL_LOG_REPO
+cd $LOCAL_LOG_REPO
+analyse_client_data
+analyse_brick_data
+rm ops
+
+if [ "$cmpdir" != "" ]
+then
+	CMP_PLOT_OUTPUT=$cmpdir-$rundir-cmp.png
+	CMP_DATAFILE=$rundir/client/cmp-with-$cmpdir-data.dat
+
+	XLABEL="Time in seconds"
+	YLABEL="Operations"
+	LEGEND_A="$cmpdir"
+	LEGEND_B="$rundir"
+	PLOT_TITLE="Performance comparison - $cmpdir vs $rundir"
+	do_comparison $cmpdir/client/perf-test.log $rundir/client/perf-test.log
+fi
+
+# Create tarball of the plots
+echo "Creating plots.tar..."
+cd $rundir
+tar cf plots.tar *.png client/*.png
diff --git a/perf-framework/automate_pwl_ssh b/perf-framework/automate_pwl_ssh
new file mode 100755
index 0000000..78e4c47
--- /dev/null
+++ b/perf-framework/automate_pwl_ssh
@@ -0,0 +1,43 @@
+#!/bin/bash -u
+
+expect_log=./expectlog
+
+if [ ! -f /root/.ssh/id_rsa.pub ]
+then
+	ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa
+fi
+
+if [ ! -f /usr/bin/expect ]
+then
+	echo "expect not found. Attempting to install..."
+	if grep -i centos /etc/issue > /dev/null 2>&1
+	then
+		echo "CentOS. Using yum"
+		yum install -y `yum whatprovides expect | grep -o "expect.*x86_64"` > /dev/null 2>&1
+		if [ $? -ne 0 ]
+		then
+			echo "Failed installing expect. Exiting..."
+			exit 1
+		fi
+	elif grep -i ubuntu /etc/issue > /dev/null 2>&1
+	then
+		echo "Ubuntu. Using apt-get"
+		apt-get install -y expect > /dev/null 2>&1
+		if [ $? -ne 0 ]
+		then
+			echo "Failed installing expect. Exiting..."
+			exit 1
+		fi
+	else
+		echo "Unknown distribution"
+		echo "Install expect and try again."
+		exit 1
+	fi
+fi
+
+echo "Attempting passwordless ssh setup on multiple hosts."
+echo "log can be found in $expect_log"
+
+export ROOT_PW1
+export ROOT_PW2
+./pwl_ssh $expect_log $SSH_HOSTS
diff --git a/perf-framework/batchrun b/perf-framework/batchrun
new file mode 100755
index 0000000..d45455e
--- /dev/null
+++ b/perf-framework/batchrun
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+RUNLIST="glusterfs-3git.tar.gz"
+
+function do_run()
+{
+        FIRST=`cat .runfile`
+	./start_perf_measure
+#	sed -i 's/^\.\/create_gluster_vol/#&/' start_perf_measure
+#	sleep 10
+#	if [ $MINOR -gt 1 ]; then
+#	    LAST=$(($FIRST+3))
+#	    ./quota_gsync_run $run
+#	    sed -i 's/^#\.\/create_gluster_vol/\.\/create_gluster_vol/' start_perf_measure
+#	fi
+}
+
+function extract_release()
+{
+    MAJOR=`echo $runlabel | cut -f1 -d'.' | grep -o "^[0-9]"`
+    MINOR=`echo $runlabel | cut -f2 -d'.' | grep -o "^[0-9]"`
+}
+
+for run in $RUNLIST
+do
+	cp tarballs/$run .
+	./setrun $run
+	./deploy_gluster
+	./check_install.new
+        if [ $? -ne 0 ]
+        then
+                echo "Installation of run $run failed. Continuing with next run"
+                continue
+        fi
+	echo "Sleeping for 10 seconds.."
+	runlabel=`echo $run|sed -e 's/^glusterfs-//' -e 's/\.tar\.gz//'`
+	echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota off, gsync off)" >> $RUNLOG
+	sleep 10
+	extract_release
+	do_run
+	rm $run
+	sed -i 's/ENABLE_ACL=no/ENABLE_ACL=yes/' gf_perf_config
+ 	if [ $MINOR -gt 1 ]; then
+ 	    ./send_mail `seq $FIRST $LAST`
+ 	else
+ 	    ./send_mail $FIRST
+	fi
+done
diff --git a/perf-framework/buildit b/perf-framework/buildit
new file mode 100755
index 0000000..df3f176
--- /dev/null
+++ b/perf-framework/buildit
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+./autogen.sh > autogen.out.remote 2> autogen.err.remote
+./configure CFLAGS="-g -O0 -DDEBUG" > configure.out.remote 2> configure.err.remote
+make > make.out.remote 2> make.err.remote
+make install > make_install.out.remote 2> make_install.err.remote
diff --git a/perf-framework/calc_all b/perf-framework/calc_all
new file mode 100755
index 0000000..01e4c91
--- /dev/null
+++ b/perf-framework/calc_all
@@ -0,0 +1,49 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+num=""
+for run in "$@"
+do
+	egrep_patt+="^`echo run$run\|`"
+	PERFLOG="log_repo/run$run/client/perf-test.log"
+	if [ $run -lt 9 ]
+	then
+		run="0$run"
+	fi
+	echo "RUN$run               " > /tmp/run$run.$$
+	echo "----------------" >> /tmp/run$run.$$
+        for op in `cat ops`
+        do
+                printf "%-6s"  `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | tr '\n' '\t'|sed 's/	$//'` >> /tmp/run$run.$$
+		printf "\n" >> /tmp/run$run.$$
+        done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations               " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+if [ $# -lt 2 ]; then
+    paste /tmp/tmp_ops.$$ /tmp/run*.$$;
+else
+    for list in "$@"; do
+        if [ $list -lt 9 ]; then
+                list="0$list"
+        fi
+        if [ "$1" -eq $list ]; then
+            num="{$list"
+        else
+            num="$num,$list";
+        fi
+
+        if [ "${@: -1}" -eq $list ]; then
+            num="$num}"
+        fi
+    done
+    eval "paste /tmp/tmp_ops.$$ /tmp/run$num.$$";
+fi
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_avg b/perf-framework/calc_avg
new file mode 100755
index 0000000..596554f
--- /dev/null
+++ b/perf-framework/calc_avg
@@ -0,0 +1,54 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+num=""
+for run in "$@"
+do
+	egrep_patt+="^`echo run$run\|`"
+	PERFLOG="log_repo/run$run/client/perf-test.log"
+	if [ $run -lt 9 ]
+	then
+		run="0$run"
+	fi
+	echo "RUN$run" > /tmp/run$run.$$
+	echo "-------" >> /tmp/run$run.$$
+        for op in `cat ops`
+        do
+                time=0;
+                for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+                do
+                        time=$((time+$i))
+                done;
+                time=$((time/3)) # Average over three runs
+                echo $time >> /tmp/run$run.$$
+        done
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations               " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+if [ $# -lt 2 ]; then
+    paste /tmp/tmp_ops.$$ /tmp/run*.$$;
+else
+    for list in "$@"; do
+        if [ $list -lt 9 ]; then
+                list="0$list"
+        fi
+        if [ "$1" -eq $list ]; then
+            num="{$list"
+        else
+            num="$num,$list";
+        fi
+
+        if [ "${@: -1}" -eq $list ]; then
+            num="$num}"
+        fi
+    done
+    eval "paste /tmp/tmp_ops.$$ /tmp/run$num.$$";
+fi
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_best b/perf-framework/calc_best
new file mode 100755
index 0000000..fed9fda
--- /dev/null
+++ b/perf-framework/calc_best
@@ -0,0 +1,31 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+
+for run in "$@"
+do
+	egrep_patt+="^`echo run$run\|`"
+	PERFLOG="log_repo/run$run/client/perf-test.log"
+	if [ $run -lt 9 ]
+	then
+		run="0$run"
+	fi
+	echo "RUN$run    " > /tmp/run$run.$$
+	echo "------------" >> /tmp/run$run.$$
+        for op in `cat ops`
+        do
+                printf "%-6s"  `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | cat -n | sort -n -k2 | head -1` >> /tmp/run$run.$$
+		printf "\n" >> /tmp/run$run.$$
+        done 
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations               " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+paste /tmp/tmp_ops.$$ /tmp/run*.$$
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/calc_worst b/perf-framework/calc_worst
new file mode 100755
index 0000000..363ac58
--- /dev/null
+++ b/perf-framework/calc_worst
@@ -0,0 +1,31 @@
+#!/bin/bash -ue
+
+egrep_patt=""
+
+for run in "$@"
+do
+	egrep_patt+="^`echo run$run\|`"
+	PERFLOG="log_repo/run$run/client/perf-test.log"
+	if [ $run -lt 9 ]
+	then
+		run="0$run"
+	fi
+	echo "RUN$run    " > /tmp/run$run.$$
+	echo "------------" >> /tmp/run$run.$$
+        for op in `cat ops`
+        do
+                printf "%-6s"  `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.' | cat -n | sort -n -k2 | tail -1` >> /tmp/run$run.$$
+		printf "\n" >> /tmp/run$run.$$
+        done 
+done
+egrep_patt+="^zzz"
+
+echo ""
+egrep -w "$egrep_patt" log_repo/runlog
+echo ""
+echo "Operations               " > /tmp/tmp_ops.$$
+echo "-------------------------" >> /tmp/tmp_ops.$$
+awk '{ printf("%-25s\n", $0) }' ops >> /tmp/tmp_ops.$$
+
+paste /tmp/tmp_ops.$$ /tmp/run*.$$
+rm /tmp/run*.$$ /tmp/tmp_ops.$$
diff --git a/perf-framework/check_install b/perf-framework/check_install
new file mode 100755
index 0000000..aa72447
--- /dev/null
+++ b/perf-framework/check_install
@@ -0,0 +1,13 @@
+#!/bin/bash -ue
+
+source gf_perf_config
+
+gluster_build_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+def_gfd_loc=/usr/local/sbin/glusterfsd
+
+echo "Checking server installation"
+./run cksum $SERVER_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd $def_gfd_loc
+
+echo ""
+echo "Checking client installation"
+cksum $CLIENT_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd $def_gfd_loc
diff --git a/perf-framework/check_install.new b/perf-framework/check_install.new
new file mode 100755
index 0000000..d571ae1
--- /dev/null
+++ b/perf-framework/check_install.new
@@ -0,0 +1,33 @@
+#!/bin/bash -ue
+
+source gf_perf_config
+
+gluster_build_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+def_gfd_loc=/usr/local/sbin/glusterfsd
+
+echo "Checking server installation"
+for brick in $BRICK_IP_ADDRS
+do
+	build_file_cksum=`ssh -l root $brick "cksum $SERVER_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd" | cut -f1 -d' '`
+	current_installed_file_cksum=`ssh -l root $brick cksum $def_gfd_loc | cut -f1 -d' '`
+	if [ "$build_file_cksum" != "$current_installed_file_cksum" ]
+	then
+		echo "Checksum on $brick do not match. Aborting..."
+		exit 1
+	else
+		echo "Checksum on $brick OK"
+	fi
+done
+
+echo ""
+echo "Checking client installation"
+build_file_cksum=`cksum $CLIENT_BUILD_DIR/$gluster_build_dir/glusterfsd/src/.libs/glusterfsd | cut -f1 -d' '`
+current_installed_file_cksum=`cksum $def_gfd_loc | cut -f1 -d' '`
+
+if [ "$build_file_cksum" != "$current_installed_file_cksum" ]
+then
+	echo "Checksum on client do not match. Aborting..."
+	exit 1
+else
+	echo "Checksum on client OK"
+fi
diff --git a/perf-framework/create_gluster_vol b/perf-framework/create_gluster_vol
new file mode 100755
index 0000000..fcb91ce
--- /dev/null
+++ b/perf-framework/create_gluster_vol
@@ -0,0 +1,122 @@
+#!/bin/bash -ue
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+GF_BIN=/usr/local/sbin
+
+function mgmt_vol_create()
+{
+        if [ $ENABLE_ACL == "yes" ]
+        then
+                ./acl on
+        else
+                ./acl off
+        fi
+	for brick in $BRICK_IP_ADDRS
+	do
+		ssh  -l root $MGMT_NODE "$GF_BIN/gluster peer probe $brick"
+	done
+        ssh -l root $MGMT_NODE "$@"
+	ssh -l root $MGMT_NODE "$GF_BIN/gluster volume start $VOLNAME"
+	if [ $GF_CONFIG == "stripe" ]; then
+	    echo "Setting cache-size to 128MB for stripe tests..."
+	    ssh -l root $MGMT_NODE "$GF_BIN/gluster volume set $VOLNAME cache-size 128MB"
+	fi
+	#mem-factor is set for mount type of nfs to increase performance.
+	if [ $MOUNT_TYPE == "nfs" ]; then
+	    echo "Setting mem-factor to 20 since mount type is nfs..."
+	    ssh -l root $MGMT_NODE "$GF_BIN/gluster volume set $VOLNAME nfs.mem-factor 20"
+	fi
+}
+
+function cleanup_brick()
+{
+        ssh -l root $1 "killall glusterd glusterfs glusterfsd > /dev/null 2>&1; rm -rf /etc/glusterd"
+
+	if [ "$SERVER_EXPORT_DIR" != "" ]
+	then
+        	echo "Deleting export dir $SERVER_EXPORT_DIR on $brick..."
+        	ssh -l root $1 "rm -rf $SERVER_EXPORT_DIR"
+	else
+        	echo "************* Empty export dir in config *************"
+		exit
+	fi
+}
+
+echo "Cleaning bricks..."
+for brick in $BRICK_IP_ADDRS
+do 
+        cleanup_brick $brick &
+done
+
+wait
+
+echo "Creating export directory..."
+for brick in $BRICK_IP_ADDRS
+do 
+        ssh -l root $brick "mkdir -p $SERVER_EXPORT_DIR"
+done
+
+# Start glusterd on all the bricks
+echo "Starting glusterd..."
+for brick in $BRICK_IP_ADDRS
+do 
+	if [ $ENABLE_MEM_ACCT == "yes" ]
+	then
+        	ssh -l root $brick "GLUSTERFS_DISABLE_MEM_ACCT=0 $GF_BIN/glusterd"
+	else
+        	ssh -l root $brick "$GF_BIN/glusterd"
+	fi
+done
+
+# Create the gluster volume according to the config
+
+bricklist=""
+count=0
+for brick in $BRICK_IP_ADDRS
+do 
+        bricklist+="$brick:$SERVER_EXPORT_DIR "
+        count=$((count+1))
+done
+
+if [ "$GF_CONFIG" == "replicate" -a "$count" -gt "2" ]
+then
+        echo "NOTE:"
+        echo "Replicate volume with more than 2 bricks created."
+        echo "This will result in a distributed replicate config."
+fi
+
+echo "Creating $GF_CONFIG volume..."
+if [ "$GF_CONFIG" == "distribute" ]
+then
+        mgmt_vol_create "gluster volume create $VOLNAME $bricklist"
+elif [ "$GF_CONFIG" == "replicate" ]
+then
+        mgmt_vol_create "gluster volume create $VOLNAME replica 2 $bricklist"
+elif [ "$GF_CONFIG" == "distrep" ]
+then
+        mgmt_vol_create "gluster volume create $VOLNAME replica 2 $bricklist"
+elif [ "$GF_CONFIG" == "stripe" ]
+then
+        mgmt_vol_create "gluster volume create $VOLNAME stripe 4 $bricklist"
+else
+        echo "Unknown configuration. Exiting..."
+        exit 1
+fi
+
+sleep 5
+
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+	for brick in $BRICK_IP_ADDRS
+	do 
+		echo "Memory accounting status on : $brick -"
+        	ssh -l root $brick "\
+                	echo \"x/x &gf_mem_acct_enable\" > gf_gdb_commands;\
+                	echo \"quit\" >> gf_gdb_commands;\
+                	gdb -q --command=gf_gdb_commands -p \`pidof glusterfsd\` 2> /dev/null | \
+                	grep gf_mem_acct_enable | awk '{print \$(NF-1) \$NF}';\
+                	rm gf_gdb_commands > /dev/null 2>&1"
+		echo ""
+	done
+fi
diff --git a/perf-framework/deploy_gluster b/perf-framework/deploy_gluster
new file mode 100755
index 0000000..49543f7
--- /dev/null
+++ b/perf-framework/deploy_gluster
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+function prepare_server()
+{
+	ip_addr=$1
+	ssh -l root $ip_addr mkdir -p $SERVER_BUILD_DIR
+	scp -p $RELEASE_TARBALL $BUILD_SCRIPT root@$ip_addr:$SERVER_BUILD_DIR
+	ssh -l root $ip_addr "cd $SERVER_BUILD_DIR && tar xf $RELEASE_TARBALL"
+	ssh -l root $ip_addr "cd $SERVER_BUILD_DIR/$release_dir && ../$BUILD_SCRIPT"
+}
+
+function prepare_client()
+{
+	mkdir -p $CLIENT_BUILD_DIR
+	cp $RELEASE_TARBALL $BUILD_SCRIPT $CLIENT_BUILD_DIR
+	cd $CLIENT_BUILD_DIR && tar xf $RELEASE_TARBALL
+	cd $CLIENT_BUILD_DIR/$release_dir && ../$BUILD_SCRIPT
+}
+
+
+CONFIG_FILE=gf_perf_config
+BUILD_SCRIPT=buildit
+source $CONFIG_FILE
+
+if [ "$RELEASE_TARBALL" == "" ]
+then
+	echo "ERROR: Empty release tarball"
+	exit 1
+fi
+
+if [ ! -f $RELEASE_TARBALL ]
+then
+	echo "ERROR: Unable to find source tarball $RELEASE_TARBALL"
+	exit 1
+fi
+
+release_dir=`tar tvf $RELEASE_TARBALL | head -1 | awk '{print $NF}'`
+
+# Build gluster on the bricks
+for brick in `echo $BRICK_IP_ADDRS`
+do
+	echo "Starting build on server $brick..."
+	prepare_server $brick &
+done
+
+echo "Waiting for jobs to finish..."
+sleep 5
+
+# Build gluster on the client
+echo "Starting build on client..."
+prepare_client &
+
+wait
diff --git a/perf-framework/diff_perfrun b/perf-framework/diff_perfrun
new file mode 100755
index 0000000..65ea316
--- /dev/null
+++ b/perf-framework/diff_perfrun
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+echo ""
+echo "Q - Quota ON, q - Quota OFF, G - Gsync ON, g - Gsync  OFF"
+echo ""
+echo "======================================================================"
+echo "OPERATIONS                      q-g       q-G       Q-G       Q-g"
+echo "======================================================================"
+for i in `cat /tmp/ops`
+do
+    printf "%-25s" $i
+    for j in 2 3 4 5
+    do
+        printf "%10.1f" `echo \`grep -w ^$i /tmp/$1 | awk -v col=$j '{print $col}'\` | awk '{print ($2 - $1)*100/$1}' | tr '\n' '\t'`
+    done
+    echo ""
+done
diff --git a/perf-framework/file_open_analyse b/perf-framework/file_open_analyse
new file mode 100755
index 0000000..f7c98f5
--- /dev/null
+++ b/perf-framework/file_open_analyse
@@ -0,0 +1,430 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+if [ $# -le 0 -o $# -gt 2 ]
+then
+	echo "Usage : $0 <run directory>"
+	exit 1
+fi
+
+rundir=$1
+cmpdir=""
+
+if [ $# -eq 2 ]
+then
+	cmpdir=$2
+fi
+
+function gen_cpu_data()
+{
+	time=0
+	sum=0
+	
+	num_procs=`grep -w ^processor $SYSINFO | wc -l`
+	echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+	idle_col_tweak=$?
+	echo `grep CPU $MPSTAT_LOG | head -1 | awk '{print $3}'` | grep -o CPU > /dev/null
+	cpu_col_tweak=$?
+	for i in "all" 0 `seq $((num_procs-1))`
+	do
+	        egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$cpu_col_tweak -v v2=$idle_col_tweak '{print $(3-v1) " " $(NF-v2)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+	        cat -n cpu-$i > cpu-$i.new
+	done
+}
+
+function plot_cpu_usage()
+{
+	mpstat_interval=5
+        plot_info=pinfo.$$
+        num_procs=`grep -w ^processor $SYSINFO | wc -l`
+        ltype=2
+	identity=$1
+	
+        for i in 0 `seq $((num_procs-1))`
+        do
+                echo -ne "\"cpu-$i.new\" using (\$1*$mpstat_interval):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+                ltype=$((ltype+1))
+        done
+        echo -ne "\"cpu-all.new\" using (\$1*$mpstat_interval):2 title 'cpu-all' with lines lt 1 lw 2" >> $plot_info
+
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "CPU utilization : All CPUs ($identity)"
+	set xlabel "Time"
+	set ylabel "% CPU utilization"
+	set yr [0:100]
+	set terminal png nocrop size 1024,768
+	set output "$CPU_PLOT_OUTPUT"
+        plot `cat $plot_info`
+EOF
+        rm $plot_info
+}
+
+function gen_vm_data()
+{
+	egrep -v 'memory|free|^$' $VMSTAT_LOG  | awk '{print $4}' > vm_datafile
+	totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+	cat -n memfile > memfile.new
+}
+
+function plot_vm_usage()
+{
+	vmstat_interval=5
+	total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	identity=$1
+	
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "Memory utilization ($identity)"
+	set xlabel "Time"
+	set ylabel "Memory utilization in bytes"
+	set yr [0:$total_mem]
+	set terminal png nocrop size 1024,768
+	set output "$VM_PLOT_OUTPUT"
+	plot  "memfile.new" using (\$1*$vmstat_interval):2 title 'memory-usage' with lines lt 2 lw 2
+EOF
+}
+
+function gen_iostats()
+{
+        brick=$1
+        dev=`ssh $brick "df -h $SERVER_EXPORT_DIR" | tail -1 | awk '{print $1}' | cut -f3 -d'/'`
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $10}' | cut -f1 -d'.' > io_await
+	cat -n io_await > io_await.new
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $6}' | cut -f1 -d'.' > read_tput
+	cat -n read_tput > read_tput.new
+	egrep -v 'Device|^$' $IOSTAT_LOG |grep -w ^$dev | awk '{print $7}' | cut -f1 -d'.' > write_tput
+	cat -n write_tput > write_tput.new
+}
+
+function plot_iostats()
+{
+	iostat_interval=5
+	max_wait=$((`sort -n io_await | tail -1` + 50))
+	max_read=$(((`sort -n read_tput | tail -1`) / 2 + 100))
+	max_write=$(((`sort -n write_tput | tail -1`) / 2 + 100))
+        max_io=$max_write
+        if [ $max_read -gt $max_write ]
+        then
+                max_io=$max_read;
+        fi
+	identity=$1
+	
+	gnuplot <<EOF1
+	set autoscale 
+	set grid
+	set title "IO Wait times ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "IO Wait times in milliseconds"
+	set yr [0:$max_wait]
+	set terminal png nocrop size 1024,768
+	set output "$IO_TIMES_PLOT_OUTPUT"
+	plot  "io_await.new" using (\$1*$iostat_interval):2 title 'IO wait times' with lines lt 3 lw 2
+EOF1
+	gnuplot <<EOF2
+	set autoscale 
+	set grid
+	set title "Disk Read-Write throughput ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "Throughput in KB/sec"
+	set yr [0:$max_io]
+	set terminal png nocrop size 1024,768
+	set output "$IO_TPUT_PLOT_OUTPUT"
+	plot  "read_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Read throughput' with lines lt 4 lw 2,\
+	      "write_tput.new" using (\$1*$iostat_interval):(\$2/2) title 'Write throughput' with lines lt 5 lw 2
+EOF2
+}
+
+function gen_cmp_data()
+{
+	perflog_baseline=$1
+	perflog_current=$2
+	
+	time=0
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> btimes.$$
+	done
+	
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> ctimes.$$
+	done
+	
+	paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+	rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+	a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+	yrange=`echo $a + $a/5 | bc`
+	b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+	xrange=`echo $b - 0.5 | bc` 
+	
+	gnuplot <<EOF
+	reset
+	set key at graph 0.15, 0.85 horizontal samplen 0.1
+	set style data histogram
+	set style histogram cluster gap 1
+	set style fill solid border -1
+	set boxwidth 0.8
+	set xtic rotate by 90 scale 0
+	unset ytics
+	set y2tics rotate by 90
+	set terminal png nocrop size 1024,768
+	set xlabel ' '
+	set size 0.6, 1
+	set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+	set y2label '$XLABEL' offset -2
+	set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+	set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+	set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+	set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+	set output "tmp.$$.png"
+	p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+        convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+        rm tmp.$$.png
+}
+
+function gen_intr_csw_stats()
+{
+	egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $11}' > intrstat
+	cat -n intrstat > intrstat.new
+	egrep -v 'memory|free|^$' $VMSTAT_LOG | awk '{print $12}' > cswstat
+	cat -n cswstat > cswstat.new
+}
+
+function plot_intr_csw_stats()
+{
+	vmstat_interval=5
+	max_intr=$((`sort -n intrstat | tail -1` + 100))
+	max_csw=$((`sort -n cswstat | tail -1` + 100))
+        max_val=$max_csw
+        if [ $max_intr -gt $max_csw ]
+        then
+                max_val=$max_intr;
+        fi
+	identity=$1
+	
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "Interrupts and context switches ($identity)"
+	set xlabel "Time in seconds"
+	set ylabel "Interrupts/Context Switches"
+	set yr [0:$max_val]
+	set terminal png nocrop size 1024,768
+	set output "$INTR_CSW_PLOT_OUTPUT"
+	plot  "intrstat.new" using (\$1*$vmstat_interval):2 title 'Interrupts' with lines lt 4 lw 2,\
+	      "cswstat.new" using (\$1*$vmstat_interval):2 title 'Context Switches' with lines lt 5 lw 2
+EOF
+}
+
+function gen_netstats()
+{
+	ip_addr=$1
+        dev=`ssh $ip_addr ifconfig | grep -B1 $ip_addr | head -1 | cut -f1 -d' '`
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $3}' | cut -f1 -d'.' > rpkts
+        cat -n rpkts > rpkts.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $4}' | cut -f1 -d'.' > wpkts
+        cat -n wpkts > wpkts.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $5}' | cut -f1 -d'.' > rkbytes
+        cat -n rkbytes > rkbytes.new
+        egrep -v 'IFACE|^$' $SAR_NETSTAT_LOG |grep -w $dev | awk '{print $6}' | cut -f1 -d'.' > wkbytes
+        cat -n wkbytes > wkbytes.new
+}
+
+function plot_netstats()
+{
+        sar_netstat_interval=5
+        max_read_pkts=$((`sort -n rpkts | tail -1` + 50))
+        max_write_pkts=$((`sort -n wpkts | tail -1` + 50))
+        max_read_kbytes=$(((`sort -n rkbytes | tail -1`)/1024 + 100))
+        max_write_kbytes=$(((`sort -n wkbytes | tail -1`)/1024 + 100))
+
+        max_pkts=$max_write_pkts
+        if [ $max_read_pkts -gt $max_write_pkts ]
+        then   
+                max_pkts=$max_read_pkts;
+        fi
+
+        max_kbytes=$max_write_kbytes
+        if [ $max_read_kbytes -gt $max_write_kbytes ]
+        then   
+                max_kbytes=$max_read_kbytes;
+        fi
+        identity=$1
+
+        gnuplot <<EOF1
+        set autoscale
+        set grid
+        set title "Network statistics - Packet Read/Write ($identity)"
+        set xlabel "Time in seconds"
+        set ylabel "Number of Packets"
+        set yr [0:$max_pkts]
+        set terminal png nocrop size 1024,768
+        set output "$NET_PKTS_PLOT_OUTPUT"
+        plot  "rpkts.new" using (\$1*$sar_netstat_interval):2 title 'Read Packets' with lines lt 3 lw 2,\
+              "wpkts.new" using (\$1*$sar_netstat_interval):2 title 'Write Packets' with lines lt 4 lw 2
+EOF1
+        gnuplot <<EOF2
+        set autoscale
+        set grid
+        set title "Network Read-Write throughput ($identity)"
+        set xlabel "Time in seconds"
+        set ylabel "Throughput in KB/sec"
+        set yr [0:$max_kbytes]
+        set terminal png nocrop size 1024,768
+        set output "$NET_TPUT_PLOT_OUTPUT"
+        plot  "rkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Read throughput' with lines lt 3 lw 2,\
+              "wkbytes.new" using (\$1*$sar_netstat_interval):(\$2/1024) title 'Write throughput' with lines lt 4 lw 2
+EOF2
+}
+
+function analyse_plot_data()
+{
+	identity=$1
+
+	if [ $identity != "client" ]
+	then
+        	brick=$2
+	fi
+	# Generate CPU data
+	gen_cpu_data
+	# plot CPU data
+	plot_cpu_usage $identity
+
+	# Generate VM data
+	gen_vm_data
+	# plot VM data
+	plot_vm_usage $identity
+
+	if [ $identity != "client" ]
+	then
+		# Generate io-times and io-throughput data
+		# This makes sense only for the bricks since the client is not involved in disk IO
+		gen_iostats $brick
+		# plot io-times and io-throughput data
+		plot_iostats $identity
+	fi
+
+        # Generate interrupt and context switch data
+        gen_intr_csw_stats
+        # plot interrupt and context switch data
+        plot_intr_csw_stats $identity
+
+	# Generate network packet statistics and throughput data
+	if [ $identity != "client" ]
+	then
+		gen_netstats $brick
+	else
+		gen_netstats $CLIENT_IP_ADDR
+	fi
+	# Generate network packet statistics and throughput data
+	plot_netstats $identity
+
+	# cleanup tmp files
+	rm vm_datafile memfile*
+	rm cpu-*
+	if [ $identity != "client" ]
+	then
+		rm io_await* read_tput* write_tput*
+	fi
+	rm cswstat* intrstat*
+	rm rpkts* wpkts* rkbytes* wkbytes*
+}
+
+function analyse_client_data()
+{
+	MPSTAT_LOG=$rundir/client/mpstat_log
+	VMSTAT_LOG=$rundir/client/vmstat_log
+	IOSTAT_LOG=$rundir/client/iostat_log
+	SAR_NETSTAT_LOG=$rundir/client/sar_netstat_log
+	SYSINFO=$rundir/client/sysinfo
+	PERFLOG=$rundir/client/perf-test.log
+	CPU_PLOT_OUTPUT=$rundir/client/cpu.png
+	VM_PLOT_OUTPUT=$rundir/client/vm.png
+        IO_TIMES_PLOT_OUTPUT=$rundir/client/io_times.png
+        IO_TPUT_PLOT_OUTPUT=$rundir/client/io_tput.png
+        INTR_CSW_PLOT_OUTPUT=$rundir/client/intr_csw.png
+	NET_PKTS_PLOT_OUTPUT=$rundir/client/net-pkts.png
+	NET_TPUT_PLOT_OUTPUT=$rundir/client/net-tput.png
+	identity="client"
+	analyse_plot_data $identity
+}
+
+function analyse_brick_data()
+{
+	ind=0
+	for b in $BRICK_IP_ADDRS
+	do
+		ind=$((ind+1))
+		MPSTAT_LOG=$rundir/brick$ind*mpstat-log
+		VMSTAT_LOG=$rundir/brick$ind*vmstat-log
+		IOSTAT_LOG=$rundir/brick$ind*iostat-log
+		SAR_NETSTAT_LOG=$rundir/brick$ind-*sar_netstat-log
+		SYSINFO=$rundir/brick$ind*sysinfo-log
+		PERFLOG=$rundir/client/perf-test.log
+		CPU_PLOT_OUTPUT=$rundir/brick$ind-cpu.png
+		VM_PLOT_OUTPUT=$rundir/brick$ind-vm.png
+                IO_TIMES_PLOT_OUTPUT=$rundir/brick$ind-io_times.png
+                IO_TPUT_PLOT_OUTPUT=$rundir/brick$ind-io_tput.png
+                INTR_CSW_PLOT_OUTPUT=$rundir/brick$ind-intr_csw.png
+		NET_PKTS_PLOT_OUTPUT=$rundir/brick$ind-net-pkts.png
+		NET_TPUT_PLOT_OUTPUT=$rundir/brick$ind-net-tput.png
+
+		identity="brick$ind"
+		analyse_plot_data $identity $b
+	done
+}
+
+function do_comparison()
+{
+	# Generate comparison data
+	gen_cmp_data $1 $2
+
+	# plot perf comparison
+	plot_comparison
+}
+
+cp ops $LOCAL_LOG_REPO
+cd $LOCAL_LOG_REPO
+analyse_client_data
+analyse_brick_data
+rm ops
+
+if [ "$cmpdir" != "" ]
+then
+	CMP_PLOT_OUTPUT=$cmpdir-$rundir-cmp.png
+	CMP_DATAFILE=$rundir/client/cmp-with-$cmpdir-data.dat
+
+	XLABEL="Time in seconds"
+	YLABEL="Operations"
+	LEGEND_A="$cmpdir"
+	LEGEND_B="$rundir"
+	PLOT_TITLE="Performance comparison - $cmpdir vs $rundir"
+	do_comparison $cmpdir/client/perf-test.log $rundir/client/perf-test.log
+fi
+
+# Create tarball of the plots
+echo "Creating plots.tar..."
+cd $rundir
+tar cf plots.tar *.png client/*.png
diff --git a/perf-framework/file_open_test b/perf-framework/file_open_test
new file mode 100755
index 0000000..83cbe97
--- /dev/null
+++ b/perf-framework/file_open_test
@@ -0,0 +1,159 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+SETTLE_TIME=10
+RUNFILE=.runfile
+STAT_COLLECTOR=stat_collect
+
+# Generate current run, update runfile
+if [ ! -f $RUNFILE ]
+then
+        run=1
+else
+        run=`cat $RUNFILE`
+fi
+echo $((run+1)) > $RUNFILE
+
+# Drop vm caches on all the bricks before starting the runs
+for brick in $BRICK_IP_ADDRS
+do
+        ssh -l root $brick "echo 3 > /proc/sys/vm/drop_caches"
+done
+
+# Create the gluster volume
+./create_gluster_vol
+
+mount | grep $MOUNT_POINT > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+	umount $MOUNT_POINT
+fi
+
+if [ ! -d $MOUNT_POINT ]
+then
+	mkdir -p $MOUNT_POINT
+fi
+
+# Make sure that the fuse kernel module is loaded
+/sbin/lsmod | grep -w fuse > /dev/null 2>&1
+if [ $? -ne 0 ]
+then
+	/sbin/modprobe fuse > /dev/null 2>&1
+fi
+
+ps -eaf | egrep -w 'glusterfs|glusterfsd|glusterd' > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+	killall glusterfsd glusterd glusterfs  > /dev/null 2>&1
+fi
+
+# Mount the client
+# Sleep for a while. Sometimes, NFS mounts fail if attempted soon after creating the volume
+sleep $SETTLE_TIME
+
+if [ $ENABLE_ACL == "yes" ]
+then
+        acl_opts="-o acl"
+else
+        acl_opts=""
+fi
+
+echo "Mounting volume..."
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+	echo "Memory accounting status on client -"
+	echo "x/x &gf_mem_acct_enable" > commands.$$
+	echo "quit" >> commands.$$
+	GLUSTERFS_DISABLE_MEM_ACCT=0 mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+	mount_status=$?
+	gdb -q --command=commands.$$ -p `pidof glusterfs` | grep gf_mem_acct_enable | awk '{print $(NF-1) $NF}'
+	rm commands.$$ > /dev/null 2>&1
+else
+	mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+	mount_status=$?
+fi
+
+if [ $mount_status -ne 0 ]
+then
+        echo "mount -t $MOUNT_TYPE $acl_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT failed..."
+        echo "Exiting..."
+        exit 1
+fi
+
+# Copy statistics collection scripts to the server
+
+echo ""
+echo "Copying stat collection script to bricks..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "mkdir -p $SERVER_SCRIPTS_DIR"
+	scp -p $STAT_COLLECTOR root@$brick:$SERVER_SCRIPTS_DIR > /dev/null 2>&1
+done
+
+# Run statistics collection scripts on the server
+
+echo ""
+echo "Starting server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "mkdir -p $SERVER_LOG_DIR"
+	ssh -l root $brick "$SERVER_SCRIPTS_DIR/$STAT_COLLECTOR $SERVER_LOG_DIR" &
+done
+
+# Run statistics collection on client
+
+mkdir -p $LOCAL_LOG_REPO
+./$STAT_COLLECTOR $LOCAL_LOG_REPO/run$run/client &
+
+# Start perf test
+
+echo ""
+echo "Starting run $run..."
+
+sleep $SETTLE_TIME
+./parallel_create 50 10000
+sleep $SETTLE_TIME
+
+# Stop statistics collection scripts on the client
+
+killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+
+# Stop statistics collection scripts on the server
+
+echo ""
+echo "Stopping server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+done
+
+# Since the ssh processes were backgrounded, they will be hanging around.
+# Kill them
+kill `jobs -l | awk '{print $2}'` > /dev/null 2>&1
+
+# Copy statistics from the server
+
+echo ""
+echo "Copying server logfiles for run $run..."
+cur_log_dump_dir=$LOCAL_LOG_REPO/run$run
+mkdir -p $cur_log_dump_dir
+count=1
+for brick in $BRICK_IP_ADDRS
+do
+        for statf in mpstat vmstat iostat sysinfo sar_netstat
+        do
+                scp root@$brick:$SERVER_LOG_DIR/*$statf* $cur_log_dump_dir/brick$count-$brick-$statf-log > /dev/null 2>&1
+        done
+        count=$((count + 1))
+done
+
+# Cleanup statistics collected for this run on the server
+
+echo ""
+echo "Cleaning server logfiles..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "cd $SERVER_LOG_DIR; rm mpstat_log vmstat_log iostat_log sysinfo sar_netstat_log"
+done
diff --git a/perf-framework/gen_perf_data b/perf-framework/gen_perf_data
new file mode 100755
index 0000000..31110f6
--- /dev/null
+++ b/perf-framework/gen_perf_data
@@ -0,0 +1,191 @@
+#!/bin/bash -u
+
+MPSTAT_LOG=mpstat_log1908
+VMSTAT_LOG=vmstat_log1908
+SYSINFO=sysinfo1908
+PERFLOG=perf-fuse-ami.log
+
+source gnuplotrc
+
+function gen_cpu_data()
+{
+	time=0
+	sum=0
+	
+	if [ -f times ]
+	then
+	        rm times
+	fi
+	
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $PERFLOG | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        sum=$((sum + $time))
+	        echo $sum >> times
+	done
+	sed -i 's/$/  200/g' times
+	
+	num_procs=`grep -w ^processor $SYSINFO | wc -l`
+	echo `grep idle $MPSTAT_LOG | head -1 | awk '{print $NF}'` | grep -o idle
+	col_tweak=$?
+	for i in "all" 0 `seq $((num_procs-1))`
+	do
+	        egrep -v 'Linux|^$|idle' $MPSTAT_LOG | awk -v v1=$col_tweak '{print $3 " " $(NF-v1)}' | grep -w ^$i | cut -f 2 -d' '| sed 's/^/100-/g' | bc -l > cpu-$i;
+	        cat -n cpu-$i > cpu-$i.new
+	done
+}
+
+function plot_cpu_usage()
+{
+	xrange=$((`tail -1 times | awk '{print $1}'`+50))
+	MPSTAT_INTERVAL=5
+        plot_info=pinfo.$$
+        num_procs=`grep -w ^processor $SYSINFO | wc -l`
+        ltype=1
+	
+        for i in "all" 0 `seq $((num_procs-1))`
+        do
+                echo -ne "\"cpu-$i.new\" using (\$1*$MPSTAT_INTERVAL):2 title 'cpu-$i' with lines lt $ltype lw 2,\\c" >> $plot_info
+                ltype=$((ltype+1))
+        done
+        echo "\"times\" using 1:2 title '' with impulse lt 2 lw 1 lc rgb \"#FF0000\"" >> $plot_info
+
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "CPU utilization : All CPUs"
+	set xlabel "Time"
+	set ylabel "% CPU utilization"
+	set xr [0:$xrange]
+	set yr [0:100]
+	set terminal png nocrop size 1024,768
+	set output "$CPU_PLOT_OUTPUT"
+        plot `cat $plot_info`
+EOF
+        rm $plot_info
+}
+
+function gen_vm_data()
+{
+	egrep -v 'memory|free' $VMSTAT_LOG  | awk '{print $4}' > vm_datafile
+	totalmem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	cat vm_datafile | sed "s/^/$totalmem-/g" | bc > memfile
+}
+
+function plot_vm_usage()
+{
+	vmstat_interval=5
+	total_mem=`grep -w ^MemTotal $SYSINFO | awk '{print $2}'`
+	xrange=$((`tail -1 times | awk '{print $1}'`+50))
+	
+	gnuplot <<EOF
+	set autoscale 
+	set grid
+	set title "Memory utilization"
+	set xlabel "Time"
+	set ylabel "Memory utilization in bytes"
+	set xr [0:$xrange]
+	set yr [0:$total_mem]
+	set terminal png nocrop size 1024,768
+	set output "$VM_PLOT_OUTPUT"
+	plot  "vm_datafile" using 1 title 'free-memory' with lines lt 1 lw 2, \
+	      "memfile" using 1 title 'memory-usage' with lines lt 2 lw 2,\
+	      "times" using 1:(\$2*$total_mem/100) title '' with impulse lt 2 lw 1 lc rgb "#FF0000"
+EOF
+}
+
+function gen_cmp_data()
+{
+	perflog_baseline=$1
+	perflog_current=$2
+	
+	time=0
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_baseline | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> btimes.$$
+	done
+	
+	for op in `cat ops`
+	do
+	        time=0;
+	        for i in `grep -w ^$op $perflog_current | awk '{print $2}'| cut -f1 -d'.'`
+	        do
+	                time=$((time+$i))
+	        done;
+	        time=$((time/3)) # Average over three runs
+	        echo $time >> ctimes.$$
+	done
+	
+	paste -d " " ops btimes.$$ ctimes.$$ > $CMP_DATAFILE
+	rm btimes.$$ ctimes.$$
+}
+
+function plot_comparison()
+{
+	a=`cat $CMP_DATAFILE | awk '{print $2"\n"$3}' | sort -n | tail -1`
+	yrange=`echo $a + $a/5 | bc`
+	b=`wc -l $CMP_DATAFILE | awk '{print $1}'`
+	xrange=`echo $b - 0.5 | bc` 
+	
+	gnuplot <<EOF
+	reset
+	set key at graph 0.15, 0.85 horizontal samplen 0.1
+	set style data histogram
+	set style histogram cluster gap 1
+	set style fill solid border -1
+	set boxwidth 0.8
+	set xtic rotate by 90 scale 0
+	unset ytics
+	set y2tics rotate by 90
+	set terminal png nocrop size 1024,768
+	set xlabel ' '
+	set size 0.6, 1
+	set yrange [0:$yrange]; set xrange [-0.5:$xrange]
+	set y2label '$XLABEL' offset -2
+	set label 1 '$YLABEL' at graph 0.5, -0.4 centre rotate by 180
+	set label 2 '$LEGEND_A' at graph 0.05, 0.85 left rotate by 90
+	set label 3 '$LEGEND_B' at graph 0.12, 0.85 left rotate by 90
+	set label 4 '$PLOT_TITLE' at graph -0.01, 0.5 center rotate by 90
+	set output "tmp.$$.png"
+	p '$CMP_DATAFILE' u 2 title ' ', '' u 3 title ' ', '' u 0:(0):xticlabel(1) w l title ''
+EOF
+        convert tmp.$$.png -rotate 90 $CMP_PLOT_OUTPUT
+        rm tmp.$$.png
+}
+
+# Generate CPU data
+gen_cpu_data
+
+# plot CPU data
+plot_cpu_usage
+
+# Generate VM data
+gen_vm_data
+
+# plot VM data
+plot_vm_usage
+
+# Generate comparison data - baseline vs current run
+gen_cmp_data perf-fuse-ami.log perf-fuse-aws.log
+
+# plot perf comparison - baseline vs current run
+plot_comparison
+
+# cleanup tmp files
+rm vm_datafile memfile
+rm cpu-* times
+
+display cpu.png &
+display vm.png &
+display cmp.png &
diff --git a/perf-framework/get_date b/perf-framework/get_date
new file mode 100755
index 0000000..335359d
--- /dev/null
+++ b/perf-framework/get_date
@@ -0,0 +1,5 @@
+date +%s
+ssh -l root 10.1.10.30 "date +%s"
+ssh -l root 10.1.10.31 "date +%s"
+ssh -l root 10.1.10.35 "date +%s"
+ssh -l root 10.1.10.36 "date +%s"
diff --git a/perf-framework/gf_perf_config b/perf-framework/gf_perf_config
new file mode 100644
index 0000000..017feb8
--- /dev/null
+++ b/perf-framework/gf_perf_config
@@ -0,0 +1,34 @@
+# Common config variables
+RELEASE_TARBALL=glusterfs-3git.tar.gz
+BRICK_IP_ADDRS="10.1.10.30 10.1.10.31 10.1.10.35 10.1.10.36"
+GF_CONFIG="stripe"
+VOLNAME="testvol"
+MGMT_NODE="10.1.10.30"
+
+# Server specific config variables
+SERVER_BUILD_DIR=/data/rahul/gluster_builds
+SERVER_LOG_DIR=/data/rahul/PERF/server_logs
+SERVER_SCRIPTS_DIR=/data/rahul/PERF/scripts
+SERVER_EXPORT_DIR=/data/rahul/datastore
+
+# Client specific config variables
+MOUNT_TYPE="glusterfs"
+#MOUNT_TYPE="nfs"
+MOUNT_POINT=/mnt/perfmount
+LOCAL_LOG_REPO=/root/rahul/test_deploy/log_repo
+CLIENT_BUILD_DIR=/root/rahul/test_deploy/gluster_builds
+CLIENT_IP_ADDR=10.1.10.29
+RUNLOG=$LOCAL_LOG_REPO/runlog
+
+# Special runs
+ENABLE_ACL=yes
+ENABLE_MEM_ACCT=no
+
+# For passwordless ssh setup
+ROOT_PW1=abc123
+ROOT_PW2=abc321
+# SSH_HOSTS is a space separated list of hosts
+SSH_HOSTS="0"
+
+# Setup automated emails
+EMAIL="rahul@shell.gluster.com:/home/rahul/perf_logs"
\ No newline at end of file
diff --git a/perf-framework/glusterfs-precreate.sh b/perf-framework/glusterfs-precreate.sh
new file mode 100755
index 0000000..d1c92a3
--- /dev/null
+++ b/perf-framework/glusterfs-precreate.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e;
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <dir1> [dir2 [dir3 [dir4 ...]]]";
+    exit 1
+fi
+
+for dir in "$@"; do
+    if [ ! -d "$dir" ]; then
+	echo "$0: $dir not a directory"
+	exit 1;
+    fi
+done
+
+subdirs="{00"
+for i in {1..255}; do
+    n=$(printf "%02x" $i);
+    subdirs="$subdirs,$n";
+done
+subdirs="$subdirs}"
+
+mkdir -v "$dir/.glusterfs";
+
+for dir in $@; do
+    for i in {0..255}; do
+	n=$(printf "%02x" $i);
+	mkdir -v "$dir/.glusterfs/$n"
+	eval "mkdir -v $dir/.glusterfs/$n/$subdirs"
+    done
+done
\ No newline at end of file
diff --git a/perf-framework/motd b/perf-framework/motd
new file mode 100644
index 0000000..dd934b4
--- /dev/null
+++ b/perf-framework/motd
@@ -0,0 +1,25 @@
+/*
+ * System being used by Rahul for performance runs.
+ * It is part of the setup described below.
+ * Please contact tcp@gluster.com before running anything on this system.
+ *
+ * DO NOT DISTURB THE SETUP SINCE IT MIGHT AFFECT THE PERFORMANCE RUNS.
+ *
+ * /etc/motd last updated:
+ * Mon Jul  4 04:04:29 PDT 2011
+ */
+
++-------------+      +-------------+      +-------------+      +-------------+
+| 10.1.10.30  |<---->| 10.1.10.31  |<---->| 10.1.10.35  |<---->| 10.1.10.36  |
+| (client-10) |      | (client-11) |      | (client-15) |      | (client-16) |
+| (Brick-1)   |      | (Brick-2)   |      | (Brick-3)   |      | (Brick-4)   |
++------+------+      +------+------+      +------+------+      +------+------+
+       |                    |                    |                    |
+       +--------------------+---------+----------+--------------------+
+                                      |
+                                      v
+                               +--------------+
+                               |  10.1.10.29  |
+                               |  (client-9)  |
+                               |  (client)    |
+                               +--------------+
diff --git a/perf-framework/newtest.sh b/perf-framework/newtest.sh
new file mode 100755
index 0000000..789c784
--- /dev/null
+++ b/perf-framework/newtest.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+   mkdir -p $PG/emptyfiles;
+   for i in $(seq 1 $smallfilecount); do
+       : > $PG/emptyfiles/file.$i;
+   done
+}
+
+
+function emptyfiles_delete()
+{
+    rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+    mkdir -p $PG/emptydirs;
+
+    eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+    for top in $(seq 1 $emptytops); do
+        eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+    done
+}
+
+
+function emptydirs_delete()
+{
+    rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+    mkdir -p $PG/smallfiles;
+    for i in $(seq 1 $smallfilecount); do
+	echo -n $smallblob >$PG/smallfiles/file.$i
+    done
+}
+
+
+function smallfiles_rewrite()
+{
+    smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+    for i in $(seq 1 $smallfilecount); do
+	cat $PG/smallfiles/file.$i > /dev/null
+    done
+}
+
+
+function smallfiles_reread()
+{
+    smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+    rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+    mkdir -p $PG/largefile;
+    dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+    largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+    dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+     largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+    rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+    local subpath;
+    local depth;
+
+    subpath="$1";
+    depth="$2";
+
+    if [ $depth -eq 0 ]; then
+	for i in $(seq 1 $leafcount); do
+	    : > $subpath/file.$i;
+	done
+	return
+    fi
+
+    depth=$(($depth - 1));
+
+    eval "echo  $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+    for i in $(seq 1 $crawlwidth); do
+	crawl_create_recurse "$subpath/dir.$i" $depth;
+    done
+}
+
+
+function directory_crawl_create ()
+{
+    crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+    ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+    directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+    rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+    chmod -R 777 "$PG/crawl";
+    chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+    #run emptyfiles_create;
+    #run emptyfiles_delete;
+
+
+    # run emptydirs_create;
+    # run emptydirs_delete;
+
+
+    #run smallfiles_create;
+    #run smallfiles_rewrite;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run smallfiles_read;
+    #run smallfiles_reread;
+    #run smallfiles_delete;
+
+
+    #run largefile_create;
+    #run largefile_rewrite;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run largefile_read;
+    #run largefile_reread;
+    #run largefile_delete;
+
+
+    run directory_crawl_create;
+    echo 3 > /proc/sys/vm/drop_caches;
+    sleep 10;
+    run directory_crawl;
+    run directory_recrawl;
+    run metadata_modify;
+    run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+    rm -rvf $PG;
+    mkdir -p $PG;
+}
+
+
+function params_init()
+{
+    emptytops=10;
+    emptydirs=10000;
+
+    smallfilecount=100000;
+    smallblock=4096;
+
+    largeblock=64K;
+    largecount=16K;
+
+    crawlwidth=10;
+    crawldepth=3;
+    leafcount=100;
+
+    smallblob=;
+    for i in $(seq 1 $smallblock); do
+	smallblob=a$smallblob
+    done
+}
+
+
+function _init()
+{
+    params_init;
+
+    TSTDOUT=251
+    TSTDERR=252
+    LOGFD=253
+    LOGFILE=/tmp/perf$$
+
+    eval "exec $TSTDOUT>&1"
+    eval "exec $TSTDERR>&2"
+    eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+    MOUNT=;
+
+    if [ "x$1" == "x" ] ; then
+        echo "Usage: $0 /gluster/mount"
+        exit 1
+    fi
+
+    MOUNT="$1";
+    PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+    "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+    set -o pipefail;
+    (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+    local t;
+    local rest;
+
+    t=$1;
+    shift;
+    rest="$@";
+
+    echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+    local t;
+
+    echo -n "running $@ ... "
+    t=$(measure "$@");
+
+    if [ $? -eq 0 ]; then
+        echo "done ($t secs)";
+        log "$t" "$@";
+    else
+        echo "FAILED!!!"
+    fi
+}
+
+
+function verify_mount()
+{
+    if [ ! -d "$MOUNT" ] ; then
+        echo "Can't access '$MOUNT'"
+        exit 1
+    fi
+}
+
+
+function show_report()
+{
+    (echo "Testname Time"; cat $LOGFILE) | column -t;
+    rm -f $LOGFILE;
+}
+
+
+function main()
+{
+    parse_cmdline "$@";
+
+    verify_mount;
+
+    cleanup_playground;
+
+    run_tests;
+
+    show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/nfs-reread.sh b/perf-framework/nfs-reread.sh
new file mode 100755
index 0000000..1e2f51b
--- /dev/null
+++ b/perf-framework/nfs-reread.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+   mkdir -p $PG/emptyfiles;
+   for i in $(seq 1 $smallfilecount); do
+       : > $PG/emptyfiles/file.$i;
+   done
+}
+
+
+function emptyfiles_delete()
+{
+    rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+    mkdir -p $PG/emptydirs;
+
+    eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+    for top in $(seq 1 $emptytops); do
+        eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+    done
+}
+
+
+function emptydirs_delete()
+{
+    rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+    mkdir -p $PG/smallfiles;
+    for i in $(seq 1 $smallfilecount); do
+	echo -n $smallblob >$PG/smallfiles/file.$i
+    done
+}
+
+
+function smallfiles_rewrite()
+{
+    smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+    for i in $(seq 1 $smallfilecount); do
+	cat $PG/smallfiles/file.$i > /dev/null
+    done
+}
+
+
+function smallfiles_reread()
+{
+    smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+    rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+    mkdir -p $PG/largefile;
+    dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+    largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+    dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+     largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+    rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+    local subpath;
+    local depth;
+
+    subpath="$1";
+    depth="$2";
+
+    if [ $depth -eq 0 ]; then
+	for i in $(seq 1 $leafcount); do
+	    : > $subpath/file.$i;
+	done
+	return
+    fi
+
+    depth=$(($depth - 1));
+
+    eval "echo  $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+    for i in $(seq 1 $crawlwidth); do
+	crawl_create_recurse "$subpath/dir.$i" $depth;
+    done
+}
+
+
+function directory_crawl_create ()
+{
+    crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+    ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+    directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+    rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+    chmod -R 777 "$PG/crawl";
+    chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+    #run emptyfiles_create;
+    #run emptyfiles_delete;
+
+
+    # run emptydirs_create;
+    # run emptydirs_delete;
+
+
+    run smallfiles_create;
+    run smallfiles_rewrite;
+    echo 3 > /proc/sys/vm/drop_caches;
+    sleep 10;
+    run smallfiles_read;
+    run smallfiles_reread;
+    run smallfiles_delete;
+
+
+    #run largefile_create;
+    #run largefile_rewrite;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run largefile_read;
+    #run largefile_reread;
+    #run largefile_delete;
+
+
+    #run directory_crawl_create;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run directory_crawl;
+    #run directory_recrawl;
+    #run metadata_modify;
+    #run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+    rm -rvf $PG;
+    mkdir -p $PG;
+}
+
+
+function params_init()
+{
+    emptytops=10;
+    emptydirs=10000;
+
+    smallfilecount=100000;
+    smallblock=4096;
+
+    largeblock=64K;
+    largecount=16K;
+
+    crawlwidth=10;
+    crawldepth=3;
+    leafcount=100;
+
+    smallblob=;
+    for i in $(seq 1 $smallblock); do
+	smallblob=a$smallblob
+    done
+}
+
+
+function _init()
+{
+    params_init;
+
+    TSTDOUT=251
+    TSTDERR=252
+    LOGFD=253
+    LOGFILE=/tmp/perf$$
+
+    eval "exec $TSTDOUT>&1"
+    eval "exec $TSTDERR>&2"
+    eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+    MOUNT=;
+
+    if [ "x$1" == "x" ] ; then
+        echo "Usage: $0 /gluster/mount"
+        exit 1
+    fi
+
+    MOUNT="$1";
+    PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+    "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+    set -o pipefail;
+    (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+    local t;
+    local rest;
+
+    t=$1;
+    shift;
+    rest="$@";
+
+    echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+    local t;
+
+    echo -n "running $@ ... "
+    t=$(measure "$@");
+
+    if [ $? -eq 0 ]; then
+        echo "done ($t secs)";
+        log "$t" "$@";
+    else
+        echo "FAILED!!!"
+    fi
+}
+
+
+function verify_mount()
+{
+    if [ ! -d "$MOUNT" ] ; then
+        echo "Can't access '$MOUNT'"
+        exit 1
+    fi
+}
+
+
+function show_report()
+{
+    (echo "Testname Time"; cat $LOGFILE) | column -t;
+    rm -f $LOGFILE;
+}
+
+
+function main()
+{
+    parse_cmdline "$@";
+
+    verify_mount;
+
+    cleanup_playground;
+
+    run_tests;
+
+    show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/nfs_delete.sh b/perf-framework/nfs_delete.sh
new file mode 100755
index 0000000..a6cb5bc
--- /dev/null
+++ b/perf-framework/nfs_delete.sh
@@ -0,0 +1,331 @@
+#!/bin/bash
+
+
+function emptyfiles_create()
+{
+   mkdir -p $PG/emptyfiles;
+   for i in $(seq 1 $smallfilecount); do
+       : > $PG/emptyfiles/file.$i;
+   done
+}
+
+
+function emptyfiles_delete()
+{
+    rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+    mkdir -p $PG/emptydirs;
+
+    eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+    for top in $(seq 1 $emptytops); do
+        eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+    done
+}
+
+
+function emptydirs_delete()
+{
+    rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+    mkdir -p $PG/smallfiles;
+    for i in $(seq 1 $smallfilecount); do
+	echo -n $smallblob >$PG/smallfiles/file.$i
+    done
+}
+
+
+function smallfiles_rewrite()
+{
+    smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+    for i in $(seq 1 $smallfilecount); do
+	cat $PG/smallfiles/file.$i > /dev/null
+    done
+}
+
+
+function smallfiles_reread()
+{
+    smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+    rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+    mkdir -p $PG/largefile;
+    dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+    largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+    dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+     largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+    rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+    local subpath;
+    local depth;
+
+    subpath="$1";
+    depth="$2";
+
+    if [ $depth -eq 0 ]; then
+	for i in $(seq 1 $leafcount); do
+	    : > $subpath/file.$i;
+	done
+	return
+    fi
+
+    depth=$(($depth - 1));
+
+    eval "echo  $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+    for i in $(seq 1 $crawlwidth); do
+	crawl_create_recurse "$subpath/dir.$i" $depth;
+    done
+}
+
+
+function directory_crawl_create ()
+{
+    crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+    ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+    directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+    rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+    chmod -R 777 "$PG/crawl";
+    chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+    run emptyfiles_create;
+    #run emptyfiles_delete;
+
+
+    # run emptydirs_create;
+    # run emptydirs_delete;
+
+
+    #run smallfiles_create;
+    #run smallfiles_rewrite;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run smallfiles_read;
+    #run smallfiles_reread;
+    #run smallfiles_delete;
+
+
+    #run largefile_create;
+    #run largefile_rewrite;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run largefile_read;
+    #run largefile_reread;
+    #run largefile_delete;
+
+
+    #run directory_crawl_create;
+    #echo 3 > /proc/sys/vm/drop_caches;
+    #sleep 10;
+    #run directory_crawl;
+    #run directory_recrawl;
+    #run metadata_modify;
+    #run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+    rm -rvf $PG;
+    mkdir -p $PG;
+}
+
+
+function params_init()
+{
+    emptytops=10;
+    emptydirs=10000;
+
+    smallfilecount=80000;
+    smallblock=4096;
+
+    largeblock=64K;
+    largecount=16K;
+
+    crawlwidth=10;
+    crawldepth=3;
+    leafcount=100;
+
+    smallblob=;
+    for i in $(seq 1 $smallblock); do
+	smallblob=a$smallblob
+    done
+}
+
+
+function _init()
+{
+    params_init;
+
+    TSTDOUT=251
+    TSTDERR=252
+    LOGFD=253
+    LOGFILE=/tmp/perf$$
+
+    eval "exec $TSTDOUT>&1"
+    eval "exec $TSTDERR>&2"
+    eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+    MOUNT=;
+
+    if [ "x$1" == "x" ] ; then
+        echo "Usage: $0 /gluster/mount"
+        exit 1
+    fi
+
+    MOUNT="$1";
+    PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+    "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+    set -o pipefail;
+    (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+    local t;
+    local rest;
+
+    t=$1;
+    shift;
+    rest="$@";
+
+    echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+    local t;
+
+    echo -n "running $@ ... "
+    t=$(measure "$@");
+
+    if [ $? -eq 0 ]; then
+        echo "done ($t secs)";
+        log "$t" "$@";
+    else
+        echo "FAILED!!!"
+    fi
+}
+
+
+function verify_mount()
+{
+    if [ ! -d "$MOUNT" ] ; then
+        echo "Can't access '$MOUNT'"
+        exit 1
+    fi
+}
+
+
+function show_report()
+{
+    (echo "Testname Time"; cat $LOGFILE) | column -t;
+    rm -f $LOGFILE;
+}
+
+
+function main()
+{
+    parse_cmdline "$@";
+
+    verify_mount;
+
+    cleanup_playground;
+
+    run_tests;
+
+    show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/ops b/perf-framework/ops
new file mode 100644
index 0000000..74e1939
--- /dev/null
+++ b/perf-framework/ops
@@ -0,0 +1,17 @@
+emptyfiles_create
+emptyfiles_delete
+smallfiles_create
+smallfiles_rewrite
+smallfiles_read
+smallfiles_reread
+smallfiles_delete
+largefile_create
+largefile_rewrite
+largefile_read
+largefile_reread
+largefile_delete
+directory_crawl_create
+directory_crawl
+directory_recrawl
+metadata_modify
+directory_crawl_delete
diff --git a/perf-framework/parallel_create b/perf-framework/parallel_create
new file mode 100755
index 0000000..0bd9406
--- /dev/null
+++ b/perf-framework/parallel_create
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+FILE_REPO="/mnt/perfmount/file_repo"
+TEST=0
+DD_BS=512
+RAND_SCALE=20
+
+function usage()
+{
+        echo "usage: $1 <thread_count> <file_count>"
+        exit 1
+}
+
+function create_files()
+{
+        tid=$1
+        count=$2
+        start=$((tid*count))
+
+        if [ $TEST -eq "1" ]; then
+                for ((i=0; i<${count}; i++)); do
+                        outfile=$FILE_REPO/tid${tid}_file$((start+i))
+                        echo "dd if=/dev/urandom of=$outfile bs=1b count=$((RANDOM*20))"
+                done
+        else
+                for ((i=0; i<${count}; i++)); do
+                        filesize=$((RANDOM*RAND_SCALE))
+                        dd_count=$((filesize/DD_BS))
+                        outfile=$FILE_REPO/tid${tid}_file$((start+i))
+                        dd if=/dev/zero of=$outfile bs=1b count=$dd_count > /dev/null 2>&1
+                        echo "$tid $i"
+                done
+        fi
+}
+
+function Not_A_Number()
+{
+        echo $1$2 | grep ^[0-9]*$ > /dev/null
+        return $?
+}
+
+function main()
+{
+        if [ $# -ne 2 ]; then
+                usage $0
+        fi
+
+        if ! Not_A_Number $1 $2; then
+                echo "Invalid input. Enter only numbers."
+                usage
+        fi
+
+        if [ $2 -lt $1 ]; then
+                echo "Number of files cannot be less than number of threads"
+                usage $0
+        fi
+
+        mkdir -p $FILE_REPO
+
+        nthr=$1
+        fc=$2
+        files_per_thr=$((fc/nthr))
+        
+        for i in `seq 0 $((nthr-1))`; do
+                create_files $i $files_per_thr &
+        done    
+
+        wait
+}
+
+main $@
diff --git a/perf-framework/perf-test.sh b/perf-framework/perf-test.sh
new file mode 100755
index 0000000..6ce5b55
--- /dev/null
+++ b/perf-framework/perf-test.sh
@@ -0,0 +1,337 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+function emptyfiles_create()
+{
+   mkdir -p $PG/emptyfiles;
+   for i in $(seq 1 $smallfilecount); do
+       : > $PG/emptyfiles/file.$i;
+   done
+}
+
+
+function emptyfiles_delete()
+{
+    rm -rf $PG/emptyfiles;
+}
+
+
+function emptydirs_create()
+{
+    mkdir -p $PG/emptydirs;
+
+    eval "echo $PG/emptydirs/top.{1..$emptytops}" | xargs mkdir -p;
+    for top in $(seq 1 $emptytops); do
+        eval "echo $PG/emptydirs/top.$top/dir.{1..$emptydirs}" | xargs mkdir -p;
+    done
+}
+
+
+function emptydirs_delete()
+{
+    rm -rf $PG/emptydirs;
+}
+
+
+function smallfiles_create()
+{
+    mkdir -p $PG/smallfiles;
+    for i in $(seq 1 $smallfilecount); do
+	echo -n $smallblob >$PG/smallfiles/file.$i
+    done
+}
+
+
+function smallfiles_rewrite()
+{
+    smallfiles_create "$@";
+}
+
+
+function smallfiles_read()
+{
+    for i in $(seq 1 $smallfilecount); do
+	cat $PG/smallfiles/file.$i > /dev/null
+    done
+}
+
+
+function smallfiles_reread()
+{
+    smallfiles_read "$@";
+}
+
+
+function smallfiles_delete()
+{
+    rm -rf $PG/smallfiles;
+}
+
+
+function largefile_create()
+{
+    mkdir -p $PG/largefile;
+    dd if=/dev/zero of=$PG/largefile/large_file bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_rewrite()
+{
+    largefile_create "$@";
+}
+
+
+function largefile_read()
+{
+    dd if=$PG/largefile/large_file of=/dev/null bs=$largeblock count=$largecount 2>/dev/null;
+}
+
+
+function largefile_reread()
+{
+     largefile_read "$@";
+}
+
+
+function largefile_delete()
+{
+    rm -rf $PG/largefile
+}
+
+
+function crawl_create_recurse()
+{
+    local subpath;
+    local depth;
+
+    subpath="$1";
+    depth="$2";
+
+    if [ $depth -eq 0 ]; then
+	for i in $(seq 1 $leafcount); do
+	    : > $subpath/file.$i;
+	done
+	return
+    fi
+
+    depth=$(($depth - 1));
+
+    eval "echo  $subpath/dir.{1..$crawlwidth}" | xargs mkdir -p;
+
+    for i in $(seq 1 $crawlwidth); do
+	crawl_create_recurse "$subpath/dir.$i" $depth;
+    done
+}
+
+
+function directory_crawl_create ()
+{
+    crawl_create_recurse "$PG/crawl" $crawldepth;
+}
+
+
+function directory_crawl()
+{
+    ls -lR "$PG/crawl" >/dev/null
+}
+
+
+function directory_recrawl()
+{
+    directory_crawl "$@";
+}
+
+
+function directory_crawl_delete()
+{
+    rm -rf "$PG/crawl";
+}
+
+
+function metadata_modify ()
+{
+    chmod -R 777 "$PG/crawl";
+    chown -R 1234 "$PG/crawl";
+}
+
+
+function run_tests()
+{
+    run emptyfiles_create;
+    run emptyfiles_delete;
+
+
+    # run emptydirs_create;
+    # run emptydirs_delete;
+
+
+    run smallfiles_create;
+    run smallfiles_rewrite;
+    echo 3 > /proc/sys/vm/drop_caches;
+    sleep 10;
+    run smallfiles_read;
+    run smallfiles_reread;
+    run smallfiles_delete;
+
+
+    run largefile_create;
+    run largefile_rewrite;
+    echo 3 > /proc/sys/vm/drop_caches;
+    sleep 10;
+    run largefile_read;
+    run largefile_reread;
+    run largefile_delete;
+
+
+    run directory_crawl_create;
+    echo 3 > /proc/sys/vm/drop_caches;
+    sleep 10;
+    run directory_crawl;
+    run directory_recrawl;
+    run metadata_modify;
+    run directory_crawl_delete;
+}
+
+
+#####################################################
+############ Framework code #########################
+#####################################################
+
+
+function cleanup_playground()
+{
+    rm -rvf $PG;
+    mkdir -p $PG;
+}
+
+
+function params_init()
+{
+    emptytops=10;
+    emptydirs=10000;
+
+    if [ $MOUNT_TYPE == "nfs" ]; then
+	smallfilecount=85000;
+    else
+	smallfilecount=100000;
+    fi
+    smallblock=4096;
+
+    largeblock=64K;
+    largecount=16K;
+
+    crawlwidth=10;
+    crawldepth=3;
+    leafcount=100;
+
+    smallblob=;
+    for i in $(seq 1 $smallblock); do
+	smallblob=a$smallblob
+    done
+}
+
+
+function _init()
+{
+    params_init;
+
+    TSTDOUT=251
+    TSTDERR=252
+    LOGFD=253
+    LOGFILE=/tmp/perf$$
+
+    eval "exec $TSTDOUT>&1"
+    eval "exec $TSTDERR>&2"
+    eval "exec $LOGFD<>$LOGFILE";
+}
+
+
+function parse_cmdline()
+{
+    MOUNT=;
+
+    if [ "x$1" == "x" ] ; then
+        echo "Usage: $0 /gluster/mount"
+        exit 1
+    fi
+
+    MOUNT="$1";
+    PG=$MOUNT/playground;
+}
+
+
+function wrap()
+{
+    "$@" 1>&$TSTDOUT 2>&$TSTDERR;
+}
+
+
+function measure()
+{
+    set -o pipefail;
+    (time -p wrap "$@") 2>&1 >/dev/null | tail -n 3 | head -n 1 | cut -f2 -d' '
+}
+
+
+function log()
+{
+    local t;
+    local rest;
+
+    t=$1;
+    shift;
+    rest="$@";
+
+    echo "$rest $t" >&$LOGFD;
+}
+
+
+function run()
+{
+    local t;
+
+    echo -n "running $@ ... "
+    t=$(measure "$@");
+
+    if [ $? -eq 0 ]; then
+        echo "done ($t secs)";
+        log "$t" "$@";
+    else
+        echo "FAILED!!!"
+    fi
+}
+
+
+function verify_mount()
+{
+    if [ ! -d "$MOUNT" ] ; then
+        echo "Can't access '$MOUNT'"
+        exit 1
+    fi
+}
+
+
+function show_report()
+{
+    (echo "Testname Time"; cat $LOGFILE) | column -t;
+    rm -f $LOGFILE;
+}
+
+
+function main()
+{
+    parse_cmdline "$@";
+
+    verify_mount;
+
+    cleanup_playground;
+
+    run_tests;
+
+    show_report;
+}
+
+
+_init && main "$@"
diff --git a/perf-framework/perf.sh b/perf-framework/perf.sh
new file mode 100755
index 0000000..1a4bdc6
--- /dev/null
+++ b/perf-framework/perf.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+perf_test ()
+{
+    ./perf-test.sh $mount_point >> $logfile
+}
+
+_init ()
+{
+    mount_point=$1;
+    logfile=$2
+}
+
+main ()
+{
+    echo "" > $logfile
+    perf_test;
+    perf_test;
+    perf_test;
+}
+
+
+_init "$@" && main "$@";
diff --git a/perf-framework/pwl_ssh b/perf-framework/pwl_ssh
new file mode 100755
index 0000000..fc837f1
--- /dev/null
+++ b/perf-framework/pwl_ssh
@@ -0,0 +1,50 @@
+#!/usr/bin/expect -f
+
+proc setup_pwl_ssh { host pw1 pw2 } {
+	spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$host
+	set timeout 50
+	expect {
+	"yes/no" {
+		send "yes\r" 
+		exp_continue
+	} "password: " {
+	send "$pw1\r"
+	expect {
+		"#" {
+			# Success. Nothing to do
+		} "password: " {
+			send "$pw2\r";
+			expect {
+			"#" {
+				# Success. Nothing to do
+			} (.*) {
+				catch {close -i $spawn_id}
+				wait -nowait -i $spawn_id
+			}
+			}
+		}
+		}
+	} timeout {
+		catch {close -i $spawn_id}
+		wait -nowait -i $spawn_id
+	} eof {
+	}
+	}
+}
+
+if { $argc < 2 } {
+	puts "usage $argv0 <logfile> <list-of-hosts>"
+	exit
+}
+
+log_user 0
+set logfile [ lindex $argv 0 ]
+exp_log_file -a -noappend $logfile
+
+set pw1 $env(ROOT_PW1)
+set pw2 $env(ROOT_PW2)
+
+for {set i 1} {$i<[llength $argv]} {incr i} {
+	set host [lindex $argv $i]
+	setup_pwl_ssh $host $pw1 $pw2
+}
diff --git a/perf-framework/quota_gsync_run b/perf-framework/quota_gsync_run
new file mode 100755
index 0000000..a090b66
--- /dev/null
+++ b/perf-framework/quota_gsync_run
@@ -0,0 +1,42 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+run=$1
+runlabel=`echo $run|sed -e 's/^glusterfs-//' -e 's/\.tar\.gz//'`
+
+a=`cat .runfile`
+echo "run$a - Quota off; gsync on..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume set $VOLNAME geo-replication.indexing on"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota off, gsync on)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+echo "run$((a+1)) - Quota on; gsync on..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume quota $VOLNAME enable"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume set $VOLNAME geo-replication.indexing on"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota on, gsync on)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+echo "run$((a+2)) - Quota on; gsync off..."
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+ssh -l root $MGMT_NODE "gluster volume quota $VOLNAME enable"
+sleep 5
+echo "run`cat .runfile` - $MOUNT_TYPE - $runlabel - $GF_CONFIG - (quota on, gsync off)" >> $RUNLOG
+./start_perf_measure
+sleep 10
+
+# Restore
+ssh -l root $MGMT_NODE "gluster volume reset $VOLNAME force"
+sleep 5
+
diff --git a/perf-framework/remove_export_dirs b/perf-framework/remove_export_dirs
new file mode 100755
index 0000000..c84d881
--- /dev/null
+++ b/perf-framework/remove_export_dirs
@@ -0,0 +1,10 @@
+#!/bin/bash -ue
+
+SERVER_EXPORT_DIR=$1
+
+if [ "$SERVER_EXPORT_DIR" != "" -a "$SERVER_EXPORT_DIR" != "/" ]
+then
+	echo "$SERVER_EXPORT_DIR"
+else
+	echo "Empty export dir"
+fi
diff --git a/perf-framework/run b/perf-framework/run
new file mode 100755
index 0000000..965d7bb
--- /dev/null
+++ b/perf-framework/run
@@ -0,0 +1,9 @@
+#!/bin/bash -u
+
+source gf_perf_config
+
+for brick in $BRICK_IP_ADDRS
+do
+	echo $brick
+	ssh -l root $brick "$@"
+done
diff --git a/perf-framework/send_mail b/perf-framework/send_mail
new file mode 100755
index 0000000..e44f29b
--- /dev/null
+++ b/perf-framework/send_mail
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+./calc_avg "$@" > /tmp/run_all
+
+scp /tmp/run_all $EMAIL/run_all;
diff --git a/perf-framework/setbatch b/perf-framework/setbatch
new file mode 100755
index 0000000..fb18585
--- /dev/null
+++ b/perf-framework/setbatch
@@ -0,0 +1,25 @@
+#!/bin/bash -u
+
+TARBALL_DIR=./tarballs
+function usage()
+{
+	echo "Usage : $0 <gluster_release_tarball_list>"
+	exit 1
+}
+
+if [ $# -lt 1 ]
+then
+	usage
+fi
+
+list=`echo $@`
+for i in $list
+do
+	if [ ! -f $TARBALL_DIR/$i ]
+	then
+		echo "$i not found in directory $TARBALL_DIR. Provide correct list."
+		exit
+	fi
+done
+
+sed -i "s/^RUNLIST=.*$/RUNLIST=\"$list\"/" batchrun
diff --git a/perf-framework/setrun b/perf-framework/setrun
new file mode 100755
index 0000000..7992051
--- /dev/null
+++ b/perf-framework/setrun
@@ -0,0 +1,14 @@
+#!/bin/bash -u
+
+function usage()
+{
+	echo "Usage : $0 <gluster_release_tarball>"
+	exit 1
+}
+
+if [ $# -ne 1 ]
+then
+	usage
+fi
+
+sed -i "s/^RELEASE_TARBALL=.*$/RELEASE_TARBALL=$1/" gf_perf_config
diff --git a/perf-framework/start_perf_measure b/perf-framework/start_perf_measure
new file mode 100755
index 0000000..2e06db5
--- /dev/null
+++ b/perf-framework/start_perf_measure
@@ -0,0 +1,166 @@
+#!/bin/bash -u
+
+CONFIG_FILE=gf_perf_config
+source $CONFIG_FILE
+
+SETTLE_TIME=10
+RUNFILE=.runfile
+STAT_COLLECTOR=stat_collect
+
+# Generate current run, update runfile
+if [ ! -f $RUNFILE ]
+then
+        run=1
+else
+        run=`cat $RUNFILE`
+fi
+echo $((run+1)) > $RUNFILE
+
+# Drop vm caches on all the bricks before starting the runs
+for brick in $BRICK_IP_ADDRS
+do
+        ssh -l root $brick "echo 3 > /proc/sys/vm/drop_caches"
+done
+
+# Create the gluster volume
+./create_gluster_vol
+
+mount | grep $MOUNT_POINT > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+        umount $MOUNT_POINT
+fi
+
+if [ ! -d $MOUNT_POINT ]
+then
+	mkdir -p $MOUNT_POINT
+fi
+
+# Make sure that the fuse kernel module is loaded
+/sbin/lsmod | grep -w fuse > /dev/null 2>&1
+if [ $? -ne 0 ]
+then
+	/sbin/modprobe fuse > /dev/null 2>&1
+fi
+
+ps -eaf | egrep -w 'glusterfs|glusterfsd|glusterd' > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+	killall glusterfsd glusterd glusterfs  > /dev/null 2>&1
+fi
+
+# Mount the client
+# Sleep for a while. Sometimes, NFS mounts fail if attempted soon after creating the volume
+sleep $SETTLE_TIME
+
+if [ $ENABLE_ACL == "yes" ]
+then
+        acl_opts="-o acl"
+else
+        acl_opts=""
+fi
+
+if [ $MOUNT_TYPE == "nfs" ]
+then
+        nfs_opts="-o vers=3"
+else
+        nfs_opts=""
+fi
+
+echo "Mounting volume..."
+if [ $ENABLE_MEM_ACCT == "yes" ]
+then
+	echo "Memory accounting status on client -"
+	echo "x/x &gf_mem_acct_enable" > commands.$$
+	echo "quit" >> commands.$$
+	GLUSTERFS_DISABLE_MEM_ACCT=0 mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+	mount_status=$?
+	gdb -q --command=commands.$$ -p `pidof glusterfs` | grep gf_mem_acct_enable | awk '{print $(NF-1) $NF}'
+	rm commands.$$ > /dev/null 2>&1
+else
+        mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT
+        mount_status=$?
+fi
+
+if [ $mount_status -ne 0 ]
+then
+        echo "mount -t $MOUNT_TYPE $acl_opts $nfs_opts $MGMT_NODE:$VOLNAME $MOUNT_POINT failed..."
+        echo "Exiting..."
+        exit 1
+fi
+
+# Copy statistics collection scripts to the server
+
+echo ""
+echo "Copying stat collection script to bricks..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "mkdir -p $SERVER_SCRIPTS_DIR"
+	scp -p $STAT_COLLECTOR root@$brick:$SERVER_SCRIPTS_DIR > /dev/null 2>&1
+done
+
+# Run statistics collection scripts on the server
+
+echo ""
+echo "Starting server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "mkdir -p $SERVER_LOG_DIR"
+	ssh -l root $brick "$SERVER_SCRIPTS_DIR/$STAT_COLLECTOR $SERVER_LOG_DIR" &
+done
+
+# Run statistics collection on client
+
+mkdir -p $LOCAL_LOG_REPO
+./$STAT_COLLECTOR $LOCAL_LOG_REPO/run$run/client &
+
+# Start perf test
+
+echo ""
+echo "Starting run $run..."
+
+sleep $SETTLE_TIME
+./perf.sh $MOUNT_POINT $LOCAL_LOG_REPO/run$run/client/perf-test.log
+sleep $SETTLE_TIME
+
+# Stop statistics collection scripts on the client
+
+killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+
+# Stop statistics collection scripts on the server
+
+echo ""
+echo "Stopping server stat collection..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick killall mpstat vmstat iostat $STAT_COLLECTOR sar > /dev/null 2>&1
+done
+
+# Since the ssh processes were backgrounded, they will be hanging around.
+# Kill them
+kill `jobs -l | awk '{print $2}'` > /dev/null 2>&1
+
+# Copy statistics from the server
+
+echo ""
+echo "Copying server logfiles for run $run..."
+cur_log_dump_dir=$LOCAL_LOG_REPO/run$run
+mkdir -p $cur_log_dump_dir
+count=1
+for brick in $BRICK_IP_ADDRS
+do
+        for statf in mpstat vmstat iostat sysinfo sar_netstat
+        do
+                scp root@$brick:$SERVER_LOG_DIR/*$statf* $cur_log_dump_dir/brick$count-$brick-$statf-log > /dev/null 2>&1
+        done
+        count=$((count + 1))
+done
+
+# Cleanup statistics collected for this run on the server
+
+echo ""
+echo "Cleaning server logfiles..."
+for brick in $BRICK_IP_ADDRS
+do 
+	ssh -l root $brick "cd $SERVER_LOG_DIR; rm mpstat_log vmstat_log iostat_log sysinfo sar_netstat_log"
+done
diff --git a/perf-framework/stat_collect b/perf-framework/stat_collect
new file mode 100755
index 0000000..8b63203
--- /dev/null
+++ b/perf-framework/stat_collect
@@ -0,0 +1,36 @@
+#!/bin/bash -ue
+
+function start_stat_collect()
+{
+        echo "Interrupts details:" >> $SYSINFO
+        cat /proc/interrupts >> $SYSINFO
+        echo "CPU details:" >> $SYSINFO
+        cat /proc/cpuinfo >> $SYSINFO
+        echo "Memory details:" >> $SYSINFO
+        cat /proc/meminfo >> $SYSINFO
+        
+        iostat -xcdh 5 >> $IOSTAT_LOG &
+        mpstat -P ALL 5 >> $MPSTAT_LOG &
+        vmstat 5 >> $VMSTAT_LOG &
+	LC_TIME="POSIX" sar -n DEV 5 0 >> $SAR_NETSTAT_LOG &
+}
+
+function init()
+{
+        mkdir -p $LOGDIR
+        IOSTAT_LOG=$LOGDIR/iostat_log
+        VMSTAT_LOG=$LOGDIR/vmstat_log
+        MPSTAT_LOG=$LOGDIR/mpstat_log
+	SAR_NETSTAT_LOG=$LOGDIR/sar_netstat_log
+        SYSINFO=$LOGDIR/sysinfo
+        echo "" > $IOSTAT_LOG
+        echo "" > $VMSTAT_LOG
+        echo "" > $MPSTAT_LOG
+	echo "" > $SAR_NETSTAT_LOG
+        echo "" > $SYSINFO
+}
+
+LOGDIR=$1
+init $LOGDIR;
+start_stat_collect;
+wait
-- 
cgit