glusterfs-profiler: Add text mode support.

glusterfs-profiler works in text mode by default now. This allows it to run on systems which don't have matplotlib installed.

Modes can be selected using:

  -m graph        - Graphical mode
  -m text         - Text mode (default)

Signed-off-by: Vikas Gorur <vikas@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 268 (Add timing instrumentation code)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=268

1 files changed, 599 insertions, 42 deletions

diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler
index f843ae69a3d..7a5ca7405f1 100755
--- a/extras/profiler/glusterfs-profiler
+++ b/extras/profiler/glusterfs-profiler
@@ -1,5 +1,425 @@
 #!/usr/bin/env python
 
+# texttable - module for creating simple ASCII tables
+# Copyright (C) 2003-2009 Gerome Fournier <jefke(at)free.fr>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+# Incorporated from texttable.py downloaded from
+# http://jefke.free.fr/stuff/python/texttable/texttable-0.7.0.tar.gz
+
+import sys
+import string
+
+try:
+    if sys.version >= '2.3':
+        import textwrap
+    elif sys.version >= '2.2':
+        from optparse import textwrap
+    else:
+        from optik import textwrap
+except ImportError:
+    sys.stderr.write("Can't import textwrap module!\n")
+    raise
+
+try:
+    True, False
+except NameError:
+    (True, False) = (1, 0)
+
+def len(iterable):
+    """Redefining len here so it will be able to work with non-ASCII characters
+    """
+    if not isinstance(iterable, str):
+        return iterable.__len__()
+    
+    try:
+        return len(unicode(iterable, 'utf'))
+    except:
+        return iterable.__len__()
+
+class ArraySizeError(Exception):
+    """Exception raised when specified rows don't fit the required size
+    """
+
+    def __init__(self, msg):
+        self.msg = msg
+        Exception.__init__(self, msg, '')
+
+    def __str__(self):
+        return self.msg
+
+class Texttable:
+
+    BORDER = 1
+    HEADER = 1 << 1
+    HLINES = 1 << 2
+    VLINES = 1 << 3
+
+    def __init__(self, max_width=80):
+        """Constructor
+
+        - max_width is an integer, specifying the maximum width of the table
+        - if set to 0, size is unlimited, therefore cells won't be wrapped
+        """
+
+        if max_width <= 0:
+            max_width = False
+        self._max_width = max_width
+        self._deco = Texttable.VLINES | Texttable.HLINES | Texttable.BORDER | \
+            Texttable.HEADER
+        self.set_chars(['-', '|', '+', '='])
+        self.reset()
+
+    def reset(self):
+        """Reset the instance
+
+        - reset rows and header
+        """
+
+        self._hline_string = None
+        self._row_size = None
+        self._header = []
+        self._rows = []
+
+    def header(self, array):
+        """Specify the header of the table
+        """
+
+        self._check_row_size(array)
+        self._header = map(str, array)
+
+    def add_row(self, array):
+        """Add a row in the rows stack
+
+        - cells can contain newlines and tabs
+        """
+
+        self._check_row_size(array)
+        self._rows.append(map(str, array))
+
+    def add_rows(self, rows, header=True):
+        """Add several rows in the rows stack
+
+        - The 'rows' argument can be either an iterator returning arrays,
+          or a by-dimensional array
+        - 'header' specifies if the first row should be used as the header
+          of the table
+        """
+
+        # nb: don't use 'iter' on by-dimensional arrays, to get a 
+        #     usable code for python 2.1
+        if header:
+            if hasattr(rows, '__iter__') and hasattr(rows, 'next'):
+                self.header(rows.next())
+            else:
+                self.header(rows[0])
+                rows = rows[1:]
+        for row in rows:
+            self.add_row(row)
+
+    def set_chars(self, array):
+        """Set the characters used to draw lines between rows and columns
+
+        - the array should contain 4 fields:
+
+            [horizontal, vertical, corner, header]
+
+        - default is set to:
+
+            ['-', '|', '+', '=']
+        """
+
+        if len(array) != 4:
+            raise ArraySizeError, "array should contain 4 characters"
+        array = [ x[:1] for x in [ str(s) for s in array ] ]
+        (self._char_horiz, self._char_vert,
+            self._char_corner, self._char_header) = array
+
+    def set_deco(self, deco):
+        """Set the table decoration
+
+        - 'deco' can be a combinaison of:
+
+            Texttable.BORDER: Border around the table
+            Texttable.HEADER: Horizontal line below the header
+            Texttable.HLINES: Horizontal lines between rows
+            Texttable.VLINES: Vertical lines between columns
+
+           All of them are enabled by default
+
+        - example:
+
+            Texttable.BORDER | Texttable.HEADER
+        """
+
+        self._deco = deco
+
+    def set_cols_align(self, array):
+        """Set the desired columns alignment
+
+        - the elements of the array should be either "l", "c" or "r":
+
+            * "l": column flushed left
+            * "c": column centered
+            * "r": column flushed right
+        """
+
+        self._check_row_size(array)
+        self._align = array
+
+    def set_cols_valign(self, array):
+        """Set the desired columns vertical alignment
+
+        - the elements of the array should be either "t", "m" or "b":
+
+            * "t": column aligned on the top of the cell
+            * "m": column aligned on the middle of the cell
+            * "b": column aligned on the bottom of the cell
+        """
+
+        self._check_row_size(array)
+        self._valign = array
+
+    def set_cols_width(self, array):
+        """Set the desired columns width
+
+        - the elements of the array should be integers, specifying the
+          width of each column. For example:
+
+                [10, 20, 5]
+        """
+
+        self._check_row_size(array)
+        try:
+            array = map(int, array)
+            if reduce(min, array) <= 0:
+                raise ValueError
+        except ValueError:
+            sys.stderr.write("Wrong argument in column width specification\n")
+            raise
+        self._width = array
+
+    def draw(self):
+        """Draw the table
+
+        - the table is returned as a whole string
+        """
+
+        if not self._header and not self._rows:
+            return
+        self._compute_cols_width()
+        self._check_align()
+        out = ""
+        if self._has_border():
+            out += self._hline()
+        if self._header:
+            out += self._draw_line(self._header, isheader=True)
+            if self._has_header():
+                out += self._hline_header()
+        length = 0
+        for row in self._rows:
+            length += 1
+            out += self._draw_line(row)
+            if self._has_hlines() and length < len(self._rows):
+                out += self._hline()
+        if self._has_border():
+            out += self._hline()
+        return out[:-1]
+
+    def _check_row_size(self, array):
+        """Check that the specified array fits the previous rows size
+        """
+
+        if not self._row_size:
+            self._row_size = len(array)
+        elif self._row_size != len(array):
+            raise ArraySizeError, "array should contain %d elements" \
+                % self._row_size
+
+    def _has_vlines(self):
+        """Return a boolean, if vlines are required or not
+        """
+
+        return self._deco & Texttable.VLINES > 0
+
+    def _has_hlines(self):
+        """Return a boolean, if hlines are required or not
+        """
+
+        return self._deco & Texttable.HLINES > 0
+
+    def _has_border(self):
+        """Return a boolean, if border is required or not
+        """
+
+        return self._deco & Texttable.BORDER > 0
+
+    def _has_header(self):
+        """Return a boolean, if header line is required or not
+        """
+
+        return self._deco & Texttable.HEADER > 0
+
+    def _hline_header(self):
+        """Print header's horizontal line
+        """
+
+        return self._build_hline(True)
+
+    def _hline(self):
+        """Print an horizontal line
+        """
+
+        if not self._hline_string:
+            self._hline_string = self._build_hline()
+        return self._hline_string
+
+    def _build_hline(self, is_header=False):
+        """Return a string used to separated rows or separate header from
+        rows
+        """
+        horiz = self._char_horiz
+        if (is_header):
+            horiz = self._char_header
+        # compute cell separator
+        s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()],
+            horiz)
+        # build the line
+        l = string.join([horiz*n for n in self._width], s)
+        # add border if needed
+        if self._has_border():
+            l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz,
+                self._char_corner)
+        else:
+            l += "\n"
+        return l
+
+    def _len_cell(self, cell):
+        """Return the width of the cell
+
+        Special characters are taken into account to return the width of the
+        cell, such like newlines and tabs
+        """
+
+        cell_lines = cell.split('\n')
+        maxi = 0
+        for line in cell_lines:
+            length = 0
+            parts = line.split('\t')
+            for part, i in zip(parts, range(1, len(parts) + 1)):
+                length = length + len(part)
+                if i < len(parts):
+                    length = (length/8 + 1)*8
+            maxi = max(maxi, length)
+        return maxi
+
+    def _compute_cols_width(self):
+        """Return an array with the width of each column
+
+        If a specific width has been specified, exit. If the total of the
+        columns width exceed the table desired width, another width will be
+        computed to fit, and cells will be wrapped.
+        """
+
+        if hasattr(self, "_width"):
+            return
+        maxi = []
+        if self._header:
+            maxi = [ self._len_cell(x) for x in self._header ]
+        for row in self._rows:
+            for cell,i in zip(row, range(len(row))):
+                try:
+                    maxi[i] = max(maxi[i], self._len_cell(cell))
+                except (TypeError, IndexError):
+                    maxi.append(self._len_cell(cell))
+        items = len(maxi)
+        length = reduce(lambda x,y: x+y, maxi)
+        if self._max_width and length + items*3 + 1 > self._max_width:
+            maxi = [(self._max_width - items*3 -1) / items \
+                for n in range(items)]
+        self._width = maxi
+
+    def _check_align(self):
+        """Check if alignment has been specified, set default one if not
+        """
+
+        if not hasattr(self, "_align"):
+            self._align = ["l"]*self._row_size
+        if not hasattr(self, "_valign"):
+            self._valign = ["t"]*self._row_size
+
+    def _draw_line(self, line, isheader=False):
+        """Draw a line
+
+        Loop over a single cell length, over all the cells
+        """
+
+        line = self._splitit(line, isheader)
+        space = " "
+        out  = ""
+        for i in range(len(line[0])):
+            if self._has_border():
+                out += "%s " % self._char_vert
+            length = 0
+            for cell, width, align in zip(line, self._width, self._align):
+                length += 1
+                cell_line = cell[i]
+                fill = width - len(cell_line)
+                if isheader:
+                    align = "c"
+                if align == "r":
+                    out += "%s " % (fill * space + cell_line)
+                elif align == "c":
+                    out += "%s " % (fill/2 * space + cell_line \
+                            + (fill/2 + fill%2) * space)
+                else:
+                    out += "%s " % (cell_line + fill * space)
+                if length < len(line):
+                    out += "%s " % [space, self._char_vert][self._has_vlines()]
+            out += "%s\n" % ['', self._char_vert][self._has_border()]
+        return out
+
+    def _splitit(self, line, isheader):
+        """Split each element of line to fit the column width
+
+        Each element is turned into a list, result of the wrapping of the
+        string to the desired width
+        """
+
+        line_wrapped = []
+        for cell, width in zip(line, self._width):
+            array = []
+            for c in cell.split('\n'):
+                array.extend(textwrap.wrap(unicode(c, 'utf'), width))
+            line_wrapped.append(array)
+        max_cell_lines = reduce(max, map(len, line_wrapped))
+        for cell, valign in zip(line_wrapped, self._valign):
+            if isheader:
+                valign = "t"
+            if valign == "m":
+                missing = max_cell_lines - len(cell)
+                cell[:0] = [""] * (missing / 2)
+                cell.extend([""] * (missing / 2 + missing % 2))
+            elif valign == "b":
+                cell[:0] = [""] * (max_cell_lines - len(cell))
+            else:
+                cell.extend([""] * (max_cell_lines - len(cell)))
+        return line_wrapped
+
+
 #    Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
 #    This file is part of GlusterFS.
 
@@ -17,8 +437,14 @@
 #    along with this program.  If not, see
 #    <http://www.gnu.org/licenses/>.
 
-import numpy as np
-import matplotlib.pyplot as plt
+graph_available = True
+
+try:
+    import numpy as np
+    import matplotlib.pyplot as plt
+except ImportError:
+    graph_available = False
+
 import re
 import sys
 
@@ -87,7 +513,7 @@ def calc_latency_heights (xlator_order):
 # have sufficient number of colors
 colors = ["violet", "blue", "green", "yellow", "orange", "red"]
 
-def latency_profile (title, xlator_order):
+def latency_profile (title, xlator_order, mode):
     heights = calc_latency_heights (xlator_order)
 
     N     = len (latencies[xlator_order[0]].keys())
@@ -100,30 +526,75 @@ def latency_profile (title, xlator_order):
 
     bottoms[Nxl-1] = map (lambda x: 0, latencies[xlator_order[0]].keys())
 
+    k = latencies[xlator_order[0]].keys()
+    k.sort()
+
     for i in range (Nxl-1):
         xl = xlator_order[i+1]
-        k = latencies[xl].keys()
-        k.sort()
-
         bottoms[i] = [float(latencies[xl][key]) for key in k]
 
-    for i in range(Nxl):
-        pieces[i] = plt.bar (ind, heights[i], width, color=colors[i],
-                             bottom=bottoms[i])
+    if mode == 'text':
+        print "\n%sLatency profile for %s\n" % (' '*20, title)
+        print "Average latency (microseconds):\n"
 
-    plt.ylabel ("Average Latency (microseconds)")
-    plt.title ("Latency Profile for '%s'" % title)
-    k = latencies[xlator_order[0]].keys()
-    k.sort ()
-    plt.xticks (ind+width/2., k)
+        table = Texttable()
+
+        table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order))
+        rows = []
+
+        header = ['OP', 'OP Average (us)'] + xlator_order
+        rows = []
+
+        for op in k:
+            sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order],
+                          0)
+
+            row = [op]
+            row += ["%5.2f" % sum]
+
+            for xl in xlator_order:
+                op_index = k.index(op)
+                row += ["%5.2f" % (heights[xlator_order.index(xl)][op_index])]
+
+            rows.append(row)
+
+        def row_sort(r1, r2):
+            v1 = float(r1[1])
+            v2 = float(r2[1])
 
-    m = round (max(map (float, latencies[xlator_order[0]].values())), -2)
-    plt.yticks (np.arange(0, m + m*0.1, m/10))
-    plt.legend (map (lambda p: p[0], pieces), xlator_order)
+            if v1 < v2:
+                return -1
+            elif v1 == v2:
+                return 0
+            else:
+                return 1
 
-    plt.show ()
+        rows.sort(row_sort, reverse=True)
+        table.add_rows([header] + rows)
+        print table.draw()
 
-def fop_distribution (title, xlator_order):
+    elif mode == 'graph':
+        for i in range(Nxl):
+            pieces[i] = plt.bar (ind, heights[i], width, color=colors[i],
+                                 bottom=bottoms[i])
+
+            plt.ylabel ("Average Latency (microseconds)")
+            plt.title ("Latency Profile for '%s'" % title)
+            k = latencies[xlator_order[0]].keys()
+            k.sort ()
+            plt.xticks (ind+width/2., k)
+
+            m = round (max(map (float, latencies[xlator_order[0]].values())), -2)
+            plt.yticks (np.arange(0, m + m*0.1, m/10))
+            plt.legend (map (lambda p: p[0], pieces), xlator_order)
+
+            plt.show ()
+    else:
+        print "Unknown mode specified!"
+        sys.exit(1)
+
+
+def fop_distribution (title, xlator_order, mode):
     plt.ylabel ("Percentage of calls")
     plt.title ("FOP distribution for '%s'" % title)
     k = counts[xlator_order[0]].keys()
@@ -143,17 +614,52 @@ def fop_distribution (title, xlator_order):
     for op in k:
         heights.append (float(counts[top_xl][op])/total * 100)
 
-    bars = plt.bar (ind, heights, width, color="red")
+    if mode == 'text':
+        print "\n%sFOP distribution for %s\n" % (' '*20, title)
+        print "Total number of calls: %d\n" % total
+
+        table = Texttable()
+
+        table.set_cols_align(["l", "r", "r"])
+
+        rows = []
+        header = ["OP", "% of Calls", "Count"]
 
-    for bar in bars:
-        height = bar.get_height()
-        plt.text (bar.get_x()+bar.get_width()/2., 1.05*height,
-                  "%d%%" % int(height))
+        for op in k:
+            row = [op, "%5.2f" % (float(counts[top_xl][op])/total * 100), counts[top_xl][op]]
+            rows.append(row)
 
-    plt.xticks(ind+width/2., k)
-    plt.yticks(np.arange (0, 110, 10))
+        def row_sort(r1, r2):
+            v1 = float(r1[1])
+            v2 = float(r2[1])
+
+            if v1 < v2:
+                return -1
+            elif v1 == v2:
+                return 0
+            else:
+                return 1
+
+        rows.sort(row_sort, reverse=True)
+        table.add_rows([header] + rows)
+        print table.draw()
+
+    elif mode == 'graph':
+        bars = plt.bar (ind, heights, width, color="red")
+
+        for bar in bars:
+            height = bar.get_height()
+            plt.text (bar.get_x()+bar.get_width()/2., 1.05*height,
+                      "%d%%" % int(height))
+
+            plt.xticks(ind+width/2., k)
+            plt.yticks(np.arange (0, 110, 10))
+
+            plt.show()
+    else:
+        print "mode not specified!"
+        sys.exit(1)
 
-    plt.show()
 
 def calc_workload_heights (xlator_order, scaling):
     workload_heights = map (lambda x: [], xlator_order)
@@ -181,7 +687,7 @@ def calc_workload_heights (xlator_order, scaling):
 
     return workload_heights
 
-def workload_profile(title, xlator_order):
+def workload_profile(title, xlator_order, mode):
     plt.ylabel ("Percentage of Total Time")
     plt.title ("Workload Profile for '%s'" % title)
     k = totals[xlator_order[0]].keys()
@@ -216,20 +722,63 @@ def workload_profile(title, xlator_order):
 
         bottoms[i] = [float(totals[xl][key]) / float(totals[top_xl][key]) * p_heights[k.index(key)] for key in k]
 
-    for i in range(Nxl):
-        pieces[i] = plt.bar (ind, heights[i], width, color=colors[i],
-                             bottom=bottoms[i])
+    if mode == 'text':
+        print "\n%sWorkload profile for %s\n" % (' '*20, title)
+        print "Total Time: %d microseconds = %.1f seconds = %.1f minutes\n" % (total, total / 1000000.0, total / 6000000.0)
+
+        table = Texttable()
+        table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order))
+        rows = []
+
+        header = ['OP', 'OP Total (%)'] + xlator_order
+        rows = []
+
+        for op in k:
+            sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order],
+                          0)
+            row = [op]
+            row += ["%5.2f" % sum]
 
-    for key in k:
-        bar = pieces[Nxl-1][k.index(key)]
-        plt.text (bar.get_x() + bar.get_width()/2., 1.05*p_heights[k.index(key)],
-                  "%d%%" % int(p_heights[k.index(key)]))
+            for xl in xlator_order:
+                op_index = k.index(op)
+                row += ["%5.2f" % heights[xlator_order.index(xl)][op_index]]
 
-    plt.xticks(ind+width/2., k)
-    plt.yticks(np.arange (0, 110, 10))
-    plt.legend (map (lambda p: p[0], pieces), xlator_order)
+            rows.append(row)
+
+        def row_sort(r1, r2):
+            v1 = float(r1[1])
+            v2 = float(r2[1])
+
+            if v1 < v2:
+                return -1
+            elif v1 == v2:
+                return 0
+            else:
+                return 1
+
+        rows.sort(row_sort, reverse=True)
+        table.add_rows([header] + rows)
+        print table.draw()
+
+    elif mode == 'graph':
+        for i in range(Nxl):
+            pieces[i] = plt.bar (ind, heights[i], width, color=colors[i],
+                                 bottom=bottoms[i])
+
+        for key in k:
+            bar = pieces[Nxl-1][k.index(key)]
+            plt.text (bar.get_x() + bar.get_width()/2., 1.05*p_heights[k.index(key)],
+                      "%d%%" % int(p_heights[k.index(key)]))
+
+        plt.xticks(ind+width/2., k)
+        plt.yticks(np.arange (0, 110, 10))
+        plt.legend (map (lambda p: p[0], pieces), xlator_order)
+
+        plt.show()
+    else:
+        print "Unknown mode specified!"
+        sys.exit(1)
 
-    plt.show()
 
 def main ():
     parser = OptionParser(usage="usage: %prog [-l | -d | -w] -x <xlator order> <state dump file>")
@@ -241,6 +790,7 @@ def main ():
                       help="Produce workload profile")
     parser.add_option("-t", "--title", dest="title", help="Set the title of the graph")
     parser.add_option("-x", "--xlator-order", dest="xlator_order", help="Specify the order of xlators")
+    parser.add_option("-m", "--mode", dest="mode", help="Output format, can be text[default] or graph")
 
     (options, args) = parser.parse_args()
 
@@ -255,13 +805,20 @@ def main ():
 
     collect_data(file (args[0], 'r'))
 
+    mode = 'text'
+    if (options.mode):
+        mode = options.mode
+        if options.mode == 'graph' and graph_available == False:
+            print "matplotlib not available, falling back to text mode"
+            mode = 'text'
+
     if (options.latency):
-        latency_profile (options.title, xlator_order)
+        latency_profile (options.title, xlator_order, mode)
 
     if (options.distribution):
-        fop_distribution(options.title, xlator_order)
+        fop_distribution(options.title, xlator_order, mode)
 
     if (options.workload):
-        workload_profile(options.title, xlator_order)
+        workload_profile(options.title, xlator_order, mode)
 
 main ()