Added all files

7 files changed, 580 insertions, 0 deletions

diff --git a/xlators/bindings/python/Makefile.am b/xlators/bindings/python/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/bindings/python/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
new file mode 100644
index 000000000..c0b9141c6
--- /dev/null
+++ b/xlators/bindings/python/src/Makefile.am
@@ -0,0 +1,19 @@
+
+xlator_PROGRAMS = python.so
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
+
+python_PYTHON = gluster.py glustertypes.py glusterstack.py
+
+pythondir = $(xlatordir)/python
+
+python_so_SOURCES = python.c
+
+AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
+	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
+	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
+
+AM_LDFLAGS = $(PYTHON_LDFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
new file mode 100644
index 000000000..ee0eb1310
--- /dev/null
+++ b/xlators/bindings/python/src/gluster.py
@@ -0,0 +1,47 @@
+#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+#   This file is part of GlusterFS.
+#
+#   GlusterFS is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published
+#   by the Free Software Foundation; either version 3 of the License,
+#   or (at your option) any later version.
+#
+#   GlusterFS is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see
+#   <http://www.gnu.org/licenses/>.
+from ctypes import *
+from glustertypes import *
+from glusterstack import *
+import sys
+import inspect
+
+libglusterfs = CDLL("libglusterfs.so")
+_gf_log = libglusterfs._gf_log
+_gf_log.restype = c_int32
+_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
+
+gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
+
+GF_LOG_NONE = 0
+GF_LOG_CRITICAL = 1
+GF_LOG_ERROR = 2
+GF_LOG_WARNING = 3
+GF_LOG_DEBUG = 4
+
+def gf_log(module, level, fmt, *params):
+    if level <= gf_log_loglevel:
+        frame = sys._getframe(1)
+        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
+                frame.f_lineno, level, fmt, *params)
+
+class ComplexTranslator(object):
+    def __init__(self, xlator):
+        self.xlator = xlator_t.from_address(xlator)
+
+    def __getattr__(self, item):
+        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
new file mode 100644
index 000000000..ba24c8165
--- /dev/null
+++ b/xlators/bindings/python/src/glusterstack.py
@@ -0,0 +1,55 @@
+#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+#   This file is part of GlusterFS.
+#
+#   GlusterFS is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published
+#   by the Free Software Foundation; either version 3 of the License,
+#   or (at your option) any later version.
+#
+#   GlusterFS is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see
+#   <http://www.gnu.org/licenses/>.
+from ctypes import *
+from glustertypes import *
+
+libc = CDLL("libc.so.6")
+calloc = libc.calloc
+calloc.argtypes = [c_int, c_int]
+calloc.restype = c_void_p
+
+# TODO: Can these be done in C somehow?
+def stack_wind(frame, rfn, obj, fn, *params):
+    """Frame is a frame object"""
+    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
+    _new[0].root = frame.root
+    _new[0].next = frame.root[0].frames.next
+    _new[0].prev = pointer(frame.root[0].frames)
+    if frame.root[0].frames.next:
+        frame.root[0].frames.next[0].prev = _new
+    frame.root[0].frames.next = _new
+    _new[0].this = obj
+    # TODO: Type checking like tmp_cbk?
+    _new[0].ret = rfn
+    _new[0].parent = pointer(frame)
+    _new[0].cookie = cast(_new, c_void_p)
+    # TODO: Initialize lock
+    #_new.lock.init()
+    frame.ref_count += 1
+    fn(_new, obj, *params)
+
+def stack_unwind(frame, *params):
+    """Frame is a frame object"""
+    fn = frame[0].ret
+    parent = frame[0].parent[0]
+    parent.ref_count -= 1
+
+    op_ret = params[0]
+    op_err = params[1]
+    params = params[2:]
+    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
+            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
new file mode 100644
index 000000000..e9069d07c
--- /dev/null
+++ b/xlators/bindings/python/src/glustertypes.py
@@ -0,0 +1,167 @@
+#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+#   This file is part of GlusterFS.
+#
+#   GlusterFS is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published
+#   by the Free Software Foundation; either version 3 of the License,
+#   or (at your option) any later version.
+#
+#   GlusterFS is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see
+#   <http://www.gnu.org/licenses/>.
+from ctypes import *
+import collections
+
+#
+# Forward declaration of some gluster types
+#
+class call_frame_t(Structure):
+    pass
+
+class call_ctx_t(Structure):
+    pass
+
+class call_pool_t(Structure):
+    pass
+
+class xlator_t(Structure):
+    def _getFirstChild(self):
+        return self.children[0].xlator
+    firstChild = property(_getFirstChild)
+
+class xlator_list_t(Structure):
+    pass
+
+class xlator_fops(Structure):
+    pass
+
+class xlator_mops(Structure):
+    pass
+
+class glusterfs_ctx_t(Structure):
+    pass
+
+class list_head(Structure):
+    pass
+
+class dict_t(Structure):
+    pass
+
+class inode_table_t(Structure):
+    pass
+
+class fd_t(Structure):
+    pass
+
+class iovec(Structure):
+    _fields_ = [
+            ("iov_base", c_void_p),
+            ("iov_len", c_size_t),
+            ]
+
+    def __init__(self, s):
+        self.iov_base = cast(c_char_p(s), c_void_p)
+        self.iov_len = len(s)
+
+    def getBytes(self):
+        return string_at(self.iov_base, self.iov_len)
+
+# This is a pthread_spinlock_t
+# TODO: what happens to volatile-ness?
+gf_lock_t = c_int
+
+uid_t = c_uint32
+gid_t = c_uint32
+pid_t = c_int32
+
+off_t = c_int64
+
+#
+# Function pointer types
+#
+ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
+                              POINTER(xlator_t), c_int32, c_int32)
+
+fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
+init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
+event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
+
+list_head._fields_ = [
+        ("next", POINTER(list_head)),
+        ("prev", POINTER(list_head)),
+        ]
+
+call_frame_t._fields_ = [
+        ("root", POINTER(call_ctx_t)),
+        ("parent", POINTER(call_frame_t)),
+        ("next", POINTER(call_frame_t)),
+        ("prev", POINTER(call_frame_t)),
+        ("local", c_void_p),
+        ("this", POINTER(xlator_t)),
+        ("ret", ret_fn_t),
+        ("ref_count", c_int32),
+        ("lock", gf_lock_t),
+        ("cookie", c_void_p),
+        ("op", c_int32),
+        ("type", c_int8),
+        ]
+
+call_ctx_t._fields_ = [
+        ("all_frames", list_head),
+        ("trans", c_void_p),
+        ("pool", call_pool_t),
+        ("unique", c_uint64),
+        ("state", c_void_p),
+        ("uid", uid_t),
+        ("gid", gid_t),
+        ("pid", pid_t),
+        ("frames", call_frame_t),
+        ("req_refs", POINTER(dict_t)),
+        ("rsp_refs", POINTER(dict_t)),
+        ]
+
+xlator_t._fields_ = [
+        ("name", c_char_p),
+        ("type", c_char_p),
+        ("next", POINTER(xlator_t)),
+        ("prev", POINTER(xlator_t)),
+        ("parent", POINTER(xlator_t)),
+        ("children", POINTER(xlator_list_t)),
+        ("fops", POINTER(xlator_fops)),
+        ("mops", POINTER(xlator_mops)),
+        ("fini", fini_fn_t),
+        ("init", init_fn_t),
+        ("notify", event_notify_fn_t),
+        ("options", POINTER(dict_t)),
+        ("ctx", POINTER(glusterfs_ctx_t)),
+        ("itable", POINTER(inode_table_t)),
+        ("ready", c_char),
+        ("private", c_void_p),
+        ]
+
+xlator_list_t._fields_ = [
+        ("xlator", POINTER(xlator_t)),
+        ("next", POINTER(xlator_list_t)),
+        ]
+
+fop_functions = collections.defaultdict(lambda: c_void_p)
+fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
+        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
+        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
+        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
+        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
+        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
+        # TODO: Call backs?
+        ]
+
+fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
+                                  POINTER(fd_t), POINTER(iovec), c_int32,
+                                  off_t)
+
+fop_functions['writev'] = fop_writev_t
+xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
new file mode 100644
index 000000000..739ef7329
--- /dev/null
+++ b/xlators/bindings/python/src/python.c
@@ -0,0 +1,235 @@
+/*
+   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+   This file is part of GlusterFS.
+
+   GlusterFS is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3 of the License,
+   or (at your option) any later version.
+
+   GlusterFS is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <http://www.gnu.org/licenses/>.
+*/
+
+#include <Python.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+typedef struct
+{
+    char        *scriptname;
+    PyObject    *pXlator;
+    PyObject    *pScriptModule;
+    PyObject    *pGlusterModule;
+    PyThreadState *pInterp;
+
+    PyObject    *pFrameType, *pVectorType, *pFdType;
+} python_private_t;
+
+int32_t
+python_writev (call_frame_t *frame,
+              xlator_t *this,
+              fd_t *fd,
+              struct iovec *vector,
+              int32_t count, 
+              off_t offset)
+{
+  python_private_t *priv = (python_private_t *)this->private;
+  gf_log("python", GF_LOG_DEBUG, "In writev");
+  if (PyObject_HasAttrString(priv->pXlator, "writev"))
+  {
+
+      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
+              "O O O i l",
+              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
+              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
+              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
+              count,
+              offset);
+      if (PyErr_Occurred())
+      {
+          PyErr_Print();
+      }
+      Py_XDECREF(retval);
+  }
+  else
+  {
+      return default_writev(frame, this, fd, vector, count, offset);
+  }
+  return 0;
+}
+
+struct xlator_fops fops = {
+    .writev       = python_writev
+};
+
+struct xlator_mops mops = {
+};
+
+static PyObject *
+AnonModule_FromFile (const char* fname)
+{
+    // Get the builtins
+    PyThreadState* pThread = PyThreadState_Get();
+    PyObject *pBuiltins = pThread->interp->builtins;
+
+    if (PyErr_Occurred())
+    {
+        PyErr_Print();
+        return NULL;
+    }
+
+    // Create a new dictionary for running code in
+    PyObject *pModuleDict = PyDict_New();
+    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
+    Py_INCREF(pBuiltins);
+
+    // Run the file in the new context
+    FILE* fp = fopen(fname, "r");
+    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
+    fclose(fp);
+    if (PyErr_Occurred())
+    {
+        PyErr_Print();
+        Py_DECREF(pModuleDict);
+        Py_DECREF(pBuiltins);
+        return NULL;
+    }
+
+    // Create an object to hold the new context
+    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
+    if (PyErr_Occurred())
+    {
+        PyErr_Print();
+        Py_DECREF(pModuleDict);
+        Py_DECREF(pBuiltins);
+        return NULL;
+    }
+    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
+    if (PyErr_Occurred())
+    {
+        PyErr_Print();
+        Py_DECREF(pModuleDict);
+        Py_DECREF(pBuiltins);
+        Py_XDECREF(pModule);
+        return NULL;
+    }
+
+    // Set the new context's dictionary to the one we used to run the code
+    // inside
+    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
+    if (PyErr_Occurred())
+    {
+        PyErr_Print();
+        Py_DECREF(pModuleDict);
+        Py_DECREF(pBuiltins);
+        Py_DECREF(pModule);
+        return NULL;
+    }
+
+    return pModule;
+}
+
+int32_t
+init (xlator_t *this)
+{
+  // This is ok to call more than once per process
+  Py_InitializeEx(0);
+
+  if (!this->children) {
+    gf_log ("python", GF_LOG_ERROR, 
+            "FATAL: python should have exactly one child");
+    return -1;
+  }
+
+  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
+  ERR_ABORT (priv);
+
+  data_t *scriptname = dict_get (this->options, "scriptname");
+  if (scriptname) {
+      priv->scriptname = data_to_str(scriptname);
+  } else {
+      gf_log("python", GF_LOG_ERROR,
+              "FATAL: python requires the scriptname parameter");
+      return -1;
+  }
+
+  priv->pInterp = Py_NewInterpreter();
+    
+  // Adjust python's path
+  PyObject *syspath = PySys_GetObject("path");
+  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
+  PyList_Append(syspath, path);
+  Py_DECREF(path);
+
+  gf_log("python", GF_LOG_DEBUG,
+          "Loading gluster module");
+
+  priv->pGlusterModule = PyImport_ImportModule("gluster");
+  if (PyErr_Occurred())
+  {
+      PyErr_Print();
+      return -1;
+  }
+
+  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
+  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
+  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
+
+  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
+  
+  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
+  if (!priv->pScriptModule || PyErr_Occurred())
+  {
+      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
+      PyErr_Print();
+      return -1;
+  }
+
+  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
+  {
+      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
+      return -1;
+  }
+  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
+  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
+          PyLong_FromVoidPtr, this);
+  if (PyErr_Occurred() || !priv->pXlator)
+  {
+      PyErr_Print();
+      return -1;
+  }
+
+  this->private = priv;
+
+  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
+  return 0;
+}
+
+void 
+fini (xlator_t *this)
+{
+  python_private_t *priv = (python_private_t*)(this->private);
+  Py_DECREF(priv->pXlator);
+  Py_DECREF(priv->pScriptModule);
+  Py_DECREF(priv->pGlusterModule);
+  Py_DECREF(priv->pFrameType);
+  Py_DECREF(priv->pFdType);
+  Py_DECREF(priv->pVectorType);
+  Py_EndInterpreter(priv->pInterp);
+  return;
+}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
new file mode 100644
index 000000000..507455c85
--- /dev/null
+++ b/xlators/bindings/python/src/testxlator.py
@@ -0,0 +1,56 @@
+#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+#   This file is part of GlusterFS.
+#
+#   GlusterFS is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published
+#   by the Free Software Foundation; either version 3 of the License,
+#   or (at your option) any later version.
+#
+#   GlusterFS is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see
+#   <http://www.gnu.org/licenses/>.
+
+"""
+This is a test translator written in python.
+
+Important things to note:
+    This file must be import-able from glusterfsd.  This probably means
+    setting PYTHONPATH to where this file is located.
+
+    This file must have a top-level xlator class object that will be
+    used to instantiate individual translators.
+"""
+from gluster import *
+
+class MyXlator(ComplexTranslator):
+    name = "MyXlator"
+    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
+        stack_unwind(frame, op_ret, op_errno, buf)
+        return 0
+
+    def writev(self, frame, fd, vector, count, offset):
+        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
+        # TODO: Use cookie to pass this to writev_cbk
+        old_count = vector.iov_len
+
+        data = vector.getBytes().encode("zlib")
+
+        vector = iovec(data)
+        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
+
+        @ret_fn_t
+        def rfn(frame, prev, this, op_ret, op_errno, *params):
+            if len(params) == 0:
+                params = [0]
+            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
+
+        stack_wind(frame, rfn, self.firstChild,
+                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
+        return 0
+
+xlator = MyXlator