diff options
Diffstat (limited to 'xlators/bindings')
| -rw-r--r-- | xlators/bindings/Makefile.am | 1 | ||||
| -rw-r--r-- | xlators/bindings/python/Makefile.am | 1 | ||||
| -rw-r--r-- | xlators/bindings/python/src/Makefile.am | 19 | ||||
| -rw-r--r-- | xlators/bindings/python/src/gluster.py | 47 | ||||
| -rw-r--r-- | xlators/bindings/python/src/glusterstack.py | 55 | ||||
| -rw-r--r-- | xlators/bindings/python/src/glustertypes.py | 167 | ||||
| -rw-r--r-- | xlators/bindings/python/src/python.c | 235 | ||||
| -rw-r--r-- | xlators/bindings/python/src/testxlator.py | 56 |
8 files changed, 581 insertions, 0 deletions
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am new file mode 100644 index 00000000000..f7766580257 --- /dev/null +++ b/xlators/bindings/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = $(BINDINGS_SUBDIRS) diff --git a/xlators/bindings/python/Makefile.am b/xlators/bindings/python/Makefile.am new file mode 100644 index 00000000000..af437a64d6d --- /dev/null +++ b/xlators/bindings/python/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am new file mode 100644 index 00000000000..c0b9141c667 --- /dev/null +++ b/xlators/bindings/python/src/Makefile.am @@ -0,0 +1,19 @@ + +xlator_PROGRAMS = python.so + +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings + +python_PYTHON = gluster.py glustertypes.py glusterstack.py + +pythondir = $(xlatordir)/python + +python_so_SOURCES = python.c + +AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \ + -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \ + $(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\" + +AM_LDFLAGS = $(PYTHON_LDFLAGS) + +CLEANFILES = + diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py new file mode 100644 index 00000000000..ee0eb131011 --- /dev/null +++ b/xlators/bindings/python/src/gluster.py @@ -0,0 +1,47 @@ +# Copyright (c) 2007 Chris AtLee <chris@atlee.ca> +# This file is part of GlusterFS. +# +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. +# +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# <http://www.gnu.org/licenses/>. +from ctypes import * +from glustertypes import * +from glusterstack import * +import sys +import inspect + +libglusterfs = CDLL("libglusterfs.so") +_gf_log = libglusterfs._gf_log +_gf_log.restype = c_int32 +_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p] + +gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel") + +GF_LOG_NONE = 0 +GF_LOG_CRITICAL = 1 +GF_LOG_ERROR = 2 +GF_LOG_WARNING = 3 +GF_LOG_DEBUG = 4 + +def gf_log(module, level, fmt, *params): + if level <= gf_log_loglevel: + frame = sys._getframe(1) + _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name, + frame.f_lineno, level, fmt, *params) + +class ComplexTranslator(object): + def __init__(self, xlator): + self.xlator = xlator_t.from_address(xlator) + + def __getattr__(self, item): + return getattr(self.xlator, item) diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py new file mode 100644 index 00000000000..ba24c81652e --- /dev/null +++ b/xlators/bindings/python/src/glusterstack.py @@ -0,0 +1,55 @@ +# Copyright (c) 2007 Chris AtLee <chris@atlee.ca> +# This file is part of GlusterFS. +# +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. +# +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# <http://www.gnu.org/licenses/>. +from ctypes import * +from glustertypes import * + +libc = CDLL("libc.so.6") +calloc = libc.calloc +calloc.argtypes = [c_int, c_int] +calloc.restype = c_void_p + +# TODO: Can these be done in C somehow? +def stack_wind(frame, rfn, obj, fn, *params): + """Frame is a frame object""" + _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t)) + _new[0].root = frame.root + _new[0].next = frame.root[0].frames.next + _new[0].prev = pointer(frame.root[0].frames) + if frame.root[0].frames.next: + frame.root[0].frames.next[0].prev = _new + frame.root[0].frames.next = _new + _new[0].this = obj + # TODO: Type checking like tmp_cbk? + _new[0].ret = rfn + _new[0].parent = pointer(frame) + _new[0].cookie = cast(_new, c_void_p) + # TODO: Initialize lock + #_new.lock.init() + frame.ref_count += 1 + fn(_new, obj, *params) + +def stack_unwind(frame, *params): + """Frame is a frame object""" + fn = frame[0].ret + parent = frame[0].parent[0] + parent.ref_count -= 1 + + op_ret = params[0] + op_err = params[1] + params = params[2:] + fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this, + op_ret, op_err, *params) diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py new file mode 100644 index 00000000000..e9069d07c72 --- /dev/null +++ b/xlators/bindings/python/src/glustertypes.py @@ -0,0 +1,167 @@ +# Copyright (c) 2007 Chris AtLee <chris@atlee.ca> +# This file is part of GlusterFS. +# +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. +# +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# <http://www.gnu.org/licenses/>. +from ctypes import * +import collections + +# +# Forward declaration of some gluster types +# +class call_frame_t(Structure): + pass + +class call_ctx_t(Structure): + pass + +class call_pool_t(Structure): + pass + +class xlator_t(Structure): + def _getFirstChild(self): + return self.children[0].xlator + firstChild = property(_getFirstChild) + +class xlator_list_t(Structure): + pass + +class xlator_fops(Structure): + pass + +class xlator_mops(Structure): + pass + +class glusterfs_ctx_t(Structure): + pass + +class list_head(Structure): + pass + +class dict_t(Structure): + pass + +class inode_table_t(Structure): + pass + +class fd_t(Structure): + pass + +class iovec(Structure): + _fields_ = [ + ("iov_base", c_void_p), + ("iov_len", c_size_t), + ] + + def __init__(self, s): + self.iov_base = cast(c_char_p(s), c_void_p) + self.iov_len = len(s) + + def getBytes(self): + return string_at(self.iov_base, self.iov_len) + +# This is a pthread_spinlock_t +# TODO: what happens to volatile-ness? +gf_lock_t = c_int + +uid_t = c_uint32 +gid_t = c_uint32 +pid_t = c_int32 + +off_t = c_int64 + +# +# Function pointer types +# +ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t), + POINTER(xlator_t), c_int32, c_int32) + +fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t)) +init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t)) +event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p) + +list_head._fields_ = [ + ("next", POINTER(list_head)), + ("prev", POINTER(list_head)), + ] + +call_frame_t._fields_ = [ + ("root", POINTER(call_ctx_t)), + ("parent", POINTER(call_frame_t)), + ("next", POINTER(call_frame_t)), + ("prev", POINTER(call_frame_t)), + ("local", c_void_p), + ("this", POINTER(xlator_t)), + ("ret", ret_fn_t), + ("ref_count", c_int32), + ("lock", gf_lock_t), + ("cookie", c_void_p), + ("op", c_int32), + ("type", c_int8), + ] + +call_ctx_t._fields_ = [ + ("all_frames", list_head), + ("trans", c_void_p), + ("pool", call_pool_t), + ("unique", c_uint64), + ("state", c_void_p), + ("uid", uid_t), + ("gid", gid_t), + ("pid", pid_t), + ("frames", call_frame_t), + ("req_refs", POINTER(dict_t)), + ("rsp_refs", POINTER(dict_t)), + ] + +xlator_t._fields_ = [ + ("name", c_char_p), + ("type", c_char_p), + ("next", POINTER(xlator_t)), + ("prev", POINTER(xlator_t)), + ("parent", POINTER(xlator_t)), + ("children", POINTER(xlator_list_t)), + ("fops", POINTER(xlator_fops)), + ("mops", POINTER(xlator_mops)), + ("fini", fini_fn_t), + ("init", init_fn_t), + ("notify", event_notify_fn_t), + ("options", POINTER(dict_t)), + ("ctx", POINTER(glusterfs_ctx_t)), + ("itable", POINTER(inode_table_t)), + ("ready", c_char), + ("private", c_void_p), + ] + +xlator_list_t._fields_ = [ + ("xlator", POINTER(xlator_t)), + ("next", POINTER(xlator_list_t)), + ] + +fop_functions = collections.defaultdict(lambda: c_void_p) +fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod', + 'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access', + 'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink', + 'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush', + 'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir', + 'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir', + # TODO: Call backs? + ] + +fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t), + POINTER(fd_t), POINTER(iovec), c_int32, + off_t) + +fop_functions['writev'] = fop_writev_t +xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names] diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c new file mode 100644 index 00000000000..739ef732900 --- /dev/null +++ b/xlators/bindings/python/src/python.c @@ -0,0 +1,235 @@ +/* + Copyright (c) 2007 Chris AtLee <chris@atlee.ca> + This file is part of GlusterFS. + + GlusterFS is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + GlusterFS is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see + <http://www.gnu.org/licenses/>. +*/ + +#include <Python.h> + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glusterfs.h" +#include "xlator.h" +#include "logging.h" +#include "defaults.h" + +typedef struct +{ + char *scriptname; + PyObject *pXlator; + PyObject *pScriptModule; + PyObject *pGlusterModule; + PyThreadState *pInterp; + + PyObject *pFrameType, *pVectorType, *pFdType; +} python_private_t; + +int32_t +python_writev (call_frame_t *frame, + xlator_t *this, + fd_t *fd, + struct iovec *vector, + int32_t count, + off_t offset) +{ + python_private_t *priv = (python_private_t *)this->private; + gf_log("python", GF_LOG_DEBUG, "In writev"); + if (PyObject_HasAttrString(priv->pXlator, "writev")) + { + + PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev", + "O O O i l", + PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame), + PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd), + PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector), + count, + offset); + if (PyErr_Occurred()) + { + PyErr_Print(); + } + Py_XDECREF(retval); + } + else + { + return default_writev(frame, this, fd, vector, count, offset); + } + return 0; +} + +struct xlator_fops fops = { + .writev = python_writev +}; + +struct xlator_mops mops = { +}; + +static PyObject * +AnonModule_FromFile (const char* fname) +{ + // Get the builtins + PyThreadState* pThread = PyThreadState_Get(); + PyObject *pBuiltins = pThread->interp->builtins; + + if (PyErr_Occurred()) + { + PyErr_Print(); + return NULL; + } + + // Create a new dictionary for running code in + PyObject *pModuleDict = PyDict_New(); + PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins); + Py_INCREF(pBuiltins); + + // Run the file in the new context + FILE* fp = fopen(fname, "r"); + PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict); + fclose(fp); + if (PyErr_Occurred()) + { + PyErr_Print(); + Py_DECREF(pModuleDict); + Py_DECREF(pBuiltins); + return NULL; + } + + // Create an object to hold the new context + PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict); + if (PyErr_Occurred()) + { + PyErr_Print(); + Py_DECREF(pModuleDict); + Py_DECREF(pBuiltins); + return NULL; + } + PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict); + if (PyErr_Occurred()) + { + PyErr_Print(); + Py_DECREF(pModuleDict); + Py_DECREF(pBuiltins); + Py_XDECREF(pModule); + return NULL; + } + + // Set the new context's dictionary to the one we used to run the code + // inside + PyObject_SetAttrString(pModule, "__dict__", pModuleDict); + if (PyErr_Occurred()) + { + PyErr_Print(); + Py_DECREF(pModuleDict); + Py_DECREF(pBuiltins); + Py_DECREF(pModule); + return NULL; + } + + return pModule; +} + +int32_t +init (xlator_t *this) +{ + // This is ok to call more than once per process + Py_InitializeEx(0); + + if (!this->children) { + gf_log ("python", GF_LOG_ERROR, + "FATAL: python should have exactly one child"); + return -1; + } + + python_private_t *priv = CALLOC (sizeof (python_private_t), 1); + ERR_ABORT (priv); + + data_t *scriptname = dict_get (this->options, "scriptname"); + if (scriptname) { + priv->scriptname = data_to_str(scriptname); + } else { + gf_log("python", GF_LOG_ERROR, + "FATAL: python requires the scriptname parameter"); + return -1; + } + + priv->pInterp = Py_NewInterpreter(); + + // Adjust python's path + PyObject *syspath = PySys_GetObject("path"); + PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH); + PyList_Append(syspath, path); + Py_DECREF(path); + + gf_log("python", GF_LOG_DEBUG, + "Loading gluster module"); + + priv->pGlusterModule = PyImport_ImportModule("gluster"); + if (PyErr_Occurred()) + { + PyErr_Print(); + return -1; + } + + priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t"); + priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t"); + priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec"); + + gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname); + + priv->pScriptModule = AnonModule_FromFile(priv->scriptname); + if (!priv->pScriptModule || PyErr_Occurred()) + { + gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname); + PyErr_Print(); + return -1; + } + + if (!PyObject_HasAttrString(priv->pScriptModule, "xlator")) + { + gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname); + return -1; + } + gf_log("python", GF_LOG_DEBUG, "Instantiating translator"); + priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&", + PyLong_FromVoidPtr, this); + if (PyErr_Occurred() || !priv->pXlator) + { + PyErr_Print(); + return -1; + } + + this->private = priv; + + gf_log ("python", GF_LOG_DEBUG, "python xlator loaded"); + return 0; +} + +void +fini (xlator_t *this) +{ + python_private_t *priv = (python_private_t*)(this->private); + Py_DECREF(priv->pXlator); + Py_DECREF(priv->pScriptModule); + Py_DECREF(priv->pGlusterModule); + Py_DECREF(priv->pFrameType); + Py_DECREF(priv->pFdType); + Py_DECREF(priv->pVectorType); + Py_EndInterpreter(priv->pInterp); + return; +} diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py new file mode 100644 index 00000000000..507455c856a --- /dev/null +++ b/xlators/bindings/python/src/testxlator.py @@ -0,0 +1,56 @@ +# Copyright (c) 2007 Chris AtLee <chris@atlee.ca> +# This file is part of GlusterFS. +# +# GlusterFS is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3 of the License, +# or (at your option) any later version. +# +# GlusterFS is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# <http://www.gnu.org/licenses/>. + +""" +This is a test translator written in python. + +Important things to note: + This file must be import-able from glusterfsd. This probably means + setting PYTHONPATH to where this file is located. + + This file must have a top-level xlator class object that will be + used to instantiate individual translators. +""" +from gluster import * + +class MyXlator(ComplexTranslator): + name = "MyXlator" + def writev_cbk(self, frame, cookie, op_ret, op_errno, buf): + stack_unwind(frame, op_ret, op_errno, buf) + return 0 + + def writev(self, frame, fd, vector, count, offset): + gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len) + # TODO: Use cookie to pass this to writev_cbk + old_count = vector.iov_len + + data = vector.getBytes().encode("zlib") + + vector = iovec(data) + gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len) + + @ret_fn_t + def rfn(frame, prev, this, op_ret, op_errno, *params): + if len(params) == 0: + params = [0] + return self.writev_cbk(frame, prev, old_count, op_errno, *params) + + stack_wind(frame, rfn, self.firstChild, + self.firstChild[0].fops[0].writev, fd, vector, count, offset) + return 0 + +xlator = MyXlator |