From 8d55c25f158921b508bff0e7f25158991913f922 Mon Sep 17 00:00:00 2001 From: Pranith Kumar K Date: Wed, 11 Dec 2013 09:33:53 +0530 Subject: syncop: Change return value of syncop Problem: We found a day-1 bug when syncop_xxx() infra is used inside a synctask with compilation optimization (CFLAGS -O2). Detailed explanation of the Root cause: We found the bug in 'gf_defrag_migrate_data' in rebalance operation: Lets look at interesting parts of the function: int gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc, dict_t *migrate_data) { ..... code section - [ Loop ] while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL, &entries)) != 0) { ..... code section - [ ERRNO-1 ] (errno of readdirp is stored in readdir_operrno by a thread) /* Need to keep track of ENOENT errno, that means, there is no need to send more readdirp() */ readdir_operrno = errno; ..... code section - [ SYNCOP-1 ] (syncop_getxattr is called by a thread) ret = syncop_getxattr (this, &entry_loc, &dict, GF_XATTR_LINKINFO_KEY); code section - [ ERRNO-2] (checking for failures of syncop_getxattr(). This may not always be executed in same thread which executed [SYNCOP-1]) if (ret < 0) { if (errno != ENODATA) { loglevel = GF_LOG_ERROR; defrag->total_failures += 1; ..... } the function above could be executed by thread(t1) till [SYNCOP-1] and code from [ERRNO-2] can be executed by a different thread(t2) because of the way syncop-infra schedules the tasks. when the code is compiled with -O2 optimization this is the assembly code that is generated: [ERRNO-1] 1165 readdir_operrno = errno; <<---- errno gets expanded as *(__errno_location()) 0x00007fd149d48b60 <+496>: callq 0x7fd149d410c0 0x00007fd149d48b72 <+514>: mov %rax,0x50(%rsp) <<------ Address returned by __errno_location() is stored in a special location in stack for later use. 0x00007fd149d48b77 <+519>: mov (%rax),%eax 0x00007fd149d48b79 <+521>: mov %eax,0x78(%rsp) .... [ERRNO-2] 1281 if (errno != ENODATA) { 0x00007fd149d492ae <+2366>: mov 0x50(%rsp),%rax <<----- Because it already stored the address returned by __errno_location(), it just dereferences the address to get the errno value. BUT THIS CODE NEED NOT BE EXECUTED BY SAME THREAD!!! 0x00007fd149d492b3 <+2371>: mov $0x9,%ebp 0x00007fd149d492b8 <+2376>: mov (%rax),%edi 0x00007fd149d492ba <+2378>: cmp $0x3d,%edi The problem is that __errno_location() value of t1 and t2 are different. So [ERRNO-2] ends up reading errno of t1 instead of errno of t2 even though t2 is executing [ERRNO-2] code section. When code is compiled without any optimization for [ERRNO-2]: 1281 if (errno != ENODATA) { 0x00007fd58e7a326f <+2237>: callq 0x7fd58e797300 <<--- As it is calling __errno_location() again it gets the location from t2 so it works as intended. 0x00007fd58e7a3274 <+2242>: mov (%rax),%eax 0x00007fd58e7a3276 <+2244>: cmp $0x3d,%eax 0x00007fd58e7a3279 <+2247>: je 0x7fd58e7a32a1 Fix: Make syncop_xxx() return (-errno) value as the return value in case of errors and all the functions which make syncop_xxx() will need to use (-ret) to figure out the reason for failure in case of syncop_xxx() failures. Change-Id: I314d20dabe55d3e62ff66f3b4adb1cac2eaebb57 BUG: 1040356 Signed-off-by: Pranith Kumar K Reviewed-on: http://review.gluster.org/6475 Tested-by: Gluster Build System Reviewed-by: Anand Avati --- xlators/cluster/afr/src/afr-self-heald.c | 44 ++++++++++++++++++++++---------- xlators/cluster/afr/src/pump.c | 6 +++-- 2 files changed, 35 insertions(+), 15 deletions(-) (limited to 'xlators/cluster/afr/src') diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c index 5f85c3047..9e5c1b3e7 100644 --- a/xlators/cluster/afr/src/afr-self-heald.c +++ b/xlators/cluster/afr/src/afr-self-heald.c @@ -336,8 +336,9 @@ _get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child, ret = syncop_getxattr (readdir_xl, child, &xattr, GFID_TO_PATH_KEY); if (ret < 0) { - if ((errno == ENOENT || errno == ESTALE) && missing) + if ((-ret == ENOENT || -ret == ESTALE) && missing) *missing = _gf_true; + ret = -1; goto out; } ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path); @@ -437,10 +438,10 @@ _remove_stale_index (xlator_t *this, xlator_t *readdir_xl, gf_log (this->name, GF_LOG_DEBUG, "Removing stale index " "for %s on %s", index_loc.name, readdir_xl->name); ret = syncop_unlink (readdir_xl, &index_loc); - if(ret && (errno != ENOENT)) { + if((ret < 0) && (-ret != ENOENT)) { gf_log(this->name, GF_LOG_ERROR, "%s: Failed to remove index " "on %s - %s",index_loc.name, readdir_xl->name, - strerror (errno)); + strerror (-ret)); } index_loc.path = NULL; loc_wipe (&index_loc); @@ -467,8 +468,10 @@ _count_hard_links_under_base_indices_dir (xlator_t *this, child = crawl_data->child; ret = syncop_lookup (readdir_xl, childloc, NULL, iattr, NULL, &parent); - if (ret) + if (ret) { + ret = -1; goto out; + } ret = dict_get_int32 (output, this->name, &xl_id); if (ret) @@ -648,8 +651,10 @@ _self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *ent ret = syncop_lookup (this, child, xattr_req, iattr, &xattr_rsp, &parentbuf); - _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp, + _crawl_post_sh_action (this, parent, child, ret, -ret, xattr_rsp, crawl_data); + if (ret < 0) + ret = -1; if (xattr_rsp) dict_unref (xattr_rsp); if (ret == 0) @@ -1190,8 +1195,10 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data, afr_build_root_loc (this, &rootloc); ret = syncop_getxattr (readdir_xl, &rootloc, &xattr, GF_XATTROP_INDEX_GFID); - if (ret < 0) + if (ret < 0) { + ret = -1; goto out; + } ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid); if (ret < 0) { gf_log (this->name, GF_LOG_ERROR, "failed to get index " @@ -1210,11 +1217,12 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data, ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL, &parent); if (ret < 0) { - if (errno != ENOENT) { + if (-ret != ENOENT) { gf_log (this->name, GF_LOG_ERROR, "lookup " "failed on index dir on %s - (%s)", - readdir_xl->name, strerror (errno)); + readdir_xl->name, strerror (-ret)); } + ret = -1; goto out; } ret = _link_inode_update_loc (this, dirloc, &iattr); @@ -1224,8 +1232,10 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data, afr_build_root_loc (this, &rootloc); ret = syncop_getxattr (readdir_xl, &rootloc, &xattr, GF_BASE_INDICES_HOLDER_GFID); - if (ret < 0) + if (ret < 0) { + ret = -1; goto out; + } ret = dict_get_ptr (xattr, GF_BASE_INDICES_HOLDER_GFID, &base_indices_holder_vgfid); if (ret < 0) { @@ -1246,16 +1256,17 @@ afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data, ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL, &parent); if (ret < 0) { - if (errno != ENOENT) { + if (-ret != ENOENT) { gf_log (this->name, GF_LOG_ERROR, "lookup " "failed for base_indices_holder dir" " on %s - (%s)", readdir_xl->name, - strerror (errno)); + strerror (-ret)); } else { gf_log (this->name, GF_LOG_ERROR, "base_indices" "_holder is not yet created."); } + ret = -1; goto out; } ret = _link_inode_update_loc (this, dirloc, &iattr); @@ -1290,6 +1301,7 @@ afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd, if (ret < 0) { gf_log (this->name, GF_LOG_ERROR, "opendir failed on %s", dirloc->path); + ret = -1; goto out; } } else { @@ -1442,8 +1454,13 @@ _crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data) else ret = syncop_readdir (readdir_xl, fd, 131072, offset, &entries); - if (ret <= 0) + if (ret < 0) { + ret = -1; + break; + } else if (ret == 0) { break; + } + ret = 0; free_entries = _gf_true; @@ -1503,7 +1520,8 @@ afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos) GF_XATTR_NODE_UUID_KEY); if (ret < 0) { gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - " - "(%s)", priv->children[child]->name, strerror (errno)); + "(%s)", priv->children[child]->name, strerror (-ret)); + ret = -1; goto out; } diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c index a7f72fb30..9027e2a33 100644 --- a/xlators/cluster/afr/src/pump.c +++ b/xlators/cluster/afr/src/pump.c @@ -504,9 +504,10 @@ pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this, for (i = 0; i < priv->child_count; i++) { ret = syncop_removexattr (priv->children[i], &loc, PUMP_SOURCE_COMPLETE); - if (ret) + if (ret) { gf_log (this->name, GF_LOG_DEBUG, "removexattr " - "failed with %s", strerror (errno)); + "failed with %s", strerror (-ret)); + } } loc_wipe (&loc); @@ -598,6 +599,7 @@ pump_lookup_sink (loc_t *loc) if (ret) { gf_log (this->name, GF_LOG_DEBUG, "Lookup on sink child failed"); + ret = -1; goto out; } -- cgit