afr/cluster: PGFID heal support

Summary:
  PGFID healing enables heals which might otherwise fail due
  due to the lack of a entry heal to succeed by performing
  the entry healing within the same heal flow.

  It does this by leveraging the PGFID tracking feature of
  the POSIX xlator, and examining lookup replies for the
  PGFID attribute.  If detected, the pgfid will be decoded
  and stored for later use in case the heal fails for whatever
  reason.  Cascading heal failures are handled through
  recursion.

  This feature is critical for a couple reasons:
      1. General healing predictability - When the SHD
         attempts to heal a given GFID, it should be able
         to do so without having to wait for some other
         dependent heal to take place.
      2. Reliability - In some cases the parent directory
         may require healing, but the req'd entry in the
         indices/xattrop directory may not exist
         (e.g. bugs/crashes etc).  Prior to PGFID heal support
         some sort of external script would be required to
         queue up these heals by using FS specific utilities
         to lookup the parent directory by hardlink or
         worse...do a costly full heal to clean them up.
      3. Performance - In combination with multi-threaded SHD
         this feature will make SHD healing _much_ faster as
         directories with large amount of files to be healed
         will no longer have to wait for an entry heal to
         come along, the first file in that directory queued
         for healing will trigger an entry heal for the directory
         and this will allow the other files in that directory
         to be (immediatelly) healed in parallel.

Test Plan:
- run prove tests/basic/afr/shd_pgfid_heal.t
- run prove tests/basic/afr/shd*.t
- run prove tests/basic/afr/gfid*.t

Differential Revision: https://phabricator.fb.com/D2546133

Change-Id: I25f586047f8bcafa900c0cc9ee8f0e2128688c73
Signed-off-by: Jeff Darcy <jdarcy@fb.com>
Reviewed-on: https://review.gluster.org/17929
Smoke: Gluster Build System <jenkins@build.gluster.org>
Tested-by: Jeff Darcy <jeff@pl.atyp.us>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Jeff Darcy <jeff@pl.atyp.us>

6 files changed, 99 insertions, 15 deletions

diff --git a/tests/basic/gfid_unsplit_shd.t b/tests/basic/afr/gfid-unsplit-shd.t
index 25fab290177..77da5243724 100644
--- a/tests/basic/gfid_unsplit_shd.t
+++ b/tests/basic/afr/gfid-unsplit-shd.t
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 
 cleanup;
 
diff --git a/tests/basic/gfid_unsplit_type_mismatch.t b/tests/basic/afr/gfid-unsplit-type-mismatch.t
index 51e6a36445b..9e205021a0d 100644
--- a/tests/basic/gfid_unsplit_type_mismatch.t
+++ b/tests/basic/afr/gfid-unsplit-type-mismatch.t
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 
 cleanup;
 
@@ -13,9 +13,9 @@ TEST $CLI volume info;
 TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1..3};
 TEST $CLI volume set $V0 cluster.choose-local off
 TEST $CLI volume set $V0 cluster.self-heal-daemon off
-TEST $CLI volume set $V0 nfs.disable off
+TEST $CLI volume set $V0 nfs.disable on
 TEST $CLI volume set $V0 cluster.quorum-type none
-TEST $CLI volume set $V0 cluster.favorite-child-policy majority
+TEST $CLI volume set $V0 cluster.favorite-child-policy mtime
 #EST $CLI volume set $V0 cluster.favorite-child-by-majority on
 #EST $CLI volume set $V0 cluster.favorite-child-by-mtime on
 TEST $CLI volume set $V0 cluster.metadata-self-heal off
@@ -24,6 +24,8 @@ TEST $CLI volume set $V0 cluster.entry-self-heal off
 TEST $CLI volume start $V0
 sleep 5
 
+pkill -f gluster/glustershd
+
 # Part I: FUSE Test
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 \
   --attribute-timeout=0 --entry-timeout=0
@@ -69,7 +71,6 @@ sleep 1
 
 # Verify the file is readable
 TEST dd if=splitfile of=/dev/null 2>/dev/null
-
 # Verify entry healing happened on the back-end regardless of the
 # gfid-splitbrain state of the directory.
 TEST stat $B0/${V0}1/splitfile
diff --git a/tests/basic/gfid_unsplit.t b/tests/basic/afr/gfid-unsplit.t
index 0df96bd5ed6..0b883ab658f 100644
--- a/tests/basic/gfid_unsplit.t
+++ b/tests/basic/afr/gfid-unsplit.t
@@ -1,8 +1,8 @@
 #!/bin/bash
 
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
-. $(dirname $0)/../nfs.rc
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../nfs.rc
 
 cleanup;
 
@@ -19,7 +19,7 @@ TEST $CLI volume set $V0 cluster.self-heal-daemon off
 TEST $CLI volume set $V0 nfs.disable off
 #EST $CLI volume set $V0 cluster.favorite-child-by-majority on
 #EST $CLI volume set $V0 cluster.favorite-child-by-mtime on
-TEST $CLI volume set $V0 cluster.favorite-child-policy majority
+TEST $CLI volume set $V0 cluster.favorite-child-policy mtime
 TEST $CLI volume set $V0 cluster.metadata-self-heal off
 TEST $CLI volume set $V0 cluster.data-self-heal off
 TEST $CLI volume set $V0 cluster.entry-self-heal off
@@ -39,6 +39,8 @@ MD5=$(md5sum $M0/splitfile | cut -d\  -f1)
 TEST kill_brick $V0 $H0 $B0/${V0}1
 GFID_DIR_B1="$B0/${V0}1/.glusterfs/$(getfattr -n trusted.gfid -e hex $B0/${V0}1/splitfile 2>/dev/null | grep ^trusted | cut -d= -f2 | awk '{print substr($0,3,2)}')"
 rm -rf $GFID_DIR_B1
+mkdir -p $B0/${V0}1/.glusterfs/fd/55
+ln $B0/${V0}1/splitfile $B0/${V0}1/.glusterfs/fd/55/fd551a5c-fddd-4c1a-a4d0-96ef09ef5c08
 TEST setfattr -n "trusted.gfid" -v "0xfd551a5cfddd4c1aa4d096ef09ef5c08" $B0/${V0}1/splitfile
 
 GFID_DIR_B3="$B0/${V0}3/.glusterfs/$(getfattr -n trusted.gfid -e hex $B0/${V0}3/splitfile 2>/dev/null | grep ^trusted | cut -d= -f2 | awk '{print substr($0,3,2)}')"
diff --git a/tests/basic/shd_autofix_nogfid.t b/tests/basic/afr/shd-autofix-nogfid.t
index 5a6ed66f522..7c9026dce62 100644
--- a/tests/basic/shd_autofix_nogfid.t
+++ b/tests/basic/afr/shd-autofix-nogfid.t
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 
 cleanup;
 
diff --git a/tests/basic/shd_force_inspect.t b/tests/basic/afr/shd-force-inspect.t
index ebf3f7a17ad..caceb841322 100644
--- a/tests/basic/shd_force_inspect.t
+++ b/tests/basic/afr/shd-force-inspect.t
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-. $(dirname $0)/../include.rc
-. $(dirname $0)/../volume.rc
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
 
 cleanup;
 
diff --git a/tests/basic/afr/shd-pgfid-heal.t b/tests/basic/afr/shd-pgfid-heal.t
new file mode 100644
index 00000000000..d12d29e13ba
--- /dev/null
+++ b/tests/basic/afr/shd-pgfid-heal.t
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+# Setup a cluster with 3 replicas, and fav child by majority on
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{1..3};
+TEST $CLI volume set $V0 cluster.choose-local off
+TEST $CLI volume set $V0 cluster.self-heal-daemon on
+TEST $CLI volume set $V0 nfs.disable on
+TEST $CLI volume set $V0 cluster.quorum-type none
+#EST $CLI volume set $V0 cluster.favorite-child-by-majority on
+#EST $CLI volume set $V0 cluster.favorite-child-by-mtime on
+TEST $CLI volume set $V0 cluster.favorite-child-policy majority
+TEST $CLI volume set $V0 storage.build-pgfid on
+TEST $CLI volume set $V0 cluster.metadata-self-heal off
+TEST $CLI volume set $V0 cluster.data-self-heal off
+TEST $CLI volume set $V0 cluster.entry-self-heal off
+TEST $CLI volume start $V0
+sleep 5
+
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 \
+  --attribute-timeout=0 --entry-timeout=0
+
+cd $M0
+mkdir -p a/b/c
+dd if=/dev/urandom of=a/b/c/testfile bs=128k count=5 2>/dev/null
+MD5=$(md5sum a/b/c/testfile | cut -d\  -f1)
+
+# Kill the SHD while we setup the test
+pkill -f gluster/glustershd
+# Kill the brick as well such that 
+TEST kill_brick $V0 $H0 $B0/${V0}1
+
+# Grab the GFID of the file and parent dir
+GFID_PARENT_B_RAW=$(getfattr -n trusted.gfid -e hex $B0/${V0}1/a/b 2>/dev/null | grep trusted.gfid | cut -d= -f2)
+GFID_PARENT_B_FORMATTED=$(echo "$GFID_PARENT_B_RAW" | awk '{print substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')
+GFID_PARENT_B_LINK_B1="$B0/${V0}1/.glusterfs/$(echo $GFID_PARENT_B_RAW | awk '{print substr($0,3,2)"/"substr($0,5,2)"/"substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')"
+GFID_PARENT_C_RAW=$(getfattr -n trusted.gfid -e hex $B0/${V0}1/a/b/c 2>/dev/null | grep trusted.gfid | cut -d= -f2)
+GFID_PARENT_C_FORMATTED=$(echo "$GFID_PARENT_C_RAW" | awk '{print substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')
+GFID_PARENT_C_LINK_B1="$B0/${V0}1/.glusterfs/$(echo $GFID_PARENT_C_RAW | awk '{print substr($0,3,2)"/"substr($0,5,2)"/"substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')"
+GFID_RAW=$(getfattr -n trusted.gfid -e hex $B0/${V0}1/a/b/c/testfile 2>/dev/null | grep trusted.gfid | cut -d= -f2)
+GFID_FORMATTED=$(echo "$GFID_RAW" | awk '{print substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')
+GFID_LINK_B1="$B0/${V0}1/.glusterfs/$(echo $GFID_RAW | awk '{print substr($0,3,2)"/"substr($0,5,2)"/"substr($1,3,8)"-"substr($1,11,4)"-"substr($1,15,4)"-"substr($1,19,4)"-"substr($1,23,12)}')"
+
+#
+# Here we are going to create a situation such that a file 3
+# levels deep into the FS requires healing, along with 2 levels
+# of parent directories.  The only signal SHD has is that the
+# file itself needs healing.  The directory (entry) heals are
+# missing; simulating a crash or some sort of bug that we need
+# to be able to recover from.
+#
+
+# Nuke the file from brick 1, along with the parent directories
+# and all backend hard/symbolic links
+rm -f $B0/${V0}1/a/b/c/testfile
+rm -f $GFID_LINK_B1
+rmdir $B0/${V0}1/a/b/c
+rm -f $GFID_PARENT_C_LINK_B1
+rmdir $B0/${V0}1/a/b
+rm -f $GFID_PARENT_B_LINK_B1
+
+# Now manually queue up the parent directory for healing
+touch $B0/${V0}3/.glusterfs/indices/xattrop/$GFID_FORMATTED
+
+# Kick off the SHD and wait 30 seconds for healing to take place
+TEST gluster vol start patchy force
+EXPECT_WITHIN 30 "0" get_pending_heal_count $V0
+sleep 5
+
+# Verify the file was healed back to brick 1
+TEST stat $B0/${V0}1/a/b/c/testfile
+
+cleanup