[FEATURE]Request: Add lsof Support to Identify Open-but-Deleted NFS Files (ESTALE / “Stale file handle”)

[x-lugoo]

When an NFS client and server share a file, and the file is deleted on the NFS server while it is still open (referenced) by a process on the NFS client, there is currently no reliable way to use lsof on the client to filter and list those file descriptors that correspond to deleted / “stale file handle” (ESTALE) files.

This makes troubleshooting difficult: for example, du -sh on the server and df -h on the client (or across hosts) can diverge more and more over time, and it is hard to identify the root cause because the open-but-deleted NFS files cannot be easily discovered with lsof.

I wrote a patch as an experiment, but I am not an lsof expert and I do not think the implementation quality is good enough for submission. I’m sharing this problem description to ask whether the lsof community can provide a proper solution for detecting and filtering these NFS “deleted but still open” / ESTALE file descriptors on clients.

nfs client side:

root@virtme-ng:/home/jeffxie/vng-share/nfs# exec 3<./test.txt 

nfs server side:

# rm ./test.txt

root@virtme-ng:/home/jeffxie/vng-share# lsof +L1
NOTHING OUTPUT

root@virtme-ng:/home/jeffxie/vng-share# stat -L /proc/399/fd/3 
stat: cannot statx '/proc/399/fd/3': Stale file handle
root@virtme-ng:/home/jeffxie/vng-share# echo $?
1

patch:

diff --git a/Lsof.8 b/Lsof.8
index b254dd4..1da4e00 100644
--- a/Lsof.8
+++ b/Lsof.8
@@ -1302,6 +1302,10 @@ It is also useful when host name lookup is not working properly.
 .B \-N
 selects the listing of NFS files.
 .TP \w'names'u+4
+.B \-y
+selects the listing of files whose stat(2)/lstat(2) fails with
+ESTALE ("Stale file handle"), e.g., stale NFS file handles.
+.TP \w'names'u+4
 .BI \-o
 directs
 .I lsof
diff --git a/lib/common.h b/lib/common.h
index 1e6c67e..88cd4fa 100644
--- a/lib/common.h
+++ b/lib/common.h
@@ -569,14 +569,15 @@ extern int ZoneColW;
 #    define SELEVTFDINFO                                                       \
         0x200000 /* selected for evetnfd info;                                 \
                   * cleared in link_lfile() */
+#    define SELSTALE 0x400000 /* select files with ESTALE (stale file handle) */
 
 #    define SELALL                                                             \
         (SELCMD | SELCNTX | SELFD | SELNA | SELNET | SELNM | SELNFS | SELPID | \
-         SELUID | SELUNX | SELZONE | SELTASK)
+         SELUID | SELUNX | SELZONE | SELTASK | SELSTALE)
 #    define SELPROC                                                            \
         (SELCMD | SELCNTX | SELPGID | SELPID | SELUID | SELZONE | SELTASK)
 /* process selecters */
-#    define SELFILE (SELFD | SELNFS | SELNLINK | SELNM) /* file selecters */
+#    define SELFILE (SELFD | SELNFS | SELNLINK | SELNM | SELSTALE) /* file selecters */
 #    define SELNW (SELNA | SELNET | SELUNX)             /* network selecters */
 
 /*
diff --git a/lib/dialects/linux/dproc.c b/lib/dialects/linux/dproc.c
index 8dc7c27..38dc7f8 100644
--- a/lib/dialects/linux/dproc.c
+++ b/lib/dialects/linux/dproc.c
@@ -917,6 +917,8 @@ static int process_id(struct lsof_context *ctx, /* context */
     static int pathil = 0;
     char *rest;
     int txts = 0;
+    int enss_fd = 0;
+    int enls_fd = 0;
 
 #if defined(HASSELINUX)
     cntxlist_t *cntxp;
@@ -1206,6 +1208,7 @@ static int process_id(struct lsof_context *ctx, /* context */
             } else {
                 if (HasNFS) {
                     if (lstatsafely(ctx, path, &lsb)) {
+                        enls_fd = errno;
                         (void)statEx(ctx, pbuf, &lsb, &ls);
                         enls = errno;
                     } else {
@@ -1213,6 +1216,7 @@ static int process_id(struct lsof_context *ctx, /* context */
                         ls = SB_ALL;
                     }
                     if (statsafely(ctx, path, &sb)) {
+                        enss_fd = errno;
                         (void)statEx(ctx, pbuf, &sb, &ss);
                         enss = errno;
                     } else {
@@ -1352,7 +1356,11 @@ static int process_id(struct lsof_context *ctx, /* context */
                                 "[pidfd:%d]", fi.pid);
                     enter_nm(ctx, rest);
                 }
-
+                if ((Selflags & SELSTALE) &&
+                    (enss_fd == ESTALE || enls_fd == ESTALE)) {
+                    Lf->sf |= SELSTALE;
+                    (void)add_nma(ctx, " (STALE)", 8);
+                }
                 if (Lf->sf)
                     link_lfile(ctx);
             }
diff --git a/src/main.c b/src/main.c
index 5d91a6e..ec79d8a 100644
--- a/src/main.c
+++ b/src/main.c
@@ -151,7 +151,7 @@ int main(int argc, char *argv[]) {
      * Create option mask.
      */
     (void)snpf(options, sizeof(options),
-               "?a%sbc:%sD:d:%s%sf:F:g:hHi:%s%slL:%s%snNo:Op:QPr:%ss:S:tT:u:"
+               "?a%sbc:%sD:d:%s%sf:F:g:hHi:%s%slL:%s%snNyo:Op:QPr:%ss:S:tT:u:"
                "UvVwx:%s%s%s",
 
 #if defined(HAS_AFS) && defined(HASAOPT)
@@ -700,6 +700,9 @@ int main(int argc, char *argv[]) {
         case 'N':
             Fnfs = 1;
             break;
+        case 'y':
+            Selflags |= SELSTALE;
+            break;
         case 'o':
             if (!GOv || *GOv == '-' || *GOv == '+') {
                 Foffset = 1;
diff --git a/src/usage.c b/src/usage.c
index 546b3ae..df604a8 100644
--- a/src/usage.c
+++ b/src/usage.c
@@ -299,7 +299,7 @@ void usage(struct lsof_context *ctx, /* context */
         (void)fprintf(stderr, " latest FAQ: %s\n", LSOF_FAQ_URL);
         (void)fprintf(stderr, " latest (non-formatted) man page: %s\n",
                       LSOF_MAN_URL);
-        (void)fprintf(stderr, " usage: [-?ab%shH%slnNoOP%s%stUvV%s]",
+        (void)fprintf(stderr, " usage: [-?ab%shH%slnNoyOP%s%stUvV%s]",
 
 #if defined(HASNCACHE)
                       "C",
@@ -513,6 +513,7 @@ void usage(struct lsof_context *ctx, /* context */
         col = print_in_col(col, "-l list UID numbers");
         col = print_in_col(col, "-n no host names");
         col = print_in_col(col, "-N select NFS files");
+        col = print_in_col(col, "-y select stale file handles (ESTALE)");
         col = print_in_col(col, "-o list file offset");
         col = print_in_col(col, "-O no overhead *RISKY*");
         col = print_in_col(col, "-P no port names");

[jiegec]

Nice! I think it's a good addition, ready for a pull request. There is a small nitpick that, -y makes one hard to relate to ESTALE, do we have a better alternative?

[x-lugoo]

Nice! I think it's a good addition, ready for a pull request. There is a small nitpick that, -y makes one hard to relate to ESTALE, do we have a better alternative?

Based on the current option list, -s/-S would be a better mnemonic,
but they’re already taken. I can’t think of a more suitable short option name right now ;-)

jeffxie@jeffxie:~/vng-share/git/lsof$ ./lsof -h
lsof 4.99.5
latest revision: https://github.com/lsof-org/lsof
latest FAQ: https://github.com/lsof-org/lsof/blob/master/00FAQ
latest (non-formatted) man page: https://github.com/lsof-org/lsof/blob/master/Lsof.8
usage: [-?abhHKlnNoyOPRtUvVX] [+|-c c] [+|-d s] [+D D] [+|-E] [+|-e s] [+|-f[gG]]
[-F [f]] [-g [s]] [-i [i]] [+|-L [l]] [+m [m]] [+|-M] [-o [o]] [-p s]
[+|-r [t]] [-s [p:s]] [-S [t]] [-T [t]] [-u s] [+|-w] [-x [fl]] [--] [names]
Defaults in parentheses; comma-separated set (s) items; dash-separated ranges.
-?|-h list help -a AND selections (OR) -b avoid kernel blocks
-c c cmd c ^c /c/[bix] +c w COMMAND width (9) +d s dir s files
-d s select by FD set +D D dir D tree SLOW? +|-e s exempt s RISKY
-i select IPv[46] files -K [i] list|(i)gn tasKs -l list UID numbers
-n no host names -N select NFS files -y select stale file handles (ESTALE)
-o list file offset -O no overhead RISKY -P no port names
-Q allow failed search -R list paRent PID -s list file size
-t terse listing -T disable TCP/TPI info -U select Unix socket
-v list version info -V verbose search +|-w Warnings (+)
-X skip TCP&UDP* files -Z Z context [Z] -H human readable size
-- end option scan
-E display endpoint info +E display endpoint info and files
+f|-f +filesystem or -file names +|-f[gG] flaGs
-F [f] select fields; -F? for help
+|-L [l] list (+) suppress (-) link counts < l (0 = all; default = 0)
+m [m] use|create mount supplement
+|-M portMap registration (-) -o o o 0t offset digits (8)
-p s exclude(^)|select PIDs -S [t] t second stat timeout (15)
-T fqs TCP/TPI Fl,Q,St (s) info
-g [s] exclude(^)|select and print process group IDs
-i i select by IPv[46] address: [46][proto][@host|addr][:svc_list|port_list]
+|-r [t[m]] repeat every t seconds (15); + until no files, - forever.
An optional suffix to t is m; m must separate t from and
is an strftime(3) format for the marker line.
-s p:s exclude(^)|select protocol (p = TCP|UDP) states by name(s).
-u s exclude(^)|select login|UID set s
-x [fl] cross over +d|+D File systems or symbolic Links
names select named files or files on named file systems
Anyone can list all files; /dev warnings disabled; kernel ID check disabled.