Linux: ls hangs after NFS connection issues

We noticed today that ls was hanging when run on a particular directory. It didn’t seem to matter which options were used. This directory was not any directory but a directory where NFS shares were mounted.

Checking the output of dmesg, I could also see that there were some issues communicating with the other host:

host2:~ # dmesg
...

[1392590.834500] nfs: server host1 not responding, still trying

...

Then I tried running different commands to try and find out what was working and what was hanging:

host2:~ # cd /home/system/mnt/
host2:/home/system/mnt # ls

was hanging.

host2:~ # cd /home/system/mnt/
host2:/home/system/mnt # df -h .
Filesystem            Size  Used Avail Use% Mounted on
host1:/home         18G  9.3G  7.6G  56% /home

was not hanging. We can also see that the problem is not related to a huge or full share.

host2:~ # cd /home/system/mnt/
host2:/home/system/mnt # cat

pressing tab twice to get the completion was hanging

Also using CTRL-C or CTRL-Z was not working but it was possible to kill the session from another shell:

host2:~ # ps -Af | grep ls
root     19980     1  0 11:27 ?        00:00:00 ls -l /home/system/mnt/
root     22230 22196  0 11:47 pts/8    00:00:00 grep ls
host2:~ # kill -9 19980
host2:~ # ps -Af | grep ls
root     22300 22196  0 11:48 pts/8    00:00:00 grep ls

At some point I figured out that a simple was actually working. It was just hanging because ls was an alias and additional arguments were used. After removing the alias, a normal ls worked but an ls -l was hanging:

host2:~ # alias ls
alias ls='ls $LS_OPTIONS'
host2:~ # cd /home/system/mnt/
host2:/home/system/mnt # unalias ls
host2:/home/system/mnt # alias ls
-bash: alias: ls: not found
host2:/home/system/mnt # ls .
db_bck_add  ext_bck  twiddle.log  upload

Calling ls with strace didn’t show any issue when run without any option:

host2:~ # strace ls /home/system/mnt/
execve("/bin/ls", ["ls", "/home/system/mnt/"], [/* 174 vars */]) = 0
brk(0)                                  = 0x618000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ffa000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=63299, ...}) = 0
mmap(NULL, 63299, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ffff7fea000
close(3)                                = 0
open("/lib64/librt.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=47206, ...}) = 0
mmap(NULL, 2132976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7bd5000
fadvise64(3, 0, 2132976, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff7bdd000, 2093056, PROT_NONE) = 0
mmap(0x7ffff7ddc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7ffff7ddc000
close(3)                                = 0
open("/lib64/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@X\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=113904, ...}) = 0
mmap(NULL, 2213528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff79b8000
fadvise64(3, 0, 2213528, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff79d3000, 2093056, PROT_NONE) = 0
mmap(0x7ffff7bd2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7ffff7bd2000
mmap(0x7ffff7bd4000, 1688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff7bd4000
close(3)                                = 0
open("/lib64/libacl.so.1", O_RDONLY)    = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\37\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=31464, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe9000
mmap(NULL, 2126448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff77b0000
fadvise64(3, 0, 2126448, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff77b7000, 2093056, PROT_NONE) = 0
mmap(0x7ffff79b6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ffff79b6000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\354\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1661454, ...}) = 0
mmap(NULL, 3528776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7452000
fadvise64(3, 0, 3528776, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff75a6000, 2097152, PROT_NONE) = 0
mmap(0x7ffff77a6000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x154000) = 0x7ffff77a6000
mmap(0x7ffff77ab000, 18504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff77ab000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=135646, ...}) = 0
mmap(NULL, 2212736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7235000
fadvise64(3, 0, 2212736, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff724c000, 2097152, PROT_NONE) = 0
mmap(0x7ffff744c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ffff744c000
mmap(0x7ffff744e000, 13184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff744e000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19114, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe8000
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7031000
fadvise64(3, 0, 2109696, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff7033000, 2097152, PROT_NONE) = 0
mmap(0x7ffff7233000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ffff7233000
close(3)                                = 0
open("/lib64/libattr.so.1", O_RDONLY)   = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\24\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18936, ...}) = 0
mmap(NULL, 2113888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff6e2c000
fadvise64(3, 0, 2113888, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff6e30000, 2093056, PROT_NONE) = 0
mmap(0x7ffff702f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ffff702f000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe7000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe5000
arch_prctl(ARCH_SET_FS, 0x7ffff7fe5780) = 0
mprotect(0x7ffff702f000, 4096, PROT_READ) = 0
mprotect(0x7ffff7233000, 4096, PROT_READ) = 0
mprotect(0x7ffff744c000, 4096, PROT_READ) = 0
mprotect(0x7ffff77a6000, 16384, PROT_READ) = 0
mprotect(0x7ffff79b6000, 4096, PROT_READ) = 0
mprotect(0x7ffff7bd2000, 4096, PROT_READ) = 0
mprotect(0x7ffff7ddc000, 4096, PROT_READ) = 0
mprotect(0x616000, 4096, PROT_READ)     = 0
mprotect(0x7ffff7ffc000, 4096, PROT_READ) = 0
munmap(0x7ffff7fea000, 63299)           = 0
set_tid_address(0x7ffff7fe5a50)         = 22807
set_robust_list(0x7ffff7fe5a60, 0x18)   = 0
futex(0x7fffffffd01c, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7fffffffd01c, 0x189 /* FUTEX_??? */, 1, NULL, 7ffff7fe5780) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7ffff723a8b0, [], SA_RESTORER|SA_SIGINFO, 0x7ffff72445d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7ffff723a940, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7ffff72445d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
brk(0)                                  = 0x618000
brk(0x639000)                           = 0x639000
open("/etc/selinux/config", O_RDONLY)   = -1 ENOENT (No such file or directory)
statfs("/selinux", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=3352304, f_bfree=2448097, f_bavail=2277808, f_files=851968, f_ffree=714434, f_fsid={151929686, 1001588022}, f_namelen=255, f_frsize=4096}) = 0
open("/proc/mounts", O_RDONLY)          = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ff9000
read(3, "rootfs / rootfs rw 0 0\nudev /dev"..., 1024) = 1024
read(3, "600,retrans=2,sec=sys,mountaddr="..., 1024) = 1024
read(3, "e 0 0\n/dev/sdg1 /db/dump1 ext3 r"..., 1024) = 222
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7ffff7ff9000, 4096)            = 0
open("/usr/lib/locale/locale-archive", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/locale.alias", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ff9000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2512
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7ffff7ff9000, 4096)            = 0
open("/usr/lib/locale/en_US.UTF-8/LC_CTYPE", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/locale/en_US.utf8/LC_CTYPE", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=256324, ...}) = 0
mmap(NULL, 256324, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ffff7fa6000
close(3)                                = 0
open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=26050, ...}) = 0
mmap(NULL, 26050, PROT_READ, MAP_SHARED, 3, 0) = 0x7ffff7ff3000
close(3)                                = 0
futex(0x7ffff77aaf60, FUTEX_WAKE_PRIVATE, 2147483647) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=50, ws_col=120, ws_xpixel=0, ws_ypixel=0}) = 0
stat("/home/system/mnt/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/home/system/mnt/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
fcntl(3, F_GETFD)                       = 0x1 (flags FD_CLOEXEC)
getdents64(3, /* 6 entries */, 32768)   = 176
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 8), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ff2000
write(1, "db_bck_add  ext_bck  twiddle.log"..., 41db_bck_add  ext_bck  twiddle.log  upload
) = 41
close(1)                                = 0
munmap(0x7ffff7ff2000, 4096)            = 0
close(2)                                = 0
exit_group(0)                           = ?
s

But run with the -l option, it was hanging and I could see which entry was causing this:

host2:~ # strace ls -l /home/system/mnt/
execve("/bin/ls", ["ls", "-l", "/home/system/mnt/"], [/* 174 vars */]) = 0
brk(0)                                  = 0x618000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ffa000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=63299, ...}) = 0
mmap(NULL, 63299, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ffff7fea000
close(3)                                = 0
open("/lib64/librt.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=47206, ...}) = 0
mmap(NULL, 2132976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7bd5000
fadvise64(3, 0, 2132976, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff7bdd000, 2093056, PROT_NONE) = 0
mmap(0x7ffff7ddc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7ffff7ddc000
close(3)                                = 0
open("/lib64/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@X\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=113904, ...}) = 0
mmap(NULL, 2213528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff79b8000
fadvise64(3, 0, 2213528, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff79d3000, 2093056, PROT_NONE) = 0
mmap(0x7ffff7bd2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7ffff7bd2000
mmap(0x7ffff7bd4000, 1688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff7bd4000
close(3)                                = 0
open("/lib64/libacl.so.1", O_RDONLY)    = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\37\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=31464, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe9000
mmap(NULL, 2126448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff77b0000
fadvise64(3, 0, 2126448, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff77b7000, 2093056, PROT_NONE) = 0
mmap(0x7ffff79b6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7ffff79b6000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\354\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1661454, ...}) = 0
mmap(NULL, 3528776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7452000
fadvise64(3, 0, 3528776, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff75a6000, 2097152, PROT_NONE) = 0
mmap(0x7ffff77a6000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x154000) = 0x7ffff77a6000
mmap(0x7ffff77ab000, 18504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff77ab000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=135646, ...}) = 0
mmap(NULL, 2212736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7235000
fadvise64(3, 0, 2212736, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff724c000, 2097152, PROT_NONE) = 0
mmap(0x7ffff744c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ffff744c000
mmap(0x7ffff744e000, 13184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ffff744e000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19114, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe8000
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff7031000
fadvise64(3, 0, 2109696, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff7033000, 2097152, PROT_NONE) = 0
mmap(0x7ffff7233000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ffff7233000
close(3)                                = 0
open("/lib64/libattr.so.1", O_RDONLY)   = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\24\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18936, ...}) = 0
mmap(NULL, 2113888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ffff6e2c000
fadvise64(3, 0, 2113888, POSIX_FADV_WILLNEED) = 0
mprotect(0x7ffff6e30000, 2093056, PROT_NONE) = 0
mmap(0x7ffff702f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7ffff702f000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe7000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7fe5000
arch_prctl(ARCH_SET_FS, 0x7ffff7fe5780) = 0
mprotect(0x7ffff702f000, 4096, PROT_READ) = 0
mprotect(0x7ffff7233000, 4096, PROT_READ) = 0
mprotect(0x7ffff744c000, 4096, PROT_READ) = 0
mprotect(0x7ffff77a6000, 16384, PROT_READ) = 0
mprotect(0x7ffff79b6000, 4096, PROT_READ) = 0
mprotect(0x7ffff7bd2000, 4096, PROT_READ) = 0
mprotect(0x7ffff7ddc000, 4096, PROT_READ) = 0
mprotect(0x616000, 4096, PROT_READ)     = 0
mprotect(0x7ffff7ffc000, 4096, PROT_READ) = 0
munmap(0x7ffff7fea000, 63299)           = 0
set_tid_address(0x7ffff7fe5a50)         = 22977
set_robust_list(0x7ffff7fe5a60, 0x18)   = 0
futex(0x7fffffffd00c, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7fffffffd00c, 0x189 /* FUTEX_??? */, 1, NULL, 7ffff7fe5780) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7ffff723a8b0, [], SA_RESTORER|SA_SIGINFO, 0x7ffff72445d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7ffff723a940, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7ffff72445d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
brk(0)                                  = 0x618000
brk(0x639000)                           = 0x639000
open("/etc/selinux/config", O_RDONLY)   = -1 ENOENT (No such file or directory)
statfs("/selinux", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=3352304, f_bfree=2448097, f_bavail=2277808, f_file                                                                                        s=851968, f_ffree=714434, f_fsid={151929686, 1001588022}, f_namelen=255, f_frsize=4096}) = 0
open("/proc/mounts", O_RDONLY)          = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ff9000
read(3, "rootfs / rootfs rw 0 0\nudev /dev"..., 1024) = 1024
read(3, "600,retrans=2,sec=sys,mountaddr="..., 1024) = 1024
read(3, "e 0 0\n/dev/sdg1 /db/dump1 ext3 r"..., 1024) = 222
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7ffff7ff9000, 4096)            = 0
open("/usr/lib/locale/locale-archive", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/locale.alias", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ffff7ff9000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2512
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7ffff7ff9000, 4096)            = 0
open("/usr/lib/locale/en_US.UTF-8/LC_CTYPE", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/lib/locale/en_US.utf8/LC_CTYPE", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=256324, ...}) = 0
mmap(NULL, 256324, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ffff7fa6000
close(3)                                = 0
open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=26050, ...}) = 0
mmap(NULL, 26050, PROT_READ, MAP_SHARED, 3, 0) = 0x7ffff7ff3000
close(3)                                = 0
futex(0x7ffff77aaf60, FUTEX_WAKE_PRIVATE, 2147483647) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=50, ws_col=120, ws_xpixel=0, ws_ypixel=0}) = 0
lstat("/home/system/mnt/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lgetxattr("/home/system/mnt/", "security.selinux", 0x61e400, 255) = -1 EOPNOTSUPP (Operation not supported)
getxattr("/home/system/mnt/", "system.posix_acl_access", 0x0, 0) = -1 ENODATA (No data available)
getxattr("/home/system/mnt/", "system.posix_acl_default", 0x0, 0) = -1 ENODATA (No data available)
socket(PF_FILE, 0x80801 /* SOCK_??? */, 0) = 3
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
sendto(3, "\2\0\0\0\v\0\0\0\7\0\0\0passwd\0", 19, MSG_NOSIGNAL, NULL, 0) = 19
poll([{fd=3, events=POLLIN|POLLERR|POLLHUP}], 1, 5000) = 1 ([{fd=3, revents=POLLIN|POLLHUP}])
recvmsg(3, {msg_name(0)=NULL, msg_iov(2)=[{"passwd\0", 7}, {"\270O\3\0\0\0\0\0", 8}], msg_controllen=24, {cmsg_len=20, c                                                                                        msg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {4}}, msg_flags=MSG_CMSG_CLOEXEC}, MSG_CMSG_CLOEXEC) = 15
mmap(NULL, 217016, PROT_READ, MAP_SHARED, 4, 0) = 0x7ffff7f71000
close(4)                                = 0
close(3)                                = 0
socket(PF_FILE, 0x80801 /* SOCK_??? */, 0) = 3
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
sendto(3, "\2\0\0\0\f\0\0\0\6\0\0\0group\0", 18, MSG_NOSIGNAL, NULL, 0) = 18
poll([{fd=3, events=POLLIN|POLLERR|POLLHUP}], 1, 5000) = 1 ([{fd=3, revents=POLLIN|POLLHUP}])
recvmsg(3, {msg_name(0)=NULL, msg_iov(2)=[{"group\0", 6}, {"\270O\3\0\0\0\0\0", 8}], msg_controllen=24, {cmsg_len=20, cm                                                                                        sg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {4}}, msg_flags=MSG_CMSG_CLOEXEC}, MSG_CMSG_CLOEXEC) = 14
mmap(NULL, 217016, PROT_READ, MAP_SHARED, 4, 0) = 0x7ffff7f3c000
close(4)                                = 0
close(3)                                = 0
open("/home/system/mnt/", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
fcntl(3, F_GETFD)                       = 0x1 (flags FD_CLOEXEC)
getdents64(3, /* 6 entries */, 32768)   = 176
lstat("/home/system/mnt/db_bck_add",

The directory on which it was hanging is itself an NFS share mounted on the other host. This share had some problem and had to be unmounted and remounted. It looks like this caused the NFS connection to hang for some reason. NFS seems to be an extremely fragile filesystem and if either your network orĀ  the server itself are not 100% stable, you have a problem.

So the summary is that a reboot solves this issue and I at least figured out how to identify which of the directory contents was causing the issue. Unfortunately, not much more… If this happens again and I find out how to further investigate this or whether a restart of the NFS service is enough to solve the issue (without reboot), I’ll update this post.

 

Leave a Reply

Your email address will not be published. Required fields are marked *