0 Replies Latest reply: May 22, 2012 8:21 PM by lsgroup RSS

    Java 7 (oracle jdk 1.7) on CentOS 6.2 freezes hangs FUTEX_WAIT

    lsgroup
      Hello,
      Recently have had problems with Java 1.7 this combination:
      CentOS 6.2
      linux kernel 2.6.32.57 (vanilla kernel)
      GRSecurity grsecurity-2.9-2.6.32.57-201202232125.patch

      When trying to run jdk1.7.0_04, java hangs with no error messages in /var/log/messages and cannot be killed leaving CPU at 100% forever, OS unable to shutdown. Only solution is to pull the power plug!

      top output ...
      ================================================================
      top - 10:12:40 up 5:05, 2 users, load average: 42.93, 42.56, 41.41
      Tasks: 190 total, 5 running, 184 sleeping, 0 stopped, 1 zombie
      Cpu(s): 0.0%us, 22.4%sy, 0.0%ni, 33.1%id, 44.5%wa, 0.0%hi, 0.0%si, 0.0%st
      Mem: 8139472k total, 1675248k used, 6464224k free, 24488k buffers
      Swap: 10289144k total, 0k used, 10289144k free, 1327228k cached

      PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
      1753 testu 20 0 0 0 0 Z 101.2 0.0 274:19.18 java <defunct>
      ================================================================

      When I boot with the CentOS 6.2 kernel, java 1.7 runs fine.

      This same instance runs jdk 1.6 and jrockit 1.6 just fine

      ================================================================
      # ./java -version
      java version "1.6.0_31"
      Java(TM) SE Runtime Environment (build 1.6.0_31-b05)
      Oracle JRockit(R) (build R28.2.3-13-149708-1.6.0_31-20120327-1523-linux-x86_64, compiled mode)

      (jdk-6u25-linux-x64.bin)
      # ./java -version
      java version "1.6.0_25"
      Java(TM) SE Runtime Environment (build 1.6.0_25-b06)
      Java HotSpot(TM) 64-Bit Server VM (build 20.0-b11, mixed mode)
      ================================================================

      JDK 1.6 is fine for now, but sooner or later, the server will need to run JDK 1.7. Wondering if anyone has come across this or if someone might have an idea what would get jdk 1.7 working. Or what other info I might supply to try to figure this out.

      Thanks!






      Not sure if it helps, but I straced the command...

      ================================================================
      # strace ./java -verbose -version
      execve("./java", ["./java", "-verbose", "-version"], [* 22 vars */]) = 0
      brk(0) = 0x600b18
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7044000
      readlink("/proc/self/exe", "/root/store/jdk1.7.0_04/bin/java", 4096) = 32
      access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/tls/x86_64", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/tls", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/x86_64", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../lib/amd64/jli", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/tls/x86_64", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/tls", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/x86_64", 0x7fffffffcd70) = -1 ENOENT (No such file or directory)
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
      stat("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
      open("/etc/ld.so.cache", O_RDONLY) = 3
      fstat(3, {st_mode=S_IFREG|0644, st_size=57891, ...}) = 0
      mmap(NULL, 57891, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0bb7035000
      close(3) = 0
      open("/lib64/libpthread.so.0", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \\\0\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=138328, ...}) = 0
      mmap(NULL, 2208672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb6c0b000
      mprotect(0x7f0bb6c22000, 2093056, PROT_NONE) = 0
      mmap(0x7f0bb6e21000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f0bb6e21000
      mmap(0x7f0bb6e23000, 13216, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0bb6e23000
      close(3) = 0
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/libjli.so", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\"\0\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=103718, ...}) = 0
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7034000
      mmap(NULL, 2187192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb69f5000
      mprotect(0x7f0bb6a0b000, 2093056, PROT_NONE) = 0
      mmap(0x7f0bb6c0a000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f0bb6c0a000
      close(3) = 0
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
      open("/lib64/libdl.so.2", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\r\0\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=19536, ...}) = 0
      mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb67f1000
      mprotect(0x7f0bb67f3000, 2097152, PROT_NONE) = 0
      mmap(0x7f0bb69f3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f0bb69f3000
      close(3) = 0
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
      open("/lib64/libc.so.6", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\355\1\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=1973416, ...}) = 0
      mmap(NULL, 3803304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb6450000
      mprotect(0x7f0bb65e7000, 2097152, PROT_NONE) = 0
      mmap(0x7f0bb67e7000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x197000) = 0x7f0bb67e7000
      mmap(0x7f0bb67ec000, 18600, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0bb67ec000
      close(3) = 0
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7033000
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7032000
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7031000
      arch_prctl(ARCH_SET_FS, 0x7f0bb7032700) = 0
      mprotect(0x7f0bb67e7000, 16384, PROT_READ) = 0
      mprotect(0x7f0bb69f3000, 4096, PROT_READ) = 0
      mprotect(0x7f0bb6e21000, 4096, PROT_READ) = 0
      mprotect(0x7f0bb7046000, 4096, PROT_READ) = 0
      munmap(0x7f0bb7035000, 57891) = 0
      set_tid_address(0x7f0bb70329d0) = 1688
      set_robust_list(0x7f0bb70329e0, 0x18) = 0
      futex(0x7fffffffd67c, FUTEX_WAKE_PRIVATE, 1) = 0
      futex(0x7fffffffd67c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f0bb7032700) = -1 EAGAIN (Resource temporarily unavailable)
      rt_sigaction(SIGRTMIN, {0x7f0bb6c10aa0, [], SA_RESTORER|SA_SIGINFO, 0x7f0bb6c1a4a0}, NULL, 8) = 0
      rt_sigaction(SIGRT_1, {0x7f0bb6c10b30, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f0bb6c1a4a0}, NULL, 8) = 0
      rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
      getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
      brk(0) = 0x600b18
      brk(0x621b18) = 0x621b18
      brk(0x622000) = 0x622000
      readlink("/proc/self/exe", "/root/store/jdk1.7.0_04/bin/java", 4096) = 32
      access("/root/store/jdk1.7.0_04/lib/amd64/libjava.so", F_OK) = -1 ENOENT (No such file or directory)
      access("/root/store/jdk1.7.0_04/jre/lib/amd64/libjava.so", F_OK) = 0
      open("/root/store/jdk1.7.0_04/jre/lib/amd64/jvm.cfg", O_RDONLY) = 3
      fstat(3, {st_mode=S_IFREG|0644, st_size=677, ...}) = 0
      mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0bb7043000
      read(3, "# Copyright (c) 2003, Oracle and"..., 4096) = 677
      read(3, "", 4096) = 0
      close(3) = 0
      munmap(0x7f0bb7043000, 4096) = 0
      stat("/root/store/jdk1.7.0_04/jre/lib/amd64/server/libjvm.so", {st_mode=S_IFREG|0755, st_size=13498274, ...}) = 0
      futex(0x7f0bb69f40ec, FUTEX_WAKE_PRIVATE, 2147483647) = 0
      open("/root/store/jdk1.7.0_04/jre/lib/amd64/server/libjvm.so", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\223\31\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=13498274, ...}) = 0
      mmap(NULL, 13757672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb5731000
      mprotect(0x7f0bb6172000, 2093056, PROT_NONE) = 0
      mmap(0x7f0bb6371000, 671744, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa40000) = 0x7f0bb6371000
      mmap(0x7f0bb6415000, 240872, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0bb6415000
      close(3) = 0
      open("/root/store/jdk1.7.0_04/bin/../jre/lib/amd64/jli/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
      open("/etc/ld.so.cache", O_RDONLY) = 3
      fstat(3, {st_mode=S_IFREG|0644, st_size=57891, ...}) = 0
      mmap(NULL, 57891, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0bb7035000
      close(3) = 0
      open("/lib64/libm.so.6", O_RDONLY) = 3
      read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240>\0\0\0\0\0\0"..., 832) = 832
      fstat(3, {st_mode=S_IFREG|0755, st_size=595800, ...}) = 0
      mmap(NULL, 2633944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0bb54ad000
      mprotect(0x7f0bb5530000, 2093056, PROT_NONE) = 0
      mmap(0x7f0bb572f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x82000) = 0x7f0bb572f000
      close(3) = 0
      mprotect(0x7f0bb572f000, 4096, PROT_READ) = 0
      munmap(0x7f0bb7035000, 57891) = 0
      mmap(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f0bb6f30000
      mprotect(0x7f0bb6f30000, 4096, PROT_NONE) = 0
      clone(child_stack=0x7f0bb702fff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f0bb70309d0, tls=0x7f0bb7030700, child_tidptr=0x7f0bb70309d0) = 1689
      futex(0x7f0bb70309d0, FUTEX_WAIT, 1689, NULL
      ================================================================