Javaパフォーマンス問題を徹底解析｜ヒープダンプ・スレッドダンプ・コアダンプの取得と解析方法

[root@quiz ~]# kill -3 274968
[root@quiz ~]# tail -1000 /opt/tomcat/logs/catalina.out
~~Truncated~~
2025-12-02 06:29:22
Full thread dump OpenJDK 64-Bit Server VM (21.0.2+13-58 mixed mode, sharing):
Threads class SMR info:
_java_thread_list=0x00007f366c002460, length=31, elements={
0x00007f371c02ce30, 0x00007f371c0ecff0, 0x00007f371c0ee650, 0x00007f371c0f0140,
0x00007f371c0f1790, 0x00007f371c0f2d40, 0x00007f371c0f4890, 0x00007f371c0f5f60,
0x00007f371c11f7b0, 0x00007f371c17e380, 0x00007f371c1f9610, 0x00007f371c2c3210,
0x00007f371c2cda70, 0x00007f371c2d5d30, 0x00007f371c568c80, 0x00007f371db6a490,
0x00007f371db63890, 0x00007f371e9bfce0, 0x00007f3648001200, 0x00007f371c9da6a0,
0x00007f371c9dba80, 0x00007f371c9dcd20, 0x00007f371c9de3b0, 0x00007f371ca00e60,
0x00007f371ca02000, 0x00007f371ca036b0, 0x00007f371ca04d60, 0x00007f371ca06330,
0x00007f371ca07900, 0x00007f371c9f9b70, 0x00007f371c9850b0
}
"main" #1 [274969] prio=5 os_prio=0 cpu=12421.73ms elapsed=108.91s tid=0x00007f371c02ce30 nid=274969 runnable  [0x00007f3723710000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.Net.accept(java.base@21.0.2/Native Method)
        at sun.nio.ch.NioSocketImpl.accept(java.base@21.0.2/NioSocketImpl.java:748)
        at java.net.ServerSocket.implAccept(java.base@21.0.2/ServerSocket.java:698)
        at java.net.ServerSocket.platformImplAccept(java.base@21.0.2/ServerSocket.java:663)
        at java.net.ServerSocket.implAccept(java.base@21.0.2/ServerSocket.java:639)
        at java.net.ServerSocket.implAccept(java.base@21.0.2/ServerSocket.java:585)
        at java.net.ServerSocket.accept(java.base@21.0.2/ServerSocket.java:543)
        at org.apache.catalina.core.StandardServer.await(StandardServer.java:557)
        at org.apache.catalina.startup.Catalina.await(Catalina.java:851)
        at org.apache.catalina.startup.Catalina.start(Catalina.java:799)
~~Truncated~~
"G1 Conc#0" os_prio=0 cpu=203.73ms elapsed=108.91s tid=0x00007f371c06e090 nid=274972 runnable
"G1 Refine#0" os_prio=0 cpu=136.13ms elapsed=108.90s tid=0x00007f371c0b08e0 nid=274973 runnable
"G1 Service" os_prio=0 cpu=7.07ms elapsed=108.90s tid=0x00007f371c0b18a0 nid=274974 runnable
"VM Periodic Task Thread" os_prio=0 cpu=96.28ms elapsed=108.88s tid=0x00007f371c0c2930 nid=274975 waiting on condition
JNI global refs: 21, weak refs: 0
Heap
 garbage-first heap   total 98304K, used 76135K [0x00000000c6e00000, 0x0000000100000000)
  region size 1024K, 45 young (46080K), 7 survivors (7168K)
 Metaspace       used 73266K, committed 74048K, reserved 1114112K
  class space    used 9706K, committed 10048K, reserved 1048576K

[root@quiz ~]# jstack -l 1022886 > /tmp/threaddump_$(date +%Y%m%d_%H%M%S).txt
[root@quiz ~]#

2025-12-02 06:42:26
Full thread dump OpenJDK 64-Bit Server VM (21.0.2+13-58 mixed mode, sharing):
~~Truncated~~
Found one Java-level deadlock:
=============================
"Thread-0":
  waiting to lock monitor 0x00007fa568002d90 (object 0x00000000ca711b58, a java.lang.Object),
  which is held by "Thread-1"
"Thread-1":
  waiting to lock monitor 0x00007fa55c001160 (object 0x00000000ca711b48, a java.lang.Object),
  which is held by "Thread-0"
Java stack information for the threads listed above:
===================================================
"Thread-0":
        at DeadlockSample.lambda$main$0(DeadlockSample.java:16)
        - waiting to lock <0x00000000ca711b58> (a java.lang.Object)
        - locked <0x00000000ca711b48> (a java.lang.Object)
        at DeadlockSample$$Lambda/0x00007fa598000a00.run(Unknown Source)
        at java.lang.Thread.runWith(java.base@21.0.2/Thread.java:1596)
        at java.lang.Thread.run(java.base@21.0.2/Thread.java:1583)
"Thread-1":
        at DeadlockSample.lambda$main$1(DeadlockSample.java:29)
        - waiting to lock <0x00000000ca711b48> (a java.lang.Object)
        - locked <0x00000000ca711b58> (a java.lang.Object)
        at DeadlockSample$$Lambda/0x00007fa598000c18.run(Unknown Source)
        at java.lang.Thread.runWith(java.base@21.0.2/Thread.java:1596)
        at java.lang.Thread.run(java.base@21.0.2/Thread.java:1583)
Found 1 deadlock.

[root@quiz ~]# jcmd 311127 GC.heap_dump /tmp/heapdump_$(date +%Y%m%d_%H%M%S).hprof
311127:
Dumping heap to /tmp/heapdump_20251203_161828.hprof ...
Heap dump file created [67250434 bytes in 0.793 secs]
[root@quiz ~]#

[root@quiz ~]# cat /opt/tomcat/bin/setenv.sh
export JAVA_HOME=/opt/java/jdk-21.0.2
export JRE_HOME=$JAVA_HOME
export PATH=$JAVA_HOME/bin:$PATH
# heap dump setting
CATALINA_OPTS="$CATALINA_OPTS \
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/heapdump_$(date +%Y%m%d_%H%M%S).hprof"
[root@quiz ~]#
[root@quiz ~]# systemctl restart tomcat
[root@quiz ~]#

[root@quiz ~]# jmap -dump:format=b,file=/tmp/heapdump_$(date +%Y%m%d_%H%M%S).hprof 996008
Dumping heap to /tmp/heapdump_20251112_063454.hprof ...
Heap dump file created [126793933 bytes in 1.171 secs]
[root@quiz ~]#

[root@quiz ~]# gcore -o /var/tomcat 311127
[New LWP 311252]
[New LWP 311188]
~~Truncated~~
[New LWP 311129]
[New LWP 311128]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
0x00007f1d8788717a in __futex_abstimed_wait_common () from /lib64/libc.so.6
warning: Memory read failed for corefile section, 4096 bytes at 0xffffffffff600000.
Saved corefile /var/tomcat.311127
[Inferior 1 (process 311127) detached]
[root@quiz ~]#

[root@quiz ~]# vi /etc/systemd/system/tomcat.service
[root@quiz ~]# cat /etc/systemd/system/tomcat.service
[Unit]
Description=Apache Tomcat 10
After=network.target
~~Truncated~~
##Core Dump Setting
LimitCORE=infinity
~~Truncated~~
[root@quiz ~]#
[root@quiz ~]# systemctl daemon-reload
[root@quiz ~]# systemctl restart tomcat
[root@quiz ~]#

[root@quiz ~]# kill -SEGV 300774
[root@quiz ~]#
[root@quiz ~]# coredumpctl list
No coredumps found.
-- Notice: 1 systemd-coredump@.service unit is running, output may be incomplete.
[root@quiz ~]# coredumpctl list
TIME                           PID UID GID SIG     COREFILE EXE                             SIZE
Wed 2025-12-03 06:56:52 JST 300774 996 993 SIGABRT present  /opt/java/jdk-21.0.2/bin/java 107.7M
[root@quiz ~]#

[root@quiz ~]# coredumpctl dump 300774 > /tmp/core-300774
           PID: 300774 (java)
           UID: 996 (tomcat)
           GID: 993 (tomcat)
        Signal: 6 (ABRT)
     Timestamp: Wed 2025-12-03 06:56:44 JST (1min 58s ago)
  Command Line: /opt/java/jdk-21.0.2/bin/java -Djava.util.logging.config.file=/opt/tomcat/conf/logging.properties -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Djdk.tls.ephemeralDHKeySize=2048 -Djava.protocol.handler.pkgs=org.apache.catalina.webresources -Dsun.io.useCanonCaches=false -Dorg.apache.catalina.security.SecurityListener.UMASK=0027 --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.rmi.port=9011 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.access.file=/opt/tomcat/conf/jmxremote.access -Dcom.sun.management.jmxremote.password.file=/opt/tomcat/conf/jmxremote.password -Dcom.sun.management.jmxremote.registry.ssl=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=quiz.eeengineer.com $'-Xlog:gc*,safepoint:file=/opt/tomcat/logs/gc.log:time,level,tags:filecount=5,filesize=10M' -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/heapdump_20251203_064958.hprof -classpath /opt/tomcat/bin/bootstrap.jar:/opt/tomcat/bin/tomcat-juli.jar -Dcatalina.base=/opt/tomcat -Dcatalina.home=/opt/tomcat -Djava.io.tmpdir=/opt/tomcat/temp org.apache.catalina.startup.Bootstrap start
    Executable: /opt/java/jdk-21.0.2/bin/java
 Control Group: /system.slice/tomcat.service
          Unit: tomcat.service
         Slice: system.slice
       Boot ID: f945e8fb8a004dc0ba4b133ac7c2252c
    Machine ID: 49a9003f8f0e4a25a5bf4d147e21e0d4
      Hostname: quiz.eeengineer.com
       Storage: /var/lib/systemd/coredump/core.java.996.f945e8fb8a004dc0ba4b133ac7c2252c.300774.1764712604000000.zst (present)
  Size on Disk: 107.7M
       Message: Process 300774 (java) of user 996 dumped core.
                Module libverify.so without build-id.
                Module libextnet.so without build-id.
                Module libmanagement_ext.so without build-id.
                Module libmanagement.so without build-id.
                
 ~~Truncated~~
　　　　　　　   #16 0x00007fdedc029680 __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x29680)
                #17 0x0000564ffc718b75 _start (java + 0xb75)
                ELF object binary architecture: AMD x86-64
[root@quiz ~]#
[root@quiz ~]# ll /tmp/core-300774
-rw-r--r-- 1 root root 585781248 Dec  3 06:58 /tmp/core-300774
[root@quiz ~]#

[root@quiz ~]# gdb /opt/java/jdk-21.0.2/bin/java /var/tomcat.311127
GNU gdb (AlmaLinux) 16.3-2.el9
Copyright (C) 2024 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
~~Truncated~~
[New LWP 311129]
[New LWP 311128]
This GDB supports auto-downloading debuginfo from the following URLs:
  <%{dist_debuginfod_url}>
Enable debuginfod for this session? (y or [n])
Debuginfod has been disabled.
--Type <RET> for more, q to quit, c to continue without paging--
To make this setting permanent, add 'set debuginfod enabled off' to .gdbinit.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/opt/java/jdk-21.0.2/bin/java -Djava.util.logging.config.file=/opt/tomcat/conf/logging.properties -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Djdk.tls.ephemeralDHKeySize=2048 -Djava.protocol.handler.pkgs=org.apache.catalina.webresources -Dsun.io.useCanonCaches=false -Dorg.apache.catalina.security.SecurityListener.UMASK=0027 --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.rmi.port=9011 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.access.file=/opt/tomcat/conf/jmxremote.access -Dcom.sun.management.jmxremote.password.file=/opt/tomcat/conf/jmxremote.password -Dcom.sun.management.jmxremote.registry.ssl=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=quiz.eeengineer.com -Xlog:gc\*,safepoint:file=/opt/tomcat/logs/gc.log:time,level,tags:filecount=5,filesize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/heapdump_20251203_161446.hprof -classpath /opt/tomcat/bin/bootstrap.jar:/opt/tomcat/bin/tomcat-juli.jar -Dcatalina.base=/opt/tomcat -Dcatalina.home=/opt/tomcat -Djava.io.tmpdir=/opt/tomcat/temp org.apache.catalina.startup.Bootstrap start'.
#0  0x00007f1d8788717a in __futex_abstimed_wait_common () from /lib64/libc.so.6
[Current thread is 1 (Thread 0x7f1d87ae4200 (LWP 311127))]
Missing rpms, try: dnf --enablerepo='*debug*' install zlib-debuginfo-1.2.11-40.el9.x86_64 glibc-debuginfo-2.34-168.el9_6.14.alma.1.x86_64 sssd-client-debuginfo-2.9.6-4.el9_6.2.x86_64
(gdb)

(gdb) info thread
  Id   Target Id                          Frame
* 1    Thread 0x7f1d87ae4200 (LWP 311127) 0x00007f1d8788717a in __futex_abstimed_wait_common () from /lib64/libc.so.6
  2    Thread 0x7f1d599f9640 (LWP 311252) 0x00007f1d8790f91f in accept () from 
  ~~Truncated~~
　42   Thread 0x7f1d8494b640 (LWP 311129) 0x00007f1d8788717a in __futex_abstimed_wait_common () from /lib64/libc.so.6
  43   Thread 0x7f1d861ff640 (LWP 311128) 0x00007f1d8790f91f in accept () from /lib64/libc.so.6
(gdb)
(gdb) bt
#0  0x00007f1d8788717a in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f1d8788bbf4 in __pthread_clockjoin_ex () from /lib64/libc.so.6
#2  0x00007f1d87af7a2f in CallJavaMainInNewThread () from /opt/java/jdk-21.0.2/bin/../lib/libjli.so
#3  0x00007f1d87af4bbd in ContinueInNewThread () from /opt/java/jdk-21.0.2/bin/../lib/libjli.so
#4  0x00007f1d87af560d in JLI_Launch () from /opt/java/jdk-21.0.2/bin/../lib/libjli.so
#5  0x0000556d450f7adf in main ()
(gdb)

import java.util.ArrayList;
import java.util.List;
public class MemoryLeakDemo {
    private static final List<Object> leakList = new ArrayList<>();
    public static void main(String[] args) {
        System.out.println("Starting MemoryLeakDemo...");
        try {
            while (true) {
                // 1MBのbyte配列を生成してリストに追加
                byte[] data = new byte[1024 * 1024];
                leakList.add(data);
                // 現在のリストサイズを出力
                System.out.println("Objects in list: " + leakList.size());
                // 少し待つ（CPU負荷軽減）
                Thread.sleep(100);
            }
        } catch (OutOfMemoryError e) {
            System.err.println("OutOfMemoryError発生！");
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

[root@quiz tmp]# vi MemoryLeakDemo.java
[root@quiz tmp]# cat MemoryLeakDemo.java
import java.util.ArrayList;
import java.util.List;
public class MemoryLeakDemo {
    private static final List<Object> leakList = new ArrayList<>();
    public static void main(String[] args) {
        System.out.println("Starting MemoryLeakDemo...");
        try {
            while (true) {
                byte[] data = new byte[1024 * 1024];
                leakList.add(data);
                System.out.println("Objects in list: " + leakList.size());
                Thread.sleep(100);
            }
        } catch (OutOfMemoryError e) {
            System.err.println("OutOfMemoryError発生！");
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
[root@quiz tmp]#
[root@quiz tmp]# javac MemoryLeakDemo.java
[root@quiz tmp]# java MemoryLeakDemo
Starting MemoryLeakDemo...
Objects in list: 1
Objects in list: 2
Objects in list: 3
Objects in list: 4
Objects in list: 5

[root@quiz ~]# cd /tmp/
[root@quiz tmp]# vi /tmp/DeadlockSample.java
[root@quiz tmp]# cat /tmp/DeadlockSample.java
/**
 * This program intentionally causes a deadlock between two threads.
 * It is useful for testing thread dumps, heap dumps, and monitoring tools.
 */
public class DeadlockSample {
    private static final Object LOCK_A = new Object();
    private static final Object LOCK_B = new Object();
    public static void main(String[] args) {
        // Thread 1 tries to lock A then lock B
        Thread t1 = new Thread(() -> {
            synchronized (LOCK_A) {
                System.out.println("Thread-1: Acquired LOCK_A");
                sleep(100); // small delay to increase the chance of deadlock
                System.out.println("Thread-1: Waiting for LOCK_B...");
                synchronized (LOCK_B) {
                    System.out.println("Thread-1: Acquired LOCK_B");
                }
            }
        });
        // Thread 2 tries to lock B then lock A
        Thread t2 = new Thread(() -> {
            synchronized (LOCK_B) {
                System.out.println("Thread-2: Acquired LOCK_B");
                sleep(100); // small delay to increase the chance of deadlock
                System.out.println("Thread-2: Waiting for LOCK_A...");
                synchronized (LOCK_A) {
                    System.out.println("Thread-2: Acquired LOCK_A");
                }
            }
        });
        // Start both threads
        t1.start();
        t2.start();
    }
    /**
     * Helper method to sleep without throwing checked exceptions.
     */
    private static void sleep(long ms) {
        try {
            Thread.sleep(ms);
        } catch (InterruptedException e) {
            // Ignore interruptions for this test program
        }
    }
}
[root@quiz tmp]#
[root@quiz tmp]# javac DeadlockSample.java
[root@quiz tmp]# java DeadlockSample
Thread-1: Acquired LOCK_A
Thread-2: Acquired LOCK_B
Thread-1: Waiting for LOCK_B...
Thread-2: Waiting for LOCK_A...