Bir OOM Hikayesi ve İş görebilir bir Script
Basic Heapdump Script
#!/bin/bash
_JAVA_HOME="/data/jdk/latest"
_SEARCH_TXT="/my-app/"
_DUMP_DEST="/tmp"
_FILE_NAME="my_app_`hostname -i`_`date '+%Y%m%d_%H%M'`"
_HEALTH_LOG_FILE="/var/log/nginx/access-health-`date +"%Y%m%d"`.log"
_BUCKET_NAME="my-bucket-name"
_BUCKET_DEST="/java-dumps/"
_CPU_THRESHOLD=60
_HEAP_THRESHOLD=0.7
_HEALTH_THRESHOLD=1.5 # LB healtcheck threshold is 2 sec
# Get the current CPU usage (user+system) as a percentage
cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2 + $4}')
# get avg of last 10 /health api response time
health_res_avg=$(tail -10 $_HEALTH_LOG_FILE | sed 's/"//g' | awk '{sum += $(NF-2); count++} END {if (count > 0) printf "%.3f\n", sum / count}')
# Check if CPU usage is greater _CPU_THRESHOLD than & LB nginx health _HEALTH_THRESHOLD
if (( $(echo "$cpu_usage > $_CPU_THRESHOLD " | bc -l) )) || (( $(echo "$health_res_avg > $_HEALTH_THRESHOLD" | bc -l) )); then
echo "Cpu usage: $cpu_usage"
echo "Last 10 health resp. time avg: $health_res_avg"
_PID=`ps -ef | grep java | grep $_SEARCH_TXT | awk '{print $2}'`
# thread dump
x=1; while [ $x -le 3 ]; do $_JAVA_HOME/bin/jstack $_PID >> $_DUMP_DEST/$_FILE_NAME.tdump; sleep 7;(( x++ )); done
in_use_heap=$($_JAVA_HOME/bin/jstat -gc $_PID | tail -1 | awk '{printf "%10.2f",($3+$4+$6+$8)/1024}' | tr -d ' ')
max_heap=$($_JAVA_HOME/bin/jstat -gc $_PID | tail -1 | awk '{printf "%10.2f",($1+$2+$5+$7)/1024}' | tr -d ' ')
# check for non-zero division
if [[ $in_use_heap > 0 ]];then
heap_usage_rate=`awk "BEGIN {print $in_use_heap/$max_heap}"`
echo "Heap Usage Rate: $heap_usage_rate"
# if total use over the threshold
if [[ $heap_usage_rate > $_HEAP_THRESHOLD ]];then
echo "Taking the heap dump... "
$_JAVA_HOME/bin/jcmd $_PID GC.heap_dump $_DUMP_DEST/$_FILE_NAME.hprof
aws s3 cp $_DUMP_DEST/$_FILE_NAME s3://$_BUCKET_NAME$_BUCKET_DEST
rm $_DUMP_DEST/$_FILE_NAME
fi
fi
fi