脚本

show-busy-java-threads

使用方法

show-busy-java-threads
# 从所有运行的Java进程中找出最消耗CPU的线程（缺省5个），打印出其线程栈

# 缺省会自动从所有的Java进程中找出最消耗CPU的线程，这样用更方便
# 当然你可以通过 -p 选项 手动指定要分析的Java进程Id，以保证只会显示你关心的那个Java进程的信息
show-busy-java-threads -p <指定的Java进程Id>
show-busy-java-threads -p 42
show-busy-java-threads -p 42,47

show-busy-java-threads -c <要展示示的线程栈个数>

show-busy-java-threads <重复执行的间隔秒数> [<重复执行的次数>]
# 多次执行；这2个参数的使用方式类似vmstat命令

show-busy-java-threads -a <运行输出的记录到的文件>
# 记录到文件以方便回溯查看

show-busy-java-threads -S <存储jstack输出文件的目录>
# 指定jstack输出文件的存储目录，方便记录以后续分析

##############################
# 注意：
##############################
# 如果Java进程的用户 与 执行脚本的当前用户 不同，则jstack不了这个Java进程
# 为了能切换到Java进程的用户，需要加sudo来执行，即可以解决：
sudo show-busy-java-threads

show-busy-java-threads -s <指定jstack命令的全路径>
# 对于sudo方式的运行，JAVA_HOME环境变量不能传递给root，
# 而root用户往往没有配置JAVA_HOME且不方便配置，不能找到jstack命令。
# 这时显式指定jstack命令的路径就反而显得更方便了

# -m 选项：执行jstack命令时加上 -m 选项，显示上Native的栈帧，一般应用排查不需要使用
show-busy-java-threads -m
# -F 选项：执行jstack命令时加上 -F 选项（如果直接jstack无响应时，用于强制jstack），一般情况不需要使用
show-busy-java-threads -F
# -l 选项：执行jstack命令时加上 -l 选项，显示上更多相关锁的信息，一般情况不需要使用
# 注意：和 -m -F 选项一起使用时，可能会大大增加jstack操作的耗时
show-busy-java-threads -l

# 帮助信息
$ show-busy-java-threads -h
Usage: show-busy-java-threads [OPTION]... [delay [count]]
Find out the highest cpu consumed threads of java processes,
and print the stack of these threads.

Example:
  show-busy-java-threads       # show busy java threads info
  show-busy-java-threads 1     # update every 1 second, (stop by eg: CTRL+C)
  show-busy-java-threads 3 10  # update every 3 seconds, update 10 times

Output control:
  -p, --pid <java pid(s)>   find out the highest cpu consumed threads from
                            the specified java process.
                            support pid list(eg: 42,47).
                            default from all java process.
  -c, --count <num>         set the thread count to show, default is 5.
                            set count 0 to show all threads.
  -a, --append-file <file>  specifies the file to append output as log.
  -S, --store-dir <dir>     specifies the directory for storing
                            the intermediate files, and keep files.
                            default store intermediate files at tmp dir,
                            and auto remove after run. use this option to keep
                            files so as to review jstack/top/ps output later.
  delay                     the delay between updates in seconds.
  count                     the number of updates.
                            delay/count arguments imitates the style of
                            vmstat command.

jstack control:
  -s, --jstack-path <path>  specifies the path of jstack command.
  -F, --force               set jstack to force a thread dump.
                            use when jstack does not respond (process is hung).
  -m, --mix-native-frames   set jstack to print both java and
                            native frames (mixed mode).
  -l, --lock-info           set jstack with long listing.
                            prints additional information about locks.

CPU usage calculation control:
  -i, --cpu-sample-interval specifies the delay between cpu samples to get
                            thread cpu usage percentage during this interval.
                            default is 0.5 (second).
                            set interval 0 to get the percentage of time spent
                            running during the *entire lifetime* of a process.

Miscellaneous:
  -h, --help                display this help and exit.
  -V, --version             display version information and exit.

脚本代码

#!/usr/bin/env bash

readonly PROG=${0##*/}
readonly PROG_VERSION='3.x-dev'

readonly -a COMMAND_LINE=("${BASH_SOURCE[0]}" "$@")

WHOAMI=$(whoami)
UNAME=$(uname)
readonly WHOAMI UNAME

readonly NL=$'\n' # new line

colorPrint() {
  local color=$1
  shift

  if [ -t 1 ]; then
    printf '\e[1;%sm%s\e[0m\n' "$color" "$*"
  else
    printf '%s\n' "$*"
  fi
}

__appendToFile() {
  [[ -n "$append_file" && -w "$append_file" ]] && printf '%s\n' "$*" >>"$append_file"
  [[ -n "$store_dir" && -w "$store_dir" ]] && printf '%s\n' "$*" >>"$store_file_prefix$PROG"
}

colorOutput() {
  local color=$1
  shift

  colorPrint "$color" "$*"
  __appendToFile "$*"
}

# shellcheck disable=SC2120
normalOutput() {
  printf '%s\n' "$*"
  __appendToFile "$*"
}

redOutput() {
  colorOutput 31 "$*"
}

greenOutput() {
  colorOutput 32 "$*"
}

yellowOutput() {
  colorOutput 33 "$*"
}

blueOutput() {
  colorOutput 36 "$*"
}

die() {
  local prompt_help=false exit_status=2
  while (($# > 0)); do
    case "$1" in
    -h)
      prompt_help=true
      shift
      ;;
    -s)
      exit_status=$2
      shift 2
      ;;
    *)
      break
      ;;
    esac
  done

  (($# > 0)) && colorPrint "1;31" "$PROG: $*"
  $prompt_help && echo "Try '$PROG --help' for more information."

  exit "$exit_status"
} >&2

logAndRun() {
  printf '%s\n' "$*"
  echo
  "$@"
}

logAndCat() {
  printf '%s\n' "$*"
  echo
  cat
}

isNonNegativeFloatNumber() {
  [[ "$1" =~ ^[+]?[0-9]+\.?[0-9]*$ ]]
}

isNaturalNumber() {
  [[ "$1" =~ ^[+]?[0-9]+$ ]]
}

isNaturalNumberList() {
  [[ "$1" =~ ^([0-9]+)(,[0-9]+)*$ ]]
}

printCallingCommandLine() {
  local arg isFirst=true
  for arg in "${COMMAND_LINE[@]}"; do
    if $isFirst; then
      isFirst=false
    else
      printf ' '
    fi
    printf '%q' "$arg"
  done
  echo
}

usage() {
  cat <<EOF
Usage: $PROG [OPTION]... [delay [count]]
Find out the highest cpu consumed threads of java processes,
and print the stack of these threads.

Example:
  $PROG       # show busy java threads info
  $PROG 1     # update every 1 second, (stop by eg: CTRL+C)
  $PROG 3 10  # update every 3 seconds, update 10 times

Output control:
  -p, --pid <java pid(s)>   find out the highest cpu consumed threads from
                            the specified java process.
                            support pid list(eg: 42,47).
                            default from all java process.
  -c, --count <num>         set the thread count to show, default is 5.
                            set count 0 to show all threads.
  -a, --append-file <file>  specifies the file to append output as log.
  -S, --store-dir <dir>     specifies the directory for storing
                            the intermediate files, and keep files.
                            default store intermediate files at tmp dir,
                            and auto remove after run. use this option to keep
                            files so as to review jstack/top/ps output later.
  delay                     the delay between updates in seconds.
  count                     the number of updates.
                            delay/count arguments imitates the style of
                            vmstat command.

jstack control:
  -s, --jstack-path <path>  specifies the path of jstack command.
  -F, --force               set jstack to force a thread dump.
                            use when jstack does not respond (process is hung).
  -m, --mix-native-frames   set jstack to print both java and
                            native frames (mixed mode).
  -l, --lock-info           set jstack with long listing.
                            prints additional information about locks.

CPU usage calculation control:
  -i, --cpu-sample-interval specifies the delay between cpu samples to get
                            thread cpu usage percentage during this interval.
                            default is 0.5 (second).
                            set interval 0 to get the percentage of time spent
                            running during the *entire lifetime* of a process.

Miscellaneous:
  -h, --help                display this help and exit.
  -V, --version             display version information and exit.
EOF

  exit
}

progVersion() {
  printf '%s\n' "$PROG $PROG_VERSION"
  exit
}

################################################################################
# check os support
################################################################################

[[ $UNAME = Linux* ]] || die "only support Linux, not support $UNAME yet!"

################################################################################
# parse options
################################################################################

# DO NOT declare and assign var ARGS(as readonly) in ONE line!
ARGS=$(
  getopt -n "$PROG" -a -o c:p:a:s:S:i:Pd:FmlhV \
    -l count:,pid:,append-file:,jstack-path:,store-dir:,cpu-sample-interval:,use-ps,top-delay:,force,mix-native-frames,lock-info,help,version \
    -- "$@"
) || die -h
eval set -- "$ARGS"
unset ARGS

count=5
cpu_sample_interval=0.5

while true; do
  case "$1" in
  -c | --count)
    count=$2
    shift 2
    ;;
  -p | --pid)
    pid_list=$2
    shift 2
    ;;
  -a | --append-file)
    append_file=$2
    shift 2
    ;;
  -s | --jstack-path)
    jstack_path=$2
    shift 2
    ;;
  -S | --store-dir)
    store_dir=$2
    shift 2
    ;;
  # support the legacy option name -P,--use-ps for compatibility
  -P | --use-ps)
    cpu_sample_interval=0
    shift
    ;;
  # support the legacy option name -d,--top-delay for compatibility
  -i | --cpu-sample-interval | -d | --top-delay)
    cpu_sample_interval=$2
    shift 2
    ;;
  -F | --force)
    force=-F
    shift
    ;;
  -m | --mix-native-frames)
    mix_native_frames=-m
    shift
    ;;
  -l | --lock-info)
    lock_info=-l
    shift
    ;;
  -h | --help)
    usage
    ;;
  -V | --version)
    progVersion
    ;;
  --)
    shift
    break
    ;;
  esac
done

readonly count cpu_sample_interval force mix_native_frames lock_info
readonly update_delay=${1:-0}
isNonNegativeFloatNumber "$update_delay" || die "update delay($update_delay) is not a non-negative float number!"

[ -z "$1" ] && update_count=1 || update_count=${2:-0}
isNaturalNumber "$update_count" || die "update count($update_count) is not a natural number!"
readonly update_count

if [ -n "$pid_list" ]; then
  pid_list=${pid_list//[[:space:]]/} # delete white space
  isNaturalNumberList "$pid_list" || die "pid(s)($pid_list) is illegal! example: 42 or 42,99,67"
fi
readonly pid_list

# check the directory of append-file(-a) mode, create if not existed.
if [ -n "$append_file" ]; then
  if [ -e "$append_file" ]; then
    [ -f "$append_file" ] || die "$append_file(specified by option -a, for storing run output files) exists but is not a file!"
    [ -w "$append_file" ] || die "file $append_file(specified by option -a, for storing run output files) exists but is not writable!"
  else
    append_file_dir=$(dirname -- "$append_file")
    if [ -e "$append_file_dir" ]; then
      [ -d "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not a directory!"
      [ -w "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not writable!"
    else
      mkdir -p "$append_file_dir" || die "fail to create directory $append_file_dir(specified by option -a, for storing run output files)!"
    fi
  fi
fi
readonly append_file

# check store directory(-S) mode, create directory if not existed.
if [ -n "$store_dir" ]; then
  if [ -e "$store_dir" ]; then
    [ -d "$store_dir" ] || die "$store_dir(specified by option -S, for storing output files) exists but is not a directory!"
    [ -w "$store_dir" ] || die "directory $store_dir(specified by option -S, for storing output files) exists but is not writable!"
  else
    mkdir -p "$store_dir" || die "fail to create directory $store_dir(specified by option -S, for storing output files)!"
  fi
fi
readonly store_dir

isNonNegativeFloatNumber "$cpu_sample_interval" || die "cpu sample interval($cpu_sample_interval) is not a non-negative float number!"

################################################################################
# search/check the existence of jstack command
#
# search order/priority:
#    1. from -s option
#    2. from under env var JAVA_HOME
#    3. from under env var PATH
################################################################################

if [ -n "$jstack_path" ]; then
  # 1. check jstack_path set by -s option
  [ -f "$jstack_path" ] || die "$jstack_path (set by -s option) is NOT found!"
  [ -x "$jstack_path" ] || die "$jstack_path (set by -s option) is NOT executable!"
elif [ -n "$JAVA_HOME" ]; then
  # 2. search jstack under JAVA_HOME
  if [ -f "$JAVA_HOME/bin/jstack" ]; then
    [ -x "$JAVA_HOME/bin/jstack" ] || die -h "found \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) is NOT executable!${NL}Use -s option set jstack path manually."
    jstack_path="$JAVA_HOME/bin/jstack"
  elif [ -f "$JAVA_HOME/../bin/jstack" ]; then
    [ -x "$JAVA_HOME/../bin/jstack" ] || die -h "found \$JAVA_HOME/../bin/jstack($JAVA_HOME/../bin/jstack) is NOT executable!${NL}Use -s option set jstack path manually."
    jstack_path="$JAVA_HOME/../bin/jstack"
  fi
elif type -P jstack &>/dev/null; then
  # 3. search jstack under PATH
  jstack_path=$(type -P jstack)
  [ -x "$jstack_path" ] || die -h "found $jstack_path from PATH is NOT executable!${NL}Use -s option set jstack path manually."
else
  die -h "jstack NOT found by JAVA_HOME(${JAVA_HOME:-not set}) setting and PATH!${NL}Use -s option set jstack path manually."
fi
readonly jstack_path

################################################################################
# biz logic
################################################################################

# DO NOT declare and assign var run_timestamp(as readonly) in ONE line!
run_timestamp=$(date "+%Y-%m-%d_%H:%M:%S.%N")
readonly run_timestamp
readonly uuid="${PROG}_${run_timestamp}_${$}_${RANDOM}"

readonly tmp_store_dir="/tmp/$uuid"
if [ -n "$store_dir" ]; then
  readonly store_file_prefix="$store_dir/${run_timestamp}_"
else
  readonly store_file_prefix="$tmp_store_dir/${run_timestamp}_"
fi
mkdir -p "$tmp_store_dir"

cleanupWhenExit() {
  rm -rf "$tmp_store_dir" &>/dev/null
}
trap cleanupWhenExit EXIT

headInfo() {
  local timestamp=$1
  colorPrint "0;34;42" ================================================================================
  printf '%s\n' "$timestamp [$((update_round_num + 1))/$update_count]: $(printCallingCommandLine)"
  colorPrint "0;34;42" ================================================================================
  echo
}

if [ -n "$pid_list" ]; then
  readonly ps_process_select_options="-p $pid_list"
else
  readonly ps_process_select_options="-C java -C jsvc"
fi

__die_when_no_java_process_found() {
  if [ -n "$pid_list" ]; then
    die "process($pid_list) is not running, or not java process!"
  else
    die 'No java process found!'
  fi
}

# output field: pid, thread id(lwp), pcpu, user
#   order by pcpu(percentage of cpu usage)
#
# NOTE:
# use ps command to find busy thread(cpu usage)
# cpu usage of ps command is expressed as
# the percentage of time spent running during the *entire lifetime* of a process,
# this is not ideal in general.
findBusyJavaThreadsByPs() {
  # 1. sort by %cpu by ps option `--sort -pcpu`
  #    unfortunately, ps from `procps-ng 3.3.12`, `--sort` does not work properly with other options,
  #    use
  #       ps <other options>
  #    combined
  #       sort -k3,3nr
  #    instead of
  #       ps <other options> --sort -pcpu
  # 2. use wide output(unlimited width) by ps option `-ww`
  #    avoid trunk user column to username_fo+ or $uid alike

  # shellcheck disable=SC2206
  local -a ps_cmd_line=(ps $ps_process_select_options -wwLo 'pid,lwp,pcpu,user' --no-headers)
  # DO NOT combine var ps_out declaration and assignment in ONE line!
  #   more info see https://github.com/koalaman/shellcheck/wiki/SC2155
  local ps_out
  ps_out=$("${ps_cmd_line[@]}" | sort -k3,3nr)
  [ -n "$ps_out" ] || __die_when_no_java_process_found

  if [ -n "$store_dir" ]; then
    printf '%s\n' "$ps_out" | logAndCat "${ps_cmd_line[*]} | sort -k3,3nr" >"$store_file_prefix$((update_round_num + 1))_ps"
  fi

  if ((count > 0)); then
    printf '%s\n' "$ps_out" | head -n "$count"
  else
    printf '%s\n' "$ps_out"
  fi
}

# top with output field: thread id, %cpu
__top_threadId_cpu() {
  # DO NOT combine var java_pid_list declaration and assignment in ONE line!
  local java_pid_list
  # shellcheck disable=SC2086
  java_pid_list=$(ps $ps_process_select_options -o pid --no-headers)
  [ -n "$java_pid_list" ] || __die_when_no_java_process_found
  # shellcheck disable=SC2086
  java_pid_list=$(echo $java_pid_list | tr ' ' ,) # join with ,

  # 1. sort by %cpu by top option `-o %CPU`
  #    unfortunately, top version 3.2 does not support -o option(supports from top version 3.3+),
  #    use
  #       HOME=$tmp_store_dir top -H -b -n 1
  #    combined
  #       sort
  #    instead of
  #       HOME=$tmp_store_dir top -H -b -n 1 -o %CPU
  # 2. change HOME env var when run top,
  #    so as to prevent top command output format being change by .toprc user config file unexpectedly
  # 3. use option `-d 0.5`(update interval 0.5 second) and `-n 2`(update 2 times),
  #    and use second time update data to get cpu percentage of thread in 0.5 second interval
  # 4. top v3.3, there is 1 black line between 2 update;
  #    but top v3.2, there is 2 blank lines between 2 update!
  local -a top_cmd_line=(top -H -b -d "$cpu_sample_interval" -n 2 -p "$java_pid_list")
  # DO NOT combine var top_out declaration and assignment in ONE line!
  local top_out
  top_out=$(HOME=$tmp_store_dir "${top_cmd_line[@]}")
  if [ -n "$store_dir" ]; then
    printf '%s\n' "$top_out" | logAndCat "${top_cmd_line[@]}" >"$store_file_prefix$((update_round_num + 1))_top"
  fi

  # DO NOT combine var result_threads_top_info declaration and assignment in ONE line!
  local result_threads_top_info
  result_threads_top_info=$(printf '%s\n' "$top_out" | awk '{
        # from text line to empty line, increase block index
        if (previousLine && !$0) blockIndex++
        # only print 4th text block(blockIndex == 3), aka. process info of second top update
        if (blockIndex == 3 && $1 ~ /^[0-9]+$/)
          print $1, $9  # $1 is thread id field, $9 is %cpu field
        previousLine = $0
      }')
  [ -n "$result_threads_top_info" ] || __die_when_no_java_process_found

  printf '%s\n' "$result_threads_top_info" | sort -k2,2nr
}

__complete_pid_user_by_ps() {
  # ps output field: pid, thread id(lwp), user
  # shellcheck disable=SC2206
  local -a ps_cmd_line=(ps $ps_process_select_options -wwLo 'pid,lwp,user' --no-headers)
  # DO NOT combine var ps_out declaration and assignment in ONE line!
  local ps_out
  ps_out=$("${ps_cmd_line[@]}")
  if [ -n "$store_dir" ]; then
    printf '%s\n' "$ps_out" | logAndCat "${ps_cmd_line[@]}" >"$store_file_prefix$((update_round_num + 1))_ps"
  fi

  local idx=0 threadId pcpu output_fields
  while read -r threadId pcpu; do
    ((count <= 0 || idx < count)) || break

    # output field: pid, threadId, pcpu, user
    output_fields=$(printf '%s\n' "$ps_out" | awk -v "threadId=$threadId" -v "pcpu=$pcpu" '$2==threadId {
          print $1, threadId, pcpu, $3; exit
        }')
    if [ -n "$output_fields" ]; then
      ((idx++))
      printf '%s\n' "$output_fields"
    fi
  done
}

# output format is same as function findBusyJavaThreadsByPs
findBusyJavaThreadsByTop() {
  __top_threadId_cpu | __complete_pid_user_by_ps
}

printStackOfThreads() {
  local idx=0 pid threadId pcpu user threadId0x
  while read -r pid threadId pcpu user; do
    printf -v threadId0x '%#x' "$threadId"

    ((idx++ > 0)) && normalOutput
    local jstackFile="$store_file_prefix$((update_round_num + 1))_jstack_$pid"
    [ -f "$jstackFile" ] || {
      # shellcheck disable=SC2206
      local -a jstack_cmd_line=("$jstack_path" $force $mix_native_frames $lock_info $pid)
      if [ "$user" = "$WHOAMI" ]; then
        # run without sudo, when java process user is current user
        logAndRun "${jstack_cmd_line[@]}" >"$jstackFile"
      elif ((UID == 0)); then
        # if java process user is not current user, must run jstack with sudo
        logAndRun sudo -u "$user" "${jstack_cmd_line[@]}" >"$jstackFile"
      else
        # current user is not root user, so can not run with sudo; print error message and rerun suggestion
        redOutput "[$idx] Fail to jstack busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user)."
        redOutput "User of java process($user) is not current user($WHOAMI), need sudo to rerun:"
        yellowOutput "    sudo $(printCallingCommandLine)"
        continue
      fi || {
        redOutput "[$idx] Fail to jstack busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user)."
        rm "$jstackFile" &>/dev/null
        continue
      }
    }

    blueOutput "[$idx] Busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user):"

    if [ -n "$mix_native_frames" ]; then
      local sed_script="/--------------- $threadId ---------------/,/^---------------/ {
          /--------------- $threadId ---------------/b # skip first separator line
          /^---------------/d # delete second separator line
          p
        }"
    elif [ -n "$force" ]; then
      local sed_script="/^Thread $threadId:/,/^$/ {
          /^$/d; p # delete end separator line
        }"
    else
      local sed_script="/ nid=($threadId0x|$threadId) /,/^$/ {
          /^$/d; p # delete end separator line
        }"
    fi
    sed "$sed_script" -n -r "$jstackFile" | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "$store_file_prefix$PROG"}
  done
}

main() {
  local update_round_num timestamp
  # if update_count <= 0, infinite loop till user interrupted (eg: CTRL+C)
  for ((update_round_num = 0; update_count <= 0 || update_round_num < update_count; ++update_round_num)); do
    ((update_round_num > 0)) && {
      sleep "$update_delay"
      normalOutput
    }

    timestamp=$(date "+%Y-%m-%d %H:%M:%S.%N")
    [[ -n "$append_file" || -n "$store_dir" ]] && headInfo "$timestamp" |
      tee ${append_file:+-a "$append_file"} ${store_dir:+-a "$store_file_prefix$PROG"} >/dev/null
    ((update_count != 1)) && headInfo "$timestamp"

    if [ "$cpu_sample_interval" = 0 ]; then
      findBusyJavaThreadsByPs
    else
      findBusyJavaThreadsByTop
    fi | printStackOfThreads
  done
}

main