#!/bin/bash
# Original is here: https://github.com/cerebrotech/domino/blob/develop/dev/nucleus-heap-dump.sh
#
# Takes a heap dump of nucleus containers on Kubernetes pods.
#
# FOR INSTRUCTIONS AND CONTEXT SEE:
# https://dominodatalab.atlassian.net/wiki/spaces/ENG/pages/275152966/JVM+Performance+Debugging#JVMPerformanceDebugging-CapturingheapdumpsfromnucleuspodsinKubernetes
#
# This script is self-contained. It can be copied and executed without any dependencies other than kubectl.
#
# Pre-requisite: kubectl must be configured to access the target Kubernetes cluster.
# E.g. this command displays the expected namespaces:
#
# kubectl get namespaces
#
# For a manual alternative see https://dominodatalab.atlassian.net/wiki/spaces/ENG/pages/411271257/Kubernetes+Platform+Runbook#KubernetesPlatformRunbook-JavaProcesses
#
set -o nounset -o errexit -o pipefail

create_dump_script() {
  cat <<EOF >$DUMP_SCRIPT
#!/bin/bash
set -o nounset -o errexit -o pipefail

# This script must be executed as the domino user.
# In Domino up to version 4.5, the default user is root.
# If that is the case, re-run this script with the same arguments as the domino user.
if [ \$UID -eq 0 ]; then
  # Re-run this same script with the same arguments as the domino user.
  chmod 755 $DUMP_SCRIPT
  exec gosu domino $DUMP_SCRIPT "\$@"
fi

POD=\$1
TIMESTAMP=\$2
TARGET_ID=\$POD-\$TIMESTAMP
TARGET_DIR=/tmp/\$TARGET_ID
mkdir -p \$TARGET_DIR
JVM_INFO_FILE_NAME=\$TARGET_DIR/\$TARGET_ID.txt
DUMP_FILE_NAME=\$TARGET_DIR/\$TARGET_ID.hprof

capture_info() {
  echo "********************************************************************************"
  echo "*** \$@"
  echo "********************************************************************************"
  \$@ 2>&1 || echo "Command failed: \$@"
  echo ""
}

echo "Capturing JVM info"
cat <<END-TIPS  >>\$JVM_INFO_FILE_NAME
Information for \$POD at \$TIMESTAMP

Tips for analyzing:
- http://java-latte.blogspot.co.il/2014/03/metaspace-in-java-8.html
- https://docs.oracle.com/javase/7/docs/technotes/tools/share/jmap.html
- https://docs.oracle.com/javase/8/docs/technotes/tools/unix/jstat.html
- https://docs.oracle.com/javase/9/tools/jcmd.htm#JSWOR743
- https://stackoverflow.com/a/44095604/376366
- https://stackoverflow.com/a/49139834/376366
- https://www.yourkit.com/java/profiler/features/

END-TIPS

commands=(
  "cat /proc/sys/fs/file-nr"
  "date"
  "env"
  "free -m"
  "java -version"
  "jcmd 1 GC.class_histogram"
  "jcmd 1 GC.heap_info"
  "jcmd 1 Thread.print"
  "jcmd 1 VM.command_line"
  "jcmd 1 VM.dynlibs"
  "jcmd 1 VM.flags"
  "jcmd 1 VM.system_properties"
  "jcmd 1 VM.uptime"
  "jcmd 1 VM.version"
  "jmap -histo 1"
  "jmap -histo:live 1"
  "jps -lvm"
  "jstack -l 1"
  "jstat -gc 1"
  "jstat -gccapacity 1"
  "jstat -gcutil -t 1"
  "jstat -gcutil 1"
  "ulimit -a"
  "ulimit -Hn"
  "ulimit -Sn"
)
for c in "\${commands[@]}"; do
  capture_info "\$c" 2>&1 >>\$JVM_INFO_FILE_NAME
done

echo "Capturing heap dump"
jmap -dump:format=b,file=\$DUMP_FILE_NAME 1

echo Creating tarball
tar -czvf /tmp/\$TARGET_ID.tar.gz -C \$TARGET_DIR . >/dev/null
EOF
}

# Source: https://gist.github.com/fernandoacorreia/b4fa9ae88c67fa6759d271b743e96063
with_backoff() {
  local max_attempts=${ATTEMPTS-7}
  local timeout=${TIMEOUT-1}
  local attempt=1
  local exitCode=0

  while (( $attempt < $max_attempts ))
  do
    set +e
    "$@"
    exitCode=$?
    set -e

    if [[ $exitCode == 0 ]]
    then
      break
    fi

    echo "Attempt $attempt/$max_attempts failed ($@). Retrying in ${timeout}s..." 1>&2
    sleep $timeout
    attempt=$(( attempt + 1 ))
    timeout=$(( timeout * 2 ))
  done

  if [[ $exitCode != 0 ]]
  then
    echo "Attempt $attempt/$max_attempts failed ($@). Maximum retries exceeded." 1>&2
  fi

  return $exitCode
}

dump_heap_from_pod() {
  declare -r component=$1
  declare -r pod=$2
  declare -r command="bash $DUMP_SCRIPT $pod $TIMESTAMP"

  echo ""
  echo "Pod: $pod"

  echo "Copying dump script to pod."
  with_backoff kubectl cp "$DUMP_SCRIPT" "$NAMESPACE/$pod:/tmp/" -c nucleus-$component

  echo "Dumping heap and info."
  kubectl exec -it -n $NAMESPACE $pod -c nucleus-$component -- $command

  echo "Copying tarball from pod."
  with_backoff kubectl cp "$NAMESPACE/$pod:/tmp/$pod-$TIMESTAMP.tar.gz" -c nucleus-$component "$TARGET_DIR/$pod-$TIMESTAMP.tar.gz"
}

dump_heap_from_pods() {
  declare -r component=$1
  declare -r pods=$(kubectl get pods -n $NAMESPACE -l app.kubernetes.io/instance=nucleus,app.kubernetes.io/component=$component -o=jsonpath='{.items[*].metadata.name}')
  IFS=' '
  read -a pod_array <<< "$pods"
  for pod in "${pod_array[@]}"; do
    dump_heap_from_pod $component $pod || echo "ERROR: Failed to dump from pod $pod."
  done
}

dump_from_components() {
  components=(
    "dispatcher"
    "frontend"
  )
  for c in "${components[@]}"; do
    dump_heap_from_pods $c
  done
}


TIMESTAMP="$(date +%s)"

NAMESPACE=$(kubectl get namespaces -l domino-platform=true -o=jsonpath='{.items[0].metadata.name}')
echo "Platform namespace: $NAMESPACE"

TARGET_DIR="./"
mkdir -p "$TARGET_DIR"

DUMP_SCRIPT="/tmp/heap-dump-$TIMESTAMP.sh"
create_dump_script

dump_from_components

echo ""
echo "Heap dumps in $TARGET_DIR:"
ls -ldh $TARGET_DIR/*
