#!/bin/sh -eE

if [ $# -ne 2 ]; then
    echo "Usage: $(basename $0) <slurm_jobid> <exename> [/path/to/shared/dir]"
    exit 1
fi

jobid=$1
nodes=`squeue -j $jobid -o %N -h`
prog=`basename $2`

out_dir="$MTT_SCRATCH_ROOT"

mkdir -p $out_dir
script=$out_dir/mytrace_$$.sh

cat > $script <<EOF
#!/bin/sh
#set -x
jobid=$1
out_dir=$out_dir
prog=$prog

EOF

cat >> $script <<'EOF'
host=`hostname`
for pid in $(pgrep $prog); do
    trace_file=$out_dir/trace.$jobid.$host.$pid.txt
    echo == ${host} == ${pid} > $trace_file
    gstack $pid >> $trace_file
done
EOF

chmod 755 $script
pdsh -w $nodes $script

rm -f $out_dir/trace_$jobid.txt
for f in $out_dir/trace.$jobid.*; do
   cat $f >> $out_dir/trace_$jobid.txt
done
sed -i -e "/^Loaded symbols/d" -e "/^Reading symbols/d" $out_dir/trace_$jobid.txt

find ${out_dir}/ -maxdepth 1 -type f -name trace.${jobid}.\*| xargs rm -f
rm -f ${script}

echo $out_dir/trace_$jobid.txt
exit 0

