#!/bin/bash
#
# Copyright (C) Mellanox Technologies Ltd. 2016-2017.  ALL RIGHTS RESERVED.
# See file LICENSE for terms.
#

#
# If you want that env. var. will be persistently saved
# please build it the following way: [sharpd|sharp_am|sharp]_[var_name]
#

. $(dirname $0)/sharp_funcs.sh


BASE_NAME=$(basename $0)

usage()
{
	echo "$BASE_NAME helps to system administrator to manage SHARP daemons."
	echo "Most of management operations require root access to nodes."
	echo ""
	echo "Usage: $BASE_NAME <operation> [-p process] [-l hostlist] [-s am_server] [-w write_logs_to] [-x var_name] [-j] [-m] [-h]"
	echo "	Operations:"
	echo "	        'start/restart/stop/status' - apply to SHARP daemons"
	echo "	        'set'       - set SHARP daemons runtime parameters"
	echo "	        'install'   - install SHARP daemons as local service"
	echo "	        'uninstall' - uninstall SHARP daemons as local service"
	echo "	        'cleanup'   - cleanup all daemons logs and system files"
	echo "	        'logs'      - collect log from SHARP daemons"
	echo "	        'ver'       - show version of SHARP daemons"
	echo "  Parameters:"
	echo "	        -p - process name sharp_am/sharpd"
	echo "	        -l - compute nodes host list"
	echo "	        -s - sharp_am server hostname"
	echo "	        -j - just dry run"
	if [ -f "$(dirname $0)/../systemd/system/sharpd.socket" ]; then
		echo "		-b - enable socket based activation. Relevant only with 'install' option"
	fi
	echo "	        -w - write logs tarball file to dir. Relevant only with 'logs' option"
	echo "	        -m - monitoring of SHARP daemons. Relevant only with 'install' option"
	echo "	        -x - name of configuration parameter to be removed. Relevant only with 'set' option"
	echo "	        -h - display this help and exit"
	echo " Configuration:"
	echo "  There are two groups of parameters:"
	echo "       - daemon specific parameters. They go to configuration files."
	echo "         You can configure them directly in configuration files, located"
	echo "         in \"../conf\" folder or in \"\$SHARP_CONF/conf\" folder."
	echo "         Or you can set them using environment variables and the script"
	echo "         will store them into configuration files."
	echo "         Please refer to help message of \"sharpd\" and \"sharp_am\" for"
	echo "         for the list of parameters"
	echo "       - script's parameters. They control script execution"
	echo "  Naming convention: <sharpd|sharp_am|sharp>_<parameter name>"
	echo "       \"sharpd\" - sharpd parameter"
	echo "       \"sharp_am\" - sharp_am parameter"
	echo "       \"sharp\" - script parameter"
	echo "  sharp_am configuration:"
	echo "       If you use UFM 5.8 or later, please refer to UFM user guide."
	echo "       For Opensm 4.9 or later you don't need any special configuration."
	echo "       For Opensm 4.7-4.8 (Mellanox OFED 3.3-x.x.x - 4.0-x.x.x), you have to configure"
	echo "       root guids (tree like topologies) using \"fabric_load_file\" (\"sharp_am_fabric_load_file\" env. variable)."
	echo " Environment:"
	echo "  SHARP_INI_FILE - takes configuration from given file instead of $USER_INI_FILE"
	echo "  SHARP_CONF - destination folder for SHARP daemons configuration. It should be shared network location."
	echo " Examples:"
	echo "  sudo $BASE_NAME start -l ajna0[1-8]   # run SHARP with defaults/recently used settings"
	echo "  sudo SHARP_CONF=/hpc/local/work/user  $BASE_NAME start -l ajna0[1-8] # store generated configuration in non-standard place"
	echo "  sudo SHARP_CONF=/hpc/local/work/user  sharp_am_root_guids_file=\"/var/log/opensm-root-guids.cfg\" $BASE_NAME start -l ajna0[1-8]  # run SHARP with old Opensm version"
	echo " Dependencies:"
	echo "  $(print_hostlist_dependency_message)"

	exit 2
}

conn_wrapper()
{
	local cmd="$1 $2"

	if [ -n "$dry_run" ]; then
		printf "Dry run mode: "
		echo \"$cmd\"
	else
		eval "$cmd"
		conn_ret=$?
	fi
}

ssh_wrapper()
{
	conn_wrapper "ssh -t" "$1 $2 $3 $4"
}

pdsh_wrapper()
{
	conn_wrapper "pdsh -w" "$1 $2 $3 $4"
}

collect_logs()
{
	LOG_DIR=${write_log_to_dir:-$log_save_dir}/sharp_logs.$$
	TARBALL=${LOG_DIR}.tgz

	mkdir -p $LOG_DIR

	cp -fR $log_save_dir/sharp_benchmark_logs.* $LOG_DIR &> /dev/null
	cp -fR $DCONF_DIR $LOG_DIR &> /dev/null

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		scp $sharp_AM_server:/var/log/sharp_am.log $LOG_DIR &> /dev/null
		scp $sharp_AM_server:$OUT_AM $LOG_DIR &> /dev/null
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "scp /var/log/sharpd.log $HOSTNAME:$LOG_DIR/sharpd_%h.log" "&>/dev/null"
		pdsh_wrapper "$sharp_hostlist" "scp $OUT_SD $HOSTNAME:$LOG_DIR/$(basename $OUT_SD)_%h.log" "&>/dev/null"
	fi

	if [ -z "$dry_run" ]; then
		ibnetdiscover &> $LOG_DIR/ibnetdiscover.out
		ibdiagnet --sharp &> $LOG_DIR/ibdiagnet.out
		cp -fR /var/tmp/ibdiagnet2 $LOG_DIR

		tar -C $LOG_DIR -czf $TARBALL .

		echo "Logs location is $HOSTNAME:$TARBALL"
	fi

	\rm -rf $LOG_DIR $log_save_dir/sharp_benchmark_logs.* &> /dev/null
}

all_run()
{
	AM_COMM="$1 --config-file $SHARP_AM_CONFIG_FILE"
	SD_COMM="$2 --config-file $SHARP_SD_CONFIG_FILE"

	local sharp_sleep_after_cmd=${sharp_sleep_after_cmd:-3}
	local ssh_ret="0"
	local pdsh_ret="0"

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "SHARP_CONF=$SHARP_CONF nohup $AM_COMM &> $OUT_AM" "&>/dev/null" && sleep $sharp_sleep_after_cmd
		ssh_ret=$conn_ret
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "SHARP_CONF=$SHARP_CONF nohup $SD_COMM &> $OUT_SD" "&>/dev/null" && sleep $sharp_sleep_after_cmd
		pdsh_ret=$conn_ret
	fi

	ret=0

	if (( $ssh_ret != 0 )); then
		ret=$ssh_ret
	elif (( $pdsh_ret != 0 )); then
		ret=$pdsh_ret
	fi
}

start()
{
	generate_conf_files $DCONF_DIR

	echo "SHARP config dir: $(readlink -f $DCONF_DIR)"
	echo "Startup SHARP daemons..."

	all_run	"$SHARP_INSTALL/etc/sharp_am start" "$SHARP_INSTALL/etc/sharpd start"
}

stop()
{
	echo "Stopping SHARP daemons..."
	all_run	"$SHARP_INSTALL/etc/sharp_am stop" "$SHARP_INSTALL/etc/sharpd stop"

	[ -n "$ret" ] && [ $ret -ne 0 ] && echo "Some problem during daemons stop. Force killing of processes."

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "pkill sharp_am &>/dev/null; rm -f $OUT_AM /var/run/sharp_am.pid" "&>/dev/null"
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "pkill sharpd &>/dev/null; rm -f $OUT_SD /var/run/sharpd.pid" "&>/dev/null"
		pdsh_wrapper "$sharp_hostlist" "systemctl stop sharpd.socket"
		pdsh_wrapper "$sharp_hostlist" "systemctl stop sharpd.service"
	fi
}

check_on_boot()
{
	TMF_FILE=/tmp/procs.$$
	SHARP_INIT=${SHARP_INSTALL}/sbin/sharp.init

	echo -e "\nBoot state:"

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "readlink -e /etc/init.d/sharp_am" ">$TMF_FILE" "2>/dev/null"
		_list=$((grep $SHARP_INIT $TMF_FILE | cut -d: -f1) 2>/dev/null)
		[ -n "$_list" ] && on_boot="sharp_am" || on_boot=""

		if [ -n "$on_boot" ]; then
			echo " sharp_am is \"service\" on     $sharp_AM_server"
		else
			echo " sharp_am is NOT \"service\" on $sharp_AM_server"
		fi
	fi

	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "readlink -e /etc/init.d/sharpd" ">$TMF_FILE" "2>/dev/null"

		_list=$((grep $SHARP_INIT $TMF_FILE | cut -d: -f1) 2>/dev/null)
		[ -n "$_list" ] && on_boot=$($hostlist_tool $_list) || on_boot=""
		non_on_boot=$($hostlist_tool -d $sharp_hostlist $on_boot) || non_on_boot=""

		[ -n "$on_boot" ] && echo " sharpds are \"service\" on     $on_boot"
		[ -n "$non_on_boot" ] && echo " sharpds are NOT \"service\" on $non_on_boot"
	fi

	\rm -f $TMP_FILE &>/dev/null
}

status()
{
	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		cnt1=$((ssh_wrapper "$sharp_AM_server" "$SHARP_INSTALL/etc/sharp_am status" | grep "is running" | wc -l) 2>/dev/null)
		cnt2=1

		if [ -n "$dry_run" ]; then
			ssh_wrapper "$sharp_AM_server" "$SHARP_INSTALL/etc/sharp_am status" "2>/dev/null"
			ret_am=0
		elif [ $cnt1 -eq $cnt2 ]; then
			echo "sharp_am is UP"
			ret_am=0
		else
			echo "sharp_am is DOWN."
			ret_am=1
		fi
	else
		ret_am=0
	fi

	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		cnt1=$((pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/etc/sharpd status" | grep "is running" | wc -l) 2>/dev/null)
		cnt2=$($hostlist_tool -e $sharp_hostlist | wc -l)

		if [ -n "$dry_run" ]; then
			pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/etc/sharpd status" "2>/dev/null"
			ret_d=0
		elif [ $cnt1 -eq $cnt2 ]; then
			echo "sharpds are UP"
			ret_d=0
		else
			echo "One or more sharpds are DOWN:"

			TMF_FILE=/tmp/procs.$$
			pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/etc/sharpd status" ">$TMF_FILE" "2>/dev/null"

			_list=$((grep "is running" $TMF_FILE | cut -d: -f1) 2>/dev/null)
			[ -n "$_list" ] && _up=$($hostlist_tool $_list)
			_down=$($hostlist_tool -d $sharp_hostlist $_up)

			if [ -s "$TMF_FILE" ]; then
				[ -n "$_up" ] && echo "   UP   - $_up"
				[ -n "$_down" ] && echo "   DOWN - $_down"
			else
				echo "$sharp_hostlist - unreachable"
			fi

			\rm -f $TMF_FILE &> /dev/null

			ret_d=$((cnt2 - cnt1))
		fi
	else
		ret_d=0
	fi

	[ -z "$dry_run" ] && check_on_boot

	if [ $# -eq 1 ] && [ "$1" = "stop" ]; then
		if [ $ret_am -eq 1 ] && [ $ret_d -eq $cnt2 ]; then
			ret=0
		else
			ret=1
		fi
	else
		if [ $ret_am -eq 1 ] || [ $ret_d -eq 1 ]; then
			ret=1
		else
			ret=0
		fi
	fi
}

show_vers()
{
	TMF_FILE=/tmp/vers.$$

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "$SHARP_INSTALL/bin/sharp_am -v 2>/dev/null | grep '(sharp)'" ">$TMF_FILE" "2>/dev/null"
		[ -z "$dry_run" ] && echo "   $(cat $TMF_FILE | tr -d '\r') - $sharp_AM_server"
			#			str=$(cat $TMF_FILE | tr -d '\r')
			#echo "   $str - $sharp_AM_server"
		#fi
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/bin/sharpd -v 2>/dev/null | grep '(sharp)'" ">$TMF_FILE" "2>/dev/null"

		[ -n "$dry_run" ] && return

		while read ver_str; do
			[ -s $TMF_FILE ] || break

			_ver=$(echo $ver_str | cut -d: -f2 | sed 's/^ //g')
			_hosts=$($hostlist_tool $(grep "$_ver" $TMF_FILE | cut -d: -f1))

			for one_host in $($hostlist_tool -e $(echo $_hosts)); do
				sed -i "/^$one_host/ d" $TMF_FILE
			done

			echo "   $_ver - ${_hosts} "
		done < $TMF_FILE

		\rm -f $TMF_FILE &> /dev/null
	fi
}

install()
{
	TMP_FILE=/tmp/hosts.$$

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "$SHARP_INSTALL/sbin/sharp_daemons_setup.sh -s -d sharp_am $monit" ">$TMP_FILE" "2>/dev/null"
		_list=$(grep "Service sharp_am is installed" $TMP_FILE | cut -d: -f1) 2>/dev/null

		if [ -n "$_list" ]; then
			echo "Service sharp_am is installed on $sharp_AM_server"
			ret_am=0
		else
			echo "Service sharp_am is missing on $sharp_AM_server"
			ret_am=1
		fi
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/sbin/sharp_daemons_setup.sh -s -d sharpd $monit $socket_based" ">$TMP_FILE" "2>/dev/null"
		_list=$(grep "Service sharpd is installed" $TMP_FILE | cut -d: -f1) 2>/dev/null
		[ -n "$_list" ] && serv_in=$($hostlist_tool $_list)
		serv_out=$($hostlist_tool -d $sharp_hostlist $serv_in)

		if [ -n "$serv_in" ]; then
			echo "Service sharpd is installed on $serv_in"
			retd=0
		fi
		if [ -n "$serv_out" ]; then
			echo "Service sharpd is missing on $serv_out"
			retd=1
		fi
	fi

	if [ $ret_am -eq 1 ] || [ $retd -eq 1 ]; then
		ret=1
	else
		ret=0
	fi

	\rm -f $TMP_FILE &>/dev/null
}

uninstall()
{
	TMP_FILE=/tmp/hosts.$$

	if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
		ssh_wrapper "$sharp_AM_server" "$SHARP_INSTALL/sbin/sharp_daemons_setup.sh -r -d sharp_am" ">$TMP_FILE" "2>/dev/null"
		_list=$(grep "Service sharp_am is removed" $TMP_FILE | cut -d: -f1) 2>/dev/null

		if [ -n "$_list" ]; then
			echo "Service sharp_am is removed on $sharp_AM_server"
			ret_am=0
		else
			echo "Service sharp_am can't be removed on $sharp_AM_server"
			ret_am=1
		fi
	fi
	if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
		pdsh_wrapper "$sharp_hostlist" "$SHARP_INSTALL/sbin/sharp_daemons_setup.sh -r -d sharpd" ">$TMP_FILE" "2>/dev/null"
		_list=$(grep "Service sharpd is removed" $TMP_FILE | cut -d: -f1) 2>/dev/null
		[ -n "$_list" ] && serv_in=$($hostlist_tool $_list)
		serv_out=$($hostlist_tool -d $sharp_hostlist $serv_in)

		if [ -n "$serv_in" ]; then
			echo "Service sharpd is removed on $serv_in"
			retd=0
		fi
		if [ -n "$serv_out" ]; then
			echo "Service sharpd can't be removed on $serv_out"
			retd=1
		fi
	fi

	if [ $ret_am -eq 1 ] || [ $retd -eq 1 ]; then
		ret=1
	else
		ret=0
	fi

	\rm -f $TMP_FILE &>/dev/null
}

daemon_clean()
{
	dname="$1"
	cmd="$2"

	toclean="/var/lock/subsys/$dname /var/run/$dname.pid /var/log/$dname.log"

	for one_file in $toclean; do
		if [ -z "$process" ] || [ "$process" == "$dname" ]; then
			$cmd "\rm -f $one_file" "&>/dev/null"
		fi
	done
}

cleanup()
{
	daemon_clean "sharp_am" "ssh_wrapper \"$sharp_AM_server\""
	daemon_clean "sharpd"   "pdsh_wrapper \"$sharp_hostlist\""
}

#
# Main
#
OUT_AM=/tmp/d_sharp_am.log
OUT_SD=/tmp/d_sharpd.log

[ $# -eq 0 ] && usage
if [ $# -ge 1 ]; then
	operation=$1
	shift
fi
[ "$operation" = "-h" ] && usage

if [ $(whoami) != "root" ]; then
	echo "Error: root permission required. Exit."
	exit 1
fi

#
# Check dependency
#
check_hostlist_tool

load_params

while getopts "p:l:s:w:x:jmhb" flag_arg; do
	case $flag_arg in
		p) process="$OPTARG"                       ;;
		l) sharp_hostlist="$OPTARG"        ;;
		s) sharp_AM_server="$OPTARG"       ;;
		w) write_log_to_dir="$OPTARG"              ;;
		j) dry_run=1                               ;;
		m) monit='-m'                              ;;
		b) socket_based="-b"			   ;;
		x) var2remove="$OPTARG"                    ;;
		h) usage                                   ;;
		*) usage                                   ;;
	esac
done

if [ -z "$(which pdsh 2>/dev/null)" ]; then
	echo "'pdsh' not found. Dry mode will be invoked."
	dry_run=1
fi

#
# Check params
#

if [ ! -f "$(dirname $0)/../systemd/system/sharpd.socket" ] && [ ! -z "$socket_based" ]; then
	echo "Socket based activation is not compatible with this OS version. Exit."
	exit 1
fi

if [ -z "$sharp_AM_server" ]; then
	if pgrep opensm &> /dev/null; then
		sharp_AM_server=$(hostname -s)
	else
		echo "Please provide AM server. Exit."
		exit 1
	fi
fi

check_and_update_hostlist sharp_hostlist

if [ -z "$process" ] || [ "$process" == "sharp_am" ]; then
	echo -e "sharp_am host: $sharp_AM_server"
fi
if [ -z "$process" ] || [ "$process" == "sharpd" ]; then
	echo -e "sharpds hosts: $sharp_hostlist"
fi
echo ""


mkdir -p $DCONF_DIR &> /dev/null
if [ $? -ne 0 ]; then
	echo "Unable to create $DCONF_DIR dir. Exit."
	exit 1
fi


[ -n "$var2remove" ] && eval $var2remove=""
set_local_params "sharpd"
set_local_params "sharp_am"
set_local_params "sharp"

case $operation in
	"start")		start
					if [ -z "$dry_run" ]; then
						sleep 2
						status
					fi
					;;
	"restart")		stop
					start
					if [ -z "$dry_run" ]; then
						sleep 2
						status
					fi
					;;
	"stop")			stop
					status "stop"
					;;
	"status")		status
					;;
	"set")			if [ -n "$dry_run" ]; then
						echo "Nothing to do in dry run mode."
					else
						generate_conf_files $DCONF_DIR
					fi
					;;
	"install")		install
					;;
	"uninstall")	uninstall
					;;
	"cleanup")		cleanup
					;;
	"logs")			collect_logs
					;;
	"ver")			show_vers
					;;
	*)				usage
					;;
esac

echo -e "\nAll Done"
[ -n "$ret" ] && exit $ret
