#!/bin/bash
#
# Bring up/down openibd
#
# chkconfig: 2345 05 95
# description: Activates/Deactivates InfiniBand Network Interfaces
#
### BEGIN INIT INFO
# Provides:       openibd
### END INIT INFO
#
# Copyright (c) 2006 Mellanox Technologies. All rights reserved.
#
# This Software is licensed under one of the following licenses:
#
# 1) under the terms of the "Common Public License 1.0" a copy of which is
#    available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/cpl.php.
#
# 2) under the terms of the "The BSD License" a copy of which is
#    available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/bsd-license.php.
#
# 3) under the terms of the "GNU General Public License (GPL) Version 2" a
#    copy of which is available from the Open Source Initiative, see
#    http://www.opensource.org/licenses/gpl-license.php.
#
# Licensee has the right to choose one of the above licenses.
#
# Redistributions of source code must retain the above copyright
# notice and one of the license notices.
#
# Redistributions in binary form must reproduce both the above copyright
# notice, one of the license notices in the documentation
# and/or other materials provided with the distribution.
#
#
#  $Id: openib-1.0-openibd.init,v 1.12.2.1 2007/10/16 15:37:50 dledford Exp $
#

# config: /etc/ofed/openib.conf
CONFIG="/etc/ofed/openib.conf"
NETWORK_CONF_DIR=/etc/sysconfig/network-scripts

. /etc/rc.d/init.d/functions

# If module $1 is loaded return - 0 else - 1
is_module()
{
local RC

    /sbin/lsmod | grep -w "$1" > /dev/null 2>&1
    RC=$?
    
return $RC        
}

count_ib_ports()
{
    return $(/sbin/lspci | grep InfiniBand | wc -l)
}

count_ib_ports
num_ports=$?

load_hardware_modules()
{
    local RC=0

    for driver in `grep 'ib_mthca\|ipath_core\|ib_ipath\|ib_ehca\|iw_cxgb3\|iw_c2' /etc/sysconfig/hwconf | cut -f 2 -d ' '`; do
        if ! is_module $driver ; then
            /sbin/modprobe $driver
            RC=$[ $RC + $? ]
        fi
	if is_module ipath_core -a ! is_module ib_ipath; then
	    /sbin/modprobe ib_ipath
            RC=$[ $RC + $? ]
	fi
    done

    return $RC
}

if [ -f $CONFIG ]; then
    . $CONFIG

    # Setting OpenIB start parameters
    POST_LOAD_MODULES=""

    if [ "X${SDP_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES ib_sdp"
	IPOIB_LOAD="yes"
    fi

    IPOIB=0
    if [ "X${IPOIB_LOAD}" == "Xyes" ]; then
	IPOIB=1
    fi

    if [ "X${SRP_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES ib_srp"
    fi

    if [ "X${VNIC_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES ib_vnic"
    fi

    if [ "X${RDMA_CM_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES rdma_cm"
    fi

    if [ "X${UCM_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES ib_ucm"
    fi

    if [ "X${RDMA_UCM_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES rdma_ucm"
    fi

    if [ "X${ISER_LOAD}" == "Xyes" ]; then
	POST_LOAD_MODULES="$POST_LOAD_MODULES ib_iser"
    fi
else
    POST_LOAD_MODULES="ib_sdp rdma_cm ib_ucm rdma_ucm"
    IPOIB=1
    IPOIB_LOAD="yes"
fi

PRE_UNLOAD_MODULES="ib_iser ib_srp ib_sdp ib_vnic ib_ucm rdma_ucm rdma_cm"

# W/A for unloading modules
POST_UNLOAD_MODULES="$PRE_UNLOAD_MODULES ib_vnic ib_ipoib ib_cm ib_sa ib_uverbs ib_umad ib_mthca ib_ipath ipath_core ib_ehca ib_mad ib_core"

STATUS_MODULES="rdma_ucm rdma_cm ib_ucm ib_cm ib_sdp ib_srp ib_iser ib_ipoib"

interfaces=`/sbin/ifconfig | grep "^ib[0-9]*" | cut -f 1 -d ' ' | sed -e 'y/\r/ /'`
                   
errata_58()
{
# Check AMD chipset issue Errata #58
if test -x /sbin/lspci && test -x /sbin/setpci; then
     if ( /sbin/lspci -nd 1022:1100 | grep "1100" > /dev/null ) &&
        ( /sbin/lspci -nd 1022:7450 | grep "7450" > /dev/null ) &&
        ( /sbin/lspci -nd 15b3:5a46 | grep "5a46" > /dev/null ); then

        if ! ( grep FIX_AMD_8131_ERR58 $CONFIG > /dev/null ); then
            echo >> $CONFIG
            echo "# Set FIX_AMD_8131_ERR58=YES to apply AMD-8131 Errata #58 workaround" >> $CONFIG
            echo "FIX_AMD_8131_ERR58=\"YES\"" >> $CONFIG
        fi
	
	# Set default value for FIX_AMD_8131_ERR58
	if [ -z "$FIX_AMD_8131_ERR58" ]; then
		FIX_AMD_8131_ERR58="YES"
	fi
        
        CURVAL=`/sbin/setpci -d 1022:1100 69`

        for val in $CURVAL
        do
            if [ "${val}" != "c0" ]; then
                if [ "$FIX_AMD_8131_ERR58" == "YES" ]; then
                    /sbin/setpci -d 1022:1100 69=c0
                    if [ $? -eq 0 ]; then
                        # echo "AMD-8131 Errata #58 workaround applied"
                        break
                    else
                        echo "Failed to apply AMD-8131 Errata #58 workaround"
                    fi
                else
                    echo
                    echo "Invalid configuration found for PCI-X chipset AMD-8131 (Errata #58)"
                    echo "For more details see IBG2 Distribution Release Notes (IBG2_Release_Notes.pdf under docs directory)"
                    echo
                    exit 1
                fi
            fi
        done
    
    fi
fi

}

errata_56()
{
# Check AMD chipset issue Errata #56
if test -x /sbin/lspci && test -x /sbin/setpci; then
     if ( /sbin/lspci -nd 1022:1100 | grep "1100" > /dev/null ) &&
        ( /sbin/lspci -nd 1022:7450 | grep "7450" > /dev/null ) &&
	( /sbin/lspci -nd 15b3:5a46 | grep "5a46" > /dev/null ); then

        if ! ( grep FIX_AMD_8131_ERR56 $CONFIG > /dev/null ); then
            echo >> $CONFIG
            echo "# Set FIX_AMD_8131_ERR56=YES to apply AMD-8131 Errata #56 workaround" >> $CONFIG
            echo "FIX_AMD_8131_ERR56=\"YES\"" >> $CONFIG
        fi

        # Set default value for FIX_AMD_8131_ERR56
        if [ -z "$FIX_AMD_8131_ERR56" ]; then
                FIX_AMD_8131_ERR56="YES"
        fi

	bus=""

	# Look for devices AMD-8131
	for amd_dev in `/sbin/setpci -v -f -d 1022:7450 19 | cut -d':' -f1,2`
	do

		bus=`/sbin/setpci -s $amd_dev 19`
		rev=`/sbin/setpci -s $amd_dev 8`


		# Look for Tavor attach to secondary bus of this devices
		for device in `/sbin/setpci -f -s $bus: -d 15b3:5a46 19`
		do
			if [ $rev -lt 13 ]; then
				if [ "$FIX_AMD_8131_ERR56" == "YES" ]; then			
					/sbin/setpci -d 15b3:5a44 72=14
					if [ $? -eq 0 ]; then
		                        	# echo "AMD-8131 Errata #56 workaround applied"
			                        break
			                else
                        			echo "Failed to apply AMD-8131 Errata #56 workaround"
			                fi
                		else
		                	echo
			                echo "Invalid configuration found for PCI-X chipset AMD-8131 (Errata #56)"
                    			echo "For more details see IBG2 Distribution Release Notes (IBG2_Release_Notes.pdf under docs directory)"
                    			echo

		        	        exit 1
				fi

			else
				continue
			fi

			# If more than one device is on the bus the issue a
			# warning
			num=`/sbin/setpci -f -s $bus: 0 | wc -l |  sed 's/\ *//g'`

			if [ $num -gt 1 ]; then
				echo "Warning: your current PCI-X configuration might be incorrect."
				echo "see AMD-8131 Errata 56 for more details."
			fi

		done
	done

     fi
fi
}

start()
{
    local RC=0
    local loaded=0

    [ "$num_ports" -eq "0" ] && return $RC

    echo -n "Loading OpenIB kernel modules:"

    load_hardware_modules    
    RC=$[ $RC + $? ]
   
    errata_58
    
    # Add node description to sysfs
    IBSYSDIR="/sys/class/infiniband"
    if [ -d ${IBSYSDIR} ]; then
        declare -i hca_id=1
        for hca in ${IBSYSDIR}/*
        do
	    if [ -e ${hca}/node_desc ]; then
            	echo -n "$(hostname | cut -f 1 -d .) HCA-${hca_id}" >> ${hca}/node_desc
	    fi
            let hca_id++
        done
    fi
   
    if ! is_module ib_umad ; then
	/sbin/modprobe ib_umad > /dev/null 2>&1
        RC=$[ $RC + $? ]
    fi
    if ! is_module ib_uverbs ; then
        /sbin/modprobe ib_uverbs > /dev/null 2>&1
        RC=$[ $RC + $? ]
    fi
    
    if ! is_module ib_ipoib ; then
	if [ "$IPOIB" -eq 1 ]; then
	    /sbin/modprobe ib_ipoib > /dev/null 2>&1
	    RC=$[ $RC + $? ]
	fi
    fi

    if [ $RC -ne 0 ]; then
    	echo_failure
	echo
        return $RC
    fi    
    
    # Load configured modules
    if [[ "$POST_LOAD_MODULES" != "" ]]; then
        for mod in  $POST_LOAD_MODULES
        do
		if ! is_module $mod ; then
            		/sbin/modprobe $mod > /dev/null 2>&1
            		if [ $? -ne 0 ]; then
				# Failure to load any of these modules is a
				# non-fatal error, so we don't save off these
				# return values and the overall start of the
				# service gets a return code of 0.  But, we
				# do notify the user of this module's failure
				# to load.
				echo
				echo -n "Failed to load module $mod"
			fi
		fi
        done    
    fi
   
    touch /var/lock/subsys/openibd
    errata_56
    sleep 1
    echo_success
    echo
    return $RC    
}

unload()
{
	local mod=$1
	# Unload module $1
	if is_module $mod; then
            	/sbin/modprobe -r $mod > /dev/null 2>&1
            	if [ $? -ne 0 ]; then
			# Try rmmod if modprobe failed: case that previous installation included more IB modules.
			/sbin/rmmod $mod > /dev/null 2>&1
                    	if [ $? -ne 0 ]; then
			    echo
                    	    echo "Failed to unload $mod"
                    	    return 1
                    	fi    
		fi
	fi
	return 0
}

stop()
{
    # Check if applications which use infiniband are running
    local apps="opensm osmtest ibbs ibns"
    local pid
    local RC=0
    
    [ "$num_ports" -eq "0" ] && return $RC

    echo -n "Unloading OpenIB kernel modules:"

    for app in $apps
    do
    	if ( ps -ef | grep $app | grep -v grep > /dev/null 2>&1 ); then
            echo
            echo "Please stop $app and all applications running over InfiniBand"
            echo "Then run \"$0 $ACTION\""
            echo_failure
	    echo
            return 1
        fi
    done

    if ! is_module ib_core; then
        # Nothing to do, make sure lock file is gone and return
	rm -f /var/lock/subsys/openibd
	echo_success
	echo
        return 0
    fi

    # Unload ULPs modules

    if [ "$PRE_UNLOAD_MODULES" != "" ]; then
        for mod in  $PRE_UNLOAD_MODULES
        do
	    unload $mod
	    RC=$[ $RC + $? ]
        done
    fi

    # Remove srp_presistant_bind.sh before removeing ib_srp module
    PID_SCRPT_TO_KILL=`ps -efww | grep srp_persistent | grep -v grep | awk '{print $2}'`
    if ! [ "$PID_SCRPT_TO_KILL" == "" ]; then
        PID_SLEEP_TO_KILL=`ps -efww | grep $PID_SCRPT_TO_KILL | grep sleep | awk '{print $2}'`
        kill -9 $PID_SCRPT_TO_KILL
    fi
    if ! [ "$PID_SLEEP_TO_KILL" == "" ]; then
        kill -9 $PID_SLEEP_TO_KILL
    fi
    
        
    # Unload IPoIB interfaces
    if is_module ib_ipoib; then
        for i in $interfaces
        do
            ifdown ${i} > /dev/null 2>&1
        done    
	# sleep after downing interfaces to avoid a kernel timer warning
	sleep 4
    fi
        
    # Unload OpenIB modules
        
    if [ "$POST_UNLOAD_MODULES" != "" ]; then
        for mod in  $POST_UNLOAD_MODULES
        do
	    unload $mod
	    RC=$[ $RC + $? ]
        done
    fi

    rm -f /var/lock/subsys/openibd
    echo_success
    echo
    return $RC    
}

status()
{
	local cnt=0
	local modules=0

	echo -n "Low level hardware support loaded: "
	if is_module ib_mthca; then
               echo -n "ib_mthca "
	       cnt=$[ $cnt + 1 ]
	fi
	if is_module ib_ipath; then
               echo -n "ib_ipath "
	       cnt=$[ $cnt + 1 ]
	fi
	if is_module ib_ehca; then
               echo -n "ib_ehca "
	       cnt=$[ $cnt + 1 ]
	fi
	if is_module iw_cxgb3; then
               echo -n "iw_cxgb3 "
	       cnt=$[ $cnt + 1 ]
	fi
	if is_module iw_c2; then
               echo -n "iw_c2 "
	       cnt=$[ $cnt + 1 ]
	fi
	if [ $cnt -eq 0 ]; then
		echo -n "none found"
	fi
        echo
        echo
  
    if is_module ib_ipoib; then
       echo -n "Configured IPoIB interfaces: "
       for i in /etc/sysconfig/network-scripts/ifcfg-ib*
       do
            . $i
            echo -n "$DEVICE "
       done
       echo
       echo -n "Currently active IPoIB interfaces: "
       [ ! -z "$interfaces" ] && echo $interfaces || echo "none"
    fi
    
    echo
    
    for mod in  $STATUS_MODULES
    do
    	if is_module $mod; then
    	    [ $modules -eq 0 ] && echo "The following modules are also loaded:"
    	    echo "	$mod"
	    modules=$[ $modules + 1 ]
    	fi
    done
     
    [ $modules -ne 0 ] && echo
    
    if [ $cnt -eq 0 -a $modules -eq 0 ]; then
        if [ -f /var/lock/subsys/openibd ]; then
	    return 2
	else
	    return 3
	fi
    else
        return 0
    fi
}


RC=0
start_time=$(date +%s | tr -d '[:space:]')

trap_handler()
{
    let run_time=$(date +%s | tr -d '[:space:]')-${start_time}

    # Ask to wait for 5 seconds if trying to stop openibd
    if [ $run_time -gt 5 ] && [ "$ACTION" == "stop" ]; then
        printf "\nProbably some application are still using InfiniBand modules...\n"
    else
        printf "\nPlease wait ...\n"
    fi    
    return 0
}

trap 'trap_handler' 2 9 15

restart ()
{
	stop
	start
}

condrestart ()
{
	[ -e /var/lock/subsys/openibd ] && restart || return 0
}

usage ()
{
	echo
	echo "Usage: `basename $0` {start|stop|restart|condrestart|try-restart|force-reload|status}"
	echo
	return 2
}

case $1 in
	start) start; RC=$? ;;
	stop) stop; RC=$? ;;
	restart) restart; RC=$? ;;
	reload) RC=3 ;;
	condrestart) condrestart; RC=$? ;;
	try-restart) condrestart; RC=$? ;;
	force-reload) condrestart; RC=$? ;;
	status) status; RC=$? ;;
	*) usage; RC=$? ;;
esac

exit $RC
