From 2046a832bb76c89b3bb3041daa34961823deaf4c Mon Sep 17 00:00:00 2001
From: Penghe Geng <pgeng@nvidia.com>
Date: Tue, 7 Jan 2025 16:22:54 -0500
Subject: [PATCH] bfb-install: Handle PF1 bind/unbind in NIC mode

RM #4221363
---
 scripts/bfb-install | 40 +++++++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/scripts/bfb-install b/scripts/bfb-install
index aea7f04..853dc7f 100755
--- a/scripts/bfb-install
+++ b/scripts/bfb-install
@@ -392,12 +392,18 @@ check_nic_mode()
   # Get PCIE BDF
   str=`cat ${rshim_node}/misc | grep DEV_NAME | awk '{print $2}'`
   str=${str:5}
-  str=${str/.*/.0}
-  pcie_bdf="$str"
+  str=${str/.*/}
+  pcie_bd="$str"
+  num_ports=$(lspci -nn -s $pcie_bd | grep -v '\[0801\]' | wc -l || { echo "Failed to query PCIe ports for $pcie_bd"; echo 0; })
+  if [ "$num_ports" -eq 2 ]; then
+    dual_nic_port=1
+    [ $verbose -eq 1 ] && echo "Detected dual-port device: ${pcie_bd}.0 and ${pcie_bd}.1"
+  fi
 
   # Check NIC mode
-  str=`mlxconfig -d $pcie_bdf -e q INTERNAL_CPU_OFFLOAD_ENGINE 2>/dev/null | grep INTERNAL_CPU_OFFLOAD_ENGINE | awk '{print $(NF-1)}'`
-  if [ ."$str" = ."DISABLED(1)" ]; then
+  str=`mlxconfig -d ${pcie_bd}.0 -e q INTERNAL_CPU_OFFLOAD_ENGINE 2>/dev/null | grep INTERNAL_CPU_OFFLOAD_ENGINE | awk '{print $(NF-1)}'`
+  if [ ."$str" = ."DISABLED(1)" ] || grep -q "NIC mode" ${rshim_node}/misc; then
+    echo "${rshim_node} (${pcie_bd}) is in NIC mode"
     is_nic_mode=1
   else
     is_nic_mode=0
@@ -538,8 +544,16 @@ cleanup() {
   fi
 
   # Bind driver back for NIC mode.
-  if [ $is_nic_mode -eq 1 -a -n "$pcie_bdf" ]; then
-    echo "$pcie_bdf" > /sys/bus/pci/drivers/mlx5_core/bind
+  if [ $is_nic_mode -eq 1 -a -n "${pcie_bd}" ]; then
+    if [[ ! -e /sys/bus/pci/drivers/mlx5_core/${pcie_bd}.0 ]]; then
+      echo "Binding ${pcie_bd}.0"
+      run_cmd_exit local "echo \"${pcie_bd}.0\" > /sys/bus/pci/drivers/mlx5_core/bind"
+    fi
+    if [[ $dual_nic_port -eq 1 && ! -e /sys/bus/pci/drivers/mlx5_core/${pcie_bd}.1 ]]; then
+      echo "Binding ${pcie_bd}.1"
+      run_cmd_exit local "echo \"${pcie_bd}.1\" > /sys/bus/pci/drivers/mlx5_core/bind"
+    fi
+
   fi
 }
 
@@ -568,7 +582,8 @@ cleanup_started=0
 trap cleanup EXIT INT TERM
 
 is_nic_mode=0     # Flag to indicate whether DPU in NIC mod or not
-pcie_bdf=""       # PCIE BDF
+pcie_bd=""        # PCIE Bus-Device
+dual_nic_port=0   # Whether there are two NIC ports, aka PF0/PF1, are present
 
 options=`getopt -n bfb-install -o b:c:f:hkm:r:Ruv \
         -l bfb:,config:,rootfs:,help,keep-log,remote-mode:,reverse-nc,rshim:,runtime,verbose \
@@ -797,8 +812,15 @@ fi
 
 # Check NIC mode and unbind mlx5_core driver in NIC mode.
 check_nic_mode
-if [ $is_nic_mode -eq 1 -a -n "$pcie_bdf" ]; then
-  echo "$pcie_bdf" > /sys/bus/pci/drivers/mlx5_core/unbind 2>/dev/null
+if [ $is_nic_mode -eq 1 -a -n "${pcie_bd}" ]; then
+  if [[ -e /sys/bus/pci/drivers/mlx5_core/${pcie_bd}.0 ]]; then
+    echo "Unbinding ${pcie_bd}.0"
+    run_cmd_exit local "echo \"${pcie_bd}.0\" > /sys/bus/pci/drivers/mlx5_core/unbind"
+  fi
+  if [[ $dual_nic_port -eq 1 && -e /sys/bus/pci/drivers/mlx5_core/${pcie_bd}.1 ]]; then
+    echo "Unbinding ${pcie_bd}.1"
+    run_cmd_exit local "echo \"${pcie_bd}.1\" > /sys/bus/pci/drivers/mlx5_core/unbind"
+  fi
 fi
 push_boot_stream