summaryrefslogtreecommitdiffstats
path: root/drivers/accel
diff options
context:
space:
mode:
authorfarah kassabri <fkassabri@habana.ai>2023-08-08 12:56:47 +0300
committerOded Gabbay <ogabbay@kernel.org>2023-10-09 12:37:21 +0300
commit7c4130e6ddd709be2033a6635c91d445cb2baea5 (patch)
treebdf924dfe7763fad06a826e80e115c6b8e9cf390 /drivers/accel
parent72bff371b2e26a0088d4cb1b44aee20c77e423ba (diff)
downloadlinux-stable-7c4130e6ddd709be2033a6635c91d445cb2baea5.tar.gz
linux-stable-7c4130e6ddd709be2033a6635c91d445cb2baea5.tar.bz2
linux-stable-7c4130e6ddd709be2033a6635c91d445cb2baea5.zip
accel/habanalabs/gaudi2: handle eq health heartbeat check
Add mechanism for fw eq health check. this will be done using two flows: using the heartbeat mechanism and raising a dedicated interrupt to indicate an eq failure like EQ full. This patch will add implementation for the eq heartbeat for gaudi2 asic. More info about the heartbeat mechanism: Expand the heartbeat mechanism to monitor a new event that will be sent from FW upon receiving heartbeat message. that way driver can know that the eq is working or not. Signed-off-by: farah kassabri <fkassabri@habana.ai> Reviewed-by: Oded Gabbay <ogabbay@kernel.org> Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
Diffstat (limited to 'drivers/accel')
-rw-r--r--drivers/accel/habanalabs/common/device.c37
-rw-r--r--drivers/accel/habanalabs/common/habanalabs.h2
-rw-r--r--drivers/accel/habanalabs/gaudi2/gaudi2.c10
-rw-r--r--drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h14
4 files changed, 55 insertions, 8 deletions
diff --git a/drivers/accel/habanalabs/common/device.c b/drivers/accel/habanalabs/common/device.c
index e217ee6d1768..1d1ccd8d5c75 100644
--- a/drivers/accel/habanalabs/common/device.c
+++ b/drivers/accel/habanalabs/common/device.c
@@ -989,6 +989,25 @@ static bool is_pci_link_healthy(struct hl_device *hdev)
return (vendor_id == PCI_VENDOR_ID_HABANALABS);
}
+static void hl_device_eq_heartbeat(struct hl_device *hdev)
+{
+ u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ /*
+ * This feature supported in FW version 1.12.0 45.2.0 and above,
+ * only on those FW versions eq_health_check_supported will be set.
+ * Start checking eq health only after driver has enabled events from FW.
+ */
+ if (!prop->cpucp_info.eq_health_check_supported || !hdev->init_done)
+ return;
+
+ if (hdev->eq_heartbeat_received)
+ hdev->eq_heartbeat_received = false;
+ else
+ hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
+}
+
static void hl_device_heartbeat(struct work_struct *work)
{
struct hl_device *hdev = container_of(work, struct hl_device,
@@ -999,6 +1018,12 @@ static void hl_device_heartbeat(struct work_struct *work)
if (!hl_device_operational(hdev, NULL))
goto reschedule;
+ /*
+ * For EQ health check need to check if driver received the heartbeat eq event
+ * in order to validate the eq is working.
+ */
+ hl_device_eq_heartbeat(hdev);
+
if (!hdev->asic_funcs->send_heartbeat(hdev))
goto reschedule;
@@ -1055,7 +1080,15 @@ static int device_late_init(struct hl_device *hdev)
hdev->high_pll = hdev->asic_prop.high_pll;
if (hdev->heartbeat) {
+ /*
+ * Before scheduling the heartbeat driver will check if eq event has received.
+ * for the first schedule we need to set the indication as true then for the next
+ * one this indication will be true only if eq event was sent by FW.
+ */
+ hdev->eq_heartbeat_received = true;
+
INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
+
schedule_delayed_work(&hdev->work_heartbeat,
usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
}
@@ -2235,8 +2268,6 @@ int hl_device_init(struct hl_device *hdev)
"Successfully added device %s to habanalabs driver\n",
dev_name(&(hdev)->pdev->dev));
- hdev->init_done = true;
-
/* After initialization is done, we are ready to receive events from
* the F/W. We can't do it before because we will ignore events and if
* those events are fatal, we won't know about it and the device will
@@ -2244,6 +2275,8 @@ int hl_device_init(struct hl_device *hdev)
*/
hdev->asic_funcs->enable_events_from_fw(hdev);
+ hdev->init_done = true;
+
return 0;
cb_pool_fini:
diff --git a/drivers/accel/habanalabs/common/habanalabs.h b/drivers/accel/habanalabs/common/habanalabs.h
index f8c597903cac..e5b416852996 100644
--- a/drivers/accel/habanalabs/common/habanalabs.h
+++ b/drivers/accel/habanalabs/common/habanalabs.h
@@ -3314,6 +3314,7 @@ struct hl_reset_info {
* device.
* @supports_ctx_switch: true if a ctx switch is required upon first submission.
* @support_preboot_binning: true if we support read binning info from preboot.
+ * @eq_heartbeat_received: indication that eq heartbeat event has received from FW.
* @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing.
* @fw_components: Controls which f/w components to load to the device. There are multiple f/w
* stages and sometimes we want to stop at a certain stage. Used only for testing.
@@ -3474,6 +3475,7 @@ struct hl_device {
u8 reset_upon_device_release;
u8 supports_ctx_switch;
u8 support_preboot_binning;
+ u8 eq_heartbeat_received;
/* Parameters for bring-up to be upstreamed */
u64 nic_ports_mask;
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2.c b/drivers/accel/habanalabs/gaudi2/gaudi2.c
index 677900e18519..e507847bf460 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2.c
@@ -7804,6 +7804,7 @@ static inline bool is_info_event(u32 event)
* an indication to an error.
*/
case GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0 ... GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1:
+ case GAUDI2_EVENT_ARC_EQ_HEARTBEAT:
return true;
default:
return false;
@@ -9765,6 +9766,11 @@ static u16 event_id_to_engine_id(struct hl_device *hdev, u16 event_type)
return U16_MAX;
}
+static void hl_eq_heartbeat_event_handle(struct hl_device *hdev)
+{
+ hdev->eq_heartbeat_received = true;
+}
+
static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
{
struct gaudi2_device *gaudi2 = hdev->asic_specific;
@@ -10190,6 +10196,10 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
gaudi2_irq_map_table[event_type].name);
break;
+ case GAUDI2_EVENT_ARC_EQ_HEARTBEAT:
+ hl_eq_heartbeat_event_handle(hdev);
+ error_count = GAUDI2_NA_EVENT_CAUSE;
+ break;
default:
if (gaudi2_irq_map_table[event_type].valid) {
dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
diff --git a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
index 6cb0f615fc3e..57e661771b6c 100644
--- a/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
+++ b/drivers/accel/habanalabs/include/gaudi2/gaudi2_async_ids_map_extended.h
@@ -2674,17 +2674,19 @@ static struct gaudi2_async_events_ids_map gaudi2_irq_map_table[] = {
{ .fc_id = 1321, .cpu_id = 627, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_HARD,
.name = "DEV_RESET_REQ" },
{ .fc_id = 1322, .cpu_id = 628, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_BRK_ENTRY" },
+ .name = "ARC_PWR_BRK_ENTRY" },
{ .fc_id = 1323, .cpu_id = 629, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_BRK_EXT" },
+ .name = "ARC_PWR_BRK_EXT" },
{ .fc_id = 1324, .cpu_id = 630, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_RD_MODE0" },
+ .name = "ARC_PWR_RD_MODE0" },
{ .fc_id = 1325, .cpu_id = 631, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_RD_MODE1" },
+ .name = "ARC_PWR_RD_MODE1" },
{ .fc_id = 1326, .cpu_id = 632, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_RD_MODE2" },
+ .name = "ARC_PWR_RD_MODE2" },
{ .fc_id = 1327, .cpu_id = 633, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
- .name = "PWR_RD_MODE3" },
+ .name = "ARC_PWR_RD_MODE3" },
+ { .fc_id = 1328, .cpu_id = 634, .valid = 1, .msg = 1, .reset = EVENT_RESET_TYPE_NONE,
+ .name = "ARC_EQ_HEARTBEAT" },
};
#endif /* __GAUDI2_ASYNC_IDS_MAP_EVENTS_EXT_H_ */