From 814da63c557726e60b06a206ec8c1123f176d392 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:07 +0800 Subject: [PATCH 01/12] net: hns3: use HNS3_NIC_STATE_INITED to indicate the initialization state of enet Besides of module_init and module_exit, the process of reset will also uninitialize and initialize the enet client. When reset process fails with enet client uninitialized, the module_exit does not need to uninitialize the enet client, otherwise it may cause double uninitialization problem. So we need the HNS3_NIC_STATE_INITED flag to indicate whether the enet client is initialized. Also HNS3_NIC_STATE_REINITING is previously unused, so change it to HNS3_NIC_STATE_INITED. Fixes: bb6b94a896d4 ("net: hns3: Add reset interface implementation in client") Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 17 +++++++++++++++++ drivers/net/ethernet/hisilicon/hns3/hns3_enet.h | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 3f96aa30068e..d51469c6f759 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -3397,6 +3397,8 @@ static int hns3_client_init(struct hnae3_handle *handle) /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */ netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); + set_bit(HNS3_NIC_STATE_INITED, &priv->state); + return ret; out_reg_netdev_fail: @@ -3423,6 +3425,11 @@ static void hns3_client_uninit(struct hnae3_handle *handle, bool reset) if (netdev->reg_state != NETREG_UNINITIALIZED) unregister_netdev(netdev); + if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) { + netdev_warn(netdev, "already uninitialized\n"); + goto out_netdev_free; + } + hns3_del_all_fd_rules(netdev, true); hns3_force_clear_all_rx_ring(handle); @@ -3443,6 +3450,7 @@ static void hns3_client_uninit(struct hnae3_handle *handle, bool reset) priv->ring_data = NULL; +out_netdev_free: free_netdev(netdev); } @@ -3782,6 +3790,8 @@ static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) priv->ring_data = NULL; } + set_bit(HNS3_NIC_STATE_INITED, &priv->state); + return ret; } @@ -3792,6 +3802,11 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) struct hns3_nic_priv *priv = netdev_priv(netdev); int ret; + if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) { + netdev_warn(netdev, "already uninitialized\n"); + return 0; + } + hns3_force_clear_all_rx_ring(handle); ret = hns3_nic_uninit_vector_data(priv); @@ -3815,6 +3830,8 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) hns3_del_all_fd_rules(netdev, false); } + clear_bit(HNS3_NIC_STATE_INITED, &priv->state); + return ret; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index d3636d088aa3..7b759e424d54 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -15,7 +15,7 @@ extern const char hns3_driver_version[]; enum hns3_nic_state { HNS3_NIC_STATE_TESTING, HNS3_NIC_STATE_RESETTING, - HNS3_NIC_STATE_REINITING, + HNS3_NIC_STATE_INITED, HNS3_NIC_STATE_DOWN, HNS3_NIC_STATE_DISABLED, HNS3_NIC_STATE_REMOVING, From 720bd5837e3721f553a896a00da4a99ea12f0551 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:08 +0800 Subject: [PATCH 02/12] net: hns3: add set_default_reset_request in the hnae3_ae_ops Currently, when reset_event is called because of tx timeout, it will upgrade the reset level (For PF, HNAE3_FUNC_RESET -> HNAE3_CORE_RESET -> HNAE3_GLOBAL_RESET) if the time between the new reset and last reset is within 20 secs, or restore the reset level to HNAE3_FUNC_RESET if the time between the new reset and last reset is over 20 secs. There is requirement that the caller needs to decide the reset level when triggering a reset, for example, RAS recovery. So this patch adds the set_default_reset_request to meet this requirement. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 2 ++ .../hisilicon/hns3/hns3pf/hclge_main.c | 13 +++++++++ .../hisilicon/hns3/hns3pf/hclge_main.h | 1 + .../hisilicon/hns3/hns3vf/hclgevf_main.c | 29 ++++++++++++++++++- .../hisilicon/hns3/hns3vf/hclgevf_main.h | 1 + 5 files changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 055b40606dbc..7deab9270d16 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -403,6 +403,8 @@ struct hnae3_ae_ops { u16 vlan, u8 qos, __be16 proto); int (*enable_hw_strip_rxvtag)(struct hnae3_handle *handle, bool enable); void (*reset_event)(struct pci_dev *pdev, struct hnae3_handle *handle); + void (*set_default_reset_request)(struct hnae3_ae_dev *ae_dev, + enum hnae3_reset_type rst_type); void (*get_channels)(struct hnae3_handle *handle, struct ethtool_channels *ch); void (*get_tqps_and_rss_info)(struct hnae3_handle *h, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index ffdd96020860..5e6006b6ed1a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2517,6 +2517,10 @@ static void hclge_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) if (time_before(jiffies, (handle->last_reset_time + 3 * HZ))) return; + else if (hdev->default_reset_request) + handle->reset_level = + hclge_get_reset_level(hdev, + &hdev->default_reset_request); else if (time_after(jiffies, (handle->last_reset_time + 4 * 5 * HZ))) handle->reset_level = HNAE3_FUNC_RESET; @@ -2531,6 +2535,14 @@ static void hclge_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) handle->reset_level++; } +static void hclge_set_def_reset_request(struct hnae3_ae_dev *ae_dev, + enum hnae3_reset_type rst_type) +{ + struct hclge_dev *hdev = ae_dev->priv; + + set_bit(rst_type, &hdev->default_reset_request); +} + static void hclge_reset_subtask(struct hclge_dev *hdev) { /* check if there is any ongoing reset in the hardware. This status can @@ -7321,6 +7333,7 @@ static const struct hnae3_ae_ops hclge_ops = { .set_vf_vlan_filter = hclge_set_vf_vlan_filter, .enable_hw_strip_rxvtag = hclge_en_hw_strip_rxvtag, .reset_event = hclge_reset_event, + .set_default_reset_request = hclge_set_def_reset_request, .get_tqps_and_rss_info = hclge_get_tqps_and_rss_info, .set_channels = hclge_set_channels, .get_channels = hclge_get_channels, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 0d9215404269..7cfb61ebb92e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -595,6 +595,7 @@ struct hclge_dev { unsigned long state; enum hnae3_reset_type reset_type; + unsigned long default_reset_request; unsigned long reset_request; /* reset has been requested */ unsigned long reset_pending; /* client rst is pending to be served */ u32 fw_version; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 085edb945389..42982fc2b5d7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1219,6 +1219,19 @@ static int hclgevf_do_reset(struct hclgevf_dev *hdev) return status; } +static enum hnae3_reset_type hclgevf_get_reset_level(struct hclgevf_dev *hdev, + unsigned long *addr) +{ + enum hnae3_reset_type rst_level = HNAE3_NONE_RESET; + + if (test_bit(HNAE3_VF_RESET, addr)) { + rst_level = HNAE3_VF_RESET; + clear_bit(HNAE3_VF_RESET, addr); + } + + return rst_level; +} + static void hclgevf_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) { @@ -1226,7 +1239,12 @@ static void hclgevf_reset_event(struct pci_dev *pdev, dev_info(&hdev->pdev->dev, "received reset request from VF enet\n"); - handle->reset_level = HNAE3_VF_RESET; + if (!hdev->default_reset_request) + handle->reset_level = + hclgevf_get_reset_level(hdev, + &hdev->default_reset_request); + else + handle->reset_level = HNAE3_VF_RESET; /* reset of this VF requested */ set_bit(HCLGEVF_RESET_REQUESTED, &hdev->reset_state); @@ -1235,6 +1253,14 @@ static void hclgevf_reset_event(struct pci_dev *pdev, handle->last_reset_time = jiffies; } +static void hclgevf_set_def_reset_request(struct hnae3_ae_dev *ae_dev, + enum hnae3_reset_type rst_type) +{ + struct hclgevf_dev *hdev = ae_dev->priv; + + set_bit(rst_type, &hdev->default_reset_request); +} + static u32 hclgevf_get_fw_version(struct hnae3_handle *handle) { struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); @@ -2193,6 +2219,7 @@ static const struct hnae3_ae_ops hclgevf_ops = { .set_vlan_filter = hclgevf_set_vlan_filter, .enable_hw_strip_rxvtag = hclgevf_en_hw_strip_rxvtag, .reset_event = hclgevf_reset_event, + .set_default_reset_request = hclgevf_set_def_reset_request, .get_channels = hclgevf_get_channels, .get_tqps_and_rss_info = hclgevf_get_tqps_and_rss_info, .get_status = hclgevf_get_status, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index aed241e8ffab..6b2fd5a0326f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -145,6 +145,7 @@ struct hclgevf_dev { struct hclgevf_misc_vector misc_vector; struct hclgevf_rss_cfg rss_cfg; unsigned long state; + unsigned long default_reset_request; #define HCLGEVF_RESET_REQUESTED 0 #define HCLGEVF_RESET_PENDING 1 From 4d60291b6b53afd246a57817fdae0c61a4655182 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:09 +0800 Subject: [PATCH 03/12] net: hns3: provide some interface & information for the client The client needs to know if the hardware is resetting when loading or unloading itself, because client may abort the loading process or wait for the reset process to finish when unloading if hardware is resetting. So this patch provides these interfaces to do it. 1. get_hw_reset_stat, the reset status of hardware. 2. ae_dev_resetting, whether reset task is scheduling. 3. ae_dev_reset_cnt, how many reset has been done. Also, the RoCE client needs some field in the hnae3_roce_private_info to save its state, and process_hw_error interface in the hnae3_client_ops to process hardware errors. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 12 ++++++++ .../hisilicon/hns3/hns3pf/hclge_main.c | 29 +++++++++++++++++++ .../hisilicon/hns3/hns3pf/hclge_main.h | 1 + .../hisilicon/hns3/hns3vf/hclgevf_main.c | 25 ++++++++++++++++ .../hisilicon/hns3/hns3vf/hclgevf_main.h | 1 + 5 files changed, 68 insertions(+) diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 7deab9270d16..f57034d1654c 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -162,6 +162,7 @@ struct hnae3_client_ops { int (*setup_tc)(struct hnae3_handle *handle, u8 tc); int (*reset_notify)(struct hnae3_handle *handle, enum hnae3_reset_notify_type type); + enum hnae3_reset_type (*process_hw_error)(struct hnae3_handle *handle); }; #define HNAE3_CLIENT_NAME_LENGTH 16 @@ -432,6 +433,9 @@ struct hnae3_ae_ops { int (*restore_fd_rules)(struct hnae3_handle *handle); void (*enable_fd)(struct hnae3_handle *handle, bool enable); pci_ers_result_t (*process_hw_error)(struct hnae3_ae_dev *ae_dev); + bool (*get_hw_reset_stat)(struct hnae3_handle *handle); + bool (*ae_dev_resetting)(struct hnae3_handle *handle); + unsigned long (*ae_dev_reset_cnt)(struct hnae3_handle *handle); }; struct hnae3_dcb_ops { @@ -490,6 +494,14 @@ struct hnae3_roce_private_info { void __iomem *roce_io_base; int base_vector; int num_vectors; + + /* The below attributes defined for RoCE client, hnae3 gives + * initial values to them, and RoCE client can modify and use + * them. + */ + unsigned long reset_state; + unsigned long instance_state; + unsigned long state; }; struct hnae3_unic_private_info { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 5e6006b6ed1a..b6695424805b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2466,6 +2466,7 @@ static void hclge_reset(struct hclge_dev *hdev) * know if device is undergoing reset */ ae_dev->reset_type = hdev->reset_type; + hdev->reset_count++; /* perform reset of the stack & ae device for a client */ handle = &hdev->vport[0].nic; rtnl_lock(); @@ -4604,6 +4605,31 @@ static int hclge_get_all_rules(struct hnae3_handle *handle, return 0; } +static bool hclge_get_hw_reset_stat(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + return hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG) || + hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING); +} + +static bool hclge_ae_dev_resetting(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + return test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); +} + +static unsigned long hclge_ae_dev_reset_cnt(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + return hdev->reset_count; +} + static void hclge_enable_fd(struct hnae3_handle *handle, bool enable) { struct hclge_vport *vport = hclge_get_vport(handle); @@ -7350,6 +7376,9 @@ static const struct hnae3_ae_ops hclge_ops = { .restore_fd_rules = hclge_restore_fd_entries, .enable_fd = hclge_enable_fd, .process_hw_error = hclge_process_ras_hw_error, + .get_hw_reset_stat = hclge_get_hw_reset_stat, + .ae_dev_resetting = hclge_ae_dev_resetting, + .ae_dev_reset_cnt = hclge_ae_dev_reset_cnt, }; static struct hnae3_ae_algo ae_algo = { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 7cfb61ebb92e..b6a17f56c8d8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -598,6 +598,7 @@ struct hclge_dev { unsigned long default_reset_request; unsigned long reset_request; /* reset has been requested */ unsigned long reset_pending; /* client rst is pending to be served */ + unsigned long reset_count; /* the number of reset has been done */ u32 fw_version; u16 num_vmdq_vport; /* Num vmdq vport this PF has set up */ u16 num_tqps; /* Num task queue pairs of this PF */ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 42982fc2b5d7..517204bbf0b7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1165,6 +1165,7 @@ static int hclgevf_reset(struct hclgevf_dev *hdev) { int ret; + hdev->reset_count++; rtnl_lock(); /* bring down the nic to stop any ongoing TX/RX */ @@ -2185,6 +2186,27 @@ static void hclgevf_get_media_type(struct hnae3_handle *handle, *media_type = hdev->hw.mac.media_type; } +static bool hclgevf_get_hw_reset_stat(struct hnae3_handle *handle) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + + return !!hclgevf_read_dev(&hdev->hw, HCLGEVF_FUN_RST_ING); +} + +static bool hclgevf_ae_dev_resetting(struct hnae3_handle *handle) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + + return test_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state); +} + +static unsigned long hclgevf_ae_dev_reset_cnt(struct hnae3_handle *handle) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + + return hdev->reset_count; +} + static const struct hnae3_ae_ops hclgevf_ops = { .init_ae_dev = hclgevf_init_ae_dev, .uninit_ae_dev = hclgevf_uninit_ae_dev, @@ -2225,6 +2247,9 @@ static const struct hnae3_ae_ops hclgevf_ops = { .get_status = hclgevf_get_status, .get_ksettings_an_result = hclgevf_get_ksettings_an_result, .get_media_type = hclgevf_get_media_type, + .get_hw_reset_stat = hclgevf_get_hw_reset_stat, + .ae_dev_resetting = hclgevf_ae_dev_resetting, + .ae_dev_reset_cnt = hclgevf_ae_dev_reset_cnt, }; static struct hnae3_ae_algo ae_algovf = { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index 6b2fd5a0326f..ffb8c7768b88 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -150,6 +150,7 @@ struct hclgevf_dev { #define HCLGEVF_RESET_REQUESTED 0 #define HCLGEVF_RESET_PENDING 1 unsigned long reset_state; /* requested, pending */ + unsigned long reset_count; /* the number of reset has been done */ u32 reset_attempts; u32 fw_version; From 7edff5339a4a7515b5c4b72338c679c5eb402923 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:10 +0800 Subject: [PATCH 04/12] net: hns3: adjust the location of clearing the table when doing reset When doing a function reset, the hardware table should be cleared before the hardware reset. In current code, this clearing is done in hns3_reset_notify_uninit_enet, but it is too late, because the hardware reset is already done, hns3_reset_notify_down_enet is more suitable to do that. Fixes: bb6b94a896d4 ("net: hns3: Add reset interface implementation in client") Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../net/ethernet/hisilicon/hns3/hns3_enet.c | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index d51469c6f759..75e3e79fa695 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -3716,9 +3716,19 @@ static void hns3_restore_coal(struct hns3_nic_priv *priv) static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) { + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); struct hnae3_knic_private_info *kinfo = &handle->kinfo; struct net_device *ndev = kinfo->netdev; + /* it is cumbersome for hardware to pick-and-choose entries for deletion + * from table space. Hence, for function reset software intervention is + * required to delete the entries + */ + if (hns3_dev_ongoing_func_reset(ae_dev)) { + hns3_remove_hw_addr(ndev); + hns3_del_all_fd_rules(ndev, false); + } + if (!netif_running(ndev)) return 0; @@ -3797,7 +3807,6 @@ static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) { - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); struct net_device *netdev = handle->kinfo.netdev; struct hns3_nic_priv *priv = netdev_priv(netdev); int ret; @@ -3821,15 +3830,6 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) if (ret) netdev_err(netdev, "uninit ring error\n"); - /* it is cumbersome for hardware to pick-and-choose entries for deletion - * from table space. Hence, for function reset software intervention is - * required to delete the entries - */ - if (hns3_dev_ongoing_func_reset(ae_dev)) { - hns3_remove_hw_addr(netdev); - hns3_del_all_fd_rules(netdev, false); - } - clear_bit(HNS3_NIC_STATE_INITED, &priv->state); return ret; From 8df0fa91682c86403173f0813a615efb46e8be42 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:11 +0800 Subject: [PATCH 05/12] net: hns3: enable/disable ring in the enet while doing UP/DOWN While hardware gets into reset status, the firmware will not respond to driver's command request, which may cause ring not disabled problem during reset process. So this patch uses register instead of command to enable/disable the ring in the enet while doing UP/DOWN operation. Also, HNS3_RING_RX_VM_REG is previously unused, so change it to the correct meaning, and add a wrapper function for readl(). Fixes: 46a3df9f9718 ("net: hns3: Add HNS3 Acceleration Engine & Compatibility Layer Support") Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../net/ethernet/hisilicon/hns3/hns3_enet.c | 30 ++++++++++++++++ .../net/ethernet/hisilicon/hns3/hns3_enet.h | 12 ++++++- .../hisilicon/hns3/hns3pf/hclge_main.c | 8 ----- .../hisilicon/hns3/hns3vf/hclgevf_main.c | 35 ------------------- 4 files changed, 41 insertions(+), 44 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 75e3e79fa695..e4cf3247c0ac 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -312,6 +312,24 @@ static u16 hns3_get_max_available_channels(struct hnae3_handle *h) return min_t(u16, rss_size, max_rss_size); } +static void hns3_tqp_enable(struct hnae3_queue *tqp) +{ + u32 rcb_reg; + + rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG); + rcb_reg |= BIT(HNS3_RING_EN_B); + hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg); +} + +static void hns3_tqp_disable(struct hnae3_queue *tqp) +{ + u32 rcb_reg; + + rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG); + rcb_reg &= ~BIT(HNS3_RING_EN_B); + hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg); +} + static int hns3_nic_net_up(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); @@ -334,6 +352,10 @@ static int hns3_nic_net_up(struct net_device *netdev) for (i = 0; i < priv->vector_num; i++) hns3_vector_enable(&priv->tqp_vector[i]); + /* enable rcb */ + for (j = 0; j < h->kinfo.num_tqps; j++) + hns3_tqp_enable(h->kinfo.tqp[j]); + /* start the ae_dev */ ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0; if (ret) @@ -344,6 +366,9 @@ static int hns3_nic_net_up(struct net_device *netdev) return 0; out_start_err: + while (j--) + hns3_tqp_disable(h->kinfo.tqp[j]); + for (j = i - 1; j >= 0; j--) hns3_vector_disable(&priv->tqp_vector[j]); @@ -385,6 +410,7 @@ static int hns3_nic_net_open(struct net_device *netdev) static void hns3_nic_net_down(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = hns3_get_handle(netdev); const struct hnae3_ae_ops *ops; int i; @@ -395,6 +421,10 @@ static void hns3_nic_net_down(struct net_device *netdev) for (i = 0; i < priv->vector_num; i++) hns3_vector_disable(&priv->tqp_vector[i]); + /* disable rcb */ + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_tqp_disable(h->kinfo.tqp[i]); + /* stop ae_dev */ ops = priv->ae_handle->ae_algo->ops; if (ops->stop) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 7b759e424d54..3e9db7356cf2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -47,7 +47,7 @@ enum hns3_nic_state { #define HNS3_RING_PREFETCH_EN_REG 0x0007C #define HNS3_RING_CFG_VF_NUM_REG 0x00080 #define HNS3_RING_ASID_REG 0x0008C -#define HNS3_RING_RX_VM_REG 0x00090 +#define HNS3_RING_EN_REG 0x00090 #define HNS3_RING_T0_BE_RST 0x00094 #define HNS3_RING_COULD_BE_RST 0x00098 #define HNS3_RING_WRR_WEIGHT_REG 0x0009c @@ -194,6 +194,8 @@ enum hns3_nic_state { #define HNS3_VECTOR_RL_OFFSET 0x900 #define HNS3_VECTOR_RL_EN_B 6 +#define HNS3_RING_EN_B 0 + enum hns3_pkt_l3t_type { HNS3_L3T_NONE, HNS3_L3T_IPV6, @@ -577,6 +579,11 @@ static inline int is_ring_empty(struct hns3_enet_ring *ring) return ring->next_to_use == ring->next_to_clean; } +static inline u32 hns3_read_reg(void __iomem *base, u32 reg) +{ + return readl(base + reg); +} + static inline void hns3_write_reg(void __iomem *base, u32 reg, u32 value) { u8 __iomem *reg_addr = READ_ONCE(base); @@ -589,6 +596,9 @@ static inline bool hns3_dev_ongoing_func_reset(struct hnae3_ae_dev *ae_dev) return (ae_dev && (ae_dev->reset_type == HNAE3_FUNC_RESET)); } +#define hns3_read_dev(a, reg) \ + hns3_read_reg((a)->io_base, (reg)) + #define hns3_write_dev(a, reg, value) \ hns3_write_reg((a)->io_base, (reg), (value)) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index b6695424805b..b784db067290 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -4843,10 +4843,6 @@ static int hclge_ae_start(struct hnae3_handle *handle) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - int i; - - for (i = 0; i < vport->alloc_tqps; i++) - hclge_tqp_enable(hdev, i, 0, true); /* mac enable */ hclge_cfg_mac_mode(hdev, true); @@ -4866,7 +4862,6 @@ static void hclge_ae_stop(struct hnae3_handle *handle) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - int i; set_bit(HCLGE_STATE_DOWN, &hdev->state); @@ -4879,9 +4874,6 @@ static void hclge_ae_stop(struct hnae3_handle *handle) return; } - for (i = 0; i < vport->alloc_tqps; i++) - hclge_tqp_enable(hdev, i, 0, false); - /* Mac disable */ hclge_cfg_mac_mode(hdev, false); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 517204bbf0b7..7531bdde02a7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -956,13 +956,6 @@ static int hclgevf_tqp_enable(struct hclgevf_dev *hdev, int tqp_id, return status; } -static int hclgevf_get_queue_id(struct hnae3_queue *queue) -{ - struct hclgevf_tqp *tqp = container_of(queue, struct hclgevf_tqp, q); - - return tqp->index; -} - static void hclgevf_reset_tqp_stats(struct hnae3_handle *handle) { struct hnae3_knic_private_info *kinfo = &handle->kinfo; @@ -1593,21 +1586,7 @@ static int hclgevf_init_vlan_config(struct hclgevf_dev *hdev) static int hclgevf_ae_start(struct hnae3_handle *handle) { - struct hnae3_knic_private_info *kinfo = &handle->kinfo; struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); - int i, queue_id; - - for (i = 0; i < kinfo->num_tqps; i++) { - /* ring enable */ - queue_id = hclgevf_get_queue_id(kinfo->tqp[i]); - if (queue_id < 0) { - dev_warn(&hdev->pdev->dev, - "Get invalid queue id, ignore it\n"); - continue; - } - - hclgevf_tqp_enable(hdev, queue_id, 0, true); - } /* reset tqp stats */ hclgevf_reset_tqp_stats(handle); @@ -1622,24 +1601,10 @@ static int hclgevf_ae_start(struct hnae3_handle *handle) static void hclgevf_ae_stop(struct hnae3_handle *handle) { - struct hnae3_knic_private_info *kinfo = &handle->kinfo; struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); - int i, queue_id; set_bit(HCLGEVF_STATE_DOWN, &hdev->state); - for (i = 0; i < kinfo->num_tqps; i++) { - /* Ring disable */ - queue_id = hclgevf_get_queue_id(kinfo->tqp[i]); - if (queue_id < 0) { - dev_warn(&hdev->pdev->dev, - "Get invalid queue id, ignore it\n"); - continue; - } - - hclgevf_tqp_enable(hdev, queue_id, 0, false); - } - /* reset tqp stats */ hclgevf_reset_tqp_stats(handle); del_timer_sync(&hdev->service_timer); From 257e4f29946ab8e257ee9bfbc35e680726298571 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:12 +0800 Subject: [PATCH 06/12] net: hns3: use HNS3_NIC_STATE_RESETTING to indicate resetting While hclge is going to reset, it will notify its client with HNAE3_DOWN_CLIENT, so this client should get into a resetting status from this moment, other operations from the stack need to be blocked as well. And when the reset is finished, the client will be notified with HNAE3_UP_CLIENT, so this is the end of the resetting status. This patch uses HNS3_NIC_STATE_RESETTING flag to implement that, and adds hns3_nic_resetting() to indicate which operation is not allowed. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../net/ethernet/hisilicon/hns3/hns3_dcbnl.c | 12 ++++++++++++ .../net/ethernet/hisilicon/hns3/hns3_enet.c | 10 ++++++++++ .../net/ethernet/hisilicon/hns3/hns3_enet.h | 7 +++++++ .../ethernet/hisilicon/hns3/hns3_ethtool.c | 19 +++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c index ea5f8a84070d..b6fabbbdfd5b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c @@ -9,6 +9,9 @@ int hns3_dcbnl_ieee_getets(struct net_device *ndev, struct ieee_ets *ets) { struct hnae3_handle *h = hns3_get_handle(ndev); + if (hns3_nic_resetting(ndev)) + return -EBUSY; + if (h->kinfo.dcb_ops->ieee_getets) return h->kinfo.dcb_ops->ieee_getets(h, ets); @@ -20,6 +23,9 @@ int hns3_dcbnl_ieee_setets(struct net_device *ndev, struct ieee_ets *ets) { struct hnae3_handle *h = hns3_get_handle(ndev); + if (hns3_nic_resetting(ndev)) + return -EBUSY; + if (h->kinfo.dcb_ops->ieee_setets) return h->kinfo.dcb_ops->ieee_setets(h, ets); @@ -31,6 +37,9 @@ int hns3_dcbnl_ieee_getpfc(struct net_device *ndev, struct ieee_pfc *pfc) { struct hnae3_handle *h = hns3_get_handle(ndev); + if (hns3_nic_resetting(ndev)) + return -EBUSY; + if (h->kinfo.dcb_ops->ieee_getpfc) return h->kinfo.dcb_ops->ieee_getpfc(h, pfc); @@ -42,6 +51,9 @@ int hns3_dcbnl_ieee_setpfc(struct net_device *ndev, struct ieee_pfc *pfc) { struct hnae3_handle *h = hns3_get_handle(ndev); + if (hns3_nic_resetting(ndev)) + return -EBUSY; + if (h->kinfo.dcb_ops->ieee_setpfc) return h->kinfo.dcb_ops->ieee_setpfc(h, pfc); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index e4cf3247c0ac..d573f5fc9575 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -384,6 +384,9 @@ static int hns3_nic_net_open(struct net_device *netdev) struct hnae3_knic_private_info *kinfo; int i, ret; + if (hns3_nic_resetting(netdev)) + return -EBUSY; + netif_carrier_off(netdev); ret = hns3_nic_set_real_num_queue(netdev); @@ -3749,6 +3752,10 @@ static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); struct hnae3_knic_private_info *kinfo = &handle->kinfo; struct net_device *ndev = kinfo->netdev; + struct hns3_nic_priv *priv = netdev_priv(ndev); + + if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) + return 0; /* it is cumbersome for hardware to pick-and-choose entries for deletion * from table space. Hence, for function reset software intervention is @@ -3768,6 +3775,7 @@ static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) { struct hnae3_knic_private_info *kinfo = &handle->kinfo; + struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev); int ret = 0; if (netif_running(kinfo->netdev)) { @@ -3780,6 +3788,8 @@ static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) handle->last_reset_time = jiffies; } + clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state); + return ret; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 3e9db7356cf2..cfd6a7109691 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -599,6 +599,13 @@ static inline bool hns3_dev_ongoing_func_reset(struct hnae3_ae_dev *ae_dev) #define hns3_read_dev(a, reg) \ hns3_read_reg((a)->io_base, (reg)) +static inline bool hns3_nic_resetting(struct net_device *netdev) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + + return test_bit(HNS3_NIC_STATE_RESETTING, &priv->state); +} + #define hns3_write_dev(a, reg, value) \ hns3_write_reg((a)->io_base, (reg), (value)) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index a4762c2b8ba1..4563638367ac 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -291,6 +291,11 @@ static void hns3_self_test(struct net_device *ndev, int test_index = 0; u32 i; + if (hns3_nic_resetting(ndev)) { + netdev_err(ndev, "dev resetting!"); + return; + } + /* Only do offline selftest, or pass by default */ if (eth_test->flags != ETH_TEST_FL_OFFLINE) return; @@ -530,6 +535,11 @@ static void hns3_get_ringparam(struct net_device *netdev, struct hnae3_handle *h = priv->ae_handle; int queue_num = h->kinfo.num_tqps; + if (hns3_nic_resetting(netdev)) { + netdev_err(netdev, "dev resetting!"); + return; + } + param->tx_max_pending = HNS3_RING_MAX_PENDING; param->rx_max_pending = HNS3_RING_MAX_PENDING; @@ -760,6 +770,9 @@ static int hns3_set_ringparam(struct net_device *ndev, u32 old_desc_num, new_desc_num; int ret; + if (hns3_nic_resetting(ndev)) + return -EBUSY; + if (param->rx_mini_pending || param->rx_jumbo_pending) return -EINVAL; @@ -872,6 +885,9 @@ static int hns3_get_coalesce_per_queue(struct net_device *netdev, u32 queue, struct hnae3_handle *h = priv->ae_handle; u16 queue_num = h->kinfo.num_tqps; + if (hns3_nic_resetting(netdev)) + return -EBUSY; + if (queue >= queue_num) { netdev_err(netdev, "Invalid queue value %d! Queue max id=%d\n", @@ -1033,6 +1049,9 @@ static int hns3_set_coalesce(struct net_device *netdev, int ret; int i; + if (hns3_nic_resetting(netdev)) + return -EBUSY; + ret = hns3_check_coalesce_para(netdev, cmd); if (ret) return ret; From 7cea834d942c2d47ebb81fba0e44d2ce6172c0ae Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:13 +0800 Subject: [PATCH 07/12] net: hns3: ignore new coming low-level reset while doing high-level reset When processing a higher level reset, the pending lower level reset does not have to be processed anymore, because the higher level reset is the superset of the lower level reset. Therefore, when processing an higher level reset, the request of lower level reset needs to be cleared. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../hisilicon/hns3/hns3pf/hclge_main.c | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index b784db067290..62022d552f5d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2414,20 +2414,25 @@ static enum hnae3_reset_type hclge_get_reset_level(struct hclge_dev *hdev, enum hnae3_reset_type rst_level = HNAE3_NONE_RESET; /* return the highest priority reset level amongst all */ - if (test_bit(HNAE3_GLOBAL_RESET, addr)) - rst_level = HNAE3_GLOBAL_RESET; - else if (test_bit(HNAE3_CORE_RESET, addr)) - rst_level = HNAE3_CORE_RESET; - else if (test_bit(HNAE3_IMP_RESET, addr)) + if (test_bit(HNAE3_IMP_RESET, addr)) { rst_level = HNAE3_IMP_RESET; - else if (test_bit(HNAE3_FUNC_RESET, addr)) + clear_bit(HNAE3_IMP_RESET, addr); + clear_bit(HNAE3_GLOBAL_RESET, addr); + clear_bit(HNAE3_CORE_RESET, addr); + clear_bit(HNAE3_FUNC_RESET, addr); + } else if (test_bit(HNAE3_GLOBAL_RESET, addr)) { + rst_level = HNAE3_GLOBAL_RESET; + clear_bit(HNAE3_GLOBAL_RESET, addr); + clear_bit(HNAE3_CORE_RESET, addr); + clear_bit(HNAE3_FUNC_RESET, addr); + } else if (test_bit(HNAE3_CORE_RESET, addr)) { + rst_level = HNAE3_CORE_RESET; + clear_bit(HNAE3_CORE_RESET, addr); + clear_bit(HNAE3_FUNC_RESET, addr); + } else if (test_bit(HNAE3_FUNC_RESET, addr)) { rst_level = HNAE3_FUNC_RESET; - - /* now, clear all other resets */ - clear_bit(HNAE3_GLOBAL_RESET, addr); - clear_bit(HNAE3_CORE_RESET, addr); - clear_bit(HNAE3_IMP_RESET, addr); - clear_bit(HNAE3_FUNC_RESET, addr); + clear_bit(HNAE3_FUNC_RESET, addr); + } return rst_level; } From 0742ed7c240ceadc4472ab5a80bcb2a5f05e7ff8 Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:14 +0800 Subject: [PATCH 08/12] net: hns3: move some reset information from hnae3_handle into hclge_dev/hclgevf_dev Saving reset related information in the hclge_dev/hclgevf_dev structure is more suitable than the hnae3_handle, since hardware related information is kept in these two structure. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 3 --- .../net/ethernet/hisilicon/hns3/hns3_enet.c | 16 +++---------- .../hisilicon/hns3/hns3pf/hclge_main.c | 23 ++++++++++--------- .../hisilicon/hns3/hns3pf/hclge_main.h | 2 ++ .../hisilicon/hns3/hns3vf/hclgevf_main.c | 10 ++++---- .../hisilicon/hns3/hns3vf/hclgevf_main.h | 6 +++-- .../hisilicon/hns3/hns3vf/hclgevf_mbx.c | 2 +- 7 files changed, 28 insertions(+), 34 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index f57034d1654c..5b3b10479cd4 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -534,9 +534,6 @@ struct hnae3_handle { struct hnae3_ae_algo *ae_algo; /* the class who provides this handle */ u64 flags; /* Indicate the capabilities for this handle*/ - unsigned long last_reset_time; - enum hnae3_reset_type reset_level; - union { struct net_device *netdev; /* first member */ struct hnae3_knic_private_info kinfo; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index d573f5fc9575..183fd8344b80 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -379,7 +379,6 @@ static int hns3_nic_net_up(struct net_device *netdev) static int hns3_nic_net_open(struct net_device *netdev) { - struct hns3_nic_priv *priv = netdev_priv(netdev); struct hnae3_handle *h = hns3_get_handle(netdev); struct hnae3_knic_private_info *kinfo; int i, ret; @@ -406,7 +405,6 @@ static int hns3_nic_net_open(struct net_device *netdev) kinfo->prio_tc[i]); } - priv->ae_handle->last_reset_time = jiffies; return 0; } @@ -1648,10 +1646,9 @@ static void hns3_nic_net_timeout(struct net_device *ndev) priv->tx_timeout_count++; - if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo))) - return; - - /* request the reset */ + /* request the reset, and let the hclge to determine + * which reset level should be done + */ if (h->ae_algo->ops->reset_event) h->ae_algo->ops->reset_event(h->pdev, h); } @@ -3370,7 +3367,6 @@ static int hns3_client_init(struct hnae3_handle *handle) priv->dev = &pdev->dev; priv->netdev = netdev; priv->ae_handle = handle; - priv->ae_handle->last_reset_time = jiffies; priv->tx_timeout_count = 0; handle->kinfo.netdev = netdev; @@ -3390,11 +3386,6 @@ static int hns3_client_init(struct hnae3_handle *handle) /* Carrier off reporting is important to ethtool even BEFORE open */ netif_carrier_off(netdev); - if (handle->flags & HNAE3_SUPPORT_VF) - handle->reset_level = HNAE3_VF_RESET; - else - handle->reset_level = HNAE3_FUNC_RESET; - ret = hns3_get_ring_config(priv); if (ret) { ret = -ENOMEM; @@ -3785,7 +3776,6 @@ static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) "hns net up fail, ret=%d!\n", ret); return ret; } - handle->last_reset_time = jiffies; } clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 62022d552f5d..a90c19e6f2ca 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2465,15 +2465,14 @@ static void hclge_clear_reset_cause(struct hclge_dev *hdev) static void hclge_reset(struct hclge_dev *hdev) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); - struct hnae3_handle *handle; /* Initialize ae_dev reset status as well, in case enet layer wants to * know if device is undergoing reset */ ae_dev->reset_type = hdev->reset_type; hdev->reset_count++; + hdev->last_reset_time = jiffies; /* perform reset of the stack & ae device for a client */ - handle = &hdev->vport[0].nic; rtnl_lock(); hclge_notify_client(hdev, HNAE3_DOWN_CLIENT); rtnl_unlock(); @@ -2493,7 +2492,6 @@ static void hclge_reset(struct hclge_dev *hdev) } hclge_notify_client(hdev, HNAE3_UP_CLIENT); - handle->last_reset_time = jiffies; rtnl_unlock(); ae_dev->reset_type = HNAE3_NONE_RESET; } @@ -2521,24 +2519,24 @@ static void hclge_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) if (!handle) handle = &hdev->vport[0].nic; - if (time_before(jiffies, (handle->last_reset_time + 3 * HZ))) + if (time_before(jiffies, (hdev->last_reset_time + 3 * HZ))) return; else if (hdev->default_reset_request) - handle->reset_level = + hdev->reset_level = hclge_get_reset_level(hdev, &hdev->default_reset_request); - else if (time_after(jiffies, (handle->last_reset_time + 4 * 5 * HZ))) - handle->reset_level = HNAE3_FUNC_RESET; + else if (time_after(jiffies, (hdev->last_reset_time + 4 * 5 * HZ))) + hdev->reset_level = HNAE3_FUNC_RESET; dev_info(&hdev->pdev->dev, "received reset event , reset type is %d", - handle->reset_level); + hdev->reset_level); /* request reset & schedule reset task */ - set_bit(handle->reset_level, &hdev->reset_request); + set_bit(hdev->reset_level, &hdev->reset_request); hclge_reset_task_schedule(hdev); - if (handle->reset_level < HNAE3_GLOBAL_RESET) - handle->reset_level++; + if (hdev->reset_level < HNAE3_GLOBAL_RESET) + hdev->reset_level++; } static void hclge_set_def_reset_request(struct hnae3_ae_dev *ae_dev, @@ -2560,6 +2558,7 @@ static void hclge_reset_subtask(struct hclge_dev *hdev) * b. else, we can come back later to check this status so re-sched * now. */ + hdev->last_reset_time = jiffies; hdev->reset_type = hclge_get_reset_level(hdev, &hdev->reset_pending); if (hdev->reset_type != HNAE3_NONE_RESET) hclge_reset(hdev); @@ -6670,6 +6669,7 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) hdev->pdev = pdev; hdev->ae_dev = ae_dev; hdev->reset_type = HNAE3_NONE_RESET; + hdev->reset_level = HNAE3_FUNC_RESET; ae_dev->priv = hdev; ret = hclge_pci_init(hdev); @@ -6814,6 +6814,7 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) hclge_enable_vector(&hdev->misc_vector, true); hclge_state_init(hdev); + hdev->last_reset_time = jiffies; pr_info("%s driver initialization finished.\n", HCLGE_DRIVER_NAME); return 0; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index b6a17f56c8d8..69bdb7f646ee 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -593,8 +593,10 @@ struct hclge_dev { struct hclge_misc_vector misc_vector; struct hclge_hw_stats hw_stats; unsigned long state; + unsigned long last_reset_time; enum hnae3_reset_type reset_type; + enum hnae3_reset_type reset_level; unsigned long default_reset_request; unsigned long reset_request; /* reset has been requested */ unsigned long reset_pending; /* client rst is pending to be served */ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 7531bdde02a7..4a2148bde35a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1234,17 +1234,17 @@ static void hclgevf_reset_event(struct pci_dev *pdev, dev_info(&hdev->pdev->dev, "received reset request from VF enet\n"); if (!hdev->default_reset_request) - handle->reset_level = + hdev->reset_level = hclgevf_get_reset_level(hdev, &hdev->default_reset_request); else - handle->reset_level = HNAE3_VF_RESET; + hdev->reset_level = HNAE3_VF_RESET; /* reset of this VF requested */ set_bit(HCLGEVF_RESET_REQUESTED, &hdev->reset_state); hclgevf_reset_task_schedule(hdev); - handle->last_reset_time = jiffies; + hdev->last_reset_time = jiffies; } static void hclgevf_set_def_reset_request(struct hnae3_ae_dev *ae_dev, @@ -1372,7 +1372,7 @@ static void hclgevf_reset_service_task(struct work_struct *work) */ if (hdev->reset_attempts > 3) { /* prepare for full reset of stack + pcie interface */ - hdev->nic.reset_level = HNAE3_VF_FULL_RESET; + hdev->reset_level = HNAE3_VF_FULL_RESET; /* "defer" schedule the reset task again */ set_bit(HCLGEVF_RESET_PENDING, &hdev->reset_state); @@ -1985,6 +1985,7 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) } hclgevf_state_init(hdev); + hdev->reset_level = HNAE3_VF_RESET; ret = hclgevf_misc_irq_init(hdev); if (ret) { @@ -2026,6 +2027,7 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) goto err_config; } + hdev->last_reset_time = jiffies; pr_info("finished initializing %s driver\n", HCLGEVF_DRIVER_NAME); return 0; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index ffb8c7768b88..0ea4c9b3ae0a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -146,6 +146,8 @@ struct hclgevf_dev { struct hclgevf_rss_cfg rss_cfg; unsigned long state; unsigned long default_reset_request; + unsigned long last_reset_time; + enum hnae3_reset_type reset_level; #define HCLGEVF_RESET_REQUESTED 0 #define HCLGEVF_RESET_PENDING 1 @@ -198,14 +200,14 @@ static inline bool hclgevf_dev_ongoing_reset(struct hclgevf_dev *hdev) { return (hdev && (test_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state)) && - (hdev->nic.reset_level == HNAE3_VF_RESET)); + (hdev->reset_level == HNAE3_VF_RESET)); } static inline bool hclgevf_dev_ongoing_full_reset(struct hclgevf_dev *hdev) { return (hdev && (test_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state)) && - (hdev->nic.reset_level == HNAE3_VF_FULL_RESET)); + (hdev->reset_level == HNAE3_VF_FULL_RESET)); } int hclgevf_send_mbx_msg(struct hclgevf_dev *hdev, u16 code, u16 subcode, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c index e9d5a4f96304..558e3b838676 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c @@ -267,7 +267,7 @@ void hclgevf_mbx_async_handler(struct hclgevf_dev *hdev) * has been completely reset. After this stack should * eventually be re-initialized. */ - hdev->nic.reset_level = HNAE3_VF_RESET; + hdev->reset_level = HNAE3_VF_RESET; set_bit(HCLGEVF_RESET_PENDING, &hdev->reset_state); hclgevf_reset_task_schedule(hdev); From 35d93a30040c3884d8752c35a3fc1200d8f96b1b Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:15 +0800 Subject: [PATCH 09/12] net: hns3: adjust the process of PF reset When doing PF reset, the driver needs to do some preparatory work before asserting PF reset. Since when hardware is resetting, it is necessary to stop tx/rx queue, clear hardware table, etc, otherwise hardware may run into unrecoverable state if there is still IO running when the hardware is resetting. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../hisilicon/hns3/hns3pf/hclge_main.c | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index a90c19e6f2ca..3c327f8cb865 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2396,7 +2396,6 @@ static void hclge_do_reset(struct hclge_dev *hdev) break; case HNAE3_FUNC_RESET: dev_info(&pdev->dev, "PF Reset requested\n"); - hclge_func_reset_cmd(hdev, 0); /* schedule again to check later */ set_bit(HNAE3_FUNC_RESET, &hdev->reset_pending); hclge_reset_task_schedule(hdev); @@ -2462,6 +2461,35 @@ static void hclge_clear_reset_cause(struct hclge_dev *hdev) hclge_enable_vector(&hdev->misc_vector, true); } +static int hclge_reset_prepare_wait(struct hclge_dev *hdev) +{ + int ret = 0; + + switch (hdev->reset_type) { + case HNAE3_FUNC_RESET: + ret = hclge_func_reset_cmd(hdev, 0); + if (ret) { + dev_err(&hdev->pdev->dev, + "assertting function reset fail %d!\n", ret); + return ret; + } + + /* After performaning pf reset, it is not necessary to do the + * mailbox handling or send any command to firmware, because + * any mailbox handling or command to firmware is only valid + * after hclge_cmd_init is called. + */ + set_bit(HCLGE_STATE_CMD_DISABLE, &hdev->state); + break; + default: + break; + } + + dev_info(&hdev->pdev->dev, "prepare wait ok\n"); + + return ret; +} + static void hclge_reset(struct hclge_dev *hdev) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); @@ -2477,6 +2505,8 @@ static void hclge_reset(struct hclge_dev *hdev) hclge_notify_client(hdev, HNAE3_DOWN_CLIENT); rtnl_unlock(); + hclge_reset_prepare_wait(hdev); + if (!hclge_reset_wait(hdev)) { rtnl_lock(); hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT); @@ -4873,7 +4903,11 @@ static void hclge_ae_stop(struct hnae3_handle *handle) cancel_work_sync(&hdev->service_task); clear_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state); - if (test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state)) { + /* If it is not PF reset, the firmware will disable the MAC, + * so it only need to stop phy here. + */ + if (test_bit(HCLGE_STATE_RST_HANDLING, &hdev->state) && + hdev->reset_type != HNAE3_FUNC_RESET) { hclge_mac_stop_phy(hdev); return; } From f403a84fb25ea5a49cf9d6d27f522e8f752ef36f Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:16 +0800 Subject: [PATCH 10/12] net: hns3: call roce's reset notify callback when resetting While doing resetting, roce should do its uninitailization part before nic's, and do its initialization part after nic's. Signed-off-by: Huazhong Tan Signed-off-by: Peng Li Signed-off-by: David S. Miller --- .../hisilicon/hns3/hns3pf/hclge_main.c | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 3c327f8cb865..0c0327b733a2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2315,6 +2315,34 @@ static int hclge_notify_client(struct hclge_dev *hdev, return 0; } +static int hclge_notify_roce_client(struct hclge_dev *hdev, + enum hnae3_reset_notify_type type) +{ + struct hnae3_client *client = hdev->roce_client; + int ret = 0; + u16 i; + + if (!client) + return 0; + + if (!client->ops->reset_notify) + return -EOPNOTSUPP; + + for (i = 0; i < hdev->num_vmdq_vport + 1; i++) { + struct hnae3_handle *handle = &hdev->vport[i].roce; + + ret = client->ops->reset_notify(handle, type); + if (ret) { + dev_err(&hdev->pdev->dev, + "notify roce client failed %d(%d)", + type, ret); + return ret; + } + } + + return ret; +} + static int hclge_reset_wait(struct hclge_dev *hdev) { #define HCLGE_RESET_WATI_MS 100 @@ -2501,6 +2529,7 @@ static void hclge_reset(struct hclge_dev *hdev) hdev->reset_count++; hdev->last_reset_time = jiffies; /* perform reset of the stack & ae device for a client */ + hclge_notify_roce_client(hdev, HNAE3_DOWN_CLIENT); rtnl_lock(); hclge_notify_client(hdev, HNAE3_DOWN_CLIENT); rtnl_unlock(); @@ -2508,6 +2537,7 @@ static void hclge_reset(struct hclge_dev *hdev) hclge_reset_prepare_wait(hdev); if (!hclge_reset_wait(hdev)) { + hclge_notify_roce_client(hdev, HNAE3_UNINIT_CLIENT); rtnl_lock(); hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT); hclge_reset_ae_dev(hdev->ae_dev); @@ -2524,6 +2554,9 @@ static void hclge_reset(struct hclge_dev *hdev) hclge_notify_client(hdev, HNAE3_UP_CLIENT); rtnl_unlock(); ae_dev->reset_type = HNAE3_NONE_RESET; + + hclge_notify_roce_client(hdev, HNAE3_INIT_CLIENT); + hclge_notify_roce_client(hdev, HNAE3_UP_CLIENT); } static void hclge_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) From 65e41e7e683862136a817f29a1a457ecdf37ba8a Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:17 +0800 Subject: [PATCH 11/12] net: hns3: add error handler for hclge_reset() When hclge_reset() is called, it may fail for several reasons. For example, an higher-level reset event occurs, memory allocation failure, hardware reset timeout, etc. Therefore, it is necessary to add corresponding error handling for these situations. 1. A high-level reset is required due to a high-level reset failure. 2. For memory allocation failure, a high-level reset is initiated by the timer to recover. The reason for using the timer is to prevent this new high-level reset to interrupt the reset process of other pf/vf; 3. For the case of hardware reset timeout, reschedule the reset task to wait for the hardware to complete the reset. For memory allocation failure and reset timeouts, in order to prevent an infinite number of scheduled reset tasks, the number of error recovery needs to be limited. This patch also add some reset related debug log printing. Signed-off-by: Huazhong Tan Signed-off-by: David S. Miller --- .../hisilicon/hns3/hns3pf/hclge_main.c | 141 +++++++++++++++--- .../hisilicon/hns3/hns3pf/hclge_main.h | 3 + 2 files changed, 121 insertions(+), 23 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 0c0327b733a2..579945bdcc76 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -2145,6 +2145,7 @@ static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) /* check for vector0 reset event sources */ if (BIT(HCLGE_VECTOR0_GLOBALRESET_INT_B) & rst_src_reg) { + dev_info(&hdev->pdev->dev, "global reset interrupt\n"); set_bit(HCLGE_STATE_CMD_DISABLE, &hdev->state); set_bit(HNAE3_GLOBAL_RESET, &hdev->reset_pending); *clearval = BIT(HCLGE_VECTOR0_GLOBALRESET_INT_B); @@ -2152,6 +2153,7 @@ static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) } if (BIT(HCLGE_VECTOR0_CORERESET_INT_B) & rst_src_reg) { + dev_info(&hdev->pdev->dev, "core reset interrupt\n"); set_bit(HCLGE_STATE_CMD_DISABLE, &hdev->state); set_bit(HNAE3_CORE_RESET, &hdev->reset_pending); *clearval = BIT(HCLGE_VECTOR0_CORERESET_INT_B); @@ -2159,6 +2161,7 @@ static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) } if (BIT(HCLGE_VECTOR0_IMPRESET_INT_B) & rst_src_reg) { + dev_info(&hdev->pdev->dev, "IMP reset interrupt\n"); set_bit(HNAE3_IMP_RESET, &hdev->reset_pending); *clearval = BIT(HCLGE_VECTOR0_IMPRESET_INT_B); return HCLGE_VECTOR0_EVENT_RST; @@ -2308,8 +2311,11 @@ static int hclge_notify_client(struct hclge_dev *hdev, int ret; ret = client->ops->reset_notify(handle, type); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "notify nic client failed %d(%d)\n", type, ret); return ret; + } } return 0; @@ -2518,9 +2524,49 @@ static int hclge_reset_prepare_wait(struct hclge_dev *hdev) return ret; } +static bool hclge_reset_err_handle(struct hclge_dev *hdev, bool is_timeout) +{ +#define MAX_RESET_FAIL_CNT 5 +#define RESET_UPGRADE_DELAY_SEC 10 + + if (hdev->reset_pending) { + dev_info(&hdev->pdev->dev, "Reset pending %lu\n", + hdev->reset_pending); + return true; + } else if ((hdev->reset_type != HNAE3_IMP_RESET) && + (hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG) & + BIT(HCLGE_IMP_RESET_BIT))) { + dev_info(&hdev->pdev->dev, + "reset failed because IMP Reset is pending\n"); + hclge_clear_reset_cause(hdev); + return false; + } else if (hdev->reset_fail_cnt < MAX_RESET_FAIL_CNT) { + hdev->reset_fail_cnt++; + if (is_timeout) { + set_bit(hdev->reset_type, &hdev->reset_pending); + dev_info(&hdev->pdev->dev, + "re-schedule to wait for hw reset done\n"); + return true; + } + + dev_info(&hdev->pdev->dev, "Upgrade reset level\n"); + hclge_clear_reset_cause(hdev); + mod_timer(&hdev->reset_timer, + jiffies + RESET_UPGRADE_DELAY_SEC * HZ); + + return false; + } + + hclge_clear_reset_cause(hdev); + dev_err(&hdev->pdev->dev, "Reset fail!\n"); + return false; +} + static void hclge_reset(struct hclge_dev *hdev) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + bool is_timeout = false; + int ret; /* Initialize ae_dev reset status as well, in case enet layer wants to * know if device is undergoing reset @@ -2529,34 +2575,66 @@ static void hclge_reset(struct hclge_dev *hdev) hdev->reset_count++; hdev->last_reset_time = jiffies; /* perform reset of the stack & ae device for a client */ - hclge_notify_roce_client(hdev, HNAE3_DOWN_CLIENT); + ret = hclge_notify_roce_client(hdev, HNAE3_DOWN_CLIENT); + if (ret) + goto err_reset; + rtnl_lock(); - hclge_notify_client(hdev, HNAE3_DOWN_CLIENT); + ret = hclge_notify_client(hdev, HNAE3_DOWN_CLIENT); + if (ret) + goto err_reset_lock; + rtnl_unlock(); - hclge_reset_prepare_wait(hdev); + ret = hclge_reset_prepare_wait(hdev); + if (ret) + goto err_reset; - if (!hclge_reset_wait(hdev)) { - hclge_notify_roce_client(hdev, HNAE3_UNINIT_CLIENT); - rtnl_lock(); - hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT); - hclge_reset_ae_dev(hdev->ae_dev); - hclge_notify_client(hdev, HNAE3_INIT_CLIENT); - - hclge_clear_reset_cause(hdev); - } else { - rtnl_lock(); - /* schedule again to check pending resets later */ - set_bit(hdev->reset_type, &hdev->reset_pending); - hclge_reset_task_schedule(hdev); + if (hclge_reset_wait(hdev)) { + is_timeout = true; + goto err_reset; } - hclge_notify_client(hdev, HNAE3_UP_CLIENT); - rtnl_unlock(); - ae_dev->reset_type = HNAE3_NONE_RESET; + ret = hclge_notify_roce_client(hdev, HNAE3_UNINIT_CLIENT); + if (ret) + goto err_reset; - hclge_notify_roce_client(hdev, HNAE3_INIT_CLIENT); - hclge_notify_roce_client(hdev, HNAE3_UP_CLIENT); + rtnl_lock(); + ret = hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT); + if (ret) + goto err_reset_lock; + + ret = hclge_reset_ae_dev(hdev->ae_dev); + if (ret) + goto err_reset_lock; + + ret = hclge_notify_client(hdev, HNAE3_INIT_CLIENT); + if (ret) + goto err_reset_lock; + + hclge_clear_reset_cause(hdev); + + ret = hclge_notify_client(hdev, HNAE3_UP_CLIENT); + if (ret) + goto err_reset_lock; + + rtnl_unlock(); + + ret = hclge_notify_roce_client(hdev, HNAE3_INIT_CLIENT); + if (ret) + goto err_reset; + + ret = hclge_notify_roce_client(hdev, HNAE3_UP_CLIENT); + if (ret) + goto err_reset; + + return; + +err_reset_lock: + rtnl_unlock(); +err_reset: + if (hclge_reset_err_handle(hdev, is_timeout)) + hclge_reset_task_schedule(hdev); } static void hclge_reset_event(struct pci_dev *pdev, struct hnae3_handle *handle) @@ -2610,6 +2688,16 @@ static void hclge_set_def_reset_request(struct hnae3_ae_dev *ae_dev, set_bit(rst_type, &hdev->default_reset_request); } +static void hclge_reset_timer(struct timer_list *t) +{ + struct hclge_dev *hdev = from_timer(hdev, t, reset_timer); + + dev_info(&hdev->pdev->dev, + "triggering global reset in reset timer\n"); + set_bit(HNAE3_GLOBAL_RESET, &hdev->default_reset_request); + hclge_reset_event(hdev->pdev, NULL); +} + static void hclge_reset_subtask(struct hclge_dev *hdev) { /* check if there is any ongoing reset in the hardware. This status can @@ -4416,8 +4504,12 @@ static int hclge_restore_fd_entries(struct hnae3_handle *handle) struct hlist_node *node; int ret; + /* Return ok here, because reset error handling will check this + * return value. If error is returned here, the reset process will + * fail. + */ if (!hnae3_dev_fd_supported(hdev)) - return -EOPNOTSUPP; + return 0; hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) { ret = hclge_config_action(hdev, HCLGE_FD_STAGE_1, rule); @@ -6713,6 +6805,8 @@ static void hclge_state_uninit(struct hclge_dev *hdev) if (hdev->service_timer.function) del_timer_sync(&hdev->service_timer); + if (hdev->reset_timer.function) + del_timer_sync(&hdev->reset_timer); if (hdev->service_task.func) cancel_work_sync(&hdev->service_task); if (hdev->rst_service_task.func) @@ -6871,6 +6965,7 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) hclge_dcb_ops_set(hdev); timer_setup(&hdev->service_timer, hclge_service_timer, 0); + timer_setup(&hdev->reset_timer, hclge_reset_timer, 0); INIT_WORK(&hdev->service_task, hclge_service_task); INIT_WORK(&hdev->rst_service_task, hclge_reset_service_task); INIT_WORK(&hdev->mbx_service_task, hclge_mailbox_service_task); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 69bdb7f646ee..36f341368683 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -102,6 +102,7 @@ enum HLCGE_PORT_TYPE { #define HCLGE_GLOBAL_RESET_REG 0x20A00 #define HCLGE_GLOBAL_RESET_BIT 0 #define HCLGE_CORE_RESET_BIT 1 +#define HCLGE_IMP_RESET_BIT 2 #define HCLGE_FUN_RST_ING 0x20C00 #define HCLGE_FUN_RST_ING_B 0 @@ -601,6 +602,7 @@ struct hclge_dev { unsigned long reset_request; /* reset has been requested */ unsigned long reset_pending; /* client rst is pending to be served */ unsigned long reset_count; /* the number of reset has been done */ + u32 reset_fail_cnt; u32 fw_version; u16 num_vmdq_vport; /* Num vmdq vport this PF has set up */ u16 num_tqps; /* Num task queue pairs of this PF */ @@ -648,6 +650,7 @@ struct hclge_dev { unsigned long service_timer_period; unsigned long service_timer_previous; struct timer_list service_timer; + struct timer_list reset_timer; struct work_struct service_task; struct work_struct rst_service_task; struct work_struct mbx_service_task; From 8b0195a30585fefd4c181881077b4181670ca85d Mon Sep 17 00:00:00 2001 From: Huazhong Tan Date: Wed, 7 Nov 2018 12:06:18 +0800 Subject: [PATCH 12/12] net: hns3: fix for cmd queue memory not freed problem during reset It is not necessary to reallocate the descriptor and remap the descriptor memory in reset process, otherwise it may cause memory not freed problem. Also, this patch initializes the cmd queue's spinlocks in hclgevf_alloc_cmd_queue, and take the spinlocks when reinitializing cmd queue' registers. Fixes: fedd0c15d288 ("net: hns3: Add HNS3 VF IMP(Integrated Management Proc) cmd interface") Signed-off-by: Huazhong Tan Signed-off-by: Yunsheng Lin Signed-off-by: David S. Miller --- .../hisilicon/hns3/hns3vf/hclgevf_cmd.c | 159 ++++++++++-------- .../hisilicon/hns3/hns3vf/hclgevf_cmd.h | 1 + .../hisilicon/hns3/hns3vf/hclgevf_main.c | 14 +- 3 files changed, 100 insertions(+), 74 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c index 0d3b445f6799..b917acf71ee6 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c @@ -72,6 +72,45 @@ static bool hclgevf_is_special_opcode(u16 opcode) return false; } +static void hclgevf_cmd_config_regs(struct hclgevf_cmq_ring *ring) +{ + struct hclgevf_dev *hdev = ring->dev; + struct hclgevf_hw *hw = &hdev->hw; + u32 reg_val; + + if (ring->flag == HCLGEVF_TYPE_CSQ) { + reg_val = (u32)ring->desc_dma_addr; + hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_L_REG, reg_val); + reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1); + hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_H_REG, reg_val); + + reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S); + reg_val |= HCLGEVF_NIC_CMQ_ENABLE; + hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_DEPTH_REG, reg_val); + + hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_HEAD_REG, 0); + hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_TAIL_REG, 0); + } else { + reg_val = (u32)ring->desc_dma_addr; + hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_L_REG, reg_val); + reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1); + hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_H_REG, reg_val); + + reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S); + reg_val |= HCLGEVF_NIC_CMQ_ENABLE; + hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_DEPTH_REG, reg_val); + + hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_HEAD_REG, 0); + hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_TAIL_REG, 0); + } +} + +static void hclgevf_cmd_init_regs(struct hclgevf_hw *hw) +{ + hclgevf_cmd_config_regs(&hw->cmq.csq); + hclgevf_cmd_config_regs(&hw->cmq.crq); +} + static int hclgevf_alloc_cmd_desc(struct hclgevf_cmq_ring *ring) { int size = ring->desc_num * sizeof(struct hclgevf_desc); @@ -96,61 +135,23 @@ static void hclgevf_free_cmd_desc(struct hclgevf_cmq_ring *ring) } } -static int hclgevf_init_cmd_queue(struct hclgevf_dev *hdev, - struct hclgevf_cmq_ring *ring) +static int hclgevf_alloc_cmd_queue(struct hclgevf_dev *hdev, int ring_type) { struct hclgevf_hw *hw = &hdev->hw; - int ring_type = ring->flag; - u32 reg_val; + struct hclgevf_cmq_ring *ring = + (ring_type == HCLGEVF_TYPE_CSQ) ? &hw->cmq.csq : &hw->cmq.crq; int ret; - ring->desc_num = HCLGEVF_NIC_CMQ_DESC_NUM; - spin_lock_init(&ring->lock); - ring->next_to_clean = 0; - ring->next_to_use = 0; ring->dev = hdev; + ring->flag = ring_type; /* allocate CSQ/CRQ descriptor */ ret = hclgevf_alloc_cmd_desc(ring); - if (ret) { + if (ret) dev_err(&hdev->pdev->dev, "failed(%d) to alloc %s desc\n", ret, (ring_type == HCLGEVF_TYPE_CSQ) ? "CSQ" : "CRQ"); - return ret; - } - /* initialize the hardware registers with csq/crq dma-address, - * descriptor number, head & tail pointers - */ - switch (ring_type) { - case HCLGEVF_TYPE_CSQ: - reg_val = (u32)ring->desc_dma_addr; - hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_L_REG, reg_val); - reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1); - hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_H_REG, reg_val); - - reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S); - reg_val |= HCLGEVF_NIC_CMQ_ENABLE; - hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_DEPTH_REG, reg_val); - - hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_HEAD_REG, 0); - hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_TAIL_REG, 0); - return 0; - case HCLGEVF_TYPE_CRQ: - reg_val = (u32)ring->desc_dma_addr; - hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_L_REG, reg_val); - reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1); - hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_H_REG, reg_val); - - reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S); - reg_val |= HCLGEVF_NIC_CMQ_ENABLE; - hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_DEPTH_REG, reg_val); - - hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_HEAD_REG, 0); - hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_TAIL_REG, 0); - return 0; - default: - return -EINVAL; - } + return ret; } void hclgevf_cmd_setup_basic_desc(struct hclgevf_desc *desc, @@ -282,55 +283,73 @@ static int hclgevf_cmd_query_firmware_version(struct hclgevf_hw *hw, return status; } +int hclgevf_cmd_queue_init(struct hclgevf_dev *hdev) +{ + int ret; + + /* Setup the lock for command queue */ + spin_lock_init(&hdev->hw.cmq.csq.lock); + spin_lock_init(&hdev->hw.cmq.crq.lock); + + hdev->hw.cmq.tx_timeout = HCLGEVF_CMDQ_TX_TIMEOUT; + hdev->hw.cmq.csq.desc_num = HCLGEVF_NIC_CMQ_DESC_NUM; + hdev->hw.cmq.crq.desc_num = HCLGEVF_NIC_CMQ_DESC_NUM; + + ret = hclgevf_alloc_cmd_queue(hdev, HCLGEVF_TYPE_CSQ); + if (ret) { + dev_err(&hdev->pdev->dev, + "CSQ ring setup error %d\n", ret); + return ret; + } + + ret = hclgevf_alloc_cmd_queue(hdev, HCLGEVF_TYPE_CRQ); + if (ret) { + dev_err(&hdev->pdev->dev, + "CRQ ring setup error %d\n", ret); + goto err_csq; + } + + return 0; +err_csq: + hclgevf_free_cmd_desc(&hdev->hw.cmq.csq); + return ret; +} + int hclgevf_cmd_init(struct hclgevf_dev *hdev) { u32 version; int ret; - /* setup Tx write back timeout */ - hdev->hw.cmq.tx_timeout = HCLGEVF_CMDQ_TX_TIMEOUT; - - /* setup queue CSQ/CRQ rings */ - hdev->hw.cmq.csq.flag = HCLGEVF_TYPE_CSQ; - ret = hclgevf_init_cmd_queue(hdev, &hdev->hw.cmq.csq); - if (ret) { - dev_err(&hdev->pdev->dev, - "failed(%d) to initialize CSQ ring\n", ret); - return ret; - } - - hdev->hw.cmq.crq.flag = HCLGEVF_TYPE_CRQ; - ret = hclgevf_init_cmd_queue(hdev, &hdev->hw.cmq.crq); - if (ret) { - dev_err(&hdev->pdev->dev, - "failed(%d) to initialize CRQ ring\n", ret); - goto err_csq; - } + spin_lock_bh(&hdev->hw.cmq.csq.lock); + spin_lock_bh(&hdev->hw.cmq.crq.lock); /* initialize the pointers of async rx queue of mailbox */ hdev->arq.hdev = hdev; hdev->arq.head = 0; hdev->arq.tail = 0; hdev->arq.count = 0; + hdev->hw.cmq.csq.next_to_clean = 0; + hdev->hw.cmq.csq.next_to_use = 0; + hdev->hw.cmq.crq.next_to_clean = 0; + hdev->hw.cmq.crq.next_to_use = 0; + + hclgevf_cmd_init_regs(&hdev->hw); + + spin_unlock_bh(&hdev->hw.cmq.crq.lock); + spin_unlock_bh(&hdev->hw.cmq.csq.lock); /* get firmware version */ ret = hclgevf_cmd_query_firmware_version(&hdev->hw, &version); if (ret) { dev_err(&hdev->pdev->dev, "failed(%d) to query firmware version\n", ret); - goto err_crq; + return ret; } hdev->fw_version = version; dev_info(&hdev->pdev->dev, "The firmware version is %08x\n", version); return 0; -err_crq: - hclgevf_free_cmd_desc(&hdev->hw.cmq.crq); -err_csq: - hclgevf_free_cmd_desc(&hdev->hw.cmq.csq); - - return ret; } void hclgevf_cmd_uninit(struct hclgevf_dev *hdev) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h index bc294b0c8b62..090541de0c7d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h @@ -256,6 +256,7 @@ static inline u32 hclgevf_read_reg(u8 __iomem *base, u32 reg) int hclgevf_cmd_init(struct hclgevf_dev *hdev); void hclgevf_cmd_uninit(struct hclgevf_dev *hdev); +int hclgevf_cmd_queue_init(struct hclgevf_dev *hdev); int hclgevf_cmd_send(struct hclgevf_hw *hw, struct hclgevf_desc *desc, int num); void hclgevf_cmd_setup_basic_desc(struct hclgevf_desc *desc, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 4a2148bde35a..29da4806c100 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1966,6 +1966,12 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) return ret; } + ret = hclgevf_cmd_queue_init(hdev); + if (ret) { + dev_err(&pdev->dev, "Cmd queue init failed: %d\n", ret); + goto err_cmd_queue_init; + } + ret = hclgevf_cmd_init(hdev); if (ret) goto err_cmd_init; @@ -1975,13 +1981,13 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "Query vf status error, ret = %d.\n", ret); - goto err_query_vf; + goto err_cmd_init; } ret = hclgevf_init_msi(hdev); if (ret) { dev_err(&pdev->dev, "failed(%d) to init MSI/MSI-X\n", ret); - goto err_query_vf; + goto err_cmd_init; } hclgevf_state_init(hdev); @@ -2037,9 +2043,9 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) err_misc_irq_init: hclgevf_state_uninit(hdev); hclgevf_uninit_msi(hdev); -err_query_vf: - hclgevf_cmd_uninit(hdev); err_cmd_init: + hclgevf_cmd_uninit(hdev); +err_cmd_queue_init: hclgevf_pci_uninit(hdev); return ret; }