devlink: Add health recover notifications on devlink flows
Devlink health recover notifications were added only on driver direct
updates of health_state through devlink_health_reporter_state_update().
Add notifications on updates of health_state by devlink flows of report
and recover.
Moved functions devlink_nl_health_reporter_fill() and
devlink_recover_notify() to avoid forward declaration.
Fixes: 97ff3bd37f ("devlink: add devink notification when reporter update health state")
Signed-off-by: Moshe Shemesh <moshe@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Moshe Shemesh
David S. Miller
parent
53c6770095
commit
6ec8b6cd79
|
|
@ -4843,180 +4843,6 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
|
||||
|
||||
void
|
||||
devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
reporter->recovery_count++;
|
||||
reporter->last_recovery_ts = jiffies;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
|
||||
|
||||
static int
|
||||
devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
|
||||
void *priv_ctx, struct netlink_ext_ack *extack)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
|
||||
return 0;
|
||||
|
||||
if (!reporter->ops->recover)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
err = reporter->ops->recover(reporter, priv_ctx, extack);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
devlink_health_reporter_recovery_done(reporter);
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
devlink_health_dump_clear(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
if (!reporter->dump_fmsg)
|
||||
return;
|
||||
devlink_fmsg_free(reporter->dump_fmsg);
|
||||
reporter->dump_fmsg = NULL;
|
||||
}
|
||||
|
||||
static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
|
||||
void *priv_ctx,
|
||||
struct netlink_ext_ack *extack)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (!reporter->ops->dump)
|
||||
return 0;
|
||||
|
||||
if (reporter->dump_fmsg)
|
||||
return 0;
|
||||
|
||||
reporter->dump_fmsg = devlink_fmsg_alloc();
|
||||
if (!reporter->dump_fmsg) {
|
||||
err = -ENOMEM;
|
||||
return err;
|
||||
}
|
||||
|
||||
err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
err = reporter->ops->dump(reporter, reporter->dump_fmsg,
|
||||
priv_ctx, extack);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
reporter->dump_ts = jiffies;
|
||||
reporter->dump_real_ts = ktime_get_real_ns();
|
||||
|
||||
return 0;
|
||||
|
||||
dump_err:
|
||||
devlink_health_dump_clear(reporter);
|
||||
return err;
|
||||
}
|
||||
|
||||
int devlink_health_report(struct devlink_health_reporter *reporter,
|
||||
const char *msg, void *priv_ctx)
|
||||
{
|
||||
enum devlink_health_reporter_state prev_health_state;
|
||||
struct devlink *devlink = reporter->devlink;
|
||||
|
||||
/* write a log message of the current error */
|
||||
WARN_ON(!msg);
|
||||
trace_devlink_health_report(devlink, reporter->ops->name, msg);
|
||||
reporter->error_count++;
|
||||
prev_health_state = reporter->health_state;
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
|
||||
|
||||
/* abort if the previous error wasn't recovered */
|
||||
if (reporter->auto_recover &&
|
||||
(prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
|
||||
jiffies - reporter->last_recovery_ts <
|
||||
msecs_to_jiffies(reporter->graceful_period))) {
|
||||
trace_devlink_health_recover_aborted(devlink,
|
||||
reporter->ops->name,
|
||||
reporter->health_state,
|
||||
jiffies -
|
||||
reporter->last_recovery_ts);
|
||||
return -ECANCELED;
|
||||
}
|
||||
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
|
||||
|
||||
mutex_lock(&reporter->dump_lock);
|
||||
/* store current dump of current error, for later analysis */
|
||||
devlink_health_do_dump(reporter, priv_ctx, NULL);
|
||||
mutex_unlock(&reporter->dump_lock);
|
||||
|
||||
if (reporter->auto_recover)
|
||||
return devlink_health_reporter_recover(reporter,
|
||||
priv_ctx, NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(devlink_health_report);
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_attrs(struct devlink *devlink,
|
||||
struct nlattr **attrs)
|
||||
{
|
||||
struct devlink_health_reporter *reporter;
|
||||
char *reporter_name;
|
||||
|
||||
if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
|
||||
return NULL;
|
||||
|
||||
reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
|
||||
mutex_lock(&devlink->reporters_lock);
|
||||
reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
|
||||
if (reporter)
|
||||
refcount_inc(&reporter->refcount);
|
||||
mutex_unlock(&devlink->reporters_lock);
|
||||
return reporter;
|
||||
}
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_info(struct devlink *devlink,
|
||||
struct genl_info *info)
|
||||
{
|
||||
return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
|
||||
}
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
|
||||
{
|
||||
const struct genl_dumpit_info *info = genl_dumpit_info(cb);
|
||||
struct devlink_health_reporter *reporter;
|
||||
struct nlattr **attrs = info->attrs;
|
||||
struct devlink *devlink;
|
||||
|
||||
mutex_lock(&devlink_mutex);
|
||||
devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
|
||||
if (IS_ERR(devlink))
|
||||
goto unlock;
|
||||
|
||||
reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
|
||||
mutex_unlock(&devlink_mutex);
|
||||
return reporter;
|
||||
unlock:
|
||||
mutex_unlock(&devlink_mutex);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
devlink_health_reporter_put(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
refcount_dec(&reporter->refcount);
|
||||
}
|
||||
|
||||
static int
|
||||
devlink_nl_health_reporter_fill(struct sk_buff *msg,
|
||||
struct devlink *devlink,
|
||||
|
|
@ -5104,6 +4930,182 @@ static void devlink_recover_notify(struct devlink_health_reporter *reporter,
|
|||
msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
|
||||
}
|
||||
|
||||
void
|
||||
devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
reporter->recovery_count++;
|
||||
reporter->last_recovery_ts = jiffies;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
|
||||
|
||||
static int
|
||||
devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
|
||||
void *priv_ctx, struct netlink_ext_ack *extack)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
|
||||
return 0;
|
||||
|
||||
if (!reporter->ops->recover)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
err = reporter->ops->recover(reporter, priv_ctx, extack);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
devlink_health_reporter_recovery_done(reporter);
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
|
||||
devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
devlink_health_dump_clear(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
if (!reporter->dump_fmsg)
|
||||
return;
|
||||
devlink_fmsg_free(reporter->dump_fmsg);
|
||||
reporter->dump_fmsg = NULL;
|
||||
}
|
||||
|
||||
static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
|
||||
void *priv_ctx,
|
||||
struct netlink_ext_ack *extack)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (!reporter->ops->dump)
|
||||
return 0;
|
||||
|
||||
if (reporter->dump_fmsg)
|
||||
return 0;
|
||||
|
||||
reporter->dump_fmsg = devlink_fmsg_alloc();
|
||||
if (!reporter->dump_fmsg) {
|
||||
err = -ENOMEM;
|
||||
return err;
|
||||
}
|
||||
|
||||
err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
err = reporter->ops->dump(reporter, reporter->dump_fmsg,
|
||||
priv_ctx, extack);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
|
||||
if (err)
|
||||
goto dump_err;
|
||||
|
||||
reporter->dump_ts = jiffies;
|
||||
reporter->dump_real_ts = ktime_get_real_ns();
|
||||
|
||||
return 0;
|
||||
|
||||
dump_err:
|
||||
devlink_health_dump_clear(reporter);
|
||||
return err;
|
||||
}
|
||||
|
||||
int devlink_health_report(struct devlink_health_reporter *reporter,
|
||||
const char *msg, void *priv_ctx)
|
||||
{
|
||||
enum devlink_health_reporter_state prev_health_state;
|
||||
struct devlink *devlink = reporter->devlink;
|
||||
|
||||
/* write a log message of the current error */
|
||||
WARN_ON(!msg);
|
||||
trace_devlink_health_report(devlink, reporter->ops->name, msg);
|
||||
reporter->error_count++;
|
||||
prev_health_state = reporter->health_state;
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
|
||||
devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
|
||||
|
||||
/* abort if the previous error wasn't recovered */
|
||||
if (reporter->auto_recover &&
|
||||
(prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
|
||||
jiffies - reporter->last_recovery_ts <
|
||||
msecs_to_jiffies(reporter->graceful_period))) {
|
||||
trace_devlink_health_recover_aborted(devlink,
|
||||
reporter->ops->name,
|
||||
reporter->health_state,
|
||||
jiffies -
|
||||
reporter->last_recovery_ts);
|
||||
return -ECANCELED;
|
||||
}
|
||||
|
||||
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
|
||||
|
||||
mutex_lock(&reporter->dump_lock);
|
||||
/* store current dump of current error, for later analysis */
|
||||
devlink_health_do_dump(reporter, priv_ctx, NULL);
|
||||
mutex_unlock(&reporter->dump_lock);
|
||||
|
||||
if (reporter->auto_recover)
|
||||
return devlink_health_reporter_recover(reporter,
|
||||
priv_ctx, NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(devlink_health_report);
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_attrs(struct devlink *devlink,
|
||||
struct nlattr **attrs)
|
||||
{
|
||||
struct devlink_health_reporter *reporter;
|
||||
char *reporter_name;
|
||||
|
||||
if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
|
||||
return NULL;
|
||||
|
||||
reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
|
||||
mutex_lock(&devlink->reporters_lock);
|
||||
reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
|
||||
if (reporter)
|
||||
refcount_inc(&reporter->refcount);
|
||||
mutex_unlock(&devlink->reporters_lock);
|
||||
return reporter;
|
||||
}
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_info(struct devlink *devlink,
|
||||
struct genl_info *info)
|
||||
{
|
||||
return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
|
||||
}
|
||||
|
||||
static struct devlink_health_reporter *
|
||||
devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
|
||||
{
|
||||
const struct genl_dumpit_info *info = genl_dumpit_info(cb);
|
||||
struct devlink_health_reporter *reporter;
|
||||
struct nlattr **attrs = info->attrs;
|
||||
struct devlink *devlink;
|
||||
|
||||
mutex_lock(&devlink_mutex);
|
||||
devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
|
||||
if (IS_ERR(devlink))
|
||||
goto unlock;
|
||||
|
||||
reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
|
||||
mutex_unlock(&devlink_mutex);
|
||||
return reporter;
|
||||
unlock:
|
||||
mutex_unlock(&devlink_mutex);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
devlink_health_reporter_put(struct devlink_health_reporter *reporter)
|
||||
{
|
||||
refcount_dec(&reporter->refcount);
|
||||
}
|
||||
|
||||
void
|
||||
devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
|
||||
enum devlink_health_reporter_state state)
|
||||
|
|
|
|||
Loading…
Reference in New Issue