Skip to contentSkip to conversation

nvme: add thermal zone devices

PATCH Superseded Akinobu Mita 6 participants 21 messages May 15, 2019 – May 17, 2019 NVMe

v1 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

cover nvme: add thermal zone devices

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This provide these temperatures and thresholds via thermal zone devices.

Akinobu Mita (2):
nvme: add thermal zone infrastructure
nvme-pci: support thermal zone

 drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
 drivers/nvme/host/nvme.h |  24 ++++
 drivers/nvme/host/pci.c  |   5 +
 include/linux/nvme.h     |   4 +
 4 files changed, 397 insertions(+), 4 deletions(-)
Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

    |---temp: Temperature
    |---trip_point_0_temp: Over temperature threshold
    |---trip_point_0_hyst: Under temperature threshold

The thermal_zone[0-*] contains a symlink to the correspoinding nvme device. On the other hand, the following symlinks to the thermal zone devices are created in the nvme device sysfs directory.

- nvme_temp0: Composite temperature
- nvme_temp1: Temperature sensor 1
...
- nvme_temp8: Temperature sensor 8

The nvme_thermal_zones_unregister() removes the registered thermal zone devices and symlinks.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
 drivers/nvme/host/nvme.h |  24 ++++
 include/linux/nvme.h     |   4 +
 3 files changed, 392 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +364 -4
index 172551b..a915c6b 100644
@@ -1113,15 +1113,16 @@ static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
 	return id;
 }
 
-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-		      void *buffer, size_t buflen, u32 *result)
+static int nvme_features(struct nvme_ctrl *dev, u8 opcode, unsigned int fid,
+			 unsigned int dword11, void *buffer, size_t buflen,
+			 u32 *result)
 {
 	struct nvme_command c;
 	union nvme_result res;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
-	c.features.opcode = nvme_admin_set_features;
+	c.features.opcode = opcode;
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
 
@@ -1132,6 +1133,22 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword
 	return ret;
 }
 
+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)
+{
+	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
+			     buflen, result);
+}
+
+static int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)
+{
+	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
+			     buflen, result);
+}
+
 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 {
 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
@@ -1168,6 +1185,9 @@ static void nvme_enable_aen(struct nvme_ctrl *ctrl)
 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
 	int status;
 
+	if (IS_ENABLED(CONFIG_THERMAL))
+		supported_aens |= NVME_SMART_CRIT_TEMPERATURE;
+
 	if (!supported_aens)
 		return;
 
@@ -2164,6 +2184,334 @@ static void nvme_set_latency_tolerance(struct device *dev, s32 val)
 	}
 }
 
+#ifdef CONFIG_THERMAL
+
+static int nvme_get_temp(struct nvme_ctrl *ctrl, int sensor, int *temp)
+{
+	struct nvme_smart_log *log;
+	int ret;
+
+	if (sensor >= ARRAY_SIZE(log->temp_sensor))
+		return -EINVAL;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	if (sensor)
+		*temp = le16_to_cpu(log->temp_sensor[sensor - 1]);
+	else
+		*temp = get_unaligned_le16(log->temperature);
+
+	if (!*temp)
+		ret = -EINVAL;
+
+free_log:
+	kfree(log);
+
+	return ret;
+}
+
+static int nvme_tz_type_to_sensor(const char *type)
+{
+	int sensor;
+
+	if (sscanf(type, "nvme_temp%d", &sensor) != 1)
+		return -EINVAL;
+
+	if (sensor < 0 || sensor > 8)
+		return -EINVAL;
+
+	return sensor;
+}
+
+#define KELVIN_TO_MILLICELSIUS(t) DECI_KELVIN_TO_MILLICELSIUS((t) * 10)
+#define MILLICELSIUS_TO_KELVIN(t) ((MILLICELSIUS_TO_DECI_KELVIN(t) + 5) / 10)
+
+static int nvme_tz_get_temp(struct thermal_zone_device *tzdev,
+			    int *temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_temp(ctrl, sensor, temp);
+	if (!ret)
+		*temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+	return ret;
+}
+
+static int nvme_tz_get_trip_type(struct thermal_zone_device *tzdev,
+				 int trip, enum thermal_trip_type *type)
+{
+	*type = THERMAL_TRIP_ACTIVE;
+
+	return 0;
+}
+
+static int nvme_get_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+				int *temp)
+{
+	unsigned int threshold = sensor << 16;
+	int status;
+	int ret;
+
+	if (under)
+		threshold |= 0x100000;
+
+	ret = nvme_get_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+				&status);
+	if (!ret)
+		*temp = status & 0xffff;
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_get_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int *temp)
+{
+	return nvme_get_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_get_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int *temp)
+{
+	return nvme_get_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_set_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+				int temp)
+{
+	unsigned int threshold = (sensor << 16) | temp;
+	int status;
+	int ret;
+
+	if (temp > 0xffff)
+		return -EINVAL;
+
+	if (under)
+		threshold |= 0x100000;
+
+	ret = nvme_set_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+				&status);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_set_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int temp)
+{
+	return nvme_set_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_set_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int temp)
+{
+	return nvme_set_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_tz_get_trip_temp(struct thermal_zone_device *tzdev,
+				 int trip, int *temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_over_temp_thresh(ctrl, sensor, temp);
+	if (!ret)
+		*temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+	return ret;
+}
+
+static int nvme_tz_set_trip_temp(struct thermal_zone_device *tzdev,
+				 int trip, int temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	temp = MILLICELSIUS_TO_KELVIN(temp);
+
+	ret = nvme_set_over_temp_thresh(ctrl, sensor, temp);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_tz_get_trip_hyst(struct thermal_zone_device *tzdev,
+				 int trip, int *hyst)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_under_temp_thresh(ctrl, sensor, hyst);
+	if (!ret)
+		*hyst = KELVIN_TO_MILLICELSIUS(*hyst);
+
+	return ret;
+}
+
+static int nvme_tz_set_trip_hyst(struct thermal_zone_device *tzdev,
+				 int trip, int hyst)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	hyst = MILLICELSIUS_TO_KELVIN(hyst);
+
+	ret = nvme_set_under_temp_thresh(ctrl, sensor, hyst);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static struct thermal_zone_device_ops nvme_tz_ops = {
+	.get_temp = nvme_tz_get_temp,
+	.get_trip_type = nvme_tz_get_trip_type,
+	.get_trip_temp = nvme_tz_get_trip_temp,
+	.set_trip_temp = nvme_tz_set_trip_temp,
+	.get_trip_hyst = nvme_tz_get_trip_hyst,
+	.set_trip_hyst = nvme_tz_set_trip_hyst,
+};
+
+static struct thermal_zone_params nvme_tz_params = {
+	.governor_name = "user_space",
+	.no_hwmon = true,
+};
+
+static struct thermal_zone_device *
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+	struct thermal_zone_device *tzdev;
+	char type[THERMAL_NAME_LENGTH];
+	int ret;
+
+	snprintf(type, sizeof(type), "nvme_temp%d", sensor);
+
+	tzdev = thermal_zone_device_register(type, 1, 1, ctrl, &nvme_tz_ops,
+					     &nvme_tz_params, 0, 0);
+	if (IS_ERR(tzdev))
+		return tzdev;
+
+	ret = sysfs_create_link(&ctrl->ctrl_device.kobj,
+				&tzdev->device.kobj, type);
+	if (ret)
+		goto device_unregister;
+
+	ret = sysfs_create_link(&tzdev->device.kobj,
+				&ctrl->ctrl_device.kobj, "device");
+	if (ret)
+		goto remove_link;
+
+	return tzdev;
+
+remove_link:
+	sysfs_remove_link(&ctrl->ctrl_device.kobj, type);
+device_unregister:
+	thermal_zone_device_unregister(tzdev);
+
+	return ERR_PTR(ret);
+}
+
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	struct nvme_smart_log *log;
+	int ret;
+	int i;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		struct thermal_zone_device *tzdev;
+
+		if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+			continue;
+		if (ctrl->tzdev[i])
+			continue;
+
+		tzdev = nvme_thermal_zone_register(ctrl, i);
+		if (!IS_ERR(tzdev))
+			ctrl->tzdev[i] = tzdev;
+	}
+
+free_log:
+	kfree(log);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_register);
+
+void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		struct thermal_zone_device *tzdev = ctrl->tzdev[i];
+
+		if (!tzdev)
+			continue;
+
+		sysfs_remove_link(&tzdev->device.kobj, "device");
+		sysfs_remove_link(&ctrl->ctrl_device.kobj, tzdev->type);
+		thermal_zone_device_unregister(tzdev);
+
+		ctrl->tzdev[i] = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_unregister);
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+	queue_work(nvme_wq, &ctrl->thermal_work);
+}
+
+static void nvme_thermal_work(struct work_struct *work)
+{
+	struct nvme_ctrl *ctrl =
+		container_of(work, struct nvme_ctrl, thermal_work);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		if (ctrl->tzdev[i])
+			thermal_notify_framework(ctrl->tzdev[i], 0);
+	}
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+	INIT_WORK(&ctrl->thermal_work, nvme_thermal_work);
+}
+
+#else
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+}
+
+#endif /* CONFIG_THERMAL */
+
 struct nvme_core_quirk_entry {
 	/*
 	 * NVMe model and firmware strings are padded with spaces.  For
@@ -3630,6 +3978,14 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 	}
 }
 
+static void nvme_handle_aen_smart(struct nvme_ctrl *ctrl, u32 result)
+{
+	u32 aer_smart_type = (result & 0xff00) >> 8;
+
+	if (aer_smart_type == NVME_AER_SMART_TEMP_THRESH)
+		nvme_thermal_notify_framework(ctrl);
+}
+
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 		volatile union nvme_result *res)
 {
@@ -3643,8 +3999,10 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 	case NVME_AER_NOTICE:
 		nvme_handle_aen_notice(ctrl, result);
 		break;
-	case NVME_AER_ERROR:
 	case NVME_AER_SMART:
+		nvme_handle_aen_smart(ctrl, result);
+		/* fallthrough */
+	case NVME_AER_ERROR:
 	case NVME_AER_CSS:
 	case NVME_AER_VS:
 		trace_nvme_async_event(ctrl, aer_type);
@@ -3776,6 +4134,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
 
+	nvme_thermal_init(ctrl);
+
 	return 0;
 out_free_name:
 	kfree_const(ctrl->device->kobj.name);
drivers/nvme/host/nvme.h +24 -0
index 7f6f1fc..ff7bd8f 100644
@@ -15,6 +15,7 @@
 #include <linux/sed-opal.h>
 #include <linux/fault-inject.h>
 #include <linux/rcupdate.h>
+#include <linux/thermal.h>
 
 extern unsigned int nvme_io_timeout;
 #define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)
@@ -248,6 +249,11 @@ struct nvme_ctrl {
 
 	struct page *discard_page;
 	unsigned long discard_page_busy;
+
+#ifdef CONFIG_THERMAL
+	struct thermal_zone_device *tzdev[9];
+	struct work_struct thermal_work;
+#endif
 };
 
 enum nvme_iopolicy {
@@ -553,6 +559,24 @@ static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 }
 #endif /* CONFIG_NVME_MULTIPATH */
 
+#ifdef CONFIG_THERMAL
+
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl);
+void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl);
+
+#else
+
+static inline int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	return 0;
+}
+
+static inline void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl)
+{
+}
+
+#endif /* CONFIG_THERMAL */
+
 #ifdef CONFIG_NVM
 int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
 void nvme_nvm_unregister(struct nvme_ns *ns);
include/linux/nvme.h +4 -0
index 8c0b29d..7acc77d 100644
@@ -500,6 +500,10 @@ enum {
 };
 
 enum {
+	NVME_AER_SMART_TEMP_THRESH	= 0x01,
+};
+
+enum {
 	NVME_AER_NOTICE_NS_CHANGED	= 0x00,
 	NVME_AER_NOTICE_FW_ACT_STARTING = 0x01,
 	NVME_AER_NOTICE_ANA		= 0x03,
Signature · 1 line
2.7.4
#2 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

Quoted 25 lines · Akinobu Mita
drivers/nvme/host/core.c +25 -0
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	struct nvme_smart_log *log;
+	int ret;
+	int i;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		struct thermal_zone_device *tzdev;
+
+		if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+			continue;
+		if (ctrl->tzdev[i])
+			continue;
+

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2459

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+		tzdev = nvme_thermal_zone_register(ctrl, i);
+		if (!IS_ERR(tzdev))
+			ctrl->tzdev[i] = tzdev;
+	}
+
+free_log:
+	kfree(log);
+
+	return ret;
+}

Since this routine is intended for use in the device initialization path, the error returns are extra important. We have used < 0 to indicate we need to abandon initialization because we won't be able communicate with the device if we proceed. Since thermal reporting is not mandatory to manage our controllers, out-of-memory or a device that doesn't support SMART should just return 0. We should only halt init if the controller is unresponsive here.

In general, I'm okay with this feature.

#3 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?16?(?) 4:20 Keith Busch <keith.busch at intel.com>:

Quoted 38 lines · Akinobu Mita
Quoted 25 lines
drivers/nvme/host/core.c +25 -0
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+     struct nvme_smart_log *log;
+     int ret;
+     int i;
+
+     log = kzalloc(sizeof(*log), GFP_KERNEL);
+     if (!log)
+             return -ENOMEM;
+
+     ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+                        log, sizeof(*log), 0);
+     if (ret) {
+             ret = ret > 0 ? -EINVAL : ret;
+             goto free_log;
+     }
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             struct thermal_zone_device *tzdev;
+
+             if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+                     continue;
+             if (ctrl->tzdev[i])
+                     continue;
+

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2459

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+             tzdev = nvme_thermal_zone_register(ctrl, i);
+             if (!IS_ERR(tzdev))
+                     ctrl->tzdev[i] = tzdev;
+     }
+
+free_log:
+     kfree(log);
+
+     return ret;
+}
Quoted 7 lines

Since this routine is intended for use in the device initialization path, the error returns are extra important. We have used < 0 to indicate we need to abandon initialization because we won't be able communicate with the device if we proceed. Since thermal reporting is not mandatory to manage our controllers, out-of-memory or a device that doesn't support SMART should just return 0. We should only halt init if the controller is unresponsive here.

Make sense. I'll change the return type to void, and print warning in case of some errors as Minwoo said in other reply.

Quoted 1 line

In general, I'm okay with this feature.

OK. I'll prepare the next version.

maintainer
#4 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

Quoted 10 lines · Akinobu Mita
Quoted 7 lines

Since this routine is intended for use in the device initialization path, the error returns are extra important. We have used < 0 to indicate we need to abandon initialization because we won't be able communicate with the device if we proceed. Since thermal reporting is not mandatory to manage our controllers, out-of-memory or a device that doesn't support SMART should just return 0. We should only halt init if the controller is unresponsive here.

Make sense. I'll change the return type to void, and print warning in case of some errors as Minwoo said in other reply.

Oh, still needs to be an 'int' return, but just suppress non-fatal errors by returning 0. If the 'nvme_get_log' times out, though, we need to return that error since the caller will need to abort initialization.

#5 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita

Hi Akinobu,

Great feature here, I think.

Quoted 26 lines
drivers/nvme/host/core.c +10 -3
-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-		      void *buffer, size_t buflen, u32 *result)
+static int nvme_features(struct nvme_ctrl *dev, u8 opcode, unsigned int fid,
+			 unsigned int dword11, void *buffer, size_t buflen,
+			 u32 *result)
 {
 	struct nvme_command c;
 	union nvme_result res;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
-	c.features.opcode = nvme_admin_set_features;
+	c.features.opcode = opcode;
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
@@ -1132,6 +1133,22 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword
 	return ret;
 }
 
+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)
+{
+	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
+			     buflen, result);

drivers/nvme/host/core.c static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword · line 1151

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+}
+
+static int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)
+{
+	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
+			     buflen, result);
+}
+

I think it's okay to separate this part from this patch. :) (I guess I have seen this kind of patch from Keith, though)

#6 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2230

Quoted 2 lines
drivers/nvme/host/core.c +2 -0
+	if (sensor < 0 || sensor > 8)
+		return -EINVAL;

Does we really need to check the negative case here ? Am I missing something in this context ? If we really want to check it in this level, can we check the invalid case in the following function?

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2398

Quoted 8 lines
drivers/nvme/host/core.c +8 -0
+static struct thermal_zone_device *
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+	struct thermal_zone_device *tzdev;
+	char type[THERMAL_NAME_LENGTH];
+	int ret;
+
+	snprintf(type, sizeof(type), "nvme_temp%d", sensor);

Before preparing "nvme_temp%d", maybe we can make it sure here. :) What do you say?

Quoted 18 lines
drivers/nvme/host/core.c +18 -0
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	struct nvme_smart_log *log;
+	int ret;
+	int i;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2452

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+		struct thermal_zone_device *tzdev;
+
+		if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+			continue;
+		if (ctrl->tzdev[i])
+			continue;
+
+		tzdev = nvme_thermal_zone_register(ctrl, i);
+		if (!IS_ERR(tzdev))
+			ctrl->tzdev[i] = tzdev;

Quenstion here. Are we okay not to print some warnings here in case of error returned?

#7 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?16?(?) 23:32 Minwoo Im <minwoo.im.dev at gmail.com>:

Quoted 7 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2230

Quoted 3 lines
drivers/nvme/host/core.c +2 -0
+     if (sensor < 0 || sensor > 8)
+             return -EINVAL;

Does we really need to check the negative case here ? Am I missing something in this context ? If we really want to check it in this level, can we check the invalid case in the following function?

The negative case should never happen, so it can be just removed.

Quoted 11 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2398

Quoted 8 lines
drivers/nvme/host/core.c +8 -0
+static struct thermal_zone_device *
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+     struct thermal_zone_device *tzdev;
+     char type[THERMAL_NAME_LENGTH];
+     int ret;
+
+     snprintf(type, sizeof(type), "nvme_temp%d", sensor);

Before preparing "nvme_temp%d", maybe we can make it sure here. :) What do you say?

The nvme_thermal_zone_register() is only called from nvme_thermal_zones_register() which is defined just below, and it's very clear that the value of 'sensor' is from 0 to ARRAY_SIZE(ctrl->tzdev) - 1.

Quoted 23 lines
Quoted 23 lines
drivers/nvme/host/core.c +23 -0
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+     struct nvme_smart_log *log;
+     int ret;
+     int i;
+
+     log = kzalloc(sizeof(*log), GFP_KERNEL);
+     if (!log)
+             return -ENOMEM;
+
+     ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+                        log, sizeof(*log), 0);
+     if (ret) {
+             ret = ret > 0 ? -EINVAL : ret;
+             goto free_log;
+     }
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             struct thermal_zone_device *tzdev;
+
+             if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+                     continue;
+             if (ctrl->tzdev[i])
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2452

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+                     continue;
+
+             tzdev = nvme_thermal_zone_register(ctrl, i);
+             if (!IS_ERR(tzdev))
+                     ctrl->tzdev[i] = tzdev;

Quenstion here. Are we okay not to print some warnings here in case of error returned?

I'm going to print warning in case of thermal_zone_device_register() error. For sysfs_create_link() error, the warning is printed by the function itself.

#8 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

Quoted 10 lines · Akinobu Mita

2019?5?16?(?) 23:32 Minwoo Im <minwoo.im.dev at gmail.com>:

Quoted 7 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2230

Quoted 3 lines
drivers/nvme/host/core.c +2 -0
+     if (sensor < 0 || sensor > 8)
+             return -EINVAL;

Does we really need to check the negative case here ? Am I missing something in this context ? If we really want to check it in this level, can we check the invalid case in the following function?

The negative case should never happen, so it can be just removed.

Cool.

Quoted 13 lines
Quoted 12 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2398

Quoted 9 lines
drivers/nvme/host/core.c +8 -0
+static struct thermal_zone_device *
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+     struct thermal_zone_device *tzdev;
+     char type[THERMAL_NAME_LENGTH];
+     int ret;
+
+     snprintf(type, sizeof(type), "nvme_temp%d", sensor);

Before preparing "nvme_temp%d", maybe we can make it sure here. :) What do you say?

Quoted 3 lines

The nvme_thermal_zone_register() is only called from nvme_thermal_zones_register() which is defined just below, and it's very clear that the value of 'sensor' is from 0 to ARRAY_SIZE(ctrl->tzdev) - 1.

If so, we don't need to check the negative case above there.

Quoted 33 lines
Quoted 24 lines
Quoted 24 lines
drivers/nvme/host/core.c +23 -0
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+     struct nvme_smart_log *log;
+     int ret;
+     int i;
+
+     log = kzalloc(sizeof(*log), GFP_KERNEL);
+     if (!log)
+             return -ENOMEM;
+
+     ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+                        log, sizeof(*log), 0);
+     if (ret) {
+             ret = ret > 0 ? -EINVAL : ret;
+             goto free_log;
+     }
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             struct thermal_zone_device *tzdev;
+
+             if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+                     continue;
+             if (ctrl->tzdev[i])
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2452

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+                     continue;
+
+             tzdev = nvme_thermal_zone_register(ctrl, i);
+             if (!IS_ERR(tzdev))
+                     ctrl->tzdev[i] = tzdev;

Quenstion here. Are we okay not to print some warnings here in case of error returned?

Quoted 3 lines

I'm going to print warning in case of thermal_zone_device_register() error. For sysfs_create_link() error, the warning is printed by the function itself.

Sounds great.

Thanks,

#9 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita

2019?5?16?(?) 0:17 Akinobu Mita <akinobu.mita at gmail.com>:

Quoted 11 lines

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

Quoted 7 lines

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

    |---temp: Temperature
    |---trip_point_0_temp: Over temperature threshold
    |---trip_point_0_hyst: Under temperature threshold

I misunderstood the under temperature threshold. It is not hysteresis for the upper temperature threshold. It is used to notify cold temperature that the device may not correctly work.

So we should provide another trip point for under temperature threshold, but there is no suitable thermal trip type for cold temperature.

I'm going to remove trip_point_0_hyst and not utilize the under temperature threshold.

#10 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita
Quoted 6 lines · Akinobu Mita

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

s/correspoinding/corresponding/

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2501

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+	INIT_WORK(&ctrl->thermal_work, nvme_thermal_work);
+}

Does this work queue need to be destroyed at some point?

#11 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?17?(?) 6:22 Heitke, Kenneth <kenneth.heitke at intel.com>:

Quoted 11 lines · Akinobu Mita
Quoted 6 lines

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

Quoted 1 line

s/correspoinding/corresponding/

OK.

Quoted 7 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2501

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+     INIT_WORK(&ctrl->thermal_work, nvme_thermal_work);
+}

Does this work queue need to be destroyed at some point?

This is work_struct, not workqueue. So it can't be destroyed. But I noticed that we should call flush_work for thermal_work at unregistering thermal zone devices.

maintainer
#12 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

Quoted 4 lines · Akinobu Mita

This is work_struct, not workqueue. So it can't be destroyed. But I noticed that we should call flush_work for thermal_work at unregistering thermal zone devices.

Instead of creating yet-another-work_struct, let's append this event's action to the existing async_event_work.

#13 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?18?(?) 0:14 Keith Busch <kbusch at kernel.org>:

Quoted 9 lines · Akinobu Mita
Quoted 4 lines

This is work_struct, not workqueue. So it can't be destroyed. But I noticed that we should call flush_work for thermal_work at unregistering thermal zone devices.

Instead of creating yet-another-work_struct, let's append this event's action to the existing async_event_work.

Good idea.

#14 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita
Quoted 51 lines · Akinobu Mita

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

     |---temp: Temperature
     |---trip_point_0_temp: Over temperature threshold
     |---trip_point_0_hyst: Under temperature threshold

The thermal_zone[0-*] contains a symlink to the correspoinding nvme device. On the other hand, the following symlinks to the thermal zone devices are created in the nvme device sysfs directory.

- nvme_temp0: Composite temperature
- nvme_temp1: Temperature sensor 1
...
- nvme_temp8: Temperature sensor 8

The nvme_thermal_zones_unregister() removes the registered thermal zone devices and symlinks.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
  drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
  drivers/nvme/host/nvme.h |  24 ++++
  include/linux/nvme.h     |   4 +
  3 files changed, 392 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +3 -0
index 172551b..a915c6b 100644
+
+static int nvme_tz_type_to_sensor(const char *type)
+{

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2233

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+	int sensor;
+
+	if (sscanf(type, "nvme_temp%d", &sensor) != 1)
+		return -EINVAL;
+
+	if (sensor < 0 || sensor > 8)
+		return -EINVAL;
+
+	return sensor;
+}

I know this has been discussed but it bothers me that this can return a negative (error code) but then the callers of this function don't check for errors. If you can prevent the error conditions, can 'sensor' be treated as unsigned?

Quoted 17 lines
drivers/nvme/host/core.c +17 -0
+
+#define KELVIN_TO_MILLICELSIUS(t) DECI_KELVIN_TO_MILLICELSIUS((t) * 10)
+#define MILLICELSIUS_TO_KELVIN(t) ((MILLICELSIUS_TO_DECI_KELVIN(t) + 5) / 10)
+
+static int nvme_tz_get_temp(struct thermal_zone_device *tzdev,
+			    int *temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_temp(ctrl, sensor, temp);
+	if (!ret)
+		*temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+	return ret;
+}
#15 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?17?(?) 6:25 Heitke, Kenneth <kenneth.heitke at intel.com>:

Quoted 62 lines · Akinobu Mita
Quoted 58 lines

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

     |---temp: Temperature
     |---trip_point_0_temp: Over temperature threshold
     |---trip_point_0_hyst: Under temperature threshold

The thermal_zone[0-*] contains a symlink to the correspoinding nvme device. On the other hand, the following symlinks to the thermal zone devices are created in the nvme device sysfs directory.

- nvme_temp0: Composite temperature
- nvme_temp1: Temperature sensor 1
...
- nvme_temp8: Temperature sensor 8

The nvme_thermal_zones_unregister() removes the registered thermal zone devices and symlinks.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
  drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
  drivers/nvme/host/nvme.h |  24 ++++
  include/linux/nvme.h     |   4 +
  3 files changed, 392 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +10 -0
index 172551b..a915c6b 100644
+
+static int nvme_tz_type_to_sensor(const char *type)
+{
+     int sensor;
+
+     if (sscanf(type, "nvme_temp%d", &sensor) != 1)
+             return -EINVAL;
+
+     if (sensor < 0 || sensor > 8)
+             return -EINVAL;
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2233

Quoted 3 lines
drivers/nvme/host/core.c +3 -0
+
+     return sensor;
+}

I know this has been discussed but it bothers me that this can return a negative (error code) but then the callers of this function don't check for errors. If you can prevent the error conditions, can 'sensor' be treated as unsigned?

Sounds good.

#16 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

In reply to Akinobu Mita

Thanks Akinobu for this work, please see some nit comments.

Quoted 69 lines · Akinobu Mita

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

    |---temp: Temperature
    |---trip_point_0_temp: Over temperature threshold
    |---trip_point_0_hyst: Under temperature threshold

The thermal_zone[0-*] contains a symlink to the correspoinding nvme device. On the other hand, the following symlinks to the thermal zone devices are created in the nvme device sysfs directory.

- nvme_temp0: Composite temperature
- nvme_temp1: Temperature sensor 1
...
- nvme_temp8: Temperature sensor 8

The nvme_thermal_zones_unregister() removes the registered thermal zone devices and symlinks.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
 drivers/nvme/host/nvme.h |  24 ++++
 include/linux/nvme.h     |   4 +
 3 files changed, 392 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +4 -3
index 172551b..a915c6b 100644
@@ -1113,15 +1113,16 @@ static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
 	return id;
 }
 
-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-		      void *buffer, size_t buflen, u32 *result)
+static int nvme_features(struct nvme_ctrl *dev, u8 opcode, unsigned int fid,
+			 unsigned int dword11, void *buffer, size_t buflen,
+			 u32 *result)
 {
 	struct nvme_command c;
 	union nvme_result res;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
-	c.features.opcode = nvme_admin_set_features;
+	c.features.opcode = opcode;
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
 

drivers/nvme/host/core.c static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword · line 1138

Quoted 6 lines
drivers/nvme/host/core.c +3 -0
@@ -1132,6 +1133,22 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword
 	return ret;
 }
 
+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)

1. nit:- can we align the start of the line to start of the character and not to the (.

+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+ unsigned int dword11, void *buffer, size_t buflen,
+ u32 *result)

drivers/nvme/host/core.c static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword · line 1151

Quoted 13 lines
drivers/nvme/host/core.c +13 -0
+{
+	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
+			     buflen, result);
+}
+
+static int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
+			     unsigned int dword11, void *buffer, size_t buflen,
+			     u32 *result)
+{
+	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
+			     buflen, result);
+}
+

I think above code change should go into the prep patch.

Quoted 15 lines
drivers/nvme/host/core.c +3 -0
 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 {
 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
@@ -1168,6 +1185,9 @@ static void nvme_enable_aen(struct nvme_ctrl *ctrl)
 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
 	int status;
 
+	if (IS_ENABLED(CONFIG_THERMAL))
+		supported_aens |= NVME_SMART_CRIT_TEMPERATURE;
+
 	if (!supported_aens)
 		return;
 
@@ -2164,6 +2184,334 @@ static void nvme_set_latency_tolerance(struct device *dev, s32 val)
 	}

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2194

Quoted 10 lines
drivers/nvme/host/core.c +8 -0
 }
 
+#ifdef CONFIG_THERMAL
+
+static int nvme_get_temp(struct nvme_ctrl *ctrl, int sensor, int *temp)
+{
+	struct nvme_smart_log *log;
+	int ret;
+
+	if (sensor >= ARRAY_SIZE(log->temp_sensor))

Can we add a print warn or error message here ?

Quoted 25 lines
drivers/nvme/host/core.c +25 -0
+		return -EINVAL;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	if (sensor)
+		*temp = le16_to_cpu(log->temp_sensor[sensor - 1]);
+	else
+		*temp = get_unaligned_le16(log->temperature);
+
+	if (!*temp)
+		ret = -EINVAL;
+
+free_log:
+	kfree(log);
+
+	return ret;

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2229

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+}
+
+static int nvme_tz_type_to_sensor(const char *type)
+{
+	int sensor;
+
+	if (sscanf(type, "nvme_temp%d", &sensor) != 1)
+		return -EINVAL;
+
+	if (sensor < 0 || sensor > 8)

nit:- please avoid using hard coded value 8 in the above, can we please define

a macro in nvme.h ?

Quoted 29 lines
drivers/nvme/host/core.c +29 -0
+		return -EINVAL;
+
+	return sensor;
+}
+
+#define KELVIN_TO_MILLICELSIUS(t) DECI_KELVIN_TO_MILLICELSIUS((t) * 10)
+#define MILLICELSIUS_TO_KELVIN(t) ((MILLICELSIUS_TO_DECI_KELVIN(t) + 5) / 10)
+
+static int nvme_tz_get_temp(struct thermal_zone_device *tzdev,
+			    int *temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_temp(ctrl, sensor, temp);
+	if (!ret)
+		*temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+	return ret;
+}
+
+static int nvme_tz_get_trip_type(struct thermal_zone_device *tzdev,
+				 int trip, enum thermal_trip_type *type)
+{
+	*type = THERMAL_TRIP_ACTIVE;
+
+	return 0;
+}

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2268

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+
+static int nvme_get_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+				int *temp)
+{
+	unsigned int threshold = sensor << 16;
+	int status;
+	int ret;
+
+	if (under)
+		threshold |= 0x100000;

hardcoded value, possible macro ?

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2273

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+
+	ret = nvme_get_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+				&status);
+	if (!ret)
+		*temp = status & 0xffff;

hardcoded value, possible macro ?

Quoted 118 lines
drivers/nvme/host/core.c +118 -0
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_get_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int *temp)
+{
+	return nvme_get_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_get_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int *temp)
+{
+	return nvme_get_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_set_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+				int temp)
+{
+	unsigned int threshold = (sensor << 16) | temp;
+	int status;
+	int ret;
+
+	if (temp > 0xffff)
+		return -EINVAL;
+
+	if (under)
+		threshold |= 0x100000;
+
+	ret = nvme_set_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+				&status);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_set_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int temp)
+{
+	return nvme_set_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_set_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+				     int temp)
+{
+	return nvme_set_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_tz_get_trip_temp(struct thermal_zone_device *tzdev,
+				 int trip, int *temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_over_temp_thresh(ctrl, sensor, temp);
+	if (!ret)
+		*temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+	return ret;
+}
+
+static int nvme_tz_set_trip_temp(struct thermal_zone_device *tzdev,
+				 int trip, int temp)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	temp = MILLICELSIUS_TO_KELVIN(temp);
+
+	ret = nvme_set_over_temp_thresh(ctrl, sensor, temp);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_tz_get_trip_hyst(struct thermal_zone_device *tzdev,
+				 int trip, int *hyst)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	ret = nvme_get_under_temp_thresh(ctrl, sensor, hyst);
+	if (!ret)
+		*hyst = KELVIN_TO_MILLICELSIUS(*hyst);
+
+	return ret;
+}
+
+static int nvme_tz_set_trip_hyst(struct thermal_zone_device *tzdev,
+				 int trip, int hyst)
+{
+	int sensor = nvme_tz_type_to_sensor(tzdev->type);
+	struct nvme_ctrl *ctrl = tzdev->devdata;
+	int ret;
+
+	hyst = MILLICELSIUS_TO_KELVIN(hyst);
+
+	ret = nvme_set_under_temp_thresh(ctrl, sensor, hyst);
+
+	return ret > 0 ? -EINVAL : ret;
+}
+
+static struct thermal_zone_device_ops nvme_tz_ops = {
+	.get_temp = nvme_tz_get_temp,
+	.get_trip_type = nvme_tz_get_trip_type,
+	.get_trip_temp = nvme_tz_get_trip_temp,
+	.set_trip_temp = nvme_tz_set_trip_temp,
+	.get_trip_hyst = nvme_tz_get_trip_hyst,
+	.set_trip_hyst = nvme_tz_set_trip_hyst,
+};
+
+static struct thermal_zone_params nvme_tz_params = {
+	.governor_name = "user_space",
+	.no_hwmon = true,
+};
+
+static struct thermal_zone_device *

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2401

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+	struct thermal_zone_device *tzdev;
+	char type[THERMAL_NAME_LENGTH];
+	int ret;
+
+	snprintf(type, sizeof(type), "nvme_temp%d", sensor);
+
+	tzdev = thermal_zone_device_register(type, 1, 1, ctrl, &nvme_tz_ops,
+					     &nvme_tz_params, 0, 0);

The trips and type values should be #defined with a nice comment.

Quoted 33 lines
drivers/nvme/host/core.c +33 -0
+	if (IS_ERR(tzdev))
+		return tzdev;
+
+	ret = sysfs_create_link(&ctrl->ctrl_device.kobj,
+				&tzdev->device.kobj, type);
+	if (ret)
+		goto device_unregister;
+
+	ret = sysfs_create_link(&tzdev->device.kobj,
+				&ctrl->ctrl_device.kobj, "device");
+	if (ret)
+		goto remove_link;
+
+	return tzdev;
+
+remove_link:
+	sysfs_remove_link(&ctrl->ctrl_device.kobj, type);
+device_unregister:
+	thermal_zone_device_unregister(tzdev);
+
+	return ERR_PTR(ret);
+}
+
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	struct nvme_smart_log *log;
+	int ret;
+	int i;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2444

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+			   log, sizeof(*log), 0);
+	if (ret) {
+		ret = ret > 0 ? -EINVAL : ret;
+		goto free_log;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		struct thermal_zone_device *tzdev;
+

nit:- a comment will be useful here.

Quoted 74 lines
drivers/nvme/host/core.c +70 -0
+		if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+			continue;
+		if (ctrl->tzdev[i])
+			continue;
+
+		tzdev = nvme_thermal_zone_register(ctrl, i);
+		if (!IS_ERR(tzdev))
+			ctrl->tzdev[i] = tzdev;
+	}
+
+free_log:
+	kfree(log);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_register);
+
+void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		struct thermal_zone_device *tzdev = ctrl->tzdev[i];
+
+		if (!tzdev)
+			continue;
+
+		sysfs_remove_link(&tzdev->device.kobj, "device");
+		sysfs_remove_link(&ctrl->ctrl_device.kobj, tzdev->type);
+		thermal_zone_device_unregister(tzdev);
+
+		ctrl->tzdev[i] = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_unregister);
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+	queue_work(nvme_wq, &ctrl->thermal_work);
+}
+
+static void nvme_thermal_work(struct work_struct *work)
+{
+	struct nvme_ctrl *ctrl =
+		container_of(work, struct nvme_ctrl, thermal_work);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+		if (ctrl->tzdev[i])
+			thermal_notify_framework(ctrl->tzdev[i], 0);
+	}
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+	INIT_WORK(&ctrl->thermal_work, nvme_thermal_work);
+}
+
+#else
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+}
+
+#endif /* CONFIG_THERMAL */
+
 struct nvme_core_quirk_entry {
 	/*
 	 * NVMe model and firmware strings are padded with spaces.  For

drivers/nvme/host/core.c static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) · line 3980

Quoted 3 lines
drivers/nvme/host/core.c +0 -0
@@ -3630,6 +3978,14 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 	}
 }
 

nit:- I think AEN modification code needs to be split into different patch.

Quoted 24 lines
drivers/nvme/host/core.c +11 -1
+static void nvme_handle_aen_smart(struct nvme_ctrl *ctrl, u32 result)
+{
+	u32 aer_smart_type = (result & 0xff00) >> 8;
+
+	if (aer_smart_type == NVME_AER_SMART_TEMP_THRESH)
+		nvme_thermal_notify_framework(ctrl);
+}
+
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 		volatile union nvme_result *res)
 {
@@ -3643,8 +3999,10 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 	case NVME_AER_NOTICE:
 		nvme_handle_aen_notice(ctrl, result);
 		break;
-	case NVME_AER_ERROR:
 	case NVME_AER_SMART:
+		nvme_handle_aen_smart(ctrl, result);
+		/* fallthrough */
+	case NVME_AER_ERROR:
 	case NVME_AER_CSS:
 	case NVME_AER_VS:
 		trace_nvme_async_event(ctrl, aer_type);

drivers/nvme/host/core.c int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, · line 4141

Quoted 8 lines
drivers/nvme/host/core.c +2 -0
@@ -3776,6 +4134,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
 
+	nvme_thermal_init(ctrl);
+
 	return 0;
 out_free_name:
 	kfree_const(ctrl->device->kobj.name);

Also, definitions and data structures updates should go into the different patch prep patch.

Quoted 13 lines
drivers/nvme/host/nvme.h +1 -0
index 7f6f1fc..ff7bd8f 100644
@@ -15,6 +15,7 @@
 #include <linux/sed-opal.h>
 #include <linux/fault-inject.h>
 #include <linux/rcupdate.h>
+#include <linux/thermal.h>
 
 extern unsigned int nvme_io_timeout;
 #define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)

drivers/nvme/host/nvme.h struct nvme_ctrl { · line 253

Quoted 5 lines
drivers/nvme/host/nvme.h +2 -0
@@ -248,6 +249,11 @@ struct nvme_ctrl {
 
 	struct page *discard_page;
 	unsigned long discard_page_busy;
+
+#ifdef CONFIG_THERMAL

Add a macro here for tzdev[9] or refer to spec.

Quoted 46 lines
drivers/nvme/host/nvme.h +21 -0
+	struct thermal_zone_device *tzdev[9];
+	struct work_struct thermal_work;
+#endif
 };
 
 enum nvme_iopolicy {
@@ -553,6 +559,24 @@ static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 }
 #endif /* CONFIG_NVME_MULTIPATH */
 
+#ifdef CONFIG_THERMAL
+
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl);
+void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl);
+
+#else
+
+static inline int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+	return 0;
+}
+
+static inline void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl)
+{
+}
+
+#endif /* CONFIG_THERMAL */
+
 #ifdef CONFIG_NVM
 int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
 void nvme_nvm_unregister(struct nvme_ns *ns);
include/linux/nvme.h +4 -0
index 8c0b29d..7acc77d 100644
@@ -500,6 +500,10 @@ enum {
 };
 
 enum {
+	NVME_AER_SMART_TEMP_THRESH	= 0x01,
+};
+
+enum {
 	NVME_AER_NOTICE_NS_CHANGED	= 0x00,
 	NVME_AER_NOTICE_FW_ACT_STARTING = 0x01,
 	NVME_AER_NOTICE_ANA		= 0x03,
#17 lore raw

PATCH 1/2 nvme: add thermal zone infrastructure

2019?5?17?(?) 8:44 Chaitanya Kulkarni <Chaitanya.Kulkarni at wdc.com>:

Quoted 76 lines

Thanks Akinobu for this work, please see some nit comments.

Quoted 72 lines · Akinobu Mita

The NVMe controller reports up to nine temperature values in the SMART / Health log page (the composite temperature and temperature sensor 1 through temperature sensor 8).
The temperature threshold feature (Feature Identifier 04h) configures the asynchronous event request command to complete when the temperature is crossed its correspoinding temperature threshold.

This adds infrastructure to provide these temperatures and thresholds via thermal zone devices.

The nvme_thermal_zones_register() creates up to nine thermal zone devices for valid temperature sensors including composite temperature.

/sys/class/thermal/thermal_zone[0-*]:

    |---temp: Temperature
    |---trip_point_0_temp: Over temperature threshold
    |---trip_point_0_hyst: Under temperature threshold

The thermal_zone[0-*] contains a symlink to the correspoinding nvme device. On the other hand, the following symlinks to the thermal zone devices are created in the nvme device sysfs directory.

- nvme_temp0: Composite temperature
- nvme_temp1: Temperature sensor 1
...
- nvme_temp8: Temperature sensor 8

The nvme_thermal_zones_unregister() removes the registered thermal zone devices and symlinks.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/core.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++-
 drivers/nvme/host/nvme.h |  24 ++++
 include/linux/nvme.h     |   4 +
 3 files changed, 392 insertions(+), 4 deletions(-)
drivers/nvme/host/core.c +4 -3
index 172551b..a915c6b 100644
@@ -1113,15 +1113,16 @@ static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
      return id;
 }
 
-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-                   void *buffer, size_t buflen, u32 *result)
+static int nvme_features(struct nvme_ctrl *dev, u8 opcode, unsigned int fid,
+                      unsigned int dword11, void *buffer, size_t buflen,
+                      u32 *result)
 {
      struct nvme_command c;
      union nvme_result res;
      int ret;
 
      memset(&c, 0, sizeof(c));
-     c.features.opcode = nvme_admin_set_features;
+     c.features.opcode = opcode;
      c.features.fid = cpu_to_le32(fid);
      c.features.dword11 = cpu_to_le32(dword11);
 
@@ -1132,6 +1133,22 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword
      return ret;
 }
 
Quoted 6 lines

drivers/nvme/host/core.c static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword · line 1138

Quoted 3 lines
drivers/nvme/host/core.c +3 -0
+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+                          unsigned int dword11, void *buffer, size_t buflen,
+                          u32 *result)

1. nit:- can we align the start of the line to start of the character and not to the (.

This just looks unaligned in the patch form becuase of the leading '+'.

Quoted 4 lines
drivers/nvme/host/core.c +3 -0
+static int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
+                             unsigned int dword11, void *buffer, size_t buflen,
+                             u32 *result)
Quoted 14 lines

drivers/nvme/host/core.c static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword · line 1151

Quoted 13 lines
drivers/nvme/host/core.c +13 -0
+{
+     return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
+                          buflen, result);
+}
+
+static int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
+                          unsigned int dword11, void *buffer, size_t buflen,
+                          u32 *result)
+{
+     return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
+                          buflen, result);
+}
+

I think above code change should go into the prep patch.

OK.

Quoted 17 lines
Quoted 17 lines
drivers/nvme/host/core.c +3 -0
 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 {
      u32 q_count = (*count - 1) | ((*count - 1) << 16);
@@ -1168,6 +1185,9 @@ static void nvme_enable_aen(struct nvme_ctrl *ctrl)
      u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
      int status;
 
+     if (IS_ENABLED(CONFIG_THERMAL))
+             supported_aens |= NVME_SMART_CRIT_TEMPERATURE;
+
      if (!supported_aens)
              return;
 
@@ -2164,6 +2184,334 @@ static void nvme_set_latency_tolerance(struct device *dev, s32 val)
      }
 }
 
Quoted 9 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2194

Quoted 8 lines
drivers/nvme/host/core.c +8 -0
+#ifdef CONFIG_THERMAL
+
+static int nvme_get_temp(struct nvme_ctrl *ctrl, int sensor, int *temp)
+{
+     struct nvme_smart_log *log;
+     int ret;
+
+     if (sensor >= ARRAY_SIZE(log->temp_sensor))

Can we add a print warn or error message here ?

OK. I'll add WARN_ON_ONCE.

Quoted 39 lines
Quoted 25 lines
drivers/nvme/host/core.c +25 -0
+             return -EINVAL;
+
+     log = kzalloc(sizeof(*log), GFP_KERNEL);
+     if (!log)
+             return -ENOMEM;
+
+     ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+                        log, sizeof(*log), 0);
+     if (ret) {
+             ret = ret > 0 ? -EINVAL : ret;
+             goto free_log;
+     }
+
+     if (sensor)
+             *temp = le16_to_cpu(log->temp_sensor[sensor - 1]);
+     else
+             *temp = get_unaligned_le16(log->temperature);
+
+     if (!*temp)
+             ret = -EINVAL;
+
+free_log:
+     kfree(log);
+
+     return ret;

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2229

Quoted 10 lines
drivers/nvme/host/core.c +10 -0
+}
+
+static int nvme_tz_type_to_sensor(const char *type)
+{
+     int sensor;
+
+     if (sscanf(type, "nvme_temp%d", &sensor) != 1)
+             return -EINVAL;
+
+     if (sensor < 0 || sensor > 8)

nit:- please avoid using hard coded value 8 in the above, can we please define

Quoted 1 line

a macro in nvme.h ?

I'm going to remove this range check. Instead, the range check will be done with ARRAY_SIZE(ctrl->tzdev) where the value is actually used (i.e. in nvme_get_temp() and nvme_{get,set}_over_temp_thresh())

Quoted 32 lines
Quoted 32 lines
drivers/nvme/host/core.c +32 -0
+             return -EINVAL;
+
+     return sensor;
+}
+
+#define KELVIN_TO_MILLICELSIUS(t) DECI_KELVIN_TO_MILLICELSIUS((t) * 10)
+#define MILLICELSIUS_TO_KELVIN(t) ((MILLICELSIUS_TO_DECI_KELVIN(t) + 5) / 10)
+
+static int nvme_tz_get_temp(struct thermal_zone_device *tzdev,
+                         int *temp)
+{
+     int sensor = nvme_tz_type_to_sensor(tzdev->type);
+     struct nvme_ctrl *ctrl = tzdev->devdata;
+     int ret;
+
+     ret = nvme_get_temp(ctrl, sensor, temp);
+     if (!ret)
+             *temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+     return ret;
+}
+
+static int nvme_tz_get_trip_type(struct thermal_zone_device *tzdev,
+                              int trip, enum thermal_trip_type *type)
+{
+     *type = THERMAL_TRIP_ACTIVE;
+
+     return 0;
+}
+
+static int nvme_get_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+                             int *temp)
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2268

Quoted 7 lines
drivers/nvme/host/core.c +7 -0
+{
+     unsigned int threshold = sensor << 16;
+     int status;
+     int ret;
+
+     if (under)
+             threshold |= 0x100000;

hardcoded value, possible macro ?

This line will be removed, because the under temperature threshold will not be used in the next version.

Quoted 6 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2273

Quoted 5 lines
drivers/nvme/host/core.c +5 -0
+
+     ret = nvme_get_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+                             &status);
+     if (!ret)
+             *temp = status & 0xffff;

hardcoded value, possible macro ?

OK. I'll add two enums:

enum {
NVME_TEMP_THRESH_MASK = 0xffff,
NVME_TEMP_THRESH_SELECT_SHIFT = 16,
};

Quoted 121 lines
Quoted 121 lines
drivers/nvme/host/core.c +121 -0
+
+     return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_get_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+                                  int *temp)
+{
+     return nvme_get_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_get_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+                                  int *temp)
+{
+     return nvme_get_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_set_temp_thresh(struct nvme_ctrl *ctrl, int sensor, bool under,
+                             int temp)
+{
+     unsigned int threshold = (sensor << 16) | temp;
+     int status;
+     int ret;
+
+     if (temp > 0xffff)
+             return -EINVAL;
+
+     if (under)
+             threshold |= 0x100000;
+
+     ret = nvme_set_features(ctrl, NVME_FEAT_TEMP_THRESH, threshold, NULL, 0,
+                             &status);
+
+     return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_set_over_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+                                  int temp)
+{
+     return nvme_set_temp_thresh(ctrl, sensor, false, temp);
+}
+
+static int nvme_set_under_temp_thresh(struct nvme_ctrl *ctrl, int sensor,
+                                  int temp)
+{
+     return nvme_set_temp_thresh(ctrl, sensor, true, temp);
+}
+
+static int nvme_tz_get_trip_temp(struct thermal_zone_device *tzdev,
+                              int trip, int *temp)
+{
+     int sensor = nvme_tz_type_to_sensor(tzdev->type);
+     struct nvme_ctrl *ctrl = tzdev->devdata;
+     int ret;
+
+     ret = nvme_get_over_temp_thresh(ctrl, sensor, temp);
+     if (!ret)
+             *temp = KELVIN_TO_MILLICELSIUS(*temp);
+
+     return ret;
+}
+
+static int nvme_tz_set_trip_temp(struct thermal_zone_device *tzdev,
+                              int trip, int temp)
+{
+     int sensor = nvme_tz_type_to_sensor(tzdev->type);
+     struct nvme_ctrl *ctrl = tzdev->devdata;
+     int ret;
+
+     temp = MILLICELSIUS_TO_KELVIN(temp);
+
+     ret = nvme_set_over_temp_thresh(ctrl, sensor, temp);
+
+     return ret > 0 ? -EINVAL : ret;
+}
+
+static int nvme_tz_get_trip_hyst(struct thermal_zone_device *tzdev,
+                              int trip, int *hyst)
+{
+     int sensor = nvme_tz_type_to_sensor(tzdev->type);
+     struct nvme_ctrl *ctrl = tzdev->devdata;
+     int ret;
+
+     ret = nvme_get_under_temp_thresh(ctrl, sensor, hyst);
+     if (!ret)
+             *hyst = KELVIN_TO_MILLICELSIUS(*hyst);
+
+     return ret;
+}
+
+static int nvme_tz_set_trip_hyst(struct thermal_zone_device *tzdev,
+                              int trip, int hyst)
+{
+     int sensor = nvme_tz_type_to_sensor(tzdev->type);
+     struct nvme_ctrl *ctrl = tzdev->devdata;
+     int ret;
+
+     hyst = MILLICELSIUS_TO_KELVIN(hyst);
+
+     ret = nvme_set_under_temp_thresh(ctrl, sensor, hyst);
+
+     return ret > 0 ? -EINVAL : ret;
+}
+
+static struct thermal_zone_device_ops nvme_tz_ops = {
+     .get_temp = nvme_tz_get_temp,
+     .get_trip_type = nvme_tz_get_trip_type,
+     .get_trip_temp = nvme_tz_get_trip_temp,
+     .set_trip_temp = nvme_tz_set_trip_temp,
+     .get_trip_hyst = nvme_tz_get_trip_hyst,
+     .set_trip_hyst = nvme_tz_set_trip_hyst,
+};
+
+static struct thermal_zone_params nvme_tz_params = {
+     .governor_name = "user_space",
+     .no_hwmon = true,
+};
+
+static struct thermal_zone_device *
+nvme_thermal_zone_register(struct nvme_ctrl *ctrl, int sensor)
+{
+     struct thermal_zone_device *tzdev;
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2401

Quoted 7 lines
drivers/nvme/host/core.c +7 -0
+     char type[THERMAL_NAME_LENGTH];
+     int ret;
+
+     snprintf(type, sizeof(type), "nvme_temp%d", sensor);
+
+     tzdev = thermal_zone_device_register(type, 1, 1, ctrl, &nvme_tz_ops,
+                                          &nvme_tz_params, 0, 0);

The trips and type values should be #defined with a nice comment.

I'm going to add comment that explains this trip point.

Quoted 36 lines
Quoted 36 lines
drivers/nvme/host/core.c +36 -0
+     if (IS_ERR(tzdev))
+             return tzdev;
+
+     ret = sysfs_create_link(&ctrl->ctrl_device.kobj,
+                             &tzdev->device.kobj, type);
+     if (ret)
+             goto device_unregister;
+
+     ret = sysfs_create_link(&tzdev->device.kobj,
+                             &ctrl->ctrl_device.kobj, "device");
+     if (ret)
+             goto remove_link;
+
+     return tzdev;
+
+remove_link:
+     sysfs_remove_link(&ctrl->ctrl_device.kobj, type);
+device_unregister:
+     thermal_zone_device_unregister(tzdev);
+
+     return ERR_PTR(ret);
+}
+
+int nvme_thermal_zones_register(struct nvme_ctrl *ctrl)
+{
+     struct nvme_smart_log *log;
+     int ret;
+     int i;
+
+     log = kzalloc(sizeof(*log), GFP_KERNEL);
+     if (!log)
+             return -ENOMEM;
+
+     ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+                        log, sizeof(*log), 0);
+     if (ret) {
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_set_latency_tolerance(struct device *dev, s32 val) · line 2444

Quoted 7 lines
drivers/nvme/host/core.c +7 -0
+             ret = ret > 0 ? -EINVAL : ret;
+             goto free_log;
+     }
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             struct thermal_zone_device *tzdev;
+

nit:- a comment will be useful here.

OK.

/*
* All implemented temperature sensors report a non-zero value
* in temperature sensor fields in the smart log page. */

Quoted 70 lines
Quoted 70 lines
drivers/nvme/host/core.c +70 -0
+             if (i && !le16_to_cpu(log->temp_sensor[i - 1]))
+                     continue;
+             if (ctrl->tzdev[i])
+                     continue;
+
+             tzdev = nvme_thermal_zone_register(ctrl, i);
+             if (!IS_ERR(tzdev))
+                     ctrl->tzdev[i] = tzdev;
+     }
+
+free_log:
+     kfree(log);
+
+     return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_register);
+
+void nvme_thermal_zones_unregister(struct nvme_ctrl *ctrl)
+{
+     int i;
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             struct thermal_zone_device *tzdev = ctrl->tzdev[i];
+
+             if (!tzdev)
+                     continue;
+
+             sysfs_remove_link(&tzdev->device.kobj, "device");
+             sysfs_remove_link(&ctrl->ctrl_device.kobj, tzdev->type);
+             thermal_zone_device_unregister(tzdev);
+
+             ctrl->tzdev[i] = NULL;
+     }
+}
+EXPORT_SYMBOL_GPL(nvme_thermal_zones_unregister);
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+     queue_work(nvme_wq, &ctrl->thermal_work);
+}
+
+static void nvme_thermal_work(struct work_struct *work)
+{
+     struct nvme_ctrl *ctrl =
+             container_of(work, struct nvme_ctrl, thermal_work);
+     int i;
+
+     for (i = 0; i < ARRAY_SIZE(ctrl->tzdev); i++) {
+             if (ctrl->tzdev[i])
+                     thermal_notify_framework(ctrl->tzdev[i], 0);
+     }
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+     INIT_WORK(&ctrl->thermal_work, nvme_thermal_work);
+}
+
+#else
+
+static void nvme_thermal_notify_framework(struct nvme_ctrl *ctrl)
+{
+}
+
+static void nvme_thermal_init(struct nvme_ctrl *ctrl)
+{
+}
+
+#endif /* CONFIG_THERMAL */
+
Quoted 8 lines

drivers/nvme/host/core.c static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) · line 3980

Quoted 7 lines
drivers/nvme/host/core.c +0 -0
 struct nvme_core_quirk_entry {
      /*
       * NVMe model and firmware strings are padded with spaces.  For
@@ -3630,6 +3978,14 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
      }
 }
 

nit:- I think AEN modification code needs to be split into different patch.

OK.

Quoted 48 lines
Quoted 24 lines
drivers/nvme/host/core.c +11 -1
+static void nvme_handle_aen_smart(struct nvme_ctrl *ctrl, u32 result)
+{
+     u32 aer_smart_type = (result & 0xff00) >> 8;
+
+     if (aer_smart_type == NVME_AER_SMART_TEMP_THRESH)
+             nvme_thermal_notify_framework(ctrl);
+}
+
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
              volatile union nvme_result *res)
 {
@@ -3643,8 +3999,10 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
      case NVME_AER_NOTICE:
              nvme_handle_aen_notice(ctrl, result);
              break;
-     case NVME_AER_ERROR:
      case NVME_AER_SMART:
+             nvme_handle_aen_smart(ctrl, result);
+             /* fallthrough */
+     case NVME_AER_ERROR:
      case NVME_AER_CSS:
      case NVME_AER_VS:
              trace_nvme_async_event(ctrl, aer_type);

drivers/nvme/host/core.c int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, · line 4141

Quoted 8 lines
drivers/nvme/host/core.c +2 -0
@@ -3776,6 +4134,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
      dev_pm_qos_update_user_latency_tolerance(ctrl->device,
              min(default_ps_max_latency_us, (unsigned long)S32_MAX));
 
+     nvme_thermal_init(ctrl);
+
      return 0;
 out_free_name:
      kfree_const(ctrl->device->kobj.name);

Also, definitions and data structures updates should go into the different patch prep patch.

Quoted 14 lines
drivers/nvme/host/nvme.h +1 -0
index 7f6f1fc..ff7bd8f 100644
@@ -15,6 +15,7 @@
 #include <linux/sed-opal.h>
 #include <linux/fault-inject.h>
 #include <linux/rcupdate.h>
+#include <linux/thermal.h>
 
 extern unsigned int nvme_io_timeout;
 #define NVME_IO_TIMEOUT      (nvme_io_timeout * HZ)
@@ -248,6 +249,11 @@ struct nvme_ctrl {
 
Quoted 5 lines

drivers/nvme/host/nvme.h struct nvme_ctrl { · line 253

Quoted 4 lines
drivers/nvme/host/nvme.h +2 -0
      struct page *discard_page;
      unsigned long discard_page_busy;
+
+#ifdef CONFIG_THERMAL

Add a macro here for tzdev[9] or refer to spec.

I'll add comment about which index of tzdev is corresponding to which temperature sensor or composite temperature.

Quoted 4 lines
Quoted 4 lines
drivers/nvme/host/nvme.h +3 -0
+     struct thermal_zone_device *tzdev[9];
+     struct work_struct thermal_work;
+#endif
 };
#18 lore raw

PATCH 2/2 nvme-pci: support thermal zone

In reply to Akinobu Mita

This enables to use thermal zone interfaces for NVMe temperature sensors.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/pci.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/nvme/host/pci.c +5 -0
index fad5395..88a25dc 100644
@@ -2470,6 +2470,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
 	if (dev->ctrl.admin_q)
 		blk_put_queue(dev->ctrl.admin_q);
 	kfree(dev->queues);
+	nvme_thermal_zones_unregister(&dev->ctrl);
 	free_opal_dev(dev->ctrl.opal_dev);
 	mempool_destroy(dev->iod_mempool);
 	kfree(dev);
@@ -2553,6 +2554,10 @@ static void nvme_reset_work(struct work_struct *work)
 		dev->ctrl.opal_dev = NULL;
 	}
 
+	result = nvme_thermal_zones_register(&dev->ctrl);
+	if (result < 0)
+		goto out;
+
 	if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
 		result = nvme_dbbuf_dma_alloc(dev);
 		if (result)
Signature · 1 line
2.7.4
maintainer
#19 lore raw

PATCH 2/2 nvme-pci: support thermal zone

Quoted 20 lines · Akinobu Mita

This enables to use thermal zone interfaces for NVMe temperature sensors.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/pci.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/nvme/host/pci.c +0 -0
index fad5395..88a25dc 100644

drivers/nvme/host/pci.c static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) · line 2473

Quoted 4 lines
drivers/nvme/host/pci.c +1 -0
@@ -2470,6 +2470,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
 	if (dev->ctrl.admin_q)
 		blk_put_queue(dev->ctrl.admin_q);
 	kfree(dev->queues);
+	nvme_thermal_zones_unregister(&dev->ctrl);

This unregister should probably go in the nvme_remove() rather than in the last reference release.

#20 lore raw

PATCH 2/2 nvme-pci: support thermal zone

2019?5?16?(?) 2:08 Keith Busch <kbusch at kernel.org>:

Quoted 17 lines · Akinobu Mita
Quoted 15 lines

This enables to use thermal zone interfaces for NVMe temperature sensors.

Cc Zhang Rui <rui.zhang at intel.com>
Cc Eduardo Valentin <edubezval at gmail.com>
Cc Daniel Lezcano <daniel.lezcano at linaro.org>
Cc Keith Busch <keith.busch at intel.com>
Cc Jens Axboe <axboe at fb.com>
Cc Christoph Hellwig <hch at lst.de>
Cc Sagi Grimberg <sagi at grimberg.me>
Signed-off-by Akinobu Mita <akinobu.mita at gmail.com>
---
 drivers/nvme/host/pci.c | 5 +++++
 1 file changed, 5 insertions(+)
Quoted 12 lines

drivers/nvme/host/pci.c static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) · line 2473

Quoted 9 lines
drivers/nvme/host/pci.c +1 -0
index fad5395..88a25dc 100644
@@ -2470,6 +2470,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
      if (dev->ctrl.admin_q)
              blk_put_queue(dev->ctrl.admin_q);
      kfree(dev->queues);
+     nvme_thermal_zones_unregister(&dev->ctrl);

This unregister should probably go in the nvme_remove() rather than in the last reference release.

You are right. It is too late to unregister and it caused a lot of sysfs_remove_link() failures when removing driver.