nvme: ANA fixups

index 0d3ef3098063..1a8791340862 100644
@@ -421,6 +421,7 @@ int nvme_configure_ana(struct nvme_ctrl *ctrl)
 out_free_ana_log_buf:
 	kfree(ctrl->ana_log_buf);
 out_free_ana_state:
+	kfree(ctrl->ana_state);
 	return -ENOMEM;
 }
 

index b1e4f56a55ed..1be424053f47 100644
@@ -1516,6 +1516,12 @@ static int nvme_revalidate_disk(struct gendisk *disk)
 		goto out;
 	}
 
+	if (ns->anagrpid != le32_to_cpu(id->anagrpid)) {
+		dev_warn(ctrl->device, "nsid %d ANA group id changed\n",
+			 ns->head->ns_id);
+		queue_delayed_work(nvme_wq, &ctrl->ana_work, 0);
+	}
+
 	__nvme_revalidate_disk(disk, id);
 	nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
 	if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {

@@ -2390,6 +2396,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	ctrl->kas = le16_to_cpu(id->kas);
 	ctrl->max_namespaces = le32_to_cpu(id->mnan);
 	ctrl->anacap = id->anacap;
+	ctrl->anatt = id->anatt;
 	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 

@@ -3415,7 +3422,7 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
 	case NVME_AER_NOTICE_ANA:
 		if (WARN_ON_ONCE(!ctrl->ana_log_buf))
 			break;
-		queue_work(nvme_wq, &ctrl->ana_work);
+		queue_delayed_work(nvme_wq, &ctrl->ana_work, 0);
 		break;
 	default:
 		dev_warn(ctrl->device, "async event result %08x\n", result);

@@ -3452,7 +3459,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
 	nvme_stop_keep_alive(ctrl);
 	flush_work(&ctrl->async_event_work);
 	flush_work(&ctrl->scan_work);
-	cancel_work_sync(&ctrl->ana_work);
+	cancel_delayed_work_sync(&ctrl->ana_work);
 	cancel_work_sync(&ctrl->fw_act_work);
 	if (ctrl->ops->stop_ctrl)
 		ctrl->ops->stop_ctrl(ctrl);

index 1a8791340862..2fcaf50d84e2 100644
@@ -69,6 +69,8 @@ void nvme_failover_req(struct request *req)
 		 * entirely trivial..
 		 */
 		nvme_update_ana_state(ns, NVME_ANA_CHANGE);
+		queue_delayed_work(nvme_wq, &ns->ctrl->ana_work,
+				   ns->ctrl->anatt * HZ);
 		break;
 	case NVME_SC_ANA_PERSISTENT_LOSS:
 		nvme_update_ana_state(ns, NVME_ANA_PERSISTENT_LOSS);

@@ -309,7 +311,7 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 {
 	void *base = ctrl->ana_log_buf;
 	size_t offset = sizeof(struct nvme_ana_rsp_hdr);
-	int error, i;
+	int error = 0, i;
 
 	/*
 	 * If anagrpid never changes we don't need to process the namespace

@@ -323,7 +325,7 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 			ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 	if (error) {
 		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
-		return error;
+		return -EIO;
 	}
 
 	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {

@@ -345,6 +347,8 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 		dev_info(ctrl->device, "ANA group %d: %s.\n",
 				grpid, nvme_ana_state_names[desc->state]);
 		WRITE_ONCE(ctrl->ana_state[grpid], desc->state);
+		if (desc->state == NVME_ANA_CHANGE)
+			error = -EAGAIN;
 		offset += sizeof(*desc);
 		if (!nr_nsids)
 			continue;

@@ -372,14 +376,31 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 			return -EINVAL;
 	}
 
-	return 0;
+	return error;
 }
 
 static void nvme_ana_work(struct work_struct *work)
 {
-	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
+	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl,
+					      ana_work.work);
+	int ret;
+
+	if (!ctrl->ana_log_buf)
+		return;
+	if (ctrl->state != NVME_CTRL_LIVE)
+		return;
+	ret = nvme_process_ana_log(ctrl, false);
+	if (ret == -EAGAIN || ret == -EIO) {
+		unsigned long log_delay = ctrl->anatt * HZ;
 
-	nvme_process_ana_log(ctrl, false);
+		/*
+		 * In case of an I/O error just add a small delay to not hit
+		 * the target too hard
+		 */
+		if (ret == -EIO)
+			log_delay = msecs_to_jiffies(NVME_ANA_LOG_DELAY);
+		queue_delayed_work(nvme_wq, &ctrl->ana_work, log_delay);
+	}
 	nvme_kick_requeue_lists(ctrl);
 }
 

@@ -404,7 +425,7 @@ int nvme_configure_ana(struct nvme_ctrl *ctrl)
 		return 0;
 	}
 
-	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
+	INIT_DELAYED_WORK(&ctrl->ana_work, nvme_ana_work);
 	ctrl->ana_state = kcalloc(ctrl->anagrpmax, sizeof(*ctrl->ana_state),
 			GFP_KERNEL);
 	if (!ctrl->ana_state)

index 4a29c736370f..53cb7f8a6267 100644
@@ -32,6 +32,7 @@ extern unsigned int admin_timeout;
 
 #define NVME_DEFAULT_KATO	5
 #define NVME_KATO_GRACE		10
+#define NVME_ANA_LOG_DELAY	500
 
 extern struct workqueue_struct *nvme_wq;
 extern struct workqueue_struct *nvme_reset_wq;

@@ -195,12 +196,13 @@ struct nvme_ctrl {
 
 	/* asymmetric namespace access: */
 	u8 anacap;
+	u8 anatt;
 	u32 anagrpmax;
 	u32 nanagrpid;
 	enum nvme_ana_state *ana_state;
 	size_t ana_log_size;
 	struct nvme_ana_rsp_hdr *ana_log_buf;
-	struct work_struct ana_work;
+	struct delayed_work ana_work;
 
 	/* Power saving configuration */
 	u64 ps_max_latency_us;

index 1be424053f47..cc43bb685e63 100644
@@ -1457,7 +1457,9 @@ static void nvme_update_disk_info(struct gendisk *disk,
 	if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk))
 		capacity = 0;
 
-	set_capacity(disk, capacity);
+	if (nvme_ns_ana_state(ns) == NVME_ANA_OPTIMIZED ||
+	    nvme_ns_ana_state(ns) == NVME_ANA_NONOPTIMIZED)
+		set_capacity(disk, capacity);
 	nvme_config_discard(ns);
 	blk_mq_unfreeze_queue(disk->queue);
 }

index 2fcaf50d84e2..a5f43d164b19 100644
@@ -108,15 +108,6 @@ void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 	up_read(&ctrl->namespaces_rwsem);
 }
 
-static inline enum nvme_ana_state nvme_ns_ana_state(struct nvme_ns *ns)
-{
-	if (!nvme_ctrl_has_ana(ns->ctrl))
-		return NVME_ANA_OPTIMIZED;
-	if (WARN_ON_ONCE(ns->anagrpid > ns->ctrl->anagrpmax))
-		return 0;
-	return READ_ONCE(ns->ctrl->ana_state[ns->anagrpid]);
-}
-
 static const char *nvme_ana_state_names[] = {
 	[0]				= "invalid state",
 	[NVME_ANA_OPTIMIZED]		= "optimized",

index 53cb7f8a6267..8572f106d6ac 100644
@@ -485,6 +485,15 @@ static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
 		kblockd_schedule_work(&head->requeue_work);
 }
 
+static inline enum nvme_ana_state nvme_ns_ana_state(struct nvme_ns *ns)
+{
+	if (!nvme_ctrl_has_ana(ns->ctrl))
+		return NVME_ANA_OPTIMIZED;
+	if (WARN_ON_ONCE(ns->anagrpid > ns->ctrl->anagrpmax))
+		return 0;
+	return READ_ONCE(ns->ctrl->ana_state[ns->anagrpid]);
+}
+
 extern struct device_attribute dev_attr_ana_grpid;
 extern struct device_attribute dev_attr_ana_state;
 

@@ -534,6 +543,10 @@ static inline int nvme_configure_ana(struct nvme_ctrl *ctrl)
 static inline void nvme_deconfigure_ana(struct nvme_ctrl *ctrl)
 {
 }
+static inline enum nvme_ana_state nvme_ns_ana_state(struct nvme_ns *ns)
+{
+	return NVME_ANA_OPTIMIZED;
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 
 #ifdef CONFIG_NVM

index a5f43d164b19..a520582811f8 100644
@@ -44,6 +44,7 @@ void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
 static void nvme_update_ana_state(struct nvme_ns *ns, enum nvme_ana_state state)
 {
 	WRITE_ONCE(ns->ctrl->ana_state[ns->anagrpid], state);
+	nvme_mpath_clear_current_path(ns);
 }
 
 void nvme_failover_req(struct request *req)

index a520582811f8..73bd4ff6ab06 100644
@@ -19,6 +19,10 @@ module_param(multipath, bool, 0444);
 MODULE_PARM_DESC(multipath,
 	"turn on native support for multiple controllers per subsystem");
 
+static u8 ana_log_retries = 5;
+module_param(ana_log_retries, byte, 0644);
+MODULE_PARM_DESC(ana_log_retries, "max number of retries retrieving ANA log");
+
 /*
  * If multipathing is enabled we need to always use the subsystem instance
  * number for numbering our devices to avoid conflicts between subsystems that

@@ -303,7 +307,7 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 {
 	void *base = ctrl->ana_log_buf;
 	size_t offset = sizeof(struct nvme_ana_rsp_hdr);
-	int error = 0, i;
+	int error = 0, i, retries = 0;
 
 	/*
 	 * If anagrpid never changes we don't need to process the namespace

@@ -312,10 +316,13 @@ static int nvme_process_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 	if (ctrl->anacap & (1 << 6))
 		groups_only = true;
 
+resubmit:
 	error = nvme_get_log_ext(ctrl, NULL, NVME_LOG_ANA,
 			groups_only ? NVME_ANA_LOG_RGO : 0,
 			ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 	if (error) {
+		if (++retries < ana_log_retries)
+			goto resubmit;
 		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 		return -EIO;
 	}

index cc43bb685e63..194cce7e63ce 100644
@@ -2714,7 +2714,7 @@ static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
 	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
 		if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
 			return 0;
-		if (!nvme_ctrl_has_ana(nvme_get_ns_from_dev(dev)->ctrl))
+		if (!nvme_get_ns_from_dev(dev)->anagrpid)
 			return 0;
 	}
 #endif

index c2fba1c92cd8..f2eedb13482a 100644
@@ -312,7 +312,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	id->msdbd = ctrl->ops->msdbd;
 
 	id->anacap = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4);
-	id->anatt = 10; /* random value */
+	id->anatt = ctrl->subsys->anatt;
 	id->anagrpmax = cpu_to_le32(NVMET_MAX_ANAGRPS);
 	id->nanagrpid = cpu_to_le32(NVMET_MAX_ANAGRPS);
 

index ddc3fa9fd92f..2c6de9961ef4 100644
@@ -738,10 +738,38 @@ static ssize_t nvmet_subsys_attr_serial_store(struct config_item *item,
 }
 CONFIGFS_ATTR(nvmet_subsys_, attr_serial);
 
+static ssize_t nvmet_subsys_attr_anatt_show(struct config_item *item,
+					    char *page)
+{
+	struct nvmet_subsys *subsys = to_subsys(item);
+
+	return snprintf(page, PAGE_SIZE, "%d\n", subsys->anatt);
+}
+
+static ssize_t nvmet_subsys_attr_anatt_store(struct config_item *item,
+					     const char *page, size_t count)
+{
+	struct nvmet_subsys *subsys = to_subsys(item);
+	int ret;
+	u8 ana_tt;
+
+	ret = kstrtou8(page, 0, &ana_tt);
+	if (ret || ana_tt == 0) {
+		return -EINVAL;
+	}
+	down_write(&nvmet_config_sem);
+	subsys->anatt = ana_tt;
+	up_write(&nvmet_config_sem);
+
+	return count;
+}
+CONFIGFS_ATTR(nvmet_subsys_, attr_anatt);
+
 static struct configfs_attribute *nvmet_subsys_attrs[] = {
 	&nvmet_subsys_attr_attr_allow_any_host,
 	&nvmet_subsys_attr_attr_version,
 	&nvmet_subsys_attr_attr_serial,
+	&nvmet_subsys_attr_attr_anatt,
 	NULL,
 };
 

index 1301a7e8b354..f47916481d8b 100644
@@ -1103,6 +1103,7 @@ struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn,
 		return NULL;
 	}
 	subsys->type = type;
+	subsys->anatt = NVMET_DEFAULT_ANATT;
 	subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE,
 			GFP_KERNEL);
 	if (!subsys->subsysnqn) {

index 7195e313b375..04dec38ffe74 100644
@@ -166,6 +166,7 @@ struct nvmet_subsys {
 	struct list_head	namespaces;
 	unsigned int		nr_namespaces;
 	unsigned int		max_nsid;
+	u8			anatt;
 
 	struct list_head	ctrls;
 

@@ -386,6 +387,7 @@ u32 nvmet_get_log_page_len(struct nvme_command *cmd);
  */
 #define NVMET_MAX_ANAGRPS	128
 #define NVMET_DEFAULT_ANA_GRPID	1
+#define NVMET_DEFAULT_ANATT	10
 
 #define NVMET_KAS		10
 #define NVMET_DISC_KATO		120

index f2eedb13482a..a9d05eb03117 100644
@@ -217,6 +217,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	struct nvme_id_ctrl *id;
 	u16 status = 0;
 	const char model[] = "Linux";
+	u32 grpid;
+	u16 ngrps = 0;
 
 	id = kzalloc(sizeof(*id), GFP_KERNEL);
 	if (!id) {

@@ -242,8 +244,26 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	 * the safest is to leave it as zeroes.
 	 */
 
-	/* we support multiple ports, multiples hosts and ANA: */
-	id->cmic = (1 << 0) | (1 << 1) | (1 << 3);
+	/* we support multiple ports and multiple hosts: */
+	id->cmic = (1 << 0) | (1 << 1);
+
+	/* ANA Support */
+	down_read(&nvmet_ana_sem);
+	for (grpid = 1; grpid <= NVMET_MAX_ANAGRPS; grpid++) {
+		if (nvmet_ana_group_enabled[grpid])
+			ngrps++;
+	}
+	up_read(&nvmet_ana_sem);
+
+	if (ngrps) {
+		id->cmic |= (1 << 3);
+		/* All ANA states are supported */
+		id->anacap = (1 << 0) | (1 << 1) | (1 << 2) | \
+			(1 << 3) | (1 << 4);
+		id->anatt = ctrl->subsys->anatt;
+		id->anagrpmax = cpu_to_le32(NVMET_MAX_ANAGRPS);
+		id->nanagrpid = cpu_to_le32(ngrps);
+	}
 
 	/* no limit on data transfer sizes for now */
 	id->mdts = 0;

@@ -311,11 +331,6 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 
 	id->msdbd = ctrl->ops->msdbd;
 
-	id->anacap = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4);
-	id->anatt = ctrl->subsys->anatt;
-	id->anagrpmax = cpu_to_le32(NVMET_MAX_ANAGRPS);
-	id->nanagrpid = cpu_to_le32(NVMET_MAX_ANAGRPS);
-
 	/*
 	 * Meh, we don't really support any power state.  Fake up the same
 	 * values that qemu does.

index a9d05eb03117..99592c7b8bc7 100644
@@ -301,7 +301,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	id->maxcmd = cpu_to_le16(NVMET_MAX_CMD);
 
 	id->nn = cpu_to_le32(ctrl->subsys->max_nsid);
-	id->mnan = cpu_to_le32(NVMET_MAX_NAMESPACES);
+	id->mnan = cpu_to_le32(ctrl->subsys->nr_namespaces);
 	id->oncs = cpu_to_le16(NVME_CTRL_ONCS_DSM |
 			NVME_CTRL_ONCS_WRITE_ZEROES);
 

index 99592c7b8bc7..0ae25478e3eb 100644
@@ -371,6 +371,18 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req)
 	id->ncap = id->nuse = id->nsze =
 		cpu_to_le64(ns->size >> ns->blksize_shift);
 
+	if (ns->anagrpid) {
+		enum nvme_ana_state ana_state =
+			req->port->ana_state[ns->anagrpid];
+
+		/*
+		 * nuse and nsze should be zero for inaccessible or
+		 * persistent loss ANA state.
+		 */
+		if ((ana_state == NVME_ANA_INACCESSIBLE) ||
+		    (ana_state == NVME_ANA_PERSISTENT_LOSS))
+			id->nuse = id->nsze = 0;
+	}
 	/*
 	 * We just provide a single LBA format that matches what the
 	 * underlying device reports.