Skip to contentSkip to conversation

Add R10D4+ support for IMSM

PATCH Applied Mateusz Kusiak 2 participants 10 messages Apr 29, 2024 – May 7, 2024 MD RAID

lore
Share & tools
lore
#0 lore raw

cover Add R10D4+ support for IMSM

This series of patches adds support for RAID10 with more than 4 drives (R10D4+) to IMSM.

Mateusz Kusiak (8):
mdadm: pass struct context for external reshapes
mdadm: use struct context in reshape_super()
imsm: add support for literal RAID 10
imsm: refactor RAID level handling
imsm: bump minimal version
imsm: define RAID_10 attribute
imsm: simplify imsm_check_attributes()
imsm: support RAID 10 with more than 4 drives

 Assemble.c       |   7 +-
 Create.c         |   9 +-
 Grow.c           | 143 ++++++++++------
 mdadm.c          |   2 +-
 mdadm.h          |  25 ++-
 platform-intel.c |  58 +++++++
 platform-intel.h |  32 ++--
 super-intel.c    | 433 +++++++++++++++++++++++------------------------
 util.c           |  39 +++--
 9 files changed, 419 insertions(+), 329 deletions(-)
Signature · 1 line
2.39.2
#1 lore raw

PATCH 1/8 mdadm: pass struct context for external reshapes

This patch alters mutiple functions calls so the context is passed to external reshape functions.

There are two main reasons behind it:
- reduces number of arguments passed and unifies them,
- imsm code will make use of context in incoming patches.

---
 Assemble.c    |  7 ++----
 Grow.c        | 68 +++++++++++++++++++++------------------------------
 mdadm.c       |  2 +-
 mdadm.h       | 11 +++------
 super-intel.c |  6 ++---
 5 files changed, 37 insertions(+), 57 deletions(-)
Assemble.c +2 -5
index f6c5b99e25e2..f5e9ab1f0853 100644
@@ -1197,9 +1197,7 @@ static int start_array(int mdfd,
 			rv = sysfs_set_str(content, NULL,
 					   "array_state", "readonly");
 			if (rv == 0)
-				rv = Grow_continue(mdfd, st, content,
-						   c->backup_file, 0,
-						   c->freeze_reshape);
+				rv = Grow_continue(mdfd, st, content, 0, c);
 		} else if (c->readonly &&
 			   sysfs_attribute_available(content, NULL,
 						     "array_state")) {
@@ -2180,8 +2178,7 @@ int assemble_container_content(struct supertype *st, int mdfd,
 				st->update_tail = &st->updates;
 		}
 
-		err = Grow_continue(mdfd, st, content, c->backup_file,
-				    0, c->freeze_reshape);
+		err = Grow_continue(mdfd, st, content, 0, c);
 	} else switch(content->array.level) {
 		case LEVEL_LINEAR:
 		case LEVEL_MULTIPATH:
Grow.c +28 -40

173 lines not on this page · raw message

mdadm.c +1 -1
index 3f1912884d49..d18619db86bf 100644
@@ -1636,7 +1636,7 @@ int main(int argc, char *argv[])
 				c.delay = DEFAULT_BITMAP_DELAY;
 			rv = Grow_addbitmap(ident.devname, mdfd, &c, &s);
 		} else if (grow_continue)
-			rv = Grow_continue_command(ident.devname, mdfd, c.backup_file, c.verbose);
+			rv = Grow_continue_command(ident.devname, mdfd, &c);
 		else if (s.size > 0 || s.raiddisks || s.layout_str ||
 			 s.chunk != 0 || s.level != UnSet ||
 			 s.data_offset != INVALID_SECTORS) {
mdadm.h +4 -7
index 2640b39687f7..0ade4bebc400 100644
@@ -1235,9 +1235,8 @@ extern struct superswitch {
 	int (*reshape_super)(struct supertype *st,
 			     unsigned long long size, int level,
 			     int layout, int chunksize, int raid_disks,
-			     int delta_disks, char *backup, char *dev,
-			     int direction,
-			     int verbose); /* optional */
+			     int delta_disks, char *dev, int direction,
+			     struct context *c);
 	int (*manage_reshape)( /* optional */
 		int afd, struct mdinfo *sra, struct reshape *reshape,
 		struct supertype *st, unsigned long blocks,
@@ -1541,8 +1540,7 @@ extern int Grow_reshape(char *devname, int fd,
 extern int Grow_restart(struct supertype *st, struct mdinfo *info,
 			int *fdlist, int cnt, char *backup_file, int verbose);
 extern int Grow_continue(int mdfd, struct supertype *st,
-			 struct mdinfo *info, char *backup_file,
-			 int forked, int freeze_reshape);
+			 struct mdinfo *info, int forked, struct context *c);
 extern int Grow_consistency_policy(char *devname, int fd,
 				   struct context *c, struct shape *s);
 
@@ -1552,8 +1550,7 @@ extern int restore_backup(struct supertype *st,
 			  int spares,
 			  char **backup_filep,
 			  int verbose);
-extern int Grow_continue_command(char *devname, int fd,
-				 char *backup_file, int verbose);
+extern int Grow_continue_command(char *devname, int fd, struct context *c);
 
 extern int Assemble(struct supertype *st, char *mddev,
 		    struct mddev_ident *ident,
super-intel.c +2 -4
index 1faab6077f9e..417da2672504 100644
@@ -12153,10 +12153,8 @@ exit:
 }
 
 static int imsm_reshape_super(struct supertype *st, unsigned long long size,
-			      int level,
-			      int layout, int chunksize, int raid_disks,
-			      int delta_disks, char *backup, char *dev,
-			      int direction, int verbose)
+			      int level, int layout, int chunksize, int raid_disks,
+			      int delta_disks, char *dev, int direction, struct context *c)
 {
 	int ret_val = 1;
 	struct geo_params geo;
Signature · 1 line
2.39.2
#2 lore raw

PATCH 2/8 mdadm: use struct context in reshape_super()

In reply to Mateusz Kusiak

reshape_super() takes too many arguments. Change passing params in favor of single struct.

Add devname pointer and change direction members to struct shape and use it for reshape_super().

Create reshape_array_size() and reshape_array_non_size() to handle reshape_super() calls.

---
 Grow.c        | 93 +++++++++++++++++++++++++++++++++++++--------------
 mdadm.h       | 18 +++++-----
 super-intel.c | 43 +++++++++++++++---------
 3 files changed, 105 insertions(+), 49 deletions(-)
Grow.c +68 -25
index f477b438e810..87ed9214ef02 100644
@@ -862,9 +862,7 @@ static void wait_reshape(struct mdinfo *sra)
 	close(fd);
 }
 
-static int reshape_super(struct supertype *st, unsigned long long size,
-			 int level, int layout, int chunksize, int raid_disks,
-			 int delta_disks, char *dev, int direction, struct context *c)
+static int reshape_super(struct supertype *st, struct shape *shape, struct context *c)
 {
 	/* nothing extra to check in the native case */
 	if (!st->ss->external)
@@ -875,8 +873,65 @@ static int reshape_super(struct supertype *st, unsigned long long size,
 		return 1;
 	}
 
-	return st->ss->reshape_super(st, size, level, layout, chunksize, raid_disks,
-				     delta_disks, dev, direction, c);
+	return st->ss->reshape_super(st, shape, c);
+}
+
+/**
+ * reshape_super_size() - Reshape array, size only.
+ *
+ * @st: supertype.
+ * @devname: device name.
+ * @size: component size.
+ * @dir metadata changes direction
+ * Returns: 0 on success, 1 otherwise.
+ *
+ * This function is solely used to change size of the volume.
+ * Setting size is not valid for container.
+ * Size is only change that can be rolled back, thus the @dir param.
+ */
+static int reshape_super_size(struct supertype *st, char *devname,
+			      unsigned long long size, change_dir_t direction,
+			      struct context *c)
+{
+	struct shape shape = {0};
+
+	shape.level = UnSet;
+	shape.layout = UnSet;
+	shape.delta_disks = UnSet;
+	shape.dev = devname;
+	shape.size = size;
+	shape.direction = direction;
+
+	return reshape_super(st, &shape, c);
+}
+
+/**
+ * reshape_super_non_size() - Reshape array, non size changes.
+ *
+ * @st: supertype.
+ * @devname: device name.
+ * @info: superblock info.
+ * Returns: 0 on success, 1 otherwise.
+ *
+ * This function is used for any external array changes but size.
+ * It handles both volumes and containers.
+ * For changes other than size, rollback is not possible.
+ */
+static int reshape_super_non_size(struct supertype *st, char *devname,
+				  struct mdinfo *info, struct context *c)
+{
+	struct shape shape = {0};
+	/* Size already set to zero, not updating size */
+	shape.level = info->new_level;
+	shape.layout = info->new_layout;
+	shape.chunk = info->new_chunk;
+	shape.raiddisks = info->array.raid_disks;
+	shape.delta_disks = info->delta_disks;
+	shape.dev = devname;
+	/* Rollback not possible for non size changes */
+	shape.direction = APPLY_METADATA_CHANGES;
+
+	return reshape_super(st, &shape, c);
 }
 
 static void sync_metadata(struct supertype *st)
@@ -1979,9 +2034,8 @@ int Grow_reshape(char *devname, int fd,
 	}
 
 	/* ========= set size =============== */
-	if (s->size > 0 &&
-	    (s->size == MAX_SIZE || s->size != (unsigned)array.size)) {
-		unsigned long long orig_size = get_component_size(fd)/2;
+	if (s->size > 0 && (s->size == MAX_SIZE || s->size != (unsigned)array.size)) {
+		unsigned long long orig_size = get_component_size(fd) / 2;
 		unsigned long long min_csize;
 		struct mdinfo *mdi;
 		int raid0_takeover = 0;
@@ -2001,8 +2055,7 @@ int Grow_reshape(char *devname, int fd,
 			goto release;
 		}
 
-		if (reshape_super(st, s->size, UnSet, UnSet, 0, 0, UnSet,
-				  devname, APPLY_METADATA_CHANGES, c)) {
+		if (reshape_super_size(st, devname, s->size, APPLY_METADATA_CHANGES, c)) {
 			rv = 1;
 			goto release;
 		}
@@ -2120,8 +2173,8 @@ size_change_error:
 			int err = errno;
 
 			/* restore metadata */
-			if (reshape_super(st, orig_size, UnSet, UnSet, 0, 0, UnSet,
-			                  devname, ROLLBACK_METADATA_CHANGES, c) == 0)
+			if (reshape_super_size(st, devname, orig_size,
+					       ROLLBACK_METADATA_CHANGES, c) == 0)
 				sync_metadata(st);
 			pr_err("Cannot set device size for %s: %s\n",
 				devname, strerror(err));
@@ -2351,11 +2404,7 @@ size_change_error:
 		/* Impose these changes on a single array.  First
 		 * check that the metadata is OK with the change.
 		 */
-
-		if (reshape_super(st, 0, info.new_level,
-				  info.new_layout, info.new_chunk,
-				  info.array.raid_disks, info.delta_disks,
-				  devname, APPLY_METADATA_CHANGES, c)) {
+		if (reshape_super_non_size(st, devname, &info, c)) {
 			rv = 1;
 			goto release;
 		}
@@ -3668,14 +3717,8 @@ int reshape_container(char *container, char *devname,
 	int rv = restart;
 	char last_devnm[32] = "";
 
-	/* component_size is not meaningful for a container,
-	 * so pass '0' meaning 'no change'
-	 */
-	if (!restart &&
-	    reshape_super(st, 0, info->new_level,
-			  info->new_layout, info->new_chunk,
-			  info->array.raid_disks, info->delta_disks,
-			  devname, APPLY_METADATA_CHANGES, c)) {
+	/* component_size is not meaningful for a container */
+	if (!restart && reshape_super_non_size(st, devname, info, c)) {
 		unfreeze(st);
 		return 1;
 	}
mdadm.h +10 -8
index 0ade4bebc400..2ff3e46383cd 100644
@@ -594,6 +594,11 @@ enum flag_mode {
 	FlagDefault, FlagSet, FlagClear,
 };
 
+typedef enum {
+	ROLLBACK_METADATA_CHANGES,
+	APPLY_METADATA_CHANGES
+} change_dir_t;
+
 /* structures read from config file */
 /* List of mddevice names and identifiers
  * Identifiers can be:
@@ -667,7 +672,9 @@ struct context {
 };
 
 struct shape {
+	char	*dev;
 	int	raiddisks;
+	int	delta_disks;
 	int	sparedisks;
 	int	journaldisks;
 	int	level;
@@ -682,6 +689,7 @@ struct shape {
 	unsigned long long size;
 	unsigned long long data_offset;
 	int	consistency_policy;
+	change_dir_t direction;
 };
 
 /* List of device names - wildcards expanded */
@@ -1229,14 +1237,8 @@ extern struct superswitch {
 	 * initialized to indicate if reshape is being performed at the
 	 * container or subarray level
 	 */
-#define APPLY_METADATA_CHANGES		1
-#define ROLLBACK_METADATA_CHANGES	0
-
-	int (*reshape_super)(struct supertype *st,
-			     unsigned long long size, int level,
-			     int layout, int chunksize, int raid_disks,
-			     int delta_disks, char *dev, int direction,
-			     struct context *c);
+
+	int (*reshape_super)(struct supertype *st, struct shape *shape, struct context *c);
 	int (*manage_reshape)( /* optional */
 		int afd, struct mdinfo *sra, struct reshape *reshape,
 		struct supertype *st, unsigned long blocks,
super-intel.c +27 -16

67 lines not on this page · raw message

Signature · 1 line
2.39.2
#3 lore raw

PATCH 3/8 imsm: add support for literal RAID 10

In reply to Mateusz Kusiak

As for now, IMSM supports only 4 drive RAID 1+0. This patch is first in series to add support for literal RAID 10 (with more than 4 drives) to imsm.

Allow setting RAID 10 as raid level for imsm arrays.

Add update_imsm_raid_level() to handle raid level updates. Set RAID10 as default level for imsm R0 to R10 migrations. Replace magic numbers with defined values for RAID level checks/assigns.

---
 super-intel.c | 67 ++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 19 deletions(-)
super-intel.c +48 -19
index 1a8a7b125025..a7efc8df0b47 100644
@@ -166,7 +166,8 @@ struct imsm_map {
 	__u8  raid_level;
 #define IMSM_T_RAID0 0
 #define IMSM_T_RAID1 1
-#define IMSM_T_RAID5 5		/* since metadata version 1.2.02 ? */
+#define IMSM_T_RAID5 5
+#define IMSM_T_RAID10 10
 	__u8  num_members;	/* number of member disks */
 	__u8  num_domains;	/* number of parity domains */
 	__u8  failed_disk_num;  /* valid only when state is degraded */
@@ -1259,14 +1260,42 @@ static int get_imsm_disk_slot(struct imsm_map *map, const unsigned int idx)
 
 	return IMSM_STATUS_ERROR;
 }
+/**
+ * update_imsm_raid_level() - update raid level appropriately in &imsm_map.
+ * @map:	&imsm_map pointer.
+ * @new_level:	MD style level.
+ *
+ * For backward compatibility reasons we need to differentiate RAID10.
+ * In the past IMSM RAID10 was presented as RAID1.
+ * Keep compatibility unless it is not explicitly updated by UEFI driver.
+ *
+ * Routine needs num_members to be set and (optionally) raid_level.
+ */
+static void update_imsm_raid_level(struct imsm_map *map, int new_level)
+{
+	if (new_level != IMSM_T_RAID10) {
+		map->raid_level = new_level;
+		return;
+	}
+
+	if (map->num_members == 4) {
+		if (map->raid_level == IMSM_T_RAID10 || map->raid_level == IMSM_T_RAID1)
+			return;
+
+		map->raid_level = IMSM_T_RAID1;
+		return;
+	}
+
+	map->raid_level = IMSM_T_RAID10;
+}
 
 static int get_imsm_raid_level(struct imsm_map *map)
 {
-	if (map->raid_level == 1) {
+	if (map->raid_level == IMSM_T_RAID1) {
 		if (map->num_members == 2)
-			return 1;
+			return IMSM_T_RAID1;
 		else
-			return 10;
+			return IMSM_T_RAID10;
 	}
 
 	return map->raid_level;
@@ -5678,7 +5707,7 @@ static int init_super_imsm_volume(struct supertype *st, mdu_array_info_t *info,
 	set_pba_of_lba0(map, super->create_offset);
 	map->blocks_per_strip = __cpu_to_le16(info_to_blocks_per_strip(info));
 	map->failed_disk_num = ~0;
-	if (info->level > 0)
+	if (info->level > IMSM_T_RAID0)
 		map->map_state = (info->state ? IMSM_T_STATE_NORMAL
 				  : IMSM_T_STATE_UNINITIALIZED);
 	else
@@ -5686,16 +5715,15 @@ static int init_super_imsm_volume(struct supertype *st, mdu_array_info_t *info,
 						      IMSM_T_STATE_NORMAL;
 	map->ddf = 1;
 
-	if (info->level == 1 && info->raid_disks > 2) {
+	if (info->level == IMSM_T_RAID1 && info->raid_disks > 2) {
 		free(dev);
 		free(dv);
-		pr_err("imsm does not support more than 2 disksin a raid1 volume\n");
+		pr_err("imsm does not support more than 2 disks in a raid1 volume\n");
 		return 0;
 	}
+	map->num_members = info->raid_disks;
 
-	map->raid_level = info->level;
-	if (info->level == 10)
-		map->raid_level = 1;
+	update_imsm_raid_level(map, info->level);
 	set_num_domains(map);
 
 	size_per_member += NUM_BLOCKS_DIRTY_STRIPE_REGION;
@@ -5703,7 +5731,6 @@ static int init_super_imsm_volume(struct supertype *st, mdu_array_info_t *info,
 							     size_per_member /
 							     BLOCKS_PER_KB));
 
-	map->num_members = info->raid_disks;
 	update_num_data_stripes(map, array_blocks);
 	for (i = 0; i < map->num_members; i++) {
 		/* initialized in add_to_super */
@@ -8275,7 +8302,7 @@ static struct mdinfo *container_content_imsm(struct supertype *st, char *subarra
 			info_d->data_offset = pba_of_lba0(map);
 			info_d->component_size = calc_component_size(map, dev);
 
-			if (map->raid_level == 5) {
+			if (map->raid_level == IMSM_T_RAID5) {
 				info_d->ppl_sector = this->ppl_sector;
 				info_d->ppl_size = this->ppl_size;
 				if (this->consistency_policy == CONSISTENCY_POLICY_PPL &&
@@ -9533,7 +9560,7 @@ static int apply_reshape_migration_update(struct imsm_update_reshape_migration *
 			}
 
 			to_state = map->map_state;
-			if ((u->new_level == 5) && (map->raid_level == 0)) {
+			if ((u->new_level == IMSM_T_RAID5) && (map->raid_level == IMSM_T_RAID0)) {
 				map->num_members++;
 				/* this should not happen */
 				if (u->new_disks[0] < 0) {
@@ -9544,11 +9571,13 @@ static int apply_reshape_migration_update(struct imsm_update_reshape_migration *
 					to_state = IMSM_T_STATE_NORMAL;
 			}
 			migrate(new_dev, super, to_state, MIGR_GEN_MIGR);
+
 			if (u->new_level > -1)
-				map->raid_level = u->new_level;
+				update_imsm_raid_level(map, u->new_level);
+
 			migr_map = get_imsm_map(new_dev, MAP_1);
-			if ((u->new_level == 5) &&
-			    (migr_map->raid_level == 0)) {
+			if ((u->new_level == IMSM_T_RAID5) &&
+			    (migr_map->raid_level == IMSM_T_RAID0)) {
 				int ord = map->num_members - 1;
 				migr_map->num_members--;
 				if (u->new_disks[0] < 0)
@@ -9584,7 +9613,7 @@ static int apply_reshape_migration_update(struct imsm_update_reshape_migration *
 
 			/* add disk
 			 */
-			if (u->new_level != 5 || migr_map->raid_level != 0 ||
+			if (u->new_level != IMSM_T_RAID5 || migr_map->raid_level != IMSM_T_RAID0 ||
 			    migr_map->raid_level == map->raid_level)
 				goto skip_disk_add;
 
@@ -9963,7 +9992,7 @@ static int apply_takeover_update(struct imsm_update_takeover *u,
 		/* update map */
 		map->num_members /= map->num_domains;
 		map->map_state = IMSM_T_STATE_NORMAL;
-		map->raid_level = 0;
+		update_imsm_raid_level(map, IMSM_T_RAID0);
 		set_num_domains(map);
 		update_num_data_stripes(map, imsm_dev_size(dev));
 		map->failed_disk_num = -1;
@@ -10007,7 +10036,7 @@ static int apply_takeover_update(struct imsm_update_takeover *u,
 		map = get_imsm_map(dev_new, MAP_0);
 
 		map->map_state = IMSM_T_STATE_DEGRADED;
-		map->raid_level = 1;
+		update_imsm_raid_level(map, IMSM_T_RAID10);
 		set_num_domains(map);
 		map->num_members = map->num_members * map->num_domains;
 		update_num_data_stripes(map, imsm_dev_size(dev));
Signature · 1 line
2.39.2
#4 lore raw

PATCH 4/8 imsm: refactor RAID level handling

In reply to Mateusz Kusiak

Add imsm_level_ops struct for better handling and unifying raid level support. Add helper methods and move "orom_has_raid[...]" methods from header to source file.

RAID 1e is not supported under Linux, remove RAID 1e associated code.

Refactor imsm_analyze_change() and is_raid_level_supported(). Remove hardcoded check for 4 drives and make devNumChange a multiplier for RAID 10.

Refactor printing supported raid levels.

---
 platform-intel.c |  57 ++++++++++++++++++++++++
 platform-intel.h |  32 ++++++--------
 super-intel.c    | 111 ++++++++++++++++++++++++++++-------------------
 3 files changed, 138 insertions(+), 62 deletions(-)
platform-intel.c +57 -0
index ac282bc5b09b..40e8fb82da30 100644
@@ -32,6 +32,63 @@
 
 #define NVME_SUBSYS_PATH "/sys/devices/virtual/nvme-subsystem/"
 
+static bool imsm_orom_has_raid0(const struct imsm_orom *orom)
+{
+	return imsm_rlc_has_bit(orom, IMSM_OROM_RLC_RAID0);
+}
+
+static bool imsm_orom_has_raid1(const struct imsm_orom *orom)
+{
+	return imsm_rlc_has_bit(orom, IMSM_OROM_RLC_RAID1);
+}
+
+static bool imsm_orom_has_raid10(const struct imsm_orom *orom)
+{
+	return imsm_rlc_has_bit(orom, IMSM_OROM_RLC_RAID10);
+}
+
+static bool imsm_orom_has_raid5(const struct imsm_orom *orom)
+{
+	return imsm_rlc_has_bit(orom, IMSM_OROM_RLC_RAID5);
+}
+
+/* IMSM platforms do not define how many disks are allowed for each level,
+ * but there are some global limitations we need to follow.
+ */
+static bool imsm_orom_support_raid_disks_count_raid0(const int raid_disks)
+{
+	return true;
+}
+
+static bool imsm_orom_support_raid_disks_count_raid1(const int raid_disks)
+{
+	if (raid_disks == 2)
+		return true;
+	return false;
+}
+
+static bool imsm_orom_support_raid_disks_count_raid5(const int raid_disks)
+{
+	if (raid_disks > 2)
+		return true;
+	return false;
+}
+
+static bool imsm_orom_support_raid_disks_count_raid10(const int raid_disks)
+{
+	if (raid_disks == 4)
+		return true;
+	return false;
+}
+
+struct imsm_level_ops imsm_level_ops[] = {
+		{0, imsm_orom_has_raid0, imsm_orom_support_raid_disks_count_raid0, "raid0"},
+		{1, imsm_orom_has_raid1, imsm_orom_support_raid_disks_count_raid1, "raid1"},
+		{5, imsm_orom_has_raid5, imsm_orom_support_raid_disks_count_raid5, "raid5"},
+		{10, imsm_orom_has_raid10, imsm_orom_support_raid_disks_count_raid10, "raid10"},
+		{-1, NULL, NULL, NULL}
+};
+
 static int devpath_to_ll(const char *dev_path, const char *entry,
 			 unsigned long long *val);
 
platform-intel.h +14 -18
index 3c2bc595f7b5..dcc5aaa74f21 100644
@@ -109,25 +109,21 @@ struct imsm_orom {
 	#define IMSM_OROM_CAPABILITIES_TPV (1 << 10)
 } __attribute__((packed));
 
-static inline int imsm_orom_has_raid0(const struct imsm_orom *orom)
-{
-	return !!(orom->rlc & IMSM_OROM_RLC_RAID0);
-}
-static inline int imsm_orom_has_raid1(const struct imsm_orom *orom)
-{
-	return !!(orom->rlc & IMSM_OROM_RLC_RAID1);
-}
-static inline int imsm_orom_has_raid1e(const struct imsm_orom *orom)
-{
-	return !!(orom->rlc & IMSM_OROM_RLC_RAID1E);
-}
-static inline int imsm_orom_has_raid10(const struct imsm_orom *orom)
-{
-	return !!(orom->rlc & IMSM_OROM_RLC_RAID10);
-}
-static inline int imsm_orom_has_raid5(const struct imsm_orom *orom)
+/* IMSM metadata requirements for each level */
+struct imsm_level_ops {
+	int level;
+	bool (*is_level_supported)(const struct imsm_orom *);
+	bool (*is_raiddisks_count_supported)(const int);
+	char *name;
+};
+
+extern struct imsm_level_ops imsm_level_ops[];
+
+static inline bool imsm_rlc_has_bit(const struct imsm_orom *orom, const unsigned short bit)
 {
-	return !!(orom->rlc & IMSM_OROM_RLC_RAID5);
+	if (orom->rlc & bit)
+		return true;
+	return false;
 }
 
 /**
super-intel.c +67 -44

192 lines not on this page · raw message

Signature · 1 line
2.39.2
#5 lore raw

PATCH 5/8 imsm: bump minimal version

In reply to Mateusz Kusiak

IMSM version 1.3 (called ATTRIBS) brought attributes used to define array properties which require support in driver. The goal of this change was to avoid changing version when adding new features.

For some reasons migration has never been completed and currently (after 10 years of implementing) IMSM can use older versions.

It is right time to finally switch it. There is no point in using old versions, use 1.3.00 as minimal one.

Define JD_VERSION used by Windows driver.

---
 super-intel.c | 87 ++++++++++++++++++++++-----------------------------
 1 file changed, 38 insertions(+), 49 deletions(-)
super-intel.c +38 -49
index da17265d7f12..4b168add4346 100644
@@ -32,14 +32,19 @@
 /* MPB == Metadata Parameter Block */
 #define MPB_SIGNATURE "Intel Raid ISM Cfg Sig. "
 #define MPB_SIG_LEN (strlen(MPB_SIGNATURE))
-#define MPB_VERSION_RAID0 "1.0.00"
-#define MPB_VERSION_RAID1 "1.1.00"
-#define MPB_VERSION_MANY_VOLUMES_PER_ARRAY "1.2.00"
-#define MPB_VERSION_3OR4_DISK_ARRAY "1.2.01"
-#define MPB_VERSION_RAID5 "1.2.02"
-#define MPB_VERSION_5OR6_DISK_ARRAY "1.2.04"
-#define MPB_VERSION_CNG "1.2.06"
+
+/* Legacy IMSM versions:
+ * MPB_VERSION_RAID0 1.0.00
+ * MPB_VERSION_RAID1 1.1.00
+ * MPB_VERSION_MANY_VOLUMES_PER_ARRAY 1.2.00
+ * MPB_VERSION_3OR4_DISK_ARRAY 1.2.01
+ * MPB_VERSION_RAID5 1.2.02
+ * MPB_VERSION_5OR6_DISK_ARRAY 1.2.04
+ * MPB_VERSION_CNG 1.2.06
+ */
+
 #define MPB_VERSION_ATTRIBS "1.3.00"
+#define MPB_VERSION_ATTRIBS_JD "2.0.00"
 #define MAX_SIGNATURE_LENGTH  32
 #define MAX_RAID_SERIAL_LEN   16
 
@@ -5512,51 +5517,46 @@ static unsigned long long info_to_blocks_per_member(mdu_array_info_t *info,
 		return (size * 2) & ~(info_to_blocks_per_strip(info) - 1);
 }
 
+static void imsm_write_signature(struct imsm_super *mpb)
+{
+	/* It is safer to eventually truncate version rather than left it not NULL ended */
+	snprintf((char *) mpb->sig, MAX_SIGNATURE_LENGTH, MPB_SIGNATURE MPB_VERSION_ATTRIBS);
+}
+
 static void imsm_update_version_info(struct intel_super *super)
 {
 	/* update the version and attributes */
 	struct imsm_super *mpb = super->anchor;
-	char *version;
 	struct imsm_dev *dev;
 	struct imsm_map *map;
 	int i;
 
+	mpb->attributes |= MPB_ATTRIB_CHECKSUM_VERIFY;
+
 	for (i = 0; i < mpb->num_raid_devs; i++) {
 		dev = get_imsm_dev(super, i);
 		map = get_imsm_map(dev, MAP_0);
+
 		if (__le32_to_cpu(dev->size_high) > 0)
 			mpb->attributes |= MPB_ATTRIB_2TB;
 
-		/* FIXME detect when an array spans a port multiplier */
-		#if 0
-		mpb->attributes |= MPB_ATTRIB_PM;
-		#endif
-
-		if (mpb->num_raid_devs > 1 ||
-		    mpb->attributes != MPB_ATTRIB_CHECKSUM_VERIFY) {
-			version = MPB_VERSION_ATTRIBS;
-			switch (get_imsm_raid_level(map)) {
-			case 0: mpb->attributes |= MPB_ATTRIB_RAID0; break;
-			case 1: mpb->attributes |= MPB_ATTRIB_RAID1; break;
-			case 10: mpb->attributes |= MPB_ATTRIB_RAID10; break;
-			case 5: mpb->attributes |= MPB_ATTRIB_RAID5; break;
-			}
-		} else {
-			if (map->num_members >= 5)
-				version = MPB_VERSION_5OR6_DISK_ARRAY;
-			else if (dev->status == DEV_CLONE_N_GO)
-				version = MPB_VERSION_CNG;
-			else if (get_imsm_raid_level(map) == 5)
-				version = MPB_VERSION_RAID5;
-			else if (map->num_members >= 3)
-				version = MPB_VERSION_3OR4_DISK_ARRAY;
-			else if (get_imsm_raid_level(map) == 1)
-				version = MPB_VERSION_RAID1;
-			else
-				version = MPB_VERSION_RAID0;
+		switch (get_imsm_raid_level(map)) {
+		case IMSM_T_RAID0:
+			mpb->attributes |= MPB_ATTRIB_RAID0;
+			break;
+		case IMSM_T_RAID1:
+			mpb->attributes |= MPB_ATTRIB_RAID1;
+			break;
+		case IMSM_T_RAID5:
+			mpb->attributes |= MPB_ATTRIB_RAID5;
+			break;
+		case IMSM_T_RAID10:
+			mpb->attributes |= MPB_ATTRIB_RAID10;
+			break;
 		}
-		strcpy(((char *) mpb->sig) + strlen(MPB_SIGNATURE), version);
 	}
+
+	imsm_write_signature(mpb);
 }
 
 /**
@@ -5785,7 +5785,6 @@ static int init_super_imsm(struct supertype *st, mdu_array_info_t *info,
 	struct intel_super *super;
 	struct imsm_super *mpb;
 	size_t mpb_size;
-	char *version;
 
 	if (data_offset != INVALID_SECTORS) {
 		pr_err("data-offset not supported by imsm\n");
@@ -5828,13 +5827,7 @@ static int init_super_imsm(struct supertype *st, mdu_array_info_t *info,
 		return 0;
 	}
 
-	mpb->attributes = MPB_ATTRIB_CHECKSUM_VERIFY;
-
-	version = (char *) mpb->sig;
-	strcpy(version, MPB_SIGNATURE);
-	version += strlen(MPB_SIGNATURE);
-	strcpy(version, MPB_VERSION_RAID0);
-
+	imsm_update_version_info(super);
 	return 1;
 }
 
@@ -6208,7 +6201,6 @@ static union {
 
 static int write_super_imsm_spare(struct intel_super *super, struct dl *d)
 {
-	struct imsm_super *mpb = super->anchor;
 	struct imsm_super *spare = &spare_record.anchor;
 	__u32 sum;
 
@@ -6217,14 +6209,11 @@ static int write_super_imsm_spare(struct intel_super *super, struct dl *d)
 
 	spare->mpb_size = __cpu_to_le32(sizeof(struct imsm_super));
 	spare->generation_num = __cpu_to_le32(1UL);
-	spare->attributes = MPB_ATTRIB_CHECKSUM_VERIFY;
 	spare->num_disks = 1;
 	spare->num_raid_devs = 0;
-	spare->cache_size = mpb->cache_size;
 	spare->pwr_cycle_count = __cpu_to_le32(1);
 
-	snprintf((char *) spare->sig, MAX_SIGNATURE_LENGTH,
-		 MPB_SIGNATURE MPB_VERSION_RAID0);
+	imsm_write_signature(spare);
 
 	spare->disk[0] = d->disk;
 	if (__le32_to_cpu(d->disk.total_blocks_hi) > 0)
Signature · 1 line
2.39.2
#6 lore raw

PATCH 6/8 imsm: define RAID_10 attribute

In reply to Mateusz Kusiak

Add MPB_ATTRIB_RAID10_EXT attribute to support RAID 10 with more than 4 drives.

Allow more than 4 drives in imsm_orom_support_raid_disks_raid10().

This is one of last patches for introducing R10D4+ to imsm. Only small adjustments in reshape behaviours are needed.

---
 platform-intel.c | 3 ++-
 super-intel.c    | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)
platform-intel.c +2 -1
index 40e8fb82da30..15a9fa5ac160 100644
@@ -76,7 +76,8 @@ static bool imsm_orom_support_raid_disks_count_raid5(const int raid_disks)
 
 static bool imsm_orom_support_raid_disks_count_raid10(const int raid_disks)
 {
-	if (raid_disks == 4)
+	/* raid_disks count must be higher than 4 and even */
+	if (raid_disks >= 4 && (raid_disks & 1) == 0)
 		return true;
 	return false;
 }
super-intel.c +5 -0
index 4b168add4346..2d30931672cd 100644
@@ -62,6 +62,8 @@
 #define MPB_ATTRIB_RAIDCNG		__cpu_to_le32(0x00000020)
 /* supports expanded stripe sizes of  256K, 512K and 1MB */
 #define MPB_ATTRIB_EXP_STRIPE_SIZE	__cpu_to_le32(0x00000040)
+/* supports RAID10 with more than 4 drives */
+#define MPB_ATTRIB_RAID10_EXT		__cpu_to_le32(0x00000080)
 
 /* The OROM Support RST Caching of Volumes */
 #define MPB_ATTRIB_NVM			__cpu_to_le32(0x02000000)
@@ -89,6 +91,7 @@
 					MPB_ATTRIB_RAID10          | \
 					MPB_ATTRIB_RAID5           | \
 					MPB_ATTRIB_EXP_STRIPE_SIZE | \
+					MPB_ATTRIB_RAID10_EXT      | \
 					MPB_ATTRIB_BBM)
 
 /* Define attributes that are unused but not harmful */
@@ -5552,6 +5555,8 @@ static void imsm_update_version_info(struct intel_super *super)
 			break;
 		case IMSM_T_RAID10:
 			mpb->attributes |= MPB_ATTRIB_RAID10;
+			if (map->num_members > 4)
+				mpb->attributes |= MPB_ATTRIB_RAID10_EXT;
 			break;
 		}
 	}
Signature · 1 line
2.39.2
#7 lore raw

PATCH 7/8 imsm: simplify imsm_check_attributes()

In reply to Mateusz Kusiak

imsm_check_attributes() is too complex for that it really does.

Remove repeating code and simplify the function.
Fix function calls.

---
 super-intel.c | 106 ++++++++------------------------------------------
 1 file changed, 16 insertions(+), 90 deletions(-)
super-intel.c +16 -90
index 2d30931672cd..d60915e812d4 100644
@@ -2135,91 +2135,18 @@ void convert_from_4k(struct intel_super *super)
 	mpb->check_sum = __gen_imsm_checksum(mpb);
 }
 
-/*******************************************************************************
- * function: imsm_check_attributes
- * Description: Function checks if features represented by attributes flags
- *		are supported by mdadm.
- * Parameters:
- *		attributes - Attributes read from metadata
- * Returns:
- *		0 - passed attributes contains unsupported features flags
- *		1 - all features are supported
- ******************************************************************************/
-static int imsm_check_attributes(__u32 attributes)
+/**
+ * imsm_check_attributes() - Check if features represented by attributes flags are supported.
+ *
+ * @attributes: attributes read from metadata.
+ * Returns: true if all features are supported, false otherwise.
+ */
+static bool imsm_check_attributes(__u32 attributes)
 {
-	int ret_val = 1;
-	__u32 not_supported = MPB_ATTRIB_SUPPORTED^0xffffffff;
-
-	not_supported &= ~MPB_ATTRIB_IGNORED;
-
-	not_supported &= attributes;
-	if (not_supported) {
-		pr_err("(IMSM): Unsupported attributes : %x\n",
-			(unsigned)__le32_to_cpu(not_supported));
-		if (not_supported & MPB_ATTRIB_CHECKSUM_VERIFY) {
-			dprintf("\t\tMPB_ATTRIB_CHECKSUM_VERIFY \n");
-			not_supported ^= MPB_ATTRIB_CHECKSUM_VERIFY;
-		}
-		if (not_supported & MPB_ATTRIB_2TB) {
-			dprintf("\t\tMPB_ATTRIB_2TB\n");
-			not_supported ^= MPB_ATTRIB_2TB;
-		}
-		if (not_supported & MPB_ATTRIB_RAID0) {
-			dprintf("\t\tMPB_ATTRIB_RAID0\n");
-			not_supported ^= MPB_ATTRIB_RAID0;
-		}
-		if (not_supported & MPB_ATTRIB_RAID1) {
-			dprintf("\t\tMPB_ATTRIB_RAID1\n");
-			not_supported ^= MPB_ATTRIB_RAID1;
-		}
-		if (not_supported & MPB_ATTRIB_RAID10) {
-			dprintf("\t\tMPB_ATTRIB_RAID10\n");
-			not_supported ^= MPB_ATTRIB_RAID10;
-		}
-		if (not_supported & MPB_ATTRIB_RAID1E) {
-			dprintf("\t\tMPB_ATTRIB_RAID1E\n");
-			not_supported ^= MPB_ATTRIB_RAID1E;
-		}
-		if (not_supported & MPB_ATTRIB_RAID5) {
-		dprintf("\t\tMPB_ATTRIB_RAID5\n");
-			not_supported ^= MPB_ATTRIB_RAID5;
-		}
-		if (not_supported & MPB_ATTRIB_RAIDCNG) {
-			dprintf("\t\tMPB_ATTRIB_RAIDCNG\n");
-			not_supported ^= MPB_ATTRIB_RAIDCNG;
-		}
-		if (not_supported & MPB_ATTRIB_BBM) {
-			dprintf("\t\tMPB_ATTRIB_BBM\n");
-		not_supported ^= MPB_ATTRIB_BBM;
-		}
-		if (not_supported & MPB_ATTRIB_CHECKSUM_VERIFY) {
-			dprintf("\t\tMPB_ATTRIB_CHECKSUM_VERIFY (== MPB_ATTRIB_LEGACY)\n");
-			not_supported ^= MPB_ATTRIB_CHECKSUM_VERIFY;
-		}
-		if (not_supported & MPB_ATTRIB_EXP_STRIPE_SIZE) {
-			dprintf("\t\tMPB_ATTRIB_EXP_STRIP_SIZE\n");
-			not_supported ^= MPB_ATTRIB_EXP_STRIPE_SIZE;
-		}
-		if (not_supported & MPB_ATTRIB_2TB_DISK) {
-			dprintf("\t\tMPB_ATTRIB_2TB_DISK\n");
-			not_supported ^= MPB_ATTRIB_2TB_DISK;
-		}
-		if (not_supported & MPB_ATTRIB_NEVER_USE2) {
-			dprintf("\t\tMPB_ATTRIB_NEVER_USE2\n");
-			not_supported ^= MPB_ATTRIB_NEVER_USE2;
-		}
-		if (not_supported & MPB_ATTRIB_NEVER_USE) {
-			dprintf("\t\tMPB_ATTRIB_NEVER_USE\n");
-			not_supported ^= MPB_ATTRIB_NEVER_USE;
-		}
-
-		if (not_supported)
-			dprintf("(IMSM): Unknown attributes : %x\n", not_supported);
-
-		ret_val = 0;
-	}
+	if ((attributes & (MPB_ATTRIB_SUPPORTED | MPB_ATTRIB_IGNORED)) == attributes)
+		return true;
 
-	return ret_val;
+	return false;
 }
 
 static void getinfo_super_imsm(struct supertype *st, struct mdinfo *info, char *map);
@@ -2247,11 +2174,10 @@ static void examine_super_imsm(struct supertype *st, char *homehost)
 	creation_time = __le64_to_cpu(mpb->creation_time);
 	printf("  Creation Time : %.24s\n",
 		creation_time ? ctime(&creation_time) : "Unknown");
-	printf("     Attributes : ");
-	if (imsm_check_attributes(mpb->attributes))
-		printf("All supported\n");
-	else
-		printf("not supported\n");
+
+	printf("     Attributes : %08x (%s)\n", mpb->attributes,
+	       imsm_check_attributes(mpb->attributes) ? "supported" : "not supported");
+
 	getinfo_super_imsm(st, &info, NULL);
 	fname_from_uuid(&info, nbuf);
 	printf("           UUID : %s\n", nbuf + 5);
@@ -8182,9 +8108,9 @@ static struct mdinfo *container_content_imsm(struct supertype *st, char *subarra
 	int current_vol = super->current_vol;
 
 	/* do not assemble arrays when not all attributes are supported */
-	if (imsm_check_attributes(mpb->attributes) == 0) {
+	if (imsm_check_attributes(mpb->attributes) == false) {
 		sb_errors = 1;
-		pr_err("Unsupported attributes in IMSM metadata.Arrays activation is blocked.\n");
+		pr_err("Unsupported attributes in IMSM metadata. Arrays activation is blocked.\n");
 	}
 
 	/* count spare devices, not used in maps
Signature · 1 line
2.39.2
#8 lore raw

PATCH 8/8 imsm: support RAID 10 with more than 4 drives

In reply to Mateusz Kusiak

VROC UEFI driver does not support RAID 10 with more than 4 drives. Add user prompts if such layout is being created and for R0->R10 reshapes.

Refactor ask() function:
- simplify the code,
- remove dialog reattempts,
- do no pass '?' sign on function calls,
- highlight default option on output.

This patch completes adding support for R10D4+ to IMSM.

---
 Create.c      |  9 ++++++++-
 super-intel.c | 14 ++++++++++++--
 util.c        | 39 +++++++++++++++++++++++++--------------
 3 files changed, 45 insertions(+), 17 deletions(-)
Create.c +8 -1
index d94253b1ad2e..d033eb68f30c 100644
@@ -965,6 +965,13 @@ int Create(struct supertype *st, struct mddev_ident *ident, int subdevs,
 		return 1;
 	}
 
+	if (st->ss == &super_imsm && s->level == 10 && s->raiddisks > 4) {
+		/* Print no matter runstop was specifed */
+		pr_err("Warning! VROC UEFI driver does not support RAID10 in requested layout.\n");
+		pr_err("Array won't be suitable as boot device.\n");
+		warn = 1;
+	}
+
 	if (!have_container && s->level > 0 && ((maxsize-s->size)*100 > maxsize)) {
 		if (c->runstop != 1 || c->verbose >= 0)
 			pr_err("largest drive (%s) exceeds size (%lluK) by more than 1%%\n",
@@ -984,7 +991,7 @@ int Create(struct supertype *st, struct mddev_ident *ident, int subdevs,
 
 	if (warn) {
 		if (c->runstop!= 1) {
-			if (!ask("Continue creating array? ")) {
+			if (!ask("Continue creating array")) {
 				pr_err("create aborted.\n");
 				return 1;
 			}
super-intel.c +12 -2
index d60915e812d4..2b8b6fda976c 100644
@@ -523,6 +523,7 @@ enum imsm_reshape_type {
 	CH_TAKEOVER,
 	CH_MIGRATION,
 	CH_ARRAY_SIZE,
+	CH_ABORT
 };
 
 /* definition of messages passed to imsm_process_update */
@@ -11898,7 +11899,7 @@ success:
 ****************************************************************************/
 enum imsm_reshape_type imsm_analyze_change(struct supertype *st,
 					   struct geo_params *geo,
-					   int direction)
+					   int direction, struct context *c)
 {
 	struct mdinfo info;
 	int change = -1;
@@ -11925,6 +11926,14 @@ enum imsm_reshape_type imsm_analyze_change(struct supertype *st,
 				check_devs = 1;
 				raid_disks += 1; /* parity disk added */
 			} else if (geo->level == IMSM_T_RAID10) {
+				if (geo->level == IMSM_T_RAID10 && geo->raid_disks > 2 &&
+				    !c->force) {
+					pr_err("Warning! VROC UEFI driver does not support RAID10 in requested layout.\n");
+					pr_err("Array won't be suitable as boot device.\n");
+					pr_err("Note: You can omit this check with \"--force\"\n");
+					if (ask("Do you want to continue") < 1)
+						return CH_ABORT;
+				}
 				change = CH_TAKEOVER;
 				check_devs = 1;
 				raid_disks *= 2; /* mirrors added */
@@ -12219,7 +12228,7 @@ static int imsm_reshape_super(struct supertype *st, struct shape *shape, struct
 			goto exit_imsm_reshape_super;
 		}
 		super->current_vol = dev->index;
-		change = imsm_analyze_change(st, &geo, shape->direction);
+		change = imsm_analyze_change(st, &geo, shape->direction, c);
 		switch (change) {
 		case CH_TAKEOVER:
 			ret_val = imsm_takeover(st, &geo);
@@ -12262,6 +12271,7 @@ static int imsm_reshape_super(struct supertype *st, struct shape *shape, struct
 				free(u);
 		}
 		break;
+		case CH_ABORT:
 		default:
 			ret_val = 1;
 		}
util.c +25 -14
index 9e8370450a8d..4fbf11c4e2bd 100644
@@ -725,23 +725,33 @@ int stat_is_blkdev(char *devname, dev_t *rdev)
 	return 1;
 }
 
+/**
+ * ask() - prompt user for "yes/no" dialog.
+ * @mesg: message to be printed, without '?' sign.
+ * Returns: 1 if 'Y/y', 0 otherwise.
+ *
+ * The default value is 'N/n', thus the caps on "N" on prompt.
+ */
 int ask(char *mesg)
 {
-	char *add = "";
-	int i;
-	for (i = 0; i < 5; i++) {
-		char buf[100];
-		fprintf(stderr, "%s%s", mesg, add);
-		fflush(stderr);
-		if (fgets(buf, 100, stdin)==NULL)
-			return 0;
-		if (buf[0]=='y' || buf[0]=='Y')
-			return 1;
-		if (buf[0]=='n' || buf[0]=='N')
-			return 0;
-		add = "(y/n) ";
+	char buf[3] = {0};
+
+	fprintf(stderr, "%s [y/N]? ", mesg);
+	fflush(stderr);
+	if (fgets(buf, 3, stdin) == NULL)
+		return 0;
+	if (strlen(buf) == 1) {
+		pr_err("assuming no.\n");
+		return 0;
 	}
-	pr_err("assuming 'no'\n");
+	if (buf[1] != '\n')
+		goto bad_option;
+	if (toupper(buf[0]) == 'Y')
+		return 1;
+	if (toupper(buf[0]) == 'N')
+		return 0;
+bad_option:
+	pr_err("bad option.\n");
 	return 0;
 }
 
@@ -1868,6 +1878,7 @@ int set_array_info(int mdfd, struct supertype *st, struct mdinfo *info)
 
 	if (st->ss->external)
 		return sysfs_set_array(info);
+
 	memset(&inf, 0, sizeof(inf));
 	inf.major_version = info->array.major_version;
 	inf.minor_version = info->array.minor_version;
Signature · 1 line
2.39.2
#9 lore raw
In reply to Mateusz Kusiak
Quoted 13 lines · On Mon, 29 Apr 2024 15:07:12 +0200 Mateusz Kusiak <[email protected]>…

This series of patches adds support for RAID10 with more than 4 drives (R10D4+) to IMSM.

Mateusz Kusiak (8):
mdadm: pass struct context for external reshapes
mdadm: use struct context in reshape_super()
imsm: add support for literal RAID 10
imsm: refactor RAID level handling
imsm: bump minimal version
imsm: define RAID_10 attribute
imsm: simplify imsm_check_attributes()
imsm: support RAID 10 with more than 4 drives

Quoted 11 lines
 Assemble.c       |   7 +-
 Create.c         |   9 +-
 Grow.c           | 143 ++++++++++------
 mdadm.c          |   2 +-
 mdadm.h          |  25 ++-
 platform-intel.c |  58 +++++++
 platform-intel.h |  32 ++--
 super-intel.c    | 433 +++++++++++++++++++++++------------------------
 util.c           |  39 +++--
 9 files changed, 419 insertions(+), 329 deletions(-)

Applied!

Thanks,
Mariusz