Skip to contentSkip to conversation

ceph: fix multiple unsafe decodes in decode_locker()

PATCH Superseded Pavitra Jha 3 participants 9 messages May 23 – Aug 12

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ceph: fix multiple unsafe decodes in decode_locker()

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new bad: label is added to return -EINVAL on any bounds violation.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  ? __pfx__raw_spin_lock_irqsave+0x10/0x10
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ? ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  ? ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  ? __pfx_ceph_oob4_init+0x10/0x10 [ceph_oob4_poc]
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  ? __pfx_do_one_initcall+0x10/0x10
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236] Memory state around the buggy address:
  [   26.188236]  ffff888009e31f00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  [   26.188236] >ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")
---
 net/ceph/cls_lock_client.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +6 -4
index 78276273c..00f0309a6 100644
@@ -259,7 +259,7 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+	ceph_decode_copy_safe(p, end, &locker->id.name, sizeof(locker->id.name), bad);
 	s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
 	if (IS_ERR(s))
 		return PTR_ERR(s);
@@ -270,19 +270,21 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	*p += sizeof(struct ceph_timespec); /* skip expiration */
+	ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), bad); /* skip expiration */
 
 	ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
 	if (ret)
 		return ret;
 
-	len = ceph_decode_32(p);
-	*p += len; /* skip description */
+	ceph_decode_32_safe(p, end, len, bad);
+	ceph_decode_skip_n(p, end, len, bad); /* skip description */
 
 	dout("%s %s%llu cookie %s addr %s\n", __func__,
 	     ENTITY_NAME(locker->id.name), locker->id.cookie,
 	     ceph_pr_addr(&locker->info.addr));
 	return 0;
+bad:
+	return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
Signature · 1 line
2.53.0
#1 lore raw
Quoted 90 lines · Pavitra Jha

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new bad: label is added to return -EINVAL on any bounds violation.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  ? __pfx__raw_spin_lock_irqsave+0x10/0x10
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ? ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  ? ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  ? __pfx_ceph_oob4_init+0x10/0x10 [ceph_oob4_poc]
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  ? __pfx_do_one_initcall+0x10/0x10
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236] Memory state around the buggy address:
  [   26.188236]  ffff888009e31f00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  [   26.188236] >ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")
---
 net/ceph/cls_lock_client.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +0 -0
index 78276273c..00f0309a6 100644

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 262

Quoted 5 lines
net/ceph/cls_lock_client.c +1 -1
@@ -259,7 +259,7 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+	ceph_decode_copy_safe(p, end, &locker->id.name, sizeof(locker->id.name), bad);

Are you sure that this line not longer than 80 symbols?

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 273

Quoted 9 lines
net/ceph/cls_lock_client.c +1 -1
 	s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
 	if (IS_ERR(s))
 		return PTR_ERR(s);
@@ -270,19 +270,21 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	*p += sizeof(struct ceph_timespec); /* skip expiration */
+	ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), bad); /* skip expiration */

I don't think that it makes sense to keep comment at the end of line now. Let's do it in such way:

/* skip expiration */
ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), bad);

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 280

Quoted 9 lines
net/ceph/cls_lock_client.c +2 -2
 	ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
 	if (ret)
 		return ret;
 
-	len = ceph_decode_32(p);
-	*p += len; /* skip description */
+	ceph_decode_32_safe(p, end, len, bad);
+	ceph_decode_skip_n(p, end, len, bad); /* skip description */

Ditto.

/* skip description */
ceph_decode_skip_n(p, end, len, bad);

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 286

Quoted 6 lines
net/ceph/cls_lock_client.c +1 -0
 	dout("%s %s%llu cookie %s addr %s\n", __func__,
 	     ENTITY_NAME(locker->id.name), locker->id.cookie,
 	     ceph_pr_addr(&locker->info.addr));
 	return 0;
+bad:

You are not consistent. Now it's bad name but not e_inval. :)

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 287

Quoted 1 line
net/ceph/cls_lock_client.c +1 -0
+	return -EINVAL;

I still think that we have not input arguments here. I am not fully sure that EINVAL is proper error code here.

Thanks,
Slava.

Quoted 6 lines
net/ceph/cls_lock_client.c +0 -0
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
#2 lore raw

PATCH v2 1/1 ceph: fix multiple unsafe decodes in decode_locker()

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v2: Move inline comments above ceph_decode_skip_n calls to stay within
the 80-column limit, and rename label bad -> out_bad, per Viacheslav Dubeyko's review.

---
 net/ceph/cls_lock_client.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +8 -4
index 78276273c..4f27b3d15 100644
@@ -259,7 +259,7 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+	ceph_decode_copy_safe(p, end, &locker->id.name, sizeof(locker->id.name), out_bad);
 	s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
 	if (IS_ERR(s))
 		return PTR_ERR(s);
@@ -270,19 +270,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	*p += sizeof(struct ceph_timespec); /* skip expiration */
+	/* skip expiration */
+	ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
 	ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
 	if (ret)
 		return ret;
 
-	len = ceph_decode_32(p);
-	*p += len; /* skip description */
+	ceph_decode_32_safe(p, end, len, out_bad);
+	/* skip description */
+	ceph_decode_skip_n(p, end, len, out_bad);
 
 	dout("%s %s%llu cookie %s addr %s\n", __func__,
 	     ENTITY_NAME(locker->id.name), locker->id.cookie,
 	     ceph_pr_addr(&locker->info.addr));
 	return 0;
+out_bad:
+	return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
Signature · 1 line
2.53.0
#3 lore raw
Quoted 88 lines · Pavitra Jha

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v2: Move inline comments above ceph_decode_skip_n calls to stay within
the 80-column limit, and rename label bad -> out_bad, per Viacheslav Dubeyko's review.

---
 net/ceph/cls_lock_client.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +0 -0
index 78276273c..4f27b3d15 100644

net/ceph/cls_lock_client.c static int decode_locker(void **p, void *end, struct ceph_locker *locker) · line 262

Quoted 5 lines
net/ceph/cls_lock_client.c +1 -1
@@ -259,7 +259,7 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+	ceph_decode_copy_safe(p, end, &locker->id.name, sizeof(locker->id.name), out_bad);

This line contains 85 symbols. What's the point of such long line? Why not something like this?

	ceph_decode_copy_safe(p, end, &locker->id.name,
			      sizeof(locker->id.name), out_bad);

Have you run scripts/checkpatch.pl for the patch? I am sure that the check should complain about this.

Thanks,
Slava.

Quoted 30 lines
net/ceph/cls_lock_client.c +7 -3
 	s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
 	if (IS_ERR(s))
 		return PTR_ERR(s);
@@ -270,19 +270,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	*p += sizeof(struct ceph_timespec); /* skip expiration */
+	/* skip expiration */
+	ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
 	ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
 	if (ret)
 		return ret;
 
-	len = ceph_decode_32(p);
-	*p += len; /* skip description */
+	ceph_decode_32_safe(p, end, len, out_bad);
+	/* skip description */
+	ceph_decode_skip_n(p, end, len, out_bad);
 
 	dout("%s %s%llu cookie %s addr %s\n", __func__,
 	     ENTITY_NAME(locker->id.name), locker->id.cookie,
 	     ceph_pr_addr(&locker->info.addr));
 	return 0;
+out_bad:
+	return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
#4 lore raw

Re: [PATCH v2] ceph: fix bare ceph_decode_8 OOB in decode_lockers()

Hi Slava,

Sorry for the confusion here.

The original patch fixing:

*num_lockers = ceph_decode_32(p);

and the later fix for:

*type = ceph_decode_8(p);

were intended as two separate incremental fixes,

not as replacement versions of the same patch.

I mistakenly labeled the second one as "[PATCH v2]" and also

sent it as a separate thread, which made the relationship unclear.

I'll resend this properly as a clean patch series.

Thanks,
Pavitra

#5 lore raw

PATCH v3 1/1 ceph: fix multiple unsafe decodes in decode_locker()

In reply to Viacheslav Dubeyko

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v3: Split ceph_decode_copy_safe call to fit 80-column limit,
per Viacheslav Dubeyko's review of v2.
v2: Move inline comments above ceph_decode_skip_n calls, rename

    label bad -> out_bad, per Viacheslav Dubeyko's review.
---
 net/ceph/cls_lock_client.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +9 -4
index 4e6a6d3e4..79a449897 100644
@@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+	ceph_decode_copy_safe(p, end, &locker->id.name,
+			      sizeof(locker->id.name), out_bad);
 	s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
 	if (IS_ERR(s))
 		return PTR_ERR(s);
@@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
 	if (ret)
 		return ret;
 
-	*p += sizeof(struct ceph_timespec); /* skip expiration */
+	/* skip expiration */
+	ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
 	ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
 	if (ret)
 		return ret;
 
-	len = ceph_decode_32(p);
-	*p += len; /* skip description */
+	ceph_decode_32_safe(p, end, len, out_bad);
+	/* skip description */
+	ceph_decode_skip_n(p, end, len, out_bad);
 
 	dout("%s %s%llu cookie %s addr %s\n", __func__,
 	     ENTITY_NAME(locker->id.name), locker->id.cookie,
 	     ceph_pr_addr(&locker->info.addr));
 	return 0;
+out_bad:
+	return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
Signature · 1 line
2.53.0
#6 lore raw

Ilya,

This has been Reviewed-by'd by Slava since Jun 2 (v3), no outstanding comments or NACKs on this one. It's been quiet for two months now. Any blocker to applying, or should I rebase and resend?

thanks,
Pavitra

Quoted 131 lines · Pavitra Jha

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v3: Split ceph_decode_copy_safe call to fit 80-column limit,
per Viacheslav Dubeyko's review of v2.
v2: Move inline comments above ceph_decode_skip_n calls, rename

    label bad -> out_bad, per Viacheslav Dubeyko's review.
---
 net/ceph/cls_lock_client.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +9 -5
index 4e6a6d3e4..79a449897 100644
@@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+       ceph_decode_copy_safe(p, end, &locker->id.name,
+                             sizeof(locker->id.name), out_bad);
        s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
        if (IS_ERR(s))
                return PTR_ERR(s);
@@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       *p += sizeof(struct ceph_timespec); /* skip expiration */
+       /* skip expiration */
+       ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
        ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
        if (ret)
                return ret;
 
-       len = ceph_decode_32(p);
-       *p += len; /* skip description */
+       ceph_decode_32_safe(p, end, len, out_bad);
+       /* skip description */
+       ceph_decode_skip_n(p, end, len, out_bad);
 
        dout("%s %s%llu cookie %s addr %s\n", __func__,
             ENTITY_NAME(locker->id.name), locker->id.cookie,
             ceph_pr_addr(&locker->info.addr));
        return 0;
+out_bad:
+       return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
--

2.53.0

maintainer
#7 lore raw
Quoted 6 lines · Pavitra Jha

Ilya,

This has been Reviewed-by'd by Slava since Jun 2 (v3), no outstanding comments or NACKs on this one. It's been quiet for two months now. Any blocker to applying, or should I rebase and resend?

Hi Pavitra,

Sorry, no need to resend. Slava forgot to add this patch to our integration branch like he did another similar patch of yours which is already upstream -- "libceph: fix two unsafe bare decodes in decode_lockers()". I'll eventually scrub the list and pick up anything that was missed this way, at this point likely into 7.3.

Thanks,

Ilya

Quoted 136 lines

thanks,
Pavitra

Quoted 131 lines · Pavitra Jha

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v3: Split ceph_decode_copy_safe call to fit 80-column limit,
per Viacheslav Dubeyko's review of v2.
v2: Move inline comments above ceph_decode_skip_n calls, rename

    label bad -> out_bad, per Viacheslav Dubeyko's review.
---
 net/ceph/cls_lock_client.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +9 -5
index 4e6a6d3e4..79a449897 100644
@@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+       ceph_decode_copy_safe(p, end, &locker->id.name,
+                             sizeof(locker->id.name), out_bad);
        s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
        if (IS_ERR(s))
                return PTR_ERR(s);
@@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       *p += sizeof(struct ceph_timespec); /* skip expiration */
+       /* skip expiration */
+       ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
        ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
        if (ret)
                return ret;
 
-       len = ceph_decode_32(p);
-       *p += len; /* skip description */
+       ceph_decode_32_safe(p, end, len, out_bad);
+       /* skip description */
+       ceph_decode_skip_n(p, end, len, out_bad);
 
        dout("%s %s%llu cookie %s addr %s\n", __func__,
             ENTITY_NAME(locker->id.name), locker->id.cookie,
             ceph_pr_addr(&locker->info.addr));
        return 0;
+out_bad:
+       return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
--

2.53.0

maintainer
#8 lore raw
In reply to Pavitra Jha
Quoted 6 lines · Pavitra Jha

Ilya,

This has been Reviewed-by'd by Slava since Jun 2 (v3), no outstanding comments or NACKs on this one. It's been quiet for two months now. Any blocker to applying, or should I rebase and resend?

Hi Pavitra,

On a second look, I didn't see an actual Reviewed-by tag from Slava on any of the revisions, so formally this patch was still unreviewed. I went ahead and reviewed it myself, it's now applied with a small modification to use ceph_decode_skip_string() helper instead of two separate calls to ceph_decode_32_safe() and ceph_decode_skip_n():

https://github.com/ceph/ceph-client/commit/437b6551cfcc235eea1d735a874f9d421f555e17

Thanks,

Ilya

Quoted 136 lines

thanks,
Pavitra

Quoted 131 lines · Pavitra Jha

decode_locker() in cls_lock_client.c contains three unsafe decode operations that allow a malicious or compromised OSD to trigger slab-out-of-bounds reads:

1. ceph_decode_copy() at the locker_id_t name field has no preceding bounds check. With p == end after ceph_start_decoding() accepts struct_len=0, this reads sizeof(ceph_entity_name) = 9 bytes past the validated buffer boundary.

2. *p += sizeof(struct ceph_timespec) after the locker_info_t header is an unchecked pointer advance. A malicious OSD can position p past end, causing all subsequent _safe checks to pass against a bogus boundary.

3. len = ceph_decode_32(p) has no preceding bounds check, and the immediately following *p += len is uncapped. A malicious OSD can send len=0xffffffff, advancing p gigabytes past end and escaping the decode window entirely.

Fix all three by replacing bare operations with their safe variants:

  ceph_decode_copy   -> ceph_decode_copy_safe
  *p += sizeof(...)  -> ceph_decode_skip_n
  ceph_decode_32(p)  -> ceph_decode_32_safe
  *p += len          -> ceph_decode_skip_n

A new out_bad: label is added to return -EINVAL on any bounds violation. -EINVAL is appropriate here: the data received from the OSD is structurally malformed, which is an invalid argument to the decode contract regardless of whether the caller or the wire is at fault.

KASAN report (kernel 7.0.0-rc7, QEMU/x86_64, KASLR disabled):

  [   26.183969] ceph_oob4_poc: buf=ffff888009e31000 end=ffff888009e31fa0
  [   26.186087] ceph_oob4_poc: struct_v=1 struct_len=0 p==end: 1
  [   26.186738] ceph_oob4_poc: triggering bare ceph_decode_32 past slab boundary...
  [   26.187679] ==================================================================
  [   26.188236] BUG: KASAN: slab-out-of-bounds in ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236] Read of size 4 at addr ffff888009e31fa0 by task insmod/59
  [   26.188236] CPU: 0 UID: 0 PID: 59 Comm: insmod Tainted: G           O        7.0.0-rc7-g9c2abf69da83-dirty #15 PREEMPT(lazy)
  [   26.188236] Tainted: [O]=OOT_MODULE
  [   26.188236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-debian-1.17.0-1 04/01/2014
  [   26.188236] Call Trace:
  [   26.188236]  <TASK>
  [   26.188236]  dump_stack_lvl+0x4d/0x70
  [   26.188236]  print_report+0x170/0x4f3
  [   26.188236]  kasan_report+0xda/0x110
  [   26.188236]  ceph_oob4_init+0x22b/0xff0 [ceph_oob4_poc]
  [   26.188236]  do_one_initcall+0x9a/0x3a0
  [   26.188236]  do_init_module+0x27c/0x790
  [   26.188236]  load_module+0x4a9a/0x6350
  [   26.188236]  init_module_from_file+0x15c/0x180
  [   26.188236]  idempotent_init_module+0x21f/0x750
  [   26.188236]  __x64_sys_finit_module+0xba/0x120
  [   26.188236]  do_syscall_64+0xe2/0x570
  [   26.188236]  entry_SYSCALL_64_after_hwframe+0x77/0x7f
  [   26.188236]  </TASK>
  [   26.188236] The buggy address belongs to the object at ffff888009e31000
  [   26.188236]  which belongs to the cache kmalloc-4k of size 4096
  [   26.188236] The buggy address is located 0 bytes to the right of
  [   26.188236]  allocated 4000-byte region [ffff888009e31000, ffff888009e31fa0)
  [   26.188236]  ffff888009e31f80: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236]                                ^
  [   26.188236]  ffff888009e32000: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
  [   26.188236] ==================================================================
  [   26.255513] ceph_oob4_poc: len=0xcccccccc (OOB garbage from KASAN redzone)

0xCCCCCCCC is KASAN redzone poison, confirming the read landed in the slab redzone immediately past the 4000-byte allocation.

Attacker model: a malicious or compromised OSD in a multi-tenant Ceph deployment can trigger this against any kernel client that issues the lock.get_info class method (e.g. during RBD exclusive lock acquisition) without any further privileges beyond OSD session establishment.

Fixes d4ed4a530562 ("libceph: support for lock.lock_info")

---
v3: Split ceph_decode_copy_safe call to fit 80-column limit,
per Viacheslav Dubeyko's review of v2.
v2: Move inline comments above ceph_decode_skip_n calls, rename

    label bad -> out_bad, per Viacheslav Dubeyko's review.
---
 net/ceph/cls_lock_client.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
net/ceph/cls_lock_client.c +9 -5
index 4e6a6d3e4..79a449897 100644
@@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name));
+       ceph_decode_copy_safe(p, end, &locker->id.name,
+                             sizeof(locker->id.name), out_bad);
        s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO);
        if (IS_ERR(s))
                return PTR_ERR(s);
@@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker)
        if (ret)
                return ret;
 
-       *p += sizeof(struct ceph_timespec); /* skip expiration */
+       /* skip expiration */
+       ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), out_bad);
 
        ret = ceph_decode_entity_addr(p, end, &locker->info.addr);
        if (ret)
                return ret;
 
-       len = ceph_decode_32(p);
-       *p += len; /* skip description */
+       ceph_decode_32_safe(p, end, len, out_bad);
+       /* skip description */
+       ceph_decode_skip_n(p, end, len, out_bad);
 
        dout("%s %s%llu cookie %s addr %s\n", __func__,
             ENTITY_NAME(locker->id.name), locker->id.cookie,
             ceph_pr_addr(&locker->info.addr));
        return 0;
+out_bad:
+       return -EINVAL;
 }
 
 static int decode_lockers(void **p, void *end, u8 *type, char **tag,
--

2.53.0