drm/amdgpu: Update invalid PTE flag setting
Subsystems (2)
PATCH v2 1/1 drm/amdgpu: Update invalid PTE flag setting
Update the invalid PTE flag setting with TF enabled. This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault. Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.
---v1->v2:
- Update handling according to Christian's feedback.
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 3 +++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 11 +++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +7 -0
@@ -152,6 +152,10 @@ struct amdgpu_gmc_funcs {void (*override_vm_pte_flags)(struct amdgpu_device *dev,struct amdgpu_vm *vm,uint64_t addr,uint64_t *flags); +/* update no-retry flags */ +void (*update_vm_pte_noretry_flags)(struct amdgpu_device *dev, +uint64_t *flags); +/* get the amount of memory used by the vbios for pre-OS console */ unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev);
@@ -343,6 +347,9 @@ struct amdgpu_gmc {#define amdgpu_gmc_override_vm_pte_flags (adev, vm, addr, pte_flags) \ (adev)->gmc.gmc_funcs->override_vm_pte_flags \ ((adev), (vm), (addr), (pte_flags)) +#define amdgpu_gmc_update_vm_pte_noretry_flags (adev, pte_flags) \ + ((adev)->gmc.gmc_funcs->update_vm_pte_noretry_flags \ + ((adev), (pte_flags)))#define amdgpu_gmc_get_vbios_fb_size (adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev))/**
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
@@ -2583,7 +2583,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; }else if (amdgpu_vm_fault_stop ==AMDGPU_VM_FAULT_STOP_NEVER ) {/* Redirect the access to the dummy page */
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +6 -0
@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \ + AMDGPU_PTE_TF) +/* Flag combination to set no-retry with TF enabled */ +#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \ + AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +3 -0
@@ -804,6 +804,9 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, flags |=AMDGPU_PTE_EXECUTABLE ; } +if (adev->gmc.translate_further && level ==AMDGPU_VM_PTB ) +amdgpu_gmc_update_vm_pte_noretry_flags (adev, &flags); +/* APUs mapping system memory may need different MTYPEs on different * NUMA nodes. Only do this for contiguous ranges that can be assumed * to be on the same NUMA node.
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +11 -0
@@ -1307,6 +1307,16 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, mapping, flags); } +static void gmc_v9_0_update_vm_pte_noretry_flags (struct amdgpu_device *adev, +uint64_t *flags) +{ +/* Update no retry flags when TF is enabled */ +if ((*flags &AMDGPU_VM_NORETRY_FLAGS ) ==AMDGPU_VM_NORETRY_FLAGS ) { + *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |=AMDGPU_VM_NORETRY_FLAGS_TF ; + } +} +static void gmc_v9_0_override_vm_pte_flags (struct amdgpu_device *adev,struct amdgpu_vm *vm,uint64_t addr,uint64_t *flags)
@@ -1445,6 +1455,7 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { .get_vm_pde = gmc_v9_0_get_vm_pde, .get_vm_pte = gmc_v9_0_get_vm_pte, .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags, + .update_vm_pte_noretry_flags = gmc_v9_0_update_vm_pte_noretry_flags, .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, .query_mem_partition_mode = &gmc_v9_0_query_memory_partition, };
Signature · 1 line
2.35.1
Am 2023-06-12 um 12:23 schrieb Mukul Joshi:
Quoted 82 lines
Update the invalid PTE flag setting with TF enabled. This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault. Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.
Signed-off-by Mukul Joshi <[email protected]>---
v1->v2:
- Update handling according to Christian's feedback.drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 3 +++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 11 +++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +7
-0@@ -152,6 +152,10 @@ struct amdgpu_gmc_funcs {void (*override_vm_pte_flags)(struct amdgpu_device *dev,struct amdgpu_vm *vm,uint64_t addr,uint64_t *flags); +/* update no-retry flags */ +void (*update_vm_pte_noretry_flags)(struct amdgpu_device *dev, +uint64_t *flags); +/* get the amount of memory used by the vbios for pre-OS console */ unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev);@@ -343,6 +347,9 @@ struct amdgpu_gmc {#define amdgpu_gmc_override_vm_pte_flags (adev, vm, addr, pte_flags) \ (adev)->gmc.gmc_funcs->override_vm_pte_flags \ ((adev), (vm), (addr), (pte_flags)) +#define amdgpu_gmc_update_vm_pte_noretry_flags (adev, pte_flags) \ + ((adev)->gmc.gmc_funcs->update_vm_pte_noretry_flags \ + ((adev), (pte_flags)))#define amdgpu_gmc_get_vbios_fb_size (adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev))/** drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -2583,7 +2583,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; }else if (amdgpu_vm_fault_stop ==AMDGPU_VM_FAULT_STOP_NEVER ) {/* Redirect the access to the dummy page */ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +6
-0@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \ + AMDGPU_PTE_TF) +/* Flag combination to set no-retry with TF enabled */ +#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \ + AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, · line 808
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +2
-0@@ -804,6 +804,9 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, flags |=AMDGPU_PTE_EXECUTABLE ; } +if (adev->gmc.translate_further && level ==AMDGPU_VM_PTB ) +amdgpu_gmc_update_vm_pte_noretry_flags (adev, &flags);
Don't you need a check that ((adev)->gmc.gmc_funcs->update_vm_pte_noretry_flags is not NULL? But adding a new callback for this may be overkill. Since the AMDGPU_VM_NORETRY_FLAGS(_TF) are defined in a non-HW-specific header file, you can probably implement the application of those flags in amdgpu_vm_pte_update_flags directly.
Regards,
Felix
Quoted 33 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +1
-0+/* APUs mapping system memory may need different MTYPEs on different * NUMA nodes. Only do this for contiguous ranges that can be assumed * to be on the same NUMA node. drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +11
-0@@ -1307,6 +1307,16 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, mapping, flags); } +static void gmc_v9_0_update_vm_pte_noretry_flags (struct amdgpu_device *adev, +uint64_t *flags) +{ +/* Update no retry flags when TF is enabled */ +if ((*flags &AMDGPU_VM_NORETRY_FLAGS ) ==AMDGPU_VM_NORETRY_FLAGS ) { + *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |=AMDGPU_VM_NORETRY_FLAGS_TF ; + } +} +static void gmc_v9_0_override_vm_pte_flags (struct amdgpu_device *adev,struct amdgpu_vm *vm,uint64_t addr,uint64_t *flags)@@ -1445,6 +1455,7 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { .get_vm_pde = gmc_v9_0_get_vm_pde, .get_vm_pte = gmc_v9_0_get_vm_pte, .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags, + .update_vm_pte_noretry_flags = gmc_v9_0_update_vm_pte_noretry_flags, .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, .query_mem_partition_mode = &gmc_v9_0_query_memory_partition, };
[AMD Official Use Only - General]
Quoted 113 lines
-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Monday, June 12, 2023 2:15 PM
To: Joshi, Mukul <[email protected]>; [email protected]
Cc: Koenig, Christian <[email protected]>
Subject: Re: [PATCHv2] drm/amdgpu: Update invalid PTE flag settingAm 2023-06-12 um 12:23 schrieb Mukul Joshi:
Quoted 27 lines
Update the invalid PTE flag setting with TF enabled.
This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault.
Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.Signed-off-by Mukul Joshi <[email protected]>---
v1->v2:
- Update handling according to Christian's feedback.drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 3 +++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 11 +++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h struct amdgpu_gmc_funcs { · line 159
Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +4
-0@@ -152,6 +152,10 @@ struct amdgpu_gmc_funcs {void (*override_vm_pte_flags)(struct amdgpu_device *dev,struct amdgpu_vm *vm,uint64_t addr,uint64_t *flags); +/* update no-retry flags */ +void (*update_vm_pte_noretry_flags)(struct amdgpu_device *dev, +uint64_t *flags); +/* get the amount of memory used by the vbios for pre-OS console */
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h struct amdgpu_gmc { · line 347
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +0
-0unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev);@@ -343,6 +347,9 @@ struct amdgpu_gmc {#define amdgpu_gmc_override_vm_pte_flags (adev, vm, addr, pte_flags)\drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h struct amdgpu_gmc { · line 348
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +0
-0(adev)->gmc.gmc_funcs->override_vm_pte_flags\drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h struct amdgpu_gmc { · line 350
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +1
-0((adev), (vm), (addr), (pte_flags)) +#define amdgpu_gmc_update_vm_pte_noretry_flags (adev, pte_flags)\drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h struct amdgpu_gmc { · line 351
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +1
-0+ ((adev)->gmc.gmc_funcs->update_vm_pte_noretry_flags\drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c bool amdgpu_vm_handle_fault(struct
Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +1
-0+ ((adev), (pte_flags)))#define amdgpu_gmc_get_vbios_fb_size (adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size ((adev))/**
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0@@ -2583,7 +2583,7 @@ bool amdgpu_vm_handle_fault(struct
amdgpu_device*adev, u32 pasid,drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; } else if (amdgpu_vm_fault_stop ==
AMDGPU_VM_FAULT_STOP_NEVER) {drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h struct amdgpu_mem_stats; · line 88
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +0
-0/* Redirect the access to the dummy page */ diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index9c85d494f2a2..b81fcb962d8f100644drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +2
-0@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE| AMDGPU_PDE_PTE | \ > + AMDGPU_PTE_TF)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h struct amdgpu_mem_stats; · line 90
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +2
-0+/* Flag combination to set no-retry with TF enabled */ #define +AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID |
AMDGPU_PTE_SYSTEM| \drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h struct amdgpu_mem_stats;
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1
-0+ AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c static void amdgpu_vm_pte_update_flags(struct
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +0
-0@@ -804,6 +804,9 @@ static void amdgpu_vm_pte_update_flags(struct
amdgpu_vm_update_params*params,drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, · line 808
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +2
-0flags |=AMDGPU_PTE_EXECUTABLE ; } +if (adev->gmc.translate_further && level ==AMDGPU_VM_PTB ) +amdgpu_gmc_update_vm_pte_noretry_flags (adev, &flags);
Quoted 7 lines
Don't you need a check that ((adev)->gmc.gmc_funcs->update_vm_pte_noretry_flags is not NULL? But adding a new callback for this may be overkill. Since the
AMDGPU_VM_NORETRY_FLAGS(_TF)are defined in a non-HW-specific header file, you can probably implement the application of those flags inamdgpu_vm_pte_update_flagsdirectly.
Yes you are correct. Sorry I missed the check for update_vm_pte_noretry_flags is defined or not.
I agree that we can simply update the flags here instead of having another callback. I thought we wanted to update the flags only for GFX9 + TF enabled case.
I can update the patch to remove the callback and update the flags here only.
Regards,
Mukul
Quoted 47 lines
Regards,
Felixdrivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c static void amdgpu_vm_pte_update_flags(struct · line 810
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +1
-0+/* APUs mapping system memory may need different MTYPEs on different
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c static void amdgpu_vm_pte_update_flags(struct · line 811
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +0
-0* NUMA nodes. Only do this for contiguous ranges that can be assumed
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_get_vm_pte(struct
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +0
-0* to be on the same NUMA node. drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0
-0@@ -1307,6 +1307,16 @@ static void gmc_v9_0_get_vm_pte(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-0mapping, flags); } +static void gmc_v9_0_update_vm_pte_noretry_flags(struct
amdgpu_device*adev,drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +4
-0+ uint64_t *flags) +{ +/* Update no retry flags when TF is enabled */ + if ((*flags & AMDGPU_VM_NORETRY_FLAGS) ==
AMDGPU_VM_NORETRY_FLAGS) {drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +5
-0+ *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |=AMDGPU_VM_NORETRY_FLAGS_TF ; + } +} + static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device*adev,
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static void gmc_v9_0_get_vm_pte(struct · line 1322
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +0
-0struct amdgpu_vm *vm, uint64_t addr, uint64_t *flags) @@ -1445,6 +1455,7 @@ static
Quoted 5 lines
const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { .get_vm_pde = gmc_v9_0_get_vm_pde, .get_vm_pte = gmc_v9_0_get_vm_pte, .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,+ .update_vm_pte_noretry_flags =
gmc_v9_0_update_vm_pte_noretry_flags,Quoted 2 lines
.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, .query_mem_partition_mode =
&gmc_v9_0_query_memory_partition,Quoted 1 line
};