gpu: nova-core: add GA100 support
Subsystems (4)
v4 cover gpu: nova-core: add GA100 support
GA100 is an odd GPU. Architecturally, it's an Ampere. However, it uses the same GSP-RM boot process as a Turing. Its VBIOS, for whatever reason, has an IFR header that precedes the PCI ROM header and must be skipped. In addition, the FRTS window size must be set to 0 just because it's a GA100.
So add all the missing code that GA100 needs to be supported. There was some infrastructure already for GA100 (e.g. the FbHal), but it was incomplete.
Changes in v4:
1) Added a couple missing reviewed-bys
2) Removed IFR header v1/v2 "expect" comment
Timur Tabi (6):
gpu: nova-core: use correct fwsignature for GA100
gpu: nova-core: do not consider 0xBB77 as a valid PCI ROM header
signature
gpu: nova-core: only boot FRTS if its region is allocated
gpu: nova-core: add FbHal::frts_size() for GA100 support
gpu: nova-core: skip the IFR header if present
gpu: nova-core: enable GA100
drivers/gpu/nova-core/falcon/hal.rs | 2 +- drivers/gpu/nova-core/fb.rs | 6 +-- drivers/gpu/nova-core/fb/hal.rs | 3 ++ drivers/gpu/nova-core/fb/hal/ga100.rs | 6 +++ drivers/gpu/nova-core/fb/hal/ga102.rs | 4 ++ drivers/gpu/nova-core/fb/hal/tu102.rs | 11 ++++- drivers/gpu/nova-core/firmware/gsp.rs | 3 +- drivers/gpu/nova-core/gsp/boot.rs | 5 ++- drivers/gpu/nova-core/regs.rs | 21 +++++++++ drivers/gpu/nova-core/vbios.rs | 62 ++++++++++++++++++++++++++- 10 files changed, 113 insertions(+), 10 deletions(-)
base-commit: 8884dbc1fd174315b406027375bb7a4b211a8fae
Signature · 1 line
2.53.0
PATCH v4 1/6 gpu: nova-core: use correct fwsignature for GA100
Although GA100 uses the same GSP-RM firmware as Turing, it has a different signature specifically for it.
--- drivers/gpu/nova-core/firmware/gsp.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/gpu/nova-core/firmware/gsp.rs +1 -2
@@ -138,8 +138,7 @@ pub(crate) fn new<'a>( ".fwsignature_tu11x" } Architecture::Turing => ".fwsignature_tu10x", -// GA100 uses the same firmware as Turing - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", + Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", Architecture::Ampere => ".fwsignature_ga10x", Architecture::Ada => ".fwsignature_ad10x", };
Signature · 1 line
2.53.0
PATCH v4 2/6 gpu: nova-core: do not consider 0xBB77 as a valid PCI ROM header signature
In reply to Timur TabiNvidia GPUs have some PCI expansion ROM sections that have an Nvidia- specific signature instead of 0xAA55. Signature 0xBB77 is actually an internal-only value that has been deprecated for over a decade. Nova-core will never encounter a GPU with that signature, so don't look for it.
--- drivers/gpu/nova-core/vbios.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/nova-core/vbios.rs +1 -1
@@ -491,7 +491,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {// Check for valid ROM signatures. match signature { -0xAA55 |0xBB77 |0x4E56 => {} +0xAA55 |0x4E56 => {} _ => {dev_err! (dev,"ROM signature unknown {:#x}\n" , signature);return Err (EINVAL );
Signature · 1 line
2.53.0
PATCH v4 3/6 gpu: nova-core: only boot FRTS if its region is allocated
In reply to Timur TabiOn some Nvidia GPUs (i.e. GA100), the FRTS region is not allocated (its size is set to 0). In such cases, FWSEC-FRTS should not be run.
--- drivers/gpu/nova-core/gsp/boot.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/gpu/nova-core/gsp/boot.rs +4 -1
@@ -155,7 +155,10 @@ pub(crate) fn boot(let fb_layout =FbLayout ::new (chipset, bar, &gsp_fw)?;dev_dbg! (dev,"{:#x?}\n" , fb_layout); -Self ::run_fwsec_frts (dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; +// FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). +if !fb_layout.frts.is_empty () { +Self ::run_fwsec_frts (dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; + }let booter_loader =BooterFirmware ::new ( dev,
Signature · 1 line
2.53.0
PATCH v4 4/6 gpu: nova-core: add FbHal::frts_size() for GA100 support
In reply to Timur TabiIntroduce FbHal method frts_size() to return the size of the FRTS window. GA100 is a special case in that there is no FRTS, and so the size must be set to 0.
Note that we cannot use supports_display() to determine the FRTS size because there are other GPUs (e.g. GA102GL) that have display disabled (and so supports_display() returns False), but the FRTS window size still needs to be 1MB.
--- drivers/gpu/nova-core/fb.rs | 6 +++--- drivers/gpu/nova-core/fb/hal.rs | 3 +++ drivers/gpu/nova-core/fb/hal/ga100.rs | 6 ++++++ drivers/gpu/nova-core/fb/hal/ga102.rs | 4 ++++ drivers/gpu/nova-core/fb/hal/tu102.rs | 11 ++++++++++- 5 files changed, 26 insertions(+), 4 deletions(-)
drivers/gpu/nova-core/fb.rs +3 -3
@@ -216,10 +216,10 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<let frts = {const FRTS_DOWN_ALIGN :Alignment =Alignment ::new ::<SZ_128K >(); -const FRTS_SIZE :u64 =usize_as_u64 (SZ_1M ); -let frts_base = vga_workspace.start.align_down (FRTS_DOWN_ALIGN ) -FRTS_SIZE ; +let frts_size:u64 = hal.frts_size (); +let frts_base = vga_workspace.start.align_down (FRTS_DOWN_ALIGN ) - frts_size; -FbRange (frts_base..frts_base +FRTS_SIZE ) +FbRange (frts_base..frts_base + frts_size) };let boot = {
drivers/gpu/nova-core/fb/hal.rs +3 -0
@@ -25,6 +25,9 @@ pub(crate) trait FbHal {/// Returns the VRAM size, in bytes. fn vidmem_size (&self, bar: &Bar0 ) ->u64 ; + +/// Returns the FRTS size, in bytes. +fn frts_size (&self) ->u64 ; }/// Returns the HAL corresponding to `chipset`.
drivers/gpu/nova-core/fb/hal/ga100.rs +6 -0
@@ -66,6 +66,12 @@ fn supports_display(&self, bar: &Bar0) -> bool {fn vidmem_size (&self, bar: &Bar0 ) ->u64 {super ::tu102::vidmem_size_gp102 (bar) } + +// GA100 is a special case where its FRTS region exists, but is empty. We +// return a size of 0 because we still need to record where the region starts. +fn frts_size (&self) ->u64 { +0 + } }const GA100 :Ga100 =Ga100 ;
drivers/gpu/nova-core/fb/hal/ga102.rs +4 -0
@@ -35,6 +35,10 @@ fn supports_display(&self, bar: &Bar0) -> bool {fn vidmem_size (&self, bar: &Bar0 ) ->u64 {vidmem_size_ga102 (bar) } + +fn frts_size (&self) ->u64 { +super ::tu102::frts_size_tu102 () + } }const GA102 :Ga102 =Ga102 ;
drivers/gpu/nova-core/fb/hal/tu102.rs +10 -1
@@ -2,7 +2,8 @@use kernel::{ io::Io , - prelude::*,// + prelude::*, + sizes::*,// };use crate ::{
@@ -38,6 +39,10 @@ pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 { .usable_fb_size() } +pub (super )const fn frts_size_tu102 () ->u64 { + u64::SZ_1M +} +struct Tu102 ;impl FbHal for Tu102 {
@@ -56,6 +61,10 @@ fn supports_display(&self, bar: &Bar0) -> bool {fn vidmem_size (&self, bar: &Bar0 ) ->u64 {vidmem_size_gp102 (bar) } + +fn frts_size (&self) ->u64 { +frts_size_tu102 () + } }const TU102 :Tu102 =Tu102 ;
Signature · 1 line
2.53.0
PATCH v4 5/6 gpu: nova-core: skip the IFR header if present
In reply to Timur TabiThe GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes the PCI Expansion ROM images (VBIOS). When present, the PROM shadow method must parse this header to determine the offset where the PCI ROM images actually begin, and adjust all subsequent reads accordingly.
On most GPUs this is not needed because the IFR microcode has already applied the ROM offset so that PROM reads transparently skip the header. On GA100, for whatever reason, the IFR offset is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR header, if found.
--- drivers/gpu/nova-core/regs.rs | 21 ++++++++++++ drivers/gpu/nova-core/vbios.rs | 60 +++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-)
drivers/gpu/nova-core/regs.rs +21 -0
@@ -537,4 +537,25 @@ pub(crate) mod ga100 { 0:0 display_disabled => bool; } } + +// IFR Header in VBIOS + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED0 (u32 ) @0x300000 { +31 :0 signature; + } + } + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED1 (u32 ) @0x300004 { +30 :16 fixed_data_size; +15 :8 version =>u8 ; + } + } + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED2 (u32 ) @0x300008 { +19 :0 total_data_size; + } + } }
drivers/gpu/nova-core/vbios.rs +59 -1
78 lines not on this page · raw message
Signature · 1 line
2.53.0
Alexandre Courbot
maintainerRe: [PATCH v4 5/6] gpu: nova-core: skip the IFR header if present
drivers/gpu/nova-core/regs.rs
Quoted 21 lines · Timur Tabi
The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes the PCI Expansion ROM images (VBIOS). When present, the PROM shadow method must parse this header to determine the offset where the PCI ROM images actually begin, and adjust all subsequent reads accordingly.
On most GPUs this is not needed because the IFR microcode has already applied the ROM offset so that PROM reads transparently skip the header. On GA100, for whatever reason, the IFR offset is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR header, if found.
Signed-off-by Timur Tabi <[email protected]>--- drivers/gpu/nova-core/regs.rs | 21 ++++++++++++ drivers/gpu/nova-core/vbios.rs | 60 +++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-)drivers/gpu/nova-core/regs.rs +0
-0
Can you add the definitions in vbios.rs? They are not used outside of that module, and are not exactly registers like the other definitions of regs.rs.
Placing them under the ga100 module also sounds like they are exclusive to GA100, which is not the case as the vbios code reads them unconditionally.
Quoted 39 lines
drivers/gpu/nova-core/regs.rs +21
-0@@ -537,4 +537,25 @@ pub(crate) mod ga100 { 0:0 display_disabled => bool; } } + +// IFR Header in VBIOS + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED0 (u32 ) @0x300000 { +31 :0 signature; + } + } + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED1 (u32 ) @0x300004 { +30 :16 fixed_data_size; +15 :8 version =>u8 ; + } + } + +register! { +pub (crate )NV_PBUS_IFR_FMT_FIXED2 (u32 ) @0x300008 { +19 :0 total_data_size; + } + } }drivers/gpu/nova-core/vbios.rs +1
-0@@ -25,6 +25,7 @@ FalconUCodeDescV3, // }, num::FromSafeCast, + regs::ga100, // };/// The offset of the VBIOS ROM in the BAR0 space.
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 94
Quoted 5 lines
drivers/gpu/nova-core/vbios.rs +2
-0@@ -89,13 +90,70 @@ struct VbiosIterator<'a> { last_found: bool, } +/// IFR signature: ASCII "NVGI" as a little-endian u32. +const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE :u32 =0x4947564E ;
You can use
const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI");
To make the signature explicit in the code (and simplify the doccomment a bit).
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 96
Quoted 2 lines
drivers/gpu/nova-core/vbios.rs +2
-0+/// ROM directory signature: ASCII "RFRD" as a little-endian u32. +const NV_ROM_DIRECTORY_IDENTIFIER :u32 =0x44524652 ;
Same here.
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 100
Quoted 4 lines
drivers/gpu/nova-core/vbios.rs +4
-0+/// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section +const IFR_SW_EXT_ROM_ADDR_OFFSET :usize =4 ; +/// Size of Redundant Firmware Flash Status section +const RFW_FLASH_STATUS_SIZE :usize =4096 ;
SZ_4K?
Also can you move these constants into the current_offset method, as they are not used outside of it?
Quoted 4 lines
drivers/gpu/nova-core/vbios.rs +4
-0+ +/// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM. +/// +/// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 114
Quoted 10 lines
drivers/gpu/nova-core/vbios.rs +10
-0+/// the PCI Expansion ROM images (VBIOS). When present, the PROM shadow +/// method must parse this header to determine the offset where the PCI ROM +/// images actually begin, and adjust all subsequent reads accordingly. +/// +/// On most GPUs this is not needed because the IFR microcode has already +/// applied the ROM offset so that PROM reads transparently skip the header. +/// On GA100, for whatever reason, the IFR offset is not applied to PROM +/// reads. Therefore, the search for the PCI expansion must skip the IFR +/// header, if found. +fn current_offset (dev: &device::Device , bar0: &Bar0 ) ->Result <usize > {
The doccomment is very informative. The function name... not so much. :) Something like vbios_rom_offset maybe?
Also style nit: s/whatever/some
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 125
Quoted 11 lines
drivers/gpu/nova-core/vbios.rs +11
-0+let signature = bar0.read (ga100::NV_PBUS_IFR_FMT_FIXED0 ).signature (); + +if signature ==NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE { +let fixed1 = bar0.read (ga100::NV_PBUS_IFR_FMT_FIXED1 ); + +match fixed1.version () { +1 |2 => { +let fixed_data_size = usize::from (fixed1.fixed_data_size ()); +let pmgr_rom_addr_offset = fixed_data_size +IFR_SW_EXT_ROM_ADDR_OFFSET ; + bar0.try_read32 (ROM_OFFSET + pmgr_rom_addr_offset) + .map (|v| vas usize )
.map(usize::from_safe_cast)
Each as should normally come with its own justifying comment, so better use the infallible converters we introduced.
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 131
Quoted 6 lines
drivers/gpu/nova-core/vbios.rs +6
-0+ } + 3 => { +let fixed2 = bar0.read (ga100::NV_PBUS_IFR_FMT_FIXED2 ); +let total_data_size = usize::from (fixed2.total_data_size ()); +let dir_offset = + bar0.try_read32 (ROM_OFFSET + total_data_size)?as usize +RFW_FLASH_STATUS_SIZE ;
usize::from_safe_cast(bar0.try_read32(ROM_OFFSET + total_data_size)?) + RFW_FLASH_STATUS_SIZE;
(or feel free to use an intermediate value if that doesn't read well)
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 137
Quoted 6 lines
drivers/gpu/nova-core/vbios.rs +6
-0+let dir_sig = bar0.try_read32 (ROM_OFFSET + dir_offset)?; +if dir_sig !=NV_ROM_DIRECTORY_IDENTIFIER { +dev_err! (dev,"could not find IFR ROM directory\n" ); +return Err (EINVAL ); + } + bar0.try_read32 (ROM_OFFSET + dir_offset +8 )
Where does this 8 come from? This should be a constant at the very least.
drivers/gpu/nova-core/vbios.rs struct VbiosIterator<'a> { · line 138
Quoted 1 line
drivers/gpu/nova-core/vbios.rs +1
-0+ .map(|v| v as usize)
.map(usize::from_safe_cast)
With these fixed I expect v5 to be in a mergeable state.
PATCH v4 6/6 gpu: nova-core: enable GA100
In reply to Timur TabiGA100 is a compute-only variant of GA102 that boots GSP-RM like a Turing, although it also has its own unique requirements.
Now that all the pieces are in place, we can enable GA100 support. Although architecturally like an Ampere, GA100 uses the same GSP-RM firmware files as Turing, and therefore must boot it like Turing does. However, as a compute-only part, GA100 has no display engine.
--- drivers/gpu/nova-core/falcon/hal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/nova-core/falcon/hal.rs +1 -1
@@ -77,13 +77,13 @@ pub(super) fn falcon_hal<E: FalconEngine + 'static>(use Chipset ::*;let hal =match chipset { +GA100 |// GA100 boots like Turing so use Turing HAL TU102 |TU104 |TU106 |TU116 |TU117 => {KBox ::new (tu102::Tu102 ::<E >::new (),GFP_KERNEL )?as KBox <dyn FalconHal <E >> }GA102 |GA103 |GA104 |GA106 |GA107 |AD102 |AD103 |AD104 |AD106 |AD107 => {KBox ::new (ga102::Ga102 ::<E >::new (),GFP_KERNEL )?as KBox <dyn FalconHal <E >> } - _ =>return Err (ENOTSUPP ), };Ok (hal)
Signature · 1 line
2.53.0