From: Yolkfull Chow Subject: Re: [PATCH] KVM test: Add PCI pass through test Date: Thu, 15 Oct 2009 14:45:57 +0800 Message-ID: <20091015064557.GA2503@aFu.nay.redhat.com> References: <1255522080-10570-1-git-send-email-lmr@redhat.com> Reply-To: Yolkfull Chow Mime-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: QUOTED-PRINTABLE Cc: autotest@test.kernel.org, kvm@vger.kernel.org To: Lucas Meneghel Rodrigues Return-path: Received: from mx1.redhat.com ([209.132.183.28]:58097 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S932470AbZJOGq1 (ORCPT ); Thu, 15 Oct 2009 02:46:27 -0400 Content-Disposition: inline In-Reply-To: <1255522080-10570-1-git-send-email-lmr@redhat.com> Sender: kvm-owner@vger.kernel.org List-ID: On Wed, Oct 14, 2009 at 09:08:00AM -0300, Lucas Meneghel Rodrigues wrot= e: > Add a new PCI pass trough test. It supports both SR-IOV virtual > functions and physical NIC card pass through. >=20 > Single Root I/O Virtualization (SR-IOV) allows a single PCI device to > be shared amongst multiple virtual machines while retaining the > performance benefit of assigning a PCI device to a virtual machine. > A common example is where a single SR-IOV capable NIC - with perhaps > only a single physical network port - might be shared with multiple > virtual machines by assigning a virtual function to each VM. >=20 > SR-IOV support is implemented in the kernel. The core implementation = is > contained in the PCI subsystem, but there must also be driver support > for both the Physical Function (PF) and Virtual Function (VF) devices= =2E > With an SR-IOV capable device one can allocate VFs from a PF. The VFs > surface as PCI devices which are backed on the physical PCI device by > resources (queues, and register sets). >=20 > Device support: >=20 > In 2.6.30, the Intel=C2=AE 82576 Gigabit Ethernet Controller is the o= nly > SR-IOV capable device supported. The igb driver has PF support and th= e > igbvf has VF support. >=20 > In 2.6.31 the Neterion=C2=AE X3100=E2=84=A2 is supported as well. Thi= s device uses > the same vxge driver for the PF as well as the VFs. Wow, new NIC card supports SR-IOV...=20 At this rate, do we need to move the driver name and its parameter into config file so that in future if a new NIC card using different driver is supported, we could handle it without changing code ? >=20 > In order to configure the test: >=20 > * For SR-IOV virtual functions passthrough, we could specify the > module parameter 'max_vfs' in config file. > * For physical NIC card pass through, we should specify the device > name(s). >=20 > Signed-off-by: Yolkfull Chow > --- > client/tests/kvm/kvm_tests.cfg.sample | 11 ++- > client/tests/kvm/kvm_utils.py | 278 +++++++++++++++++++++++= ++++++++++ > client/tests/kvm/kvm_vm.py | 72 +++++++++ > 3 files changed, 360 insertions(+), 1 deletions(-) >=20 > diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm= /kvm_tests.cfg.sample > index cc3228a..1dad188 100644 > --- a/client/tests/kvm/kvm_tests.cfg.sample > +++ b/client/tests/kvm/kvm_tests.cfg.sample > @@ -786,13 +786,22 @@ variants: > only default > image_format =3D raw > =20 > - > variants: > - @smallpages: > - hugepages: > pre_command =3D "/usr/bin/python scripts/hugepage.py /mnt/kv= m_hugepage" > extra_params +=3D " -mem-path /mnt/kvm_hugepage" > =20 > +variants: > + - @no_passthrough: > + pass_through =3D no > + - nic_passthrough: > + pass_through =3D pf > + passthrough_devs =3D eth1 > + - vfs_passthrough: > + pass_through =3D vf > + max_vfs =3D 7 > + vfs_count =3D 7 > =20 > variants: > - @basic: > diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_uti= ls.py > index 53b664a..0e3398c 100644 > --- a/client/tests/kvm/kvm_utils.py > +++ b/client/tests/kvm/kvm_utils.py > @@ -788,3 +788,281 @@ def md5sum_file(filename, size=3DNone): > size -=3D len(data) > f.close() > return o.hexdigest() > + > + > +def get_full_id(pci_id): > + """ > + Get full PCI ID of pci_id. > + """ > + cmd =3D "lspci -D | awk '/%s/ {print $1}'" % pci_id > + status, full_id =3D commands.getstatusoutput(cmd) > + if status !=3D 0: > + return None > + return full_id > + > + > +def get_vendor_id(pci_id): > + """ > + Check out the device vendor ID according to PCI ID. > + """ > + cmd =3D "lspci -n | awk '/%s/ {print $3}'" % pci_id > + return re.sub(":", " ", commands.getoutput(cmd)) > + > + > +def release_dev(pci_id, pci_dict): > + """ > + Release a single PCI device. > + > + @param pci_id: PCI ID of a given PCI device > + @param pci_dict: Dictionary with information about PCI devices > + """ > + base_dir =3D "/sys/bus/pci" > + full_id =3D get_full_id(pci_id) > + vendor_id =3D get_vendor_id(pci_id) > + drv_path =3D os.path.join(base_dir, "devices/%s/driver" % full_i= d) > + if 'pci-stub' in os.readlink(drv_path): > + cmd =3D "echo '%s' > %s/new_id" % (vendor_id, drv_path) > + if os.system(cmd): > + return False > + > + stub_path =3D os.path.join(base_dir, "drivers/pci-stub") > + cmd =3D "echo '%s' > %s/unbind" % (full_id, stub_path) > + if os.system(cmd): > + return False > + > + prev_driver =3D pci_dict[pci_id] > + cmd =3D "echo '%s' > %s/bind" % (full_id, prev_driver) > + if os.system(cmd): > + return False > + return True > + > + > +def release_pci_devs(pci_dict): > + """ > + Release all PCI devices assigned to host. > + > + @param pci_dict: Dictionary with information about PCI devices > + """ > + for pci_id in pci_dict: > + if not release_dev(pci_id, pci_dict): > + logging.error("Failed to release device [%s] to host" % = pci_id) > + else: > + logging.info("Release device [%s] successfully" % pci_id= ) > + > + > +class PassThrough(object): > + """ > + Request passthroughable devices on host. It will check whether t= o request > + PF (physical NIC cards) or VF (Virtual Functions). > + """ > + def __init__(self, type=3D"nic_vf", max_vfs=3DNone, names=3DNone= ): > + """ > + Initialize parameter 'type' which could be: > + nic_vf: Virtual Functions > + nic_pf: Physical NIC card > + mixed: Both includes VFs and PFs > + > + If pass through Physical NIC cards, we need to specify which= devices > + to be assigned, e.g. 'eth1 eth2'. > + > + If pass through Virtual Functions, we need to specify how ma= ny vfs > + are going to be assigned, e.g. passthrough_count =3D 8 and m= ax_vfs in > + config file. > + > + @param type: Pass through device's type > + @param max_vfs: parameter of module 'igb' > + @param names: Physical NIC cards' names, e.g.'eth1 eth2 ...' > + """ > + self.type =3D type > + if max_vfs: > + self.max_vfs =3D int(max_vfs) > + if names: > + self.name_list =3D names.split() > + > + > + def sr_iov_setup(self): > + """ > + Setup SR-IOV environment, check if module 'igb' is loaded wi= th > + parameter 'max_vfs'. > + """ > + re_probe =3D False > + # Check whether the module 'igb' is loaded > + s, o =3D commands.getstatusoutput('lsmod | grep igb') > + if s: I would take extra consideration that here could also happen timeout and return status is None problem. Could change to: if s !=3D 0: > + re_probe =3D True > + elif not self.chk_vfs_count(): > + os.system("modprobe -r igb") > + re_probe =3D True > + > + # Re-probe module 'igb' > + if re_probe: > + cmd =3D "modprobe igb max_vfs=3D%d" % self.max_vfs > + s, o =3D commands.getstatusoutput(cmd) > + if s: =3D=3D> if s !=3D 0: > + return False > + if not self.chk_vfs_count(): > + return False > + return True > + > + > + def get_vfs_count(self): > + """ > + Get Vfs count number according to 'max_vfs' > + """ > + cmd =3D "lspci | grep 'Virtual Function' | wc -l" > + return int(commands.getoutput(cmd)) > + > + > + def chk_vfs_count(self): > + """ > + Check VFs count number according to parameter 'max_vfs'. > + """ > + if self.get_vfs_count() !=3D (self.max_vfs * 2): > + return False > + return True > + > + > + def _get_pci_id(self, name, search_str): > + """ > + Get the device's PCI ID according to name. > + """ > + cmd =3D "ethtool -i %s | awk '/bus-info/ {print $2}'" % name > + s, pci_id =3D commands.getstatusoutput(cmd) > + if not (s or "Cannot get driver information" in pci_id): > + return pci_id[5:] > + > + cmd =3D "lspci | awk '/%s/ {print $1}'" % search_str > + pci_ids =3D [id for id in commands.getoutput(cmd).splitlines= ()] > + nic_id =3D int(re.search('[0-9]+', name).group(0)) > + if (len(pci_ids) - 1) < nic_id: > + return None > + return pci_ids[nic_id] > + > + > + def _write_to_file(self, content, file_path): > + """ > + Write some content to a file. > + > + @param content: Content to be written. > + @param file_path: Path for the file. > + """ > + success =3D False > + try: > + file =3D open(file_path, 'w') > + except IOError: > + return success > + > + try: > + file.write(content) > + success =3D True > + except IOError: > + pass > + finally: > + file.close() > + > + return success > + > + > + def is_binded_to_stub(self, full_id): > + """ > + Verify whether the device with full_id is already binded to = pci-stub. > + """ > + base_dir =3D "/sys/bus/pci" > + stub_path =3D os.path.join(base_dir, "drivers/pci-stub") > + if os.path.exists(os.path.join(stub_path, full_id)): > + return True > + return False > + > + > + def request_devs(self, count, pt_file, setup=3DTrue): > + """ > + Implement setup process: unbind the PCI device and then bind= it > + to pci-stub driver. > + > + @return: a list of successfully requested devices' PCI IDs. > + """ > + base_dir =3D "/sys/bus/pci" > + stub_path =3D os.path.join(base_dir, "drivers/pci-stub") > + > + self.pci_ids =3D self.catch_devs(int(count)) > + logging.debug("Catch_devs has found pci_ids: %s" % self.pci_= ids) > + > + if setup: > + requested_pci_ids =3D [] > + logging.debug("Going to setup devices: %s" % self.pci_id= s) > + dev_prev_drivers =3D {} > + > + # Setup all devices specified for passthrough to guest > + for pci_id in self.pci_ids: > + full_id =3D get_full_id(pci_id) > + if not full_id: > + continue > + drv_path =3D os.path.join(base_dir, "devices/%s/driv= er" % full_id) > + dev_prev_driver=3D os.path.realpath(os.path.join(drv= _path, > + os.readlink(drv_pa= th))) > + dev_prev_drivers[pci_id] =3D dev_prev_driver > + > + # Judge whether the device driver has been binded to= stub > + if not self.is_binded_to_stub(full_id): > + logging.debug("Setting up device: %s" % full_id) > + stub_new_id =3D os.path.join(stub_path, 'new_id'= ) > + unbind_dev =3D os.path.join(drv_path, 'unbind') > + stub_bind =3D os.path.join(stub_path, 'bind') > + > + vendor_id =3D get_vendor_id(pci_id) > + if not self._write_to_file(vendor_id, stub_new_i= d): > + continue > + if not self._write_to_file(full_id, unbind_dev): > + continue > + if not self._write_to_file(full_id, stub_bind): > + continue > + if not self.is_binded_to_stub(full_id): > + logging.error("Request device failed: %s" % = pci_id) > + continue > + else: > + logging.debug("Device [%s] already binded to stu= b" % pci_id) > + requested_pci_ids.append(pci_id) > + cPickle.dump(dev_prev_drivers, pt_file) > + self.pci_ids =3D requested_pci_ids > + return self.pci_ids > + > + > + def catch_vf_devs(self): > + """ > + Catch all VFs PCI IDs and return as a list. > + """ > + if not self.sr_iov_setup(): > + return [] > + > + cmd =3D "lspci | awk '/Virtual Function/ {print $1}'" > + return commands.getoutput(cmd).split() > + > + > + def catch_pf_devs(self): > + """ > + Catch all PFs PCI IDs. > + """ > + pf_ids =3D [] > + for name in self.name_list: > + pf_id =3D self._get_pci_id(name, "Ethernet") > + if not pf_id: > + continue > + pf_ids.append(pf_id) > + return pf_ids > + > + > + def catch_devs(self, count): > + """ > + Check out all devices' PCI IDs according to their name. > + > + @param count: count number of PCI devices needed for pass th= rough > + @return: a list of all devices' PCI IDs > + """ > + if self.type =3D=3D "nic_vf": > + vf_ids =3D self.catch_vf_devs() > + elif self.type =3D=3D "nic_pf": > + vf_ids =3D self.catch_pf_devs() > + elif self.type =3D=3D "mixed": > + vf_ids =3D self.catch_vf_devs() > + vf_ids.extend(self.catch_pf_devs()) > + return vf_ids[0:count] > diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py > index 82f1eb4..a0b01fe 100755 > --- a/client/tests/kvm/kvm_vm.py > +++ b/client/tests/kvm/kvm_vm.py > @@ -298,6 +298,30 @@ class VM: > elif params.get("uuid"): > qemu_cmd +=3D " -uuid %s" % params.get("uuid") > =20 > + # Check whether pci_passthrough is turned on > + if not params.get("pass_through") =3D=3D "no": > + pt_type =3D params.get("pass_through") > + > + # Virtual Functions pass through > + if params.get("pass_through") =3D=3D "nic_vf": > + pt =3D kvm_utils.PassThrough(type=3Dpt_type, > + max_vfs=3Dparams.get("max= _vfs")) > + # Physical NIC card pass through > + elif params.get("pass_through") =3D=3D "nic_pf": > + pt =3D kvm_utils.PassThrough(type=3Dpt_type, > + names=3Dparams.get("passt= hrough_devs")) > + elif params.get("pass_through") =3D=3D "mixed": > + pt =3D kvm_utils.PassThrough(type=3Dpt_type, > + max_vfs=3Dparams.get("max= _vfs"), > + names=3Dparams.get("passt= hrough_devs")) > + pt_count =3D params.get("passthrough_count") > + pt_db_file =3D open("/tmp/passthrough_db", "w") > + self.pt_pci_ids =3D pt.request_devs(pt_count, pt_file=3D= pt_db_file, > + setup=3DFalse) > + pt_db_file.close() > + for pci_id in self.pt_pci_ids: > + qemu_cmd +=3D " -pcidevice host=3D%s" % pci_id > + > return qemu_cmd > =20 > =20 > @@ -380,6 +404,37 @@ class VM: > self.uuid =3D f.read().strip() > f.close() > =20 > + # Request specified PCI devices for assignment > + if not params.get("pass_through") =3D=3D "no": > + pt_type =3D params.get("pass_through") > + > + # Virtual Functions pass through > + if params.get("pass_through") =3D=3D "nic_vf": > + logging.debug("Going to assign VFs to guest") > + pt =3D kvm_utils.PassThrough(type=3Dpt_type, > + max_vfs =3D params.ge= t("max_vfs")) > + # Physical NIC card pass through > + elif params.get("pass_through") =3D=3D "nic_pf": > + logging.debug("Going to assign physical NICs to = guest") > + pt =3D kvm_utils.PassThrough(type=3Dpt_type, > + names=3Dparams.get("passt= hrough_devs")) > + elif params.get("pass_through") =3D=3D "mixed": > + logging.debug("Assigned NICs may contain both VF= s and PFs") > + pt =3D kvm_utils.PassThrough(type =3D pt_type, > + max_vfs =3D params.ge= t("max_vfs"), > + names =3D params.get("passt= hrough_devs")) > + pt_count =3D params.get("passthrough_count") > + pt_db_file =3D open('/tmp/passthrough_db', 'w') > + self.pt_pci_ids =3D pt.request_devs(pt_count, pt_fil= e=3Dpt_db_file, > + setup=3DTrue) > + pt_db_file.close() > + if not self.pt_pci_ids: > + logging.error("Request specified PCI device(s) f= ailed;" > + " pt_pci_ids are: %s", self.pt_pci= _ids) > + return False > + logging.debug("Requested host PCI device(s): %s", > + self.pt_pci_ids) > + > # Make qemu command > qemu_command =3D self.make_qemu_command() > =20 > @@ -504,6 +559,19 @@ class VM: > return (0, data) > =20 > =20 > + def _release_pt_devs(self): > + """ > + Release the assigned passthrough PCI devices or VFs to host. > + """ > + try: > + db_file =3D open('/tmp/passthrough_db', 'r') > + pt_db =3D cPickle.load(db_file) > + kvm_utils.release_pci_devs(pt_db) > + db_file.close() > + except: > + return > + > + > def destroy(self, gracefully=3DTrue): > """ > Destroy the VM. > @@ -520,6 +588,7 @@ class VM: > # Is it already dead? > if self.is_dead(): > logging.debug("VM is already down") > + self._release_pt_devs() > return > =20 > logging.debug("Destroying VM with PID %d..." % > @@ -540,6 +609,7 @@ class VM: > return > finally: > session.close() > + self._release_pt_devs() > =20 > # Try to destroy with a monitor command > logging.debug("Trying to kill VM with monitor command...= ") > @@ -549,6 +619,7 @@ class VM: > # Wait for the VM to be really dead > if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): > logging.debug("VM is down") > + self._release_pt_devs() > return > =20 > # If the VM isn't dead yet... > @@ -558,6 +629,7 @@ class VM: > # Wait for the VM to be really dead > if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5): > logging.debug("VM is down") > + self._release_pt_devs() > return > =20 > logging.error("Process %s is a zombie!" % self.process.g= et_pid()) > --=20 > 1.6.2.5