Skip to contentSkip to conversation

Adding AutoservInstallError to common_lib.error

PATCH In review Lucas Meneghel Rodrigues 2 participants 4 messages Nov 17, 2009 KVM

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 Adding AutoservInstallError to common_lib.error

server/autotest.py was referencing it but it didn't exist. This patch creates the error class.

---
 client/common_lib/error.py |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)
client/common_lib/error.py +5 -0
index 08401ce..9e0002d 100644
@@ -185,6 +185,11 @@ class AutoservError(Exception):
     pass
 
 
+class AutoservInstallError(Exception):
+    """Autoserv failed in installing autotest on a client machine"""
+    pass
+
+
 class AutoservSSHTimeout(AutoservError):
     """SSH experienced a connection timeout"""
     pass
Signature · 1 line
1.6.2.5
#1 lore raw

PATCH 2/3 Make a standalone client to be able to use global_config.ini

In order to make it possible the autotest client to use the global_config.ini configuration files:

* Modified global_config.py to support verifying if the configuration file is under autotest's root directory, or the client directory
* Extended the autotest run method to copy over the configuration files to the client.

Notes for revisors (changes from previous patchset and other stuff):

* Now only the config values under [CLIENT] will be effectively present on the config file that will be copied to the client. That config file is generated using new methods added to global_config.
* No shadow_config or database passwords will ever be copied to the client. Yay! :)
* common_lib.host_protections was changed accordingly.
* The new config values needed for the patch that follows are contained on this intermediate patch, so when applied this patch doesn't break autoserv.

---
 client/common_lib/global_config.py    |   61 +++++++++++++++++++++++++++++----
 client/common_lib/host_protections.py |   26 ++++++++------
 global_config.ini                     |    3 ++
 server/autotest.py                    |   33 ++++++++++++++----
 4 files changed, 98 insertions(+), 25 deletions(-)
client/common_lib/global_config.py +54 -7
index 2bbeca0..ffa8993 100644
@@ -8,12 +8,6 @@ __author__ = '[email protected] (Travis Miller)'
 import os, sys, ConfigParser
 from autotest_lib.client.common_lib import error
 
-dirname = os.path.dirname(sys.modules[__name__].__file__)
-DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(dirname,
-                                                "../../global_config.ini"))
-DEFAULT_SHADOW_FILE = os.path.abspath(os.path.join(dirname,
-                                                "../../shadow_config.ini"))
-
 
 class ConfigError(error.AutotestError):
     pass
@@ -23,12 +17,50 @@ class ConfigValueError(ConfigError):
     pass
 
 
+
+common_lib_dir = os.path.dirname(sys.modules[__name__].__file__)
+client_dir = os.path.dirname(common_lib_dir)
+root_dir = os.path.dirname(client_dir)
+
+# Check if the config files are at autotest's root dir
+# This will happen if client is executing inside a full autotest tree, or if
+# other entry points are being executed
+global_config_path_root = os.path.join(root_dir, 'global_config.ini')
+shadow_config_path_root = os.path.join(root_dir, 'shadow_config.ini')
+config_in_root = (os.path.exists(global_config_path_root) and
+                  os.path.exists(shadow_config_path_root))
+
+# Check if the config files are at autotest's client dir
+# This will happen if a client stand alone execution is happening
+global_config_path_client = os.path.join(client_dir, 'global_config.ini')
+config_in_client = os.path.exists(global_config_path_client)
+
+if config_in_root:
+    DEFAULT_CONFIG_FILE = global_config_path_root
+    DEFAULT_SHADOW_FILE = shadow_config_path_root
+    RUNNING_STAND_ALONE_CLIENT = False
+elif config_in_client:
+    DEFAULT_CONFIG_FILE = global_config_path_client
+    DEFAULT_SHADOW_FILE = None
+    RUNNING_STAND_ALONE_CLIENT = True
+else:
+    raise ConfigError("Could not find configuration files "
+                      "needed for this program to function. Please refer to "
+                      "http://autotest.kernel.org/wiki/GlobalConfig "
+                      "for more info.")
+
+
 class global_config(object):
     _NO_DEFAULT_SPECIFIED = object()
 
     config = None
     config_file = DEFAULT_CONFIG_FILE
     shadow_file = DEFAULT_SHADOW_FILE
+    running_stand_alone_client = RUNNING_STAND_ALONE_CLIENT
+
+
+    def check_stand_alone_client_run(self):
+        return self.running_stand_alone_client
 
 
     def set_config_files(self, config_file=DEFAULT_CONFIG_FILE,
@@ -47,6 +79,21 @@ class global_config(object):
             return default
 
 
+    def get_session_values(self, section):
+        """
+        Return a config parser object containing a single session of the
+        global configuration, that can be later written to a file object.
+
+        @param session: Session we want to turn into a config parser object.
+        @return: ConfigParser() object containing all the contents of session.
+        """
+        cfgparser = ConfigParser.ConfigParser()
+        cfgparser.add_section(section)
+        for option, value in self.config.items(section):
+            cfgparser.set(section, option, value)
+        return cfgparser
+
+
     def get_config_value(self, section, key, type=str,
                          default=_NO_DEFAULT_SPECIFIED, allow_blank=False):
         self._ensure_config_parsed()
@@ -106,7 +153,7 @@ class global_config(object):
         # now also read the shadow file if there is one
         # this will overwrite anything that is found in the
         # other config
-        if os.path.exists(self.shadow_file):
+        if self.shadow_file and os.path.exists(self.shadow_file):
             shadow_config = ConfigParser.ConfigParser()
             shadow_config.read(self.shadow_file)
             # now we merge shadow into global
client/common_lib/host_protections.py +15 -11
index 9457114..7c9e6a0 100644
@@ -22,19 +22,23 @@ Protection = enum.Enum('No protection',          # Repair can do anything to
                                                  # this host
                        )
 
+running_client = global_config.global_config.check_stand_alone_client_run()
+
 try:
     _bad_value = object()
-    default = Protection.get_value(
-        global_config.global_config.get_config_value(
-            'HOSTS', 'default_protection', default=_bad_value))
-    if default == _bad_value:
-        raise global_config.ConfigError(
-            'No HOSTS.default_protection defined in global_config.ini')
+    default_protection = global_config.global_config.get_config_value(
+                            'HOSTS', 'default_protection', default=_bad_value)
+    if default_protection == _bad_value:
+        if running_client:
+            logging.debug('Client stand alone run detected. '
+                          'host_protection.default will not be set.')
+        else:
+            raise global_config.ConfigError(
+                'No HOSTS.default_protection defined in global_config.ini')
+    else:
+        default = Protection.get_value(default_protection)
+
 except global_config.ConfigError:
-    # can happen if no global_config.ini exists at all, but this can actually
-    # happen in reasonable cases (e.g. on a standalone clinet) where it's
-    # safe to ignore
-    logging.debug('No global_config.ini exists so host_protection.default '
-                  'will not be defined')
+    raise global_config.ConfigError('No global_config.ini exists, aborting')
 
 choices = Protection.choices()
global_config.ini +3 -0
index 1cd3b4f..ac6baab 100644
@@ -28,6 +28,9 @@ parse_failed_repair_default: 0
 # Autotest potential install paths
 client_autodir_paths: /usr/local/autotest,/home/autotest
 
+[CLIENT]
+drop_caches: True
+drop_caches_between_iterations: True
 
 [SERVER]
 hostname: autotest
server/autotest.py +26 -7
index 3d307a9..77cef54 100644
@@ -391,9 +391,15 @@ class BaseAutotest(installable_object.InstallableObject):
         cfile += open(tmppath).read()
         open(tmppath, "w").write(cfile)
 
+        # Create and copy configuration file based on the state of the
+        # client configuration
+        client_config_file = self._create_aux_file(host.job)
+        host.send_file(client_config_file, atrun.config_file)
+        os.remove(client_config_file)
+
         # Create and copy state file to remote_control_file + '.state'
         sysinfo_state = {"__sysinfo": host.job.sysinfo.serialize()}
-        state_file = self._create_state_file(host.job, sysinfo_state)
+        state_file = self._create_aux_file(host.job, sysinfo_state)
         host.send_file(state_file, atrun.remote_control_file + '.state')
         os.remove(state_file)
 
@@ -407,13 +413,25 @@ class BaseAutotest(installable_object.InstallableObject):
                 client_disconnect_timeout=client_disconnect_timeout)
 
 
-    def _create_state_file(self, job, state_dict):
-        """ Create a state file from a dictionary. Returns the path of the
-        state file. """
+    def _create_aux_file(self, job, state_dict=None):
+        """
+        If state_dict is provided, create a state file from this dictionary. If
+        none provided, it creates a client config file based on the global
+        config session CLIENT.
+
+        @param job: Autotest job instance.
+        @param state_dict: Dictionary containing the job's state.
+        @return: The path of the config file.
+        """
         fd, path = tempfile.mkstemp(dir=job.tmpdir)
-        state_file = os.fdopen(fd, "w")
-        pickle.dump(state_dict, state_file)
-        state_file.close()
+        aux_file = os.fdopen(fd, "w")
+        if state_dict:
+            pickle.dump(state_dict, aux_file)
+        else:
+            client_config = global_config.global_config.get_session_values(
+                                                                       "CLIENT")
+            client_config.write(aux_file)
+        aux_file.close()
         return path
 
 
@@ -459,6 +477,7 @@ class _Run(object):
             control += '.' + tag
         self.manual_control_file = control
         self.remote_control_file = control + '.autoserv'
+        self.config_file = os.path.join(self.autodir, 'global_config.ini')
 
 
     def verify_machine(self):
Signature · 1 line
1.6.2.5
#2 lore raw

PATCH 3/3 Make client behavior configurable

Right now autotest will drop caches between:

* Test executions
* Same test iterations

This change turns those into configurable options on global_config.ini. The default values were added on the previous patch.

2nd attempt: Now global config support for stand alone client runs was added. Everything should work now.

---
 client/bin/autotest |    7 ++++++-
 client/bin/job.py   |   11 ++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)
client/bin/autotest +6 -1
index 282ae9a..285be4e 100755
@@ -6,6 +6,7 @@ import os, sys, shutil
 import common
 from optparse import OptionParser
 from autotest_lib.client.bin import job
+from autotest_lib.client.common_lib import global_config
 
 
 # Use the name of the binary to find the real installation directory
@@ -58,5 +59,9 @@ options, args = parser.parse_args()
 if len(args) != 1:
     usage()
 
+drop_caches = global_config.global_config.get_config_value('CLIENT',
+                                                           'drop_caches',
+                                                           type=bool)
+
 # JOB: run the specified job control file.
-job.runjob(os.path.realpath(args[0]), options)
+job.runjob(os.path.realpath(args[0]), drop_caches, options)
client/bin/job.py +8 -3
index d36784f..2a75adb 100755
@@ -16,6 +16,8 @@ from autotest_lib.client.bin import partition as partition_lib
 from autotest_lib.client.common_lib import base_job
 from autotest_lib.client.common_lib import error, barrier, log, logging_manager
 from autotest_lib.client.common_lib import base_packages, packages
+from autotest_lib.client.common_lib import global_config
+
 
 LAST_BOOT_TAG = object()
 NO_DEFAULT = object()
@@ -228,7 +230,10 @@ class base_client_job(base_job.base_job):
         """
         Perform the drop caches initialization.
         """
-        self.drop_caches_between_iterations = True
+        self.drop_caches_between_iterations = (
+                       global_config.global_config.get_config_value('CLIENT',
+                                            'drop_caches_between_iterations',
+                                            type=bool))
         self.drop_caches = drop_caches
         if self.drop_caches:
             logging.debug("Dropping caches")
@@ -1311,7 +1316,7 @@ class disk_usage_monitor:
         return decorator
 
 
-def runjob(control, options):
+def runjob(control, drop_caches, options):
     """
     Run a job using the given control file.
 
@@ -1339,7 +1344,7 @@ def runjob(control, options):
         if options.cont and not os.path.exists(state):
             raise error.JobComplete("all done")
 
-        myjob = job(control, options)
+        myjob = job(control=control, drop_caches=drop_caches, options=options)
 
         # Load in the users control file, may do any one of:
         #  1) execute in toto
Signature · 1 line
1.6.2.5
#3 lore raw

Re: [PATCH 2/3] Make a standalone client to be able to use global_config.ini

In reply to Lucas Meneghel Rodrigues
Quoted 91 lines · Lucas Meneghel Rodrigues

In order to make it possible the autotest client to use the global_config.ini configuration files:

 * Modified global_config.py to support verifying if the configuration file is under autotest's root directory, or the client directory
 * Extended the autotest run method to copy over the configuration files to the client.

Notes for revisors (changes from previous patchset and other stuff):

 * Now only the config values under [CLIENT] will be effectively present on the config file that will be copied to the client. That config file is generated using new methods added to global_config.
 * No shadow_config or database passwords will ever be copied to the client. Yay! :)
 * common_lib.host_protections was changed accordingly.
 * The new config values needed for the patch that follows are contained on this intermediate patch, so when applied this patch doesn't break autoserv.

---
 client/common_lib/global_config.py    |   61 +++++++++++++++++++++++++++++---- client/common_lib/host_protections.py |   26 ++++++++------ global_config.ini                     |    3 ++ server/autotest.py                    |   33 ++++++++++++++----

 4 files changed, 98 insertions(+), 25 deletions(-)
client/common_lib/global_config.py +0 -0
index 2bbeca0..ffa8993 100644
@@ -8,12 +8,6 @@ __author__ = '[email protected] (Travis Miller)'

 import os, sys, ConfigParser
 from autotest_lib.client.common_lib import error

client/common_lib/global_config.py +0 -6
-dirname = os.path.dirname(sys.modules[__name__].__file__)
-DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(dirname,
-                                                "../../global_config.ini"))
-DEFAULT_SHADOW_FILE = os.path.abspath(os.path.join(dirname,
-                                                "../../shadow_config.ini"))
-

 class ConfigError(error.AutotestError):
    pass

client/common_lib/global_config.py +0 -0
@@ -23,12 +17,50 @@ class ConfigValueError(ConfigError):

    pass

client/common_lib/global_config.py +33 -0
+
+common_lib_dir = os.path.dirname(sys.modules[__name__].__file__)
+client_dir = os.path.dirname(common_lib_dir)
+root_dir = os.path.dirname(client_dir)
+
+# Check if the config files are at autotest's root dir
+# This will happen if client is executing inside a full autotest tree, or if
+# other entry points are being executed
+global_config_path_root = os.path.join(root_dir, 'global_config.ini')
+shadow_config_path_root = os.path.join(root_dir, 'shadow_config.ini')
+config_in_root = (os.path.exists(global_config_path_root) and
+                  os.path.exists(shadow_config_path_root))
+
+# Check if the config files are at autotest's client dir
+# This will happen if a client stand alone execution is happening
+global_config_path_client = os.path.join(client_dir, 'global_config.ini')
+config_in_client = os.path.exists(global_config_path_client)
+
+if config_in_root:
+    DEFAULT_CONFIG_FILE = global_config_path_root
+    DEFAULT_SHADOW_FILE = shadow_config_path_root
+    RUNNING_STAND_ALONE_CLIENT = False
+elif config_in_client:
+    DEFAULT_CONFIG_FILE = global_config_path_client
+    DEFAULT_SHADOW_FILE = None
+    RUNNING_STAND_ALONE_CLIENT = True
+else:
+    raise ConfigError("Could not find configuration files "
+                      "needed for this program to function. Please refer to "
+                      "http://autotest.kernel.org/wiki/GlobalConfig "
+                      "for more info.")
+
+

 class global_config(object):
    _NO_DEFAULT_SPECIFIED = object()

    config = None
    config_file = DEFAULT_CONFIG_FILE
    shadow_file = DEFAULT_SHADOW_FILE
+    running_stand_alone_client = RUNNING_STAND_ALONE_CLIENT

client/common_lib/global_config.py class ConfigValueError(ConfigError): · line 63

Quoted 4 lines
client/common_lib/global_config.py +4 -0
+
+
+    def check_stand_alone_client_run(self):
+        return self.running_stand_alone_client

I don't know if it's really necessary to have a method wrapped around this, instead of just accessing the attribute directly. I've actually been trying to re-factor existing code that does this to get rid of getters.

Quoted 12 lines

    def set_config_files(self, config_file=DEFAULT_CONFIG_FILE,

client/common_lib/global_config.py +0 -0
@@ -47,6 +79,21 @@ class global_config(object):

            return default

+    def get_session_values(self, section):
+        """
+        Return a config parser object containing a single session of the
+        global configuration, that can be later written to a file object.

client/common_lib/global_config.py +1 -0
+

client/common_lib/global_config.py class global_config(object): · line 96

Quoted 10 lines
client/common_lib/global_config.py +10 -0
+        @param session: Session we want to turn into a config parser object.
+        @return: ConfigParser() object containing all the contents of session.
+        """
+        cfgparser = ConfigParser.ConfigParser()
+        cfgparser.add_section(section)
+        for option, value in self.config.items(section):
+            cfgparser.set(section, option, value)
+        return cfgparser
+
+

The method name, parameter and the docstring aren't consistent (session vs section). For the record, I favor section over session.

Quoted 88 lines

    def get_config_value(self, section, key, type=str, default=_NO_DEFAULT_SPECIFIED, allow_blank=False):
        self._ensure_config_parsed()

client/common_lib/global_config.py +1 -1
@@ -106,7 +153,7 @@ class global_config(object):
         # now also read the shadow file if there is one
         # this will overwrite anything that is found in the
         # other config
-        if os.path.exists(self.shadow_file):
+        if self.shadow_file and os.path.exists(self.shadow_file):
             shadow_config = ConfigParser.ConfigParser()
             shadow_config.read(self.shadow_file)
             # now we merge shadow into global
client/common_lib/host_protections.py +2 -0
index 9457114..7c9e6a0 100644
@@ -22,19 +22,23 @@ Protection = enum.Enum('No protection',          # Repair can do anything to
                                                  # this host
                        )
 
+running_client = global_config.global_config.check_stand_alone_client_run()
+

 try:
    _bad_value = object()
-    default = Protection.get_value(
-        global_config.global_config.get_config_value(
-            'HOSTS', 'default_protection', default=_bad_value))
-    if default == _bad_value:
-        raise global_config.ConfigError(
-            'No HOSTS.default_protection defined in global_config.ini')
+    default_protection = global_config.global_config.get_config_value(
+                            'HOSTS', 'default_protection', default=_bad_value)
+    if default_protection == _bad_value:
+        if running_client:
+            logging.debug('Client stand alone run detected. '
+                          'host_protection.default will not be set.')
+        else:
+            raise global_config.ConfigError(
+                'No HOSTS.default_protection defined in global_config.ini')
+    else:
+        default = Protection.get_value(default_protection)
+
 except global_config.ConfigError:
-    # can happen if no global_config.ini exists at all, but this can actually
-    # happen in reasonable cases (e.g. on a standalone clinet) where it's
-    # safe to ignore
-    logging.debug('No global_config.ini exists so host_protection.default '
-                  'will not be defined')
+    raise global_config.ConfigError('No global_config.ini exists, aborting')

 choices = Protection.choices()

global_config.ini +0 -0
index 1cd3b4f..ac6baab 100644
@@ -28,6 +28,9 @@ parse_failed_repair_default: 0
  # Autotest potential install paths

 client_autodir_paths: /usr/local/autotest,/home/autotest

+[CLIENT]

global_config.ini +2 -0
+drop_caches: True
+drop_caches_between_iterations: True

 [SERVER]
 hostname: autotest

server/autotest.py +7 -1
index 3d307a9..77cef54 100644
@@ -391,9 +391,15 @@ class BaseAutotest(installable_object.InstallableObject):
         cfile += open(tmppath).read()
         open(tmppath, "w").write(cfile)
 
+        # Create and copy configuration file based on the state of the
+        # client configuration
+        client_config_file = self._create_aux_file(host.job)
+        host.send_file(client_config_file, atrun.config_file)
+        os.remove(client_config_file)
+
         # Create and copy state file to remote_control_file + '.state'
         sysinfo_state = {"__sysinfo": host.job.sysinfo.serialize()}
-        state_file = self._create_state_file(host.job, sysinfo_state)
+        state_file = self._create_aux_file(host.job, sysinfo_state)
         host.send_file(state_file, atrun.remote_control_file + '.state')
         os.remove(state_file)
 
@@ -407,13 +413,25 @@ class BaseAutotest(installable_object.InstallableObject):
                 client_disconnect_timeout=client_disconnect_timeout)

server/autotest.py class BaseAutotest(installable_object.InstallableObject): · line 420

Quoted 10 lines
server/autotest.py +5 -3
 
 
-    def _create_state_file(self, job, state_dict):
-        """ Create a state file from a dictionary. Returns the path of the
-        state file. """
+    def _create_aux_file(self, job, state_dict=None):
+        """
+        If state_dict is provided, create a state file from this dictionary. If
+        none provided, it creates a client config file based on the global
+        config session CLIENT.

This seems kind of...magical somehow. I understand why you're doing it this way to reuse the file creation code, but I'd actually take a different approach using higher order functions. Instead of having it take a parameter and then use an if-else to decide what kind of initialization to do, pass in a function. Then make create_state_file and create_config_file wrappers that call it with the appropriate function (either doing the pickle.dump, or the client config creation).

I know that seems kind of convoluted, but I really don't like "generic" methods that do very different things depending on which parameters you pass in. If we want a generic "create a temp file and write some stuff into it" function, we should make a truly generic one.

Quoted 32 lines
server/autotest.py +13 -3
+
+        @param job: Autotest job instance.
+        @param state_dict: Dictionary containing the job's state.
+        @return: The path of the config file.
+        """
         fd, path = tempfile.mkstemp(dir=job.tmpdir)
-        state_file = os.fdopen(fd, "w")
-        pickle.dump(state_dict, state_file)
-        state_file.close()
+        aux_file = os.fdopen(fd, "w")
+        if state_dict:
+            pickle.dump(state_dict, aux_file)
+        else:
+            client_config = global_config.global_config.get_session_values(
+                                                                       "CLIENT")
+            client_config.write(aux_file)
+        aux_file.close()

        return path

server/autotest.py +0 -0
@@ -459,6 +477,7 @@ class _Run(object):

            control += '.' + tag
        self.manual_control_file = control self.remote_control_file = control + '.autoserv'
+        self.config_file = os.path.join(self.autodir, 'global_config.ini')

    def verify_machine(self):
--
1.6.2.5