feat: persistence of dynamic tuning changes

Currently, any changes made to the tuning via the `instance_*` dbus calls
are lost when tuning is stopped by the service, or when the TuneD service
itself is stopped/restarted, or when the service crashes.

This commit:

* implements a sync of Plugin Instances and Profile Units that is currently missing.
  This way, dynamic instances and device assignments are persistent across stop/start
  dbus calls to TuneD.
* calculates a hash of the current profile after loading it from disk
  (after processing all includes, so we have a "flat" representation)
* creates snapshots of the current profile whenever instances or assigned
  devices change. the snapshot includes the hash of the profile as
  it was initially loaded. for each instance it stores the devices that
  are currently attached.
* restores a snapshot found at startup, if the hashes match (i.e. there
  have been no profile switches and no changes to the profile or any of
  its includes on disk)

  snapshots are restored in case of
  - daemon restarts (systemctl restart/stop/start)
  - daemon crashes
  snapshots are NOT restored in case of
  - reboots (snapshots are stored in /var/run)
  - profile changes (snapshots are explicitly deleted when switching profiles,
    even when "switching" to the same/current profile)

Signed-off-by: Adriaan Schmidt <[email protected]>

Author: adriaan42

tuned/consts.py

@@ -19,6 +19,7 @@
 DEFAULT_STORAGE_FILE = "/run/tuned/save.pickle"
 USER_PROFILES_DIR = "/etc/tuned/profiles"
 SYSTEM_PROFILES_DIR = "/usr/lib/tuned/profiles"
+PROFILE_SNAPSHOT_FILE = "/run/tuned/profile-snapshot.conf"
 PERSISTENT_STORAGE_DIR = "/var/lib/tuned"
 PLUGIN_MAIN_UNIT_NAME = "main"
 # Magic section header because ConfigParser does not support "headerless" config

tuned/daemon/controller.py

@@ -407,6 +407,7 @@ def instance_acquire_devices(self, devices, instance_name, caller = None):
 			rets = "Ignoring devices not handled by any instance '%s'." % str(devs)
 			log.info(rets)
 			return (False, rets)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bsa(ss))")
@@ -472,6 +473,8 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 		"""
 		if caller == "":
 			return (False, "Unauthorized")
+		plugin_name = str(plugin_name)
+		instance_name = str(instance_name)
 		if not self._cmd.is_valid_name(plugin_name):
 			return (False, "Invalid plugin_name")
 		if not self._cmd.is_valid_name(instance_name):
@@ -519,6 +522,7 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 					other_instance.name, instance.name))
 				plugin._remove_devices_nocheck(other_instance, devs_moving)
 				plugin._add_devices_nocheck(instance, devs_moving)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bs)")
@@ -561,4 +565,5 @@ def instance_destroy(self, instance_name, caller = None):
 		for device in devices:
 			# _add_device() will find a suitable plugin instance
 			plugin._add_device(device)
+		self._daemon.sync_instances()
 		return (True, "OK")

tuned/daemon/daemon.py

@@ -121,6 +121,11 @@ def _load_profiles(self, profile_names, manual):
 			self._notify_profile_changed(profile_names, False, errstr)
 			raise TunedException(errstr)
 
+		# restore profile snapshot (if there is one)
+		snapshot = self._profile_loader.restore_snapshot(self._profile)
+		if snapshot is not None:
+			self._profile = snapshot
+
 	def set_profile(self, profile_names, manual):
 		if self.is_running():
 			errstr = "Cannot set profile while the daemon is running."
@@ -154,6 +159,40 @@ def set_all_profiles(self, active_profiles, manual, post_loaded_profile,
 			self._save_active_profile(active_profiles, manual)
 			self._save_post_loaded_profile(post_loaded_profile)
 
+	def sync_instances(self):
+		# NOTE: currently, Controller creates the new instances, and here in Daemon
+		#       we discover what happened, and update the profile accordingly.
+		#       a potentially better approach would be to move some of the logic
+		#       from Controller to Daemon, and create/destroy the instances here,
+		#       and at the same time update the profile.
+
+		# remove all units that don't have an instance
+		instance_names = [i.name for i in self._unit_manager.instances]
+		for unit in list(self._profile.units.keys()):
+			if unit in instance_names:
+				continue
+			log.debug("snapshot sync: removing unit '%s'" % unit)
+			del self._profile.units[unit]
+		# create units for new instances
+		for instance in self._unit_manager.instances:
+			if instance.name in self._profile.units:
+				continue
+			log.debug("snapshot sync: creating unit '%s'" % instance.name)
+			config = {
+				"priority": instance.priority,
+				"type": instance._plugin.name,
+				"enabled": instance.active,
+				"devices": instance.devices_expression,
+				"devices_udev_regex": instance.devices_udev_regex,
+				"script_pre": instance.script_pre,
+				"script_post": instance.script_post,
+			}
+			for k, v in instance.options.items():
+				config[k] = v
+			self._profile.units[instance.name] = self._profile._create_unit(instance.name, config)
+		# create profile snapshot
+		self._profile_loader.create_snapshot(self._profile, self._unit_manager.instances)
+
 	@property
 	def profile(self):
 		return self._profile
@@ -200,6 +239,8 @@ def _thread_code(self):
 		self._save_active_profile(" ".join(self._active_profiles),
 					  self._manual)
 		self._save_post_loaded_profile(self._post_loaded_profile)
+		# trigger a profile snapshot
+		self.sync_instances()
 		self._unit_manager.start_tuning()
 		self._profile_applied.set()
 		log.info("static tuning from profile '%s' applied" % self._profile.name)
@@ -368,6 +409,7 @@ def stop(self, profile_switch = False):
 			return False
 		log.info("stopping tuning")
 		if profile_switch:
+			self._profile_loader.remove_snapshot()
 			self._terminate_profile_switch.set()
 		self._terminate.set()
 		self._thread.join()

tuned/plugins/base.py

@@ -164,23 +164,35 @@ def _get_matching_devices(self, instance, devices):
 			udev_devices = self._device_matcher_udev.match_list(instance.devices_udev_regex, udev_devices)
 			return set([x.sys_name for x in udev_devices])
 
+	def restore_devices(self, instance, devices):
+		if not self._devices_supported:
+			return
+
+		log.debug("Restoring devices of instance %s: %s" % (instance.name, " ".join(devices)))
+		for device in devices:
+			if device not in self._free_devices:
+				continue
+			self._free_devices.remove(device)
+			instance.assigned_devices.add(device)
+			self._assigned_devices.add(device)
+
 	def assign_free_devices(self, instance):
 		if not self._devices_supported:
 			return
 
 		log.debug("assigning devices to instance %s" % instance.name)
 		to_assign = self._get_matching_devices(instance, self._free_devices)
-		instance.active = len(to_assign) > 0
-		if not instance.active:
-			log.warning("instance %s: no matching devices available" % instance.name)
-		else:
+		if len(to_assign) > 0:
 			name = instance.name
 			if instance.name != self.name:
 				name += " (%s)" % self.name
 			log.info("instance %s: assigning devices %s" % (name, ", ".join(to_assign)))
 			instance.assigned_devices.update(to_assign) # cannot use |=
 			self._assigned_devices |= to_assign
 			self._free_devices -= to_assign
+		instance.active = len(instance.assigned_devices) > 0
+		if not instance.active:
+			log.warning("instance %s: no matching devices available" % instance.name)
 
 	def release_devices(self, instance):
 		if not self._devices_supported:

tuned/profiles/factory.py

@@ -1,5 +1,5 @@
 import tuned.profiles.profile
 
 class Factory(object):
-	def create(self, name, config):
-		return tuned.profiles.profile.Profile(name, config)
+	def create(self, name, config, variables):
+		return tuned.profiles.profile.Profile(name, config, variables)

tuned/profiles/loader.py

@@ -24,9 +24,6 @@ def __init__(self, profile_locator, profile_factory, profile_merger, global_conf
 		self._global_config = global_config
 		self._variables = variables
 
-	def _create_profile(self, profile_name, config):
-		return tuned.profiles.profile.Profile(profile_name, config)
-
 	@classmethod
 	def safe_name(cls, profile_name):
 		return re.match(r'^[a-zA-Z0-9_.-]+$', profile_name)
@@ -57,22 +54,40 @@ def load(self, profile_names):
 			final_profile = profiles[0]
 
 		final_profile.name = " ".join(profile_names)
-		if "variables" in final_profile.units:
-			self._variables.add_from_cfg(final_profile.units["variables"].options)
-			del(final_profile.units["variables"])
-		# FIXME hack, do all variable expansions in one place
-		self._expand_vars_in_devices(final_profile)
-		self._expand_vars_in_regexes(final_profile)
+		final_profile.process_variables()
+		final_profile.calculate_hash()
 		return final_profile
 
-	def _expand_vars_in_devices(self, profile):
-		for unit in profile.units:
-			profile.units[unit].devices = self._variables.expand(profile.units[unit].devices)
-
-	def _expand_vars_in_regexes(self, profile):
-		for unit in profile.units:
-			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
-			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
+	def create_snapshot(self, profile, instances):
+		snapshot = profile.snapshot(instances)
+		log.debug("Storing profile snapshot in %s:\n%s" % (consts.PROFILE_SNAPSHOT_FILE, snapshot))
+		with open(consts.PROFILE_SNAPSHOT_FILE, "w") as f:
+			f.write(snapshot)
+
+	def restore_snapshot(self, profile):
+		snapshot = None
+		if os.path.isfile(consts.PROFILE_SNAPSHOT_FILE):
+			log.debug("Found profile snapshot '%s'" % consts.PROFILE_SNAPSHOT_FILE)
+			try:
+				config = self._load_config_data(consts.PROFILE_SNAPSHOT_FILE)
+				snapshot_hash = config.get("main", {}).get("profile_base_hash", None)
+				if snapshot_hash == profile._base_hash:
+					snapshot = self._profile_factory.create("restore", config, self._variables)
+					snapshot.name = profile.name
+					snapshot.process_variables()
+					log.info("Restored profile snapshot: %s" % snapshot.name)
+				else:
+					log.debug("Snapshot hash '%s' does not match current base hash '%s'. Not restoring." % (snapshot_hash, profile._base_hash))
+					os.remove(consts.PROFILE_SNAPSHOT_FILE)
+			except InvalidProfileException as e:
+				log.error("Could not process profile snapshot: %s" % e)
+		return snapshot
+
+	def remove_snapshot(self):
+		try:
+			os.remove(consts.PROFILE_SNAPSHOT_FILE)
+		except FileNotFoundError:
+			pass
 
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
@@ -84,9 +99,9 @@ def _load_profile(self, profile_names, profiles, processed_files):
 			processed_files.append(filename)
 
 			config = self._load_config_data(filename)
-			profile = self._profile_factory.create(name, config)
+			profile = self._profile_factory.create(name, config, self._variables)
 			if "include" in profile.options:
-				include_names = re.split(r"\s*[,;]\s*", self._variables.expand(profile.options.pop("include")))
+				include_names = re.split(r"\s*[,;]\s*", profile._variables.expand(profile.options.pop("include")))
 				self._load_profile(include_names, profiles, processed_files)
 
 			profiles.append(profile)

tuned/profiles/profile.py

@@ -1,23 +1,28 @@
 import tuned.profiles.unit
+import tuned.profiles.variables
 import tuned.consts as consts
 import collections
+import hashlib
+import json
 
 class Profile(object):
 	"""
 	Representation of a tuning profile.
 	"""
 
-	__slots__ = ["_name", "_options", "_units"]
+	__slots__ = ["_name", "_options", "_units", "_variables", "_base_hash"]
 
-	def __init__(self, name, config):
+	def __init__(self, name, config, variables):
 		self._name = name
 		self._init_options(config)
 		self._init_units(config)
+		self._variables = variables
+		self._base_hash = config.get("main", {}).get("profile_base_hash", None)
 
 	def _init_options(self, config):
 		self._options = {}
 		if consts.PLUGIN_MAIN_UNIT_NAME in config:
-			self._options = dict(config[consts.PLUGIN_MAIN_UNIT_NAME])
+			self._options = collections.OrderedDict(config[consts.PLUGIN_MAIN_UNIT_NAME])
 
 	def _init_units(self, config):
 		self._units = collections.OrderedDict()
@@ -29,6 +34,43 @@ def _init_units(self, config):
 	def _create_unit(self, name, config):
 		return tuned.profiles.unit.Unit(name, config)
 
+	def process_variables(self):
+		if "variables" in self.units:
+			self._variables.add_from_cfg(self.units["variables"].options)
+			del(self.units["variables"])
+		# FIXME hack, do all variable expansions in one place
+		for unit in self.units:
+			self.units[unit].devices = self._variables.expand(self.units[unit].devices)
+			self.units[unit].cpuinfo_regex = self._variables.expand(self.units[unit].cpuinfo_regex)
+			self.units[unit].uname_regex = self._variables.expand(self.units[unit].uname_regex)
+
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		profile_dict = collections.OrderedDict()
+		profile_dict["main"] = self.options
+		profile_dict["variables"] = self._variables.as_ordered_dict()
+		for name, unit in self._units.items():
+			profile_dict[name] = unit.as_ordered_dict()
+		return profile_dict
+
+	def calculate_hash(self):
+		serialized = json.dumps(self.as_ordered_dict())
+		self._base_hash = hashlib.md5(serialized.encode(), usedforsecurity=False).hexdigest()
+
+	def snapshot(self, instances):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[main]\n"
+		snapshot += "active_profile=%s\n" % self.name
+		snapshot += "profile_base_hash=%s\n" % self._base_hash
+		snapshot += "\n" + self._variables.snapshot()
+		for unit in self.units.values():
+			snapshot += "\n" + unit.snapshot()
+			for instance in instances:
+				if instance.name == unit.name:
+					snapshot += "__devices__=%s\n" % " ".join(instance.assigned_devices | instance.processed_devices)
+					break
+		return snapshot
+
 	@property
 	def name(self):
 		"""

tuned/profiles/unit.py

@@ -26,6 +26,46 @@ def __init__(self, name, config):
 		self._script_post = config.pop("script_post", None)
 		self._options = collections.OrderedDict(config)
 
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		ret = collections.OrderedDict()
+		ret["name"] = self.name
+		ret["priority"] = self.priority
+		ret["type"] = self.type
+		ret["enabled"] = self.enabled
+		ret["replace"] = self.replace
+		ret["drop"] = self.drop
+		ret["devices"] = self.devices
+		ret["devices_udev_regex"] = self.devices_udev_regex
+		ret["cpuinfo_regex"] = self.cpuinfo_regex
+		ret["uname_regex"] = self.uname_regex
+		ret["script_pre"] = self.script_pre
+		ret["script_post"] = self.script_post
+		for k, v in self.options.items():
+			ret[k] = v
+		return ret
+
+	def snapshot(self):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[%s]\n" % self.name
+		snapshot += "priority=%s\n" % self.priority
+		snapshot += "type=%s\n" % self.type
+		snapshot += "enabled=%s\n" % self.enabled
+		snapshot += "devices=%s\n" % self.devices
+		if self.devices_udev_regex is not None:
+			snapshot += "devices_udev_regex=%s\n" % self.devices_udev_regex
+		if self.cpuinfo_regex is not None:
+			snapshot += "cpuinfo_regex=%s\n" % self.cpuinfo_regex
+		if self.uname_regex is not None:
+			snapshot += "uname_regex=%s\n" % self.uname_regex
+		if self.script_pre is not None:
+			snapshot += "script_pre=%s\n" % self.script_pre
+		if self.script_post is not None:
+			snapshot += "script_post=%s\n" % self.script_post
+		for k, v in self.options.items():
+			snapshot += "%s=%s\n" % (k, v)
+		return snapshot
+
 	@property
 	def name(self):
 		return self._name

tuned/profiles/variables.py

@@ -1,3 +1,4 @@
+import collections
 import os
 import re
 import tuned.logs
@@ -15,6 +16,7 @@ class Variables():
 
 	def __init__(self):
 		self._cmd = commands()
+		self._raw = collections.OrderedDict()
 		self._lookup_re = {}
 		self._lookup_env = {}
 		self._functions = functions.Repository()
@@ -34,6 +36,7 @@ def add_variable(self, variable, value):
 		if not self._check_var(variable):
 			log.error("variable definition '%s' contains unallowed characters" % variable)
 			return
+		self._raw[s] = value
 		v = self.expand(value)
 		# variables referenced by ${VAR}, $ can be escaped by two $,
 		# i.e. the following will not expand: $${VAR}
@@ -77,3 +80,14 @@ def expand(self, value):
 
 	def get_env(self):
 		return self._lookup_env
+
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		return self._raw
+
+	def snapshot(self):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[variables]\n"
+		for k, v in self._raw.items():
+			snapshot += "%s=%s\n" % (k, v)
+		return snapshot