[core] 7 day eviction policy for task class cache

Author: teo

core/config.go

@@ -33,6 +33,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/AliceO2Group/Control/apricot"
 	apricotpb "github.com/AliceO2Group/Control/apricot/protos"
@@ -120,6 +121,7 @@ func setDefaults() error {
 	viper.SetDefault("concurrentIteratorRoleExpansion", true)
 	viper.SetDefault("reuseUnlockedTasks", false)
 	viper.SetDefault("configCache", true)
+	viper.SetDefault("taskClassCacheTTL", 7*24*time.Hour)
 	return nil
 }
 
@@ -183,6 +185,7 @@ func setFlags() error {
 	pflag.Bool("concurrentIteratorRoleExpansion", viper.GetBool("concurrentIteratorRoleExpansion"), "Expand iterator roles concurrently during workflow template processing")
 	pflag.Bool("reuseUnlockedTasks", viper.GetBool("reuseUnlockedTasks"), "Reuse unlocked active tasks when satisfying environment deployment requests")
 	pflag.Bool("configCache", viper.GetBool("configCache"), "Enable cache layer between AliECS core and Apricot")
+	pflag.Duration("taskClassCacheTTL", viper.GetDuration("taskClassCacheTTL"), "TTL for task class cache entries")
 
 	pflag.Parse()
 	return viper.BindPFlags(pflag.CommandLine)

core/task/manager.go

@@ -251,8 +251,18 @@ func (m *Manager) removeInactiveClasses() {
 	_ = m.classes.Do(func(classMap *map[string]*taskclass.Class) error {
 		keys := make([]string, 0)
 
+		taskClassCacheTTL := viper.GetDuration("taskClassCacheTTL")
+
 		// push keys of classes that don't appear in roster any more into a slice
-		for taskClassIdentifier := range *classMap {
+		for taskClassIdentifier, class := range *classMap {
+			if class == nil {
+				// don't really know what to do with a valid TCI but nil class
+				continue
+			}
+			if time.Since(class.UpdatedTimestamp) < taskClassCacheTTL {
+				// class is still fresh, skip
+				continue
+			}
 			if len(m.roster.filteredForClass(taskClassIdentifier)) == 0 {
 				keys = append(keys, taskClassIdentifier)
 			}

core/task/taskclass/class.go

@@ -26,6 +26,7 @@ package taskclass
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/AliceO2Group/Control/common"
 	"github.com/AliceO2Group/Control/common/controlmode"
@@ -64,13 +65,14 @@ type Class struct {
 	Control    struct {
 		Mode controlmode.ControlMode `yaml:"mode"`
 	} `yaml:"control"`
-	Command     *common.CommandInfo     `yaml:"command"`
-	Wants       ResourceWants           `yaml:"wants"`
-	Limits      *ResourceLimits         `yaml:"limits"`
-	Bind        []channel.Inbound       `yaml:"bind"`
-	Properties  gera.StringMap          `yaml:"properties"`
-	Constraints []constraint.Constraint `yaml:"constraints"`
-	Connect     []channel.Outbound      `yaml:"connect"`
+	Command          *common.CommandInfo     `yaml:"command"`
+	Wants            ResourceWants           `yaml:"wants"`
+	Limits           *ResourceLimits         `yaml:"limits"`
+	Bind             []channel.Inbound       `yaml:"bind"`
+	Properties       gera.StringMap          `yaml:"properties"`
+	Constraints      []constraint.Constraint `yaml:"constraints"`
+	Connect          []channel.Outbound      `yaml:"connect"`
+	UpdatedTimestamp time.Time               `yaml:"-"`
 }
 
 func (c *Class) UnmarshalYAML(unmarshal func(interface{}) error) (err error) {
@@ -106,17 +108,18 @@ func (c *Class) UnmarshalYAML(unmarshal func(interface{}) error) (err error) {
 			}
 		}
 		*c = Class{
-			Identifier:  aux.Identifier,
-			Defaults:    gera.MakeStringMapWithMap(aux.Defaults),
-			Vars:        gera.MakeStringMapWithMap(aux.Vars),
-			Control:     aux.Control,
-			Command:     aux.Command,
-			Wants:       aux.Wants,
-			Limits:      aux.Limits,
-			Bind:        aux.Bind,
-			Properties:  gera.MakeStringMapWithMap(aux.Properties),
-			Constraints: aux.Constraints,
-			Connect:     aux.Connect,
+			Identifier:       aux.Identifier,
+			Defaults:         gera.MakeStringMapWithMap(aux.Defaults),
+			Vars:             gera.MakeStringMapWithMap(aux.Vars),
+			Control:          aux.Control,
+			Command:          aux.Command,
+			Wants:            aux.Wants,
+			Limits:           aux.Limits,
+			Bind:             aux.Bind,
+			Properties:       gera.MakeStringMapWithMap(aux.Properties),
+			Constraints:      aux.Constraints,
+			Connect:          aux.Connect,
+			UpdatedTimestamp: time.Now(),
 		}
 	}
 	return

core/task/taskclass/classes.go

@@ -24,7 +24,10 @@
 
 package taskclass
 
-import "sync"
+import (
+	"sync"
+	"time"
+)
 
 type Classes struct {
 	mu       sync.RWMutex
@@ -75,7 +78,9 @@ func (c *Classes) DeleteKeys(keys []string) {
 func (c *Classes) UpdateClass(key string, class *Class) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if _, ok := c.classMap[key]; ok { //contains
+
+	class.UpdatedTimestamp = time.Now() // used for invalidating stale classcache entries
+	if _, ok := c.classMap[key]; ok {   //contains
 		*c.classMap[key] = *class // update
 	} else {
 		c.classMap[key] = class // else add class as new entry