APP-10065: Record job completion times; include in GetMachineStatus

go.mod

@@ -85,7 +85,7 @@ require (
 	go.uber.org/atomic v1.11.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
-	go.viam.com/api v0.1.487
+	go.viam.com/api v0.1.490
 	go.viam.com/test v1.2.4
 	go.viam.com/utils v0.1.176
 	goji.io v2.0.2+incompatible

go.sum

@@ -1587,8 +1587,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-go.viam.com/api v0.1.487 h1:KdX0vQzZ6j/3YgUX8B98a3QgQ1e3AxD/RkN+GFvmJhg=
-go.viam.com/api v0.1.487/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk=
+go.viam.com/api v0.1.490 h1:4ARXkNJOcsHFukj/LH+kDn94nClSMLTjTlXBb+S8X+4=
+go.viam.com/api v0.1.490/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk=
 go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug=
 go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI=
 go.viam.com/utils v0.1.176 h1:I5TvnuBZtE9i3e6j1VOBzWQ6W3nlUiOR/L4WpTMFhxg=

robot/client/client.go

@@ -16,6 +16,7 @@ import (
 	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/grpcreflect"
+	"github.com/samber/lo"
 	"github.com/viamrobotics/webrtc/v3"
 	"go.opencensus.io/plugin/ocgrpc"
 	"go.uber.org/multierr"
@@ -1264,6 +1265,19 @@ func (rc *RobotClient) MachineStatus(ctx context.Context) (robot.MachineStatus,
 		mStatus.State = robot.StateRunning
 	}
 
+	if resp.GetJobStatuses() != nil {
+		tspbToTime := func(tspb *timestamppb.Timestamp, _ int) time.Time {
+			return tspb.AsTime()
+		}
+		mStatus.JobStatuses = make(map[string]robot.JobStatus, len(resp.GetJobStatuses()))
+		for _, js := range resp.GetJobStatuses() {
+			mStatus.JobStatuses[js.GetJobName()] = robot.JobStatus{
+				RecentSuccessfulRuns: lo.Map(js.GetRecentSuccessfulRuns(), tspbToTime),
+				RecentFailedRuns:     lo.Map(js.GetRecentFailedRuns(), tspbToTime),
+			}
+		}
+	}
+
 	return mStatus, nil
 }
 

robot/impl/jobmanager_test.go

@@ -36,6 +36,7 @@ import (
 	"go.viam.com/rdk/ml"
 	"go.viam.com/rdk/referenceframe"
 	"go.viam.com/rdk/resource"
+	rclient "go.viam.com/rdk/robot/client"
 	"go.viam.com/rdk/services/datamanager"
 	"go.viam.com/rdk/services/discovery"
 	genSvc "go.viam.com/rdk/services/generic"
@@ -49,6 +50,7 @@ import (
 	"go.viam.com/rdk/spatialmath"
 	"go.viam.com/rdk/testutils/inject"
 	injectmotion "go.viam.com/rdk/testutils/inject/motion"
+	"go.viam.com/rdk/testutils/robottestutils"
 )
 
 func TestJobManagerDurationAndCronFromJson(t *testing.T) {
@@ -69,6 +71,163 @@ func TestJobManagerDurationAndCronFromJson(t *testing.T) {
 	})
 }
 
+func TestJobManagerHistory(t *testing.T) {
+	logger := logging.NewTestLogger(t)
+
+	fakeSensorModel := resource.DefaultModelFamily.WithModel("fakesensor")
+	injectSensor := inject.NewSensor("fakesensor")
+	injectSensor.ReadingsFunc = func(ctx context.Context, extra map[string]interface{}) (map[string]interface{}, error) {
+		return map[string]interface{}{"a": 1, "b": 2, "c": 3}, nil
+	}
+	injectSensor.DoFunc = func(ctx context.Context, cmd map[string]interface{}) (map[string]interface{}, error) {
+		if cmd["command"] == "pass" {
+			return nil, nil
+		}
+		return nil, errors.New("fail")
+	}
+	fakeSensorModelPanic := resource.DefaultModelFamily.WithModel("fakesensorPanic")
+	injectSensorPanic := inject.NewSensor("fakesensorPanic")
+	injectSensorPanic.ReadingsFunc = func(ctx context.Context, extra map[string]interface{}) (map[string]interface{}, error) {
+		panic("panic")
+	}
+	injectSensorPanic.DoFunc = func(ctx context.Context, cmd map[string]interface{}) (map[string]interface{}, error) {
+		return nil, nil
+	}
+	resource.RegisterComponent(
+		sensor.API,
+		fakeSensorModel,
+		resource.Registration[sensor.Sensor, resource.NoNativeConfig]{Constructor: func(
+			ctx context.Context,
+			deps resource.Dependencies,
+			conf resource.Config,
+			logger logging.Logger,
+		) (sensor.Sensor, error) {
+			return injectSensor, nil
+		}})
+	resource.RegisterComponent(
+		sensor.API,
+		fakeSensorModelPanic,
+		resource.Registration[sensor.Sensor, resource.NoNativeConfig]{Constructor: func(
+			ctx context.Context,
+			deps resource.Dependencies,
+			conf resource.Config,
+			logger logging.Logger,
+		) (sensor.Sensor, error) {
+			return injectSensorPanic, nil
+		}})
+
+	// test GetReadings success, DoCommand fail
+	cfg := &config.Config{
+		Components: []resource.Config{
+			{
+				Model: fakeSensorModel,
+				Name:  "sensor",
+				API:   sensor.API,
+			},
+		},
+		Jobs: []config.JobConfig{
+			{
+				config.JobConfigData{
+					Name:     "fake sensor",
+					Schedule: "200ms",
+					Resource: "sensor",
+					Method:   "GetReadings",
+				},
+			},
+			{
+				config.JobConfigData{
+					Name:     "test DoCommand",
+					Schedule: "200ms",
+					Resource: "sensor",
+					Method:   "DoCommand",
+					Command: map[string]any{
+						"command": "fail",
+					},
+				},
+			},
+		},
+	}
+	// switch to test GetReadings Panic, DoCommand success
+	cfgSwitch := &config.Config{
+		Components: []resource.Config{
+			{
+				Model: fakeSensorModelPanic,
+				Name:  "sensor",
+				API:   sensor.API,
+			},
+		},
+		Jobs: []config.JobConfig{
+			{
+				config.JobConfigData{
+					Name:     "fake sensor",
+					Schedule: "200ms",
+					Resource: "sensor",
+					Method:   "GetReadings",
+				},
+			},
+			{
+				config.JobConfigData{
+					Name:     "test DoCommand",
+					Schedule: "200ms",
+					Resource: "sensor",
+					Method:   "DoCommand",
+					Command: map[string]any{
+						"command": "pass",
+					},
+				},
+			},
+		},
+	}
+
+	ctx, ctxCancelFunc := context.WithCancel(context.Background())
+	defer ctxCancelFunc()
+	lr := setupLocalRobot(t, ctx, cfg, logger)
+	o, _, addr := robottestutils.CreateBaseOptionsAndListener(t)
+	err := lr.StartWeb(ctx, o)
+	test.That(t, err, test.ShouldBeNil)
+	robotClient, err := rclient.New(ctx, addr, logger)
+	test.That(t, err, test.ShouldBeNil)
+	defer robotClient.Close(ctx)
+
+	//nolint:dupl
+	testutils.WaitForAssertionWithSleep(t, time.Second, 5, func(tb testing.TB) {
+		tb.Helper()
+		ms, err := robotClient.MachineStatus(ctx)
+		test.That(tb, err, test.ShouldBeNil)
+		test.That(tb, len(ms.JobStatuses), test.ShouldEqual, 2)
+
+		successJob, ok := ms.JobStatuses["fake sensor"]
+		test.That(tb, ok, test.ShouldBeTrue)
+		test.That(tb, len(successJob.RecentSuccessfulRuns), test.ShouldBeGreaterThan, 0)
+		test.That(tb, len(successJob.RecentFailedRuns), test.ShouldEqual, 0)
+
+		failJob, ok := ms.JobStatuses["test DoCommand"]
+		test.That(tb, ok, test.ShouldBeTrue)
+		test.That(tb, len(failJob.RecentSuccessfulRuns), test.ShouldEqual, 0)
+		test.That(tb, len(failJob.RecentFailedRuns), test.ShouldBeGreaterThan, 0)
+	})
+
+	lr.Reconfigure(ctx, cfgSwitch)
+	//nolint:dupl
+	testutils.WaitForAssertionWithSleep(t, time.Second, 5, func(tb testing.TB) {
+		tb.Helper()
+		ms, err := robotClient.MachineStatus(ctx)
+		test.That(tb, err, test.ShouldBeNil)
+		test.That(tb, len(ms.JobStatuses), test.ShouldEqual, 2)
+
+		// both ShouldBeGreaterThan because history from previous run remains
+		panicJob, ok := ms.JobStatuses["fake sensor"]
+		test.That(tb, ok, test.ShouldBeTrue)
+		test.That(tb, len(panicJob.RecentSuccessfulRuns), test.ShouldBeGreaterThan, 0)
+		test.That(tb, len(panicJob.RecentFailedRuns), test.ShouldBeGreaterThan, 0)
+
+		successJob, ok := ms.JobStatuses["test DoCommand"]
+		test.That(tb, ok, test.ShouldBeTrue)
+		test.That(tb, len(successJob.RecentSuccessfulRuns), test.ShouldBeGreaterThan, 0)
+		test.That(tb, len(successJob.RecentFailedRuns), test.ShouldBeGreaterThan, 0)
+	})
+}
+
 func TestJobManagerConfigChanges(t *testing.T) {
 	logger := logging.NewTestLogger(t)
 	model := resource.DefaultModelFamily.WithModel(utils.RandomAlphaString(8))

robot/impl/local_robot.go

@@ -1703,6 +1703,21 @@ func (r *localRobot) MachineStatus(ctx context.Context) (robot.MachineStatus, er
 	if r.initializing.Load() {
 		result.State = robot.StateInitializing
 	}
+
+	if r.jobManager != nil {
+		if n := r.jobManager.NumJobHistories.Load(); n > 0 {
+			if result.JobStatuses == nil {
+				result.JobStatuses = make(map[string]robot.JobStatus)
+			}
+			for jobName, jobHistory := range r.jobManager.JobHistories.Range {
+				result.JobStatuses[jobName] = robot.JobStatus{
+					RecentSuccessfulRuns: jobHistory.Successes(),
+					RecentFailedRuns:     jobHistory.Failures(),
+				}
+			}
+		}
+	}
+
 	return result, nil
 }