fix(health): ignore unexecuted workflow runs

docs/src/content/docs/setup/cli.md

@@ -787,6 +787,10 @@ The `--days` flag accepts 7, 30, or 90 (default: 7). Other values produce an err
 
 For release-scoped evidence, pass the release's publication timestamp as `--start-date`; this filters by time, not compiler version. Text output and JSON `collection.start_date` / `collection.end_date` show the resolved UTC window. Health still samples current workflow revisions toward five completed first-attempt runs per scope, subject to API limits; it does not enumerate every run or reconstruct historical branch heads. A narrow window can leave insufficient evidence.
 
+Runs awaiting approval (`waiting` status or `action_required` conclusion), skipped runs, and failed runs with no jobs are excluded from health metrics, revision evidence, and failure clusters. Failed runs are checked against the Jobs API for their run attempt, including runs whose approval expired. Collection continues past ignored runs to find executed runs within the requested window and API limits. An unavailable or malformed Jobs API response is reported as an error, not treated as evidence that no jobs ran.
+
+Filtered runs do not consume the minimum sample. Health searches backward for five completed first-attempt runs per current-revision scope, stopping at the requested time boundary or API budget rather than a fixed page count. It narrows the search window to continue beyond GitHub's 1,000-run search limit, overlapping boundary timestamps and deduplicating runs. If timestamp ties prevent further progress, the report explicitly marks collection as partial.
+
 Shows success/failure rates, trend indicators (↑ improving, → stable, ↓ degrading), execution duration, token usage, costs, and warnings when success rate drops below threshold.
 
 Runs that never dispatched a job are excluded from these metrics: `skipped` runs (the activation condition evaluated to false, for example a comment that does not contain the workflow's command) and `action_required` runs (created by GitHub but held for manual approval, for example comment events authored by a bot actor). Including them would report command workflows as near-0% success even when every dispatched run succeeded.

pkg/cli/health_command.go

@@ -57,6 +57,7 @@ Use --start-date and --end-date to bound run creation times. Accepts UTC dates,
 RFC3339 timestamps, or relative times such as -24h and -1w. An explicit start
 replaces --days; otherwise --days counts backward from --end-date (default: now).
 The report remains a bounded current-revision sample, not an exhaustive history.
+Runs awaiting approval (waiting or action_required), skipped runs, and failed runs with no jobs are ignored.
 
 ` + WorkflowIDExplanation,
 		Example: `  ` + string(constants.CLIExtensionPrefix) + ` health                       # Current status of all workflows

pkg/cli/health_metrics.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"slices"
 	"time"
 
 	"github.com/github/gh-aw/pkg/console"
@@ -78,6 +79,7 @@ func (t TrendDirection) String() string {
 
 // CalculateWorkflowHealth calculates health metrics for a workflow from its runs
 func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold float64) WorkflowHealth {
+	runs = slices.DeleteFunc(slices.Clone(runs), isIgnoredHealthRun)
 	healthMetricsLog.Printf("Calculating health for workflow: %s, runs: %d", workflowName, len(runs))
 
 	if len(runs) == 0 {
@@ -102,6 +104,10 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold
 	return health
 }
 
+func isIgnoredHealthRun(run WorkflowRun) bool {
+	return run.Status == "waiting" || isNonDispatchedConclusion(run.Conclusion)
+}
+
 func accumulateWorkflowHealth(runs []WorkflowRun) WorkflowHealth {
 	health := WorkflowHealth{TotalRuns: len(runs)}
 	var durationStats, tokenStats stats.StatVar

pkg/cli/health_no_jobs_test.go

@@ -0,0 +1,153 @@
+//go:build !integration
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealthSamplingContinuesPastNoJobFailures(t *testing.T) {
+	for _, conclusion := range []string{"failure", "timed_out", "cancelled", "startup_failure"} {
+		t.Run(conclusion, func(t *testing.T) {
+			originalRate, originalGH := fetchRateLimitForHostFunc, healthRunGH
+			t.Cleanup(func() { fetchRateLimitForHostFunc, healthRunGH = originalRate, originalGH })
+			fetchRateLimitForHostFunc = func(context.Context, string) (rateLimitResource, error) {
+				return rateLimitResource{Limit: 5000, Remaining: 4000}, nil
+			}
+			pages, jobChecks := 0, 0
+			healthRunGH = func(_ context.Context, _ string, args ...string) ([]byte, error) {
+				endpoint := args[3]
+				switch {
+				case strings.HasSuffix(endpoint, "/jobs?per_page=1"):
+					jobChecks++
+					assert.Contains(t, endpoint, "/attempts/2/jobs", "check the sampled attempt, not the latest one")
+					return []byte(`{"total_count":0,"jobs":[]}`), nil
+				case strings.Contains(endpoint, "/contents/"):
+					assert.NotContains(t, endpoint, "unexecuted")
+					return []byte(`# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"settings","body_hash":"prompt"}`), nil
+				default:
+					pages++
+					count := 10
+					if pages == 2 {
+						count = 5
+						assert.Contains(t, endpoint, "page=2")
+					}
+					var runs []workflowRunAPIResponse
+					for i := range count {
+						run := workflowRunAPIResponse{ID: int64(pages*10 + i), Name: "Example",
+							Path: ".github/workflows/example.lock.yml", Status: "completed",
+							Conclusion: "success", Event: "schedule", HeadBranch: "main",
+							HeadSHA: strings.Repeat("a", 40), RunAttempt: 1}
+						if pages == 1 {
+							run.Conclusion, run.HeadSHA, run.RunAttempt = conclusion, "unexecuted", 2
+						}
+						runs = append(runs, run)
+					}
+					out, err := json.Marshal(map[string]any{"workflow_runs": runs})
+					require.NoError(t, err)
+					return out, nil
+				}
+			}
+			collection := &HealthCollection{Host: "github.com", repoOverride: "org/repo"}
+			runs, err := sampleHealthWorkflow(GitHubWorkflow{ID: 1, Name: "Example",
+				Path: ".github/workflows/example.lock.yml"}, "2026-10-08", "org/repo", false, collection)
+			require.NoError(t, err)
+			assert.Equal(t, 2, pages)
+			assert.Equal(t, 10, jobChecks)
+			assert.Len(t, runs, 5)
+			config := HealthConfig{Threshold: 80, LogsDir: t.TempDir(), RepoOverride: "org/repo", collection: collection}
+			health := calculateCurrentWorkflowHealth("Example", runs, config)
+			assert.Equal(t, "healthy", health.Status)
+			assert.InDelta(t, 100.0, health.SuccessRate, 1e-9)
+			clusters, plan := buildHealthFailureClusters(runs, config)
+			assert.Empty(t, clusters)
+			assert.Nil(t, plan)
+		})
+	}
+}
+
+func TestHealthRunHasJobsRequiresConfirmedEvidence(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		response string
+		apiError error
+		wantJobs bool
+		wantErr  string
+	}{
+		{name: "no jobs", response: `{"jobs":[]}`},
+		{name: "executed failure", response: `{"jobs":[{"id":42}]}`, wantJobs: true},
+		{name: "API unavailable", apiError: errors.New("HTTP 502"), wantErr: "failed to check jobs"},
+		{name: "invalid JSON", response: `{`, wantErr: "failed to parse jobs"},
+		{name: "missing jobs", response: `{}`, wantErr: "missing jobs array"},
+		{name: "null jobs", response: `{"jobs":null}`, wantErr: "missing jobs array"},
+		{name: "malformed jobs", response: `{"jobs":{}}`, wantErr: "failed to parse jobs"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			originalRate, originalGH := fetchRateLimitForHostFunc, healthRunGH
+			t.Cleanup(func() { fetchRateLimitForHostFunc, healthRunGH = originalRate, originalGH })
+			fetchRateLimitForHostFunc = func(context.Context, string) (rateLimitResource, error) {
+				return rateLimitResource{Limit: 5000, Remaining: 4000}, nil
+			}
+			healthRunGH = func(_ context.Context, _ string, args ...string) ([]byte, error) {
+				assert.Contains(t, args, "github.example.com")
+				assert.Contains(t, args, "repos/org/repo/actions/runs/42/attempts/2/jobs?per_page=1")
+				return []byte(tc.response), tc.apiError
+			}
+			collection := &HealthCollection{Host: "github.example.com", repoOverride: "github.example.com/org/repo"}
+			hasJobs, err := healthRunHasJobs(WorkflowRun{DatabaseID: 42, Attempt: 2}, collection)
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.wantJobs, hasJobs)
+			}
+			assert.Equal(t, 1, collection.APIRequests)
+		})
+	}
+}
+
+func TestHealthNoJobLookupHonorsRequestBudget(t *testing.T) {
+	collection := &HealthCollection{Host: "github.com", APIRequests: healthMaximumAPIRequests}
+	_, err := healthRunHasJobs(WorkflowRun{DatabaseID: 42, Attempt: 1}, collection)
+	require.ErrorIs(t, err, errHealthAPIBudget)
+	assert.True(t, collection.Partial)
+	assert.Equal(t, "request_budget", collection.Reason)
+}
+
+func TestHealthOnlyNoJobFailuresRemainInsufficient(t *testing.T) {
+	originalRate, originalGH := fetchRateLimitForHostFunc, healthRunGH
+	t.Cleanup(func() { fetchRateLimitForHostFunc, healthRunGH = originalRate, originalGH })
+	fetchRateLimitForHostFunc = func(context.Context, string) (rateLimitResource, error) {
+		return rateLimitResource{Limit: 5000, Remaining: 4000}, nil
+	}
+	healthRunGH = func(_ context.Context, _ string, args ...string) ([]byte, error) {
+		switch {
+		case strings.Contains(args[3], "/actions/workflows?"):
+			return []byte(`{"workflows":[{"id":1,"name":"Example","path":".github/workflows/example.lock.yml","state":"active"}]}`), nil
+		case strings.Contains(args[3], "/actions/workflows/1/runs?"):
+			return []byte(`{"workflow_runs":[{"id":42,"name":"Example","status":"completed","conclusion":"failure","run_attempt":1}]}`), nil
+		default:
+			assert.Contains(t, args[3], "/runs/42/attempts/1/jobs?")
+			return []byte(`{"jobs":[]}`), nil
+		}
+	}
+	output := captureAuditGroupStdout(t, func() {
+		require.NoError(t, RunHealth(HealthConfig{Days: 7, Threshold: 80, JSONOutput: true, RepoOverride: "org/repo"}))
+	})
+	var report HealthSummary
+	require.NoError(t, json.Unmarshal([]byte(output), &report))
+	assert.Zero(t, report.HealthyWorkflows)
+	assert.Zero(t, report.BelowThreshold)
+	assert.Equal(t, 1, report.InsufficientWorkflows)
+	require.Len(t, report.Workflows, 1)
+	assert.Zero(t, report.Workflows[0].TotalRuns)
+	assert.Equal(t, "no_recent_runs", report.Workflows[0].Status)
+	assert.Empty(t, report.FailureClusters)
+}

pkg/cli/health_pagination.go

@@ -0,0 +1,18 @@
+package cli
+
+import "time"
+
+const healthMaximumRunSearchResults = 1000
+
+func advanceHealthRunWindow(opts *ListWorkflowRunsOptions, oldest time.Time, collection *HealthCollection) bool {
+	// Overlap the boundary second so that tied timestamps are not skipped.
+	before := oldest.Add(time.Second).Format(time.RFC3339)
+	if oldest.IsZero() || before == opts.BeforeDate {
+		collection.Partial, collection.Reason = true, "pagination_limit"
+		collection.Warning = "Health could not advance past the GitHub run-search limit without skipping runs at the timestamp boundary"
+		healthLog.Printf("%s", collection.Warning)
+		return false
+	}
+	opts.BeforeDate = before
+	return true
+}

pkg/cli/health_pagination_test.go

@@ -0,0 +1,169 @@
+//go:build !integration
+
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealthSamplingContinuesPastFilteredPageLimit(t *testing.T) {
+	original := healthListWorkflowRuns
+	t.Cleanup(func() { healthListWorkflowRuns = original })
+	runs, collection := healthRevisionFixture()
+	calls := 0
+	healthListWorkflowRuns = func(opts ListWorkflowRunsOptions, page int, _ *HealthCollection) ([]WorkflowRun, int, error) {
+		calls++
+		assert.Equal(t, calls, page)
+		assert.Equal(t, "2026-10-08", opts.StartDate)
+		if page <= MaxIterations {
+			return nil, opts.Limit, nil
+		}
+		return runs[5:], 5, nil
+	}
+	sample, err := sampleHealthWorkflow(GitHubWorkflow{ID: 1, Name: "Example",
+		Path: runs[0].WorkflowPath}, "2026-10-08", "org/repo", false, collection)
+	require.NoError(t, err)
+	assert.Equal(t, MaxIterations+1, calls)
+	assert.Len(t, sample, healthMinimumRuns)
+	assert.False(t, collection.Partial)
+	health := calculateCurrentWorkflowHealth("Example", sample, HealthConfig{Threshold: 80, collection: collection})
+	assert.Equal(t, "healthy", health.Status)
+}
+
+func TestHealthSamplingRollsSearchWindowWithoutSkippingTimestampTies(t *testing.T) {
+	originalRate, originalGH := fetchRateLimitForHostFunc, healthRunGH
+	t.Cleanup(func() { fetchRateLimitForHostFunc, healthRunGH = originalRate, originalGH })
+	fetchRateLimitForHostFunc = func(context.Context, string) (rateLimitResource, error) {
+		return rateLimitResource{Limit: 5000, Remaining: 4000}, nil
+	}
+	end := time.Date(2026, 10, 16, 4, 0, 0, 0, time.UTC)
+	start := end.Add(-24 * time.Hour).Format(time.RFC3339)
+	boundary := end.Add(-990 * time.Second)
+	var responses []workflowRunAPIResponse
+	for i := range healthMaximumRunSearchResults + healthMinimumRuns - 1 {
+		created := end.Add(-time.Duration(i) * time.Second)
+		if i >= 990 {
+			created = boundary
+		}
+		run := workflowRunAPIResponse{ID: int64(i + 1), Name: "Example",
+			Path: ".github/workflows/example.lock.yml", Status: "completed",
+			Conclusion: "action_required", Event: "schedule", HeadBranch: "main",
+			HeadSHA: "unapproved", RunAttempt: 1, CreatedAt: created}
+		if i >= healthMaximumRunSearchResults-1 {
+			run.Conclusion, run.HeadSHA = "success", strings.Repeat("a", 40)
+		}
+		responses = append(responses, run)
+	}
+	rolledPages := []int{}
+	healthRunGH = func(_ context.Context, _ string, args ...string) ([]byte, error) {
+		endpoint := args[3]
+		if strings.Contains(endpoint, "/contents/") {
+			assert.NotContains(t, endpoint, "unapproved")
+			return []byte(`# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"settings","body_hash":"prompt"}`), nil
+		}
+		parsed, err := url.Parse(endpoint)
+		require.NoError(t, err)
+		created := strings.Split(parsed.Query().Get("created"), "..")
+		require.Len(t, created, 2)
+		assert.Equal(t, start, created[0])
+		upper, err := time.Parse(time.RFC3339, created[1])
+		require.NoError(t, err)
+		assert.False(t, upper.After(end), "the requested end bound must not expand")
+		page, err := strconv.Atoi(parsed.Query().Get("page"))
+		require.NoError(t, err)
+		limit, err := strconv.Atoi(parsed.Query().Get("per_page"))
+		require.NoError(t, err)
+		if upper.Equal(boundary) {
+			rolledPages = append(rolledPages, page)
+		}
+		var matching []workflowRunAPIResponse
+		for _, run := range responses {
+			if !run.CreatedAt.After(upper) {
+				matching = append(matching, run)
+			}
+		}
+		first := min((page-1)*limit, len(matching), healthMaximumRunSearchResults)
+		last := min(first+limit, len(matching), healthMaximumRunSearchResults)
+		out, err := json.Marshal(map[string]any{"workflow_runs": matching[first:last]})
+		require.NoError(t, err)
+		return out, nil
+	}
+	collection := &HealthCollection{Host: "github.com", repoOverride: "org/repo", EndDate: end.Format(time.RFC3339)}
+	runs, err := sampleHealthWorkflow(GitHubWorkflow{ID: 1, Name: "Example",
+		Path: ".github/workflows/example.lock.yml"}, start, "org/repo", false, collection)
+	require.NoError(t, err)
+	assert.Equal(t, []int{1, 2}, rolledPages)
+	require.Len(t, runs, healthMinimumRuns)
+	assert.Equal(t, int64(healthMaximumRunSearchResults), runs[0].DatabaseID)
+	assert.False(t, collection.Partial)
+	assert.Equal(t, end.Format(time.RFC3339), collection.EndDate)
+	health := calculateCurrentWorkflowHealth("Example", runs, HealthConfig{Threshold: 80, collection: collection})
+	assert.Equal(t, "healthy", health.Status)
+}
+
+func TestHealthUnadvanceableSearchWindowRemainsExplicitlyPartial(t *testing.T) {
+	original := healthListWorkflowRuns
+	t.Cleanup(func() { healthListWorkflowRuns = original })
+	tests := []struct {
+		name   string
+		oldest time.Time
+		pages  int
+	}{
+		{"saturated timestamp", time.Date(2026, 10, 15, 1, 0, 0, 0, time.UTC), 200},
+		{"unavailable timestamp", time.Time{}, 100},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			calls := 0
+			healthListWorkflowRuns = func(opts ListWorkflowRunsOptions, _ int, _ *HealthCollection) ([]WorkflowRun, int, error) {
+				calls++
+				*opts.OldestFetchedCreatedAt = tt.oldest
+				return nil, opts.Limit, nil
+			}
+			collection := &HealthCollection{}
+			runs, err := sampleHealthWorkflow(GitHubWorkflow{Name: "Example",
+				Path: ".github/workflows/example.lock.yml"}, "2026-10-08", "org/repo", false, collection)
+			require.NoError(t, err)
+			assert.Empty(t, runs)
+			assert.Equal(t, tt.pages, calls)
+			assert.True(t, collection.Partial)
+			assert.Equal(t, "pagination_limit", collection.Reason)
+			assert.Contains(t, collection.Warning, "timestamp boundary")
+		})
+	}
+}
+
+func TestHealthFilteredPagesStillRespectRequestBudget(t *testing.T) {
+	originalRate, originalGH := fetchRateLimitForHostFunc, healthRunGH
+	t.Cleanup(func() { fetchRateLimitForHostFunc, healthRunGH = originalRate, originalGH })
+	fetchRateLimitForHostFunc = func(context.Context, string) (rateLimitResource, error) {
+		return rateLimitResource{Limit: 5000, Remaining: 4000}, nil
+	}
+	calls := 0
+	healthRunGH = func(_ context.Context, _ string, _ ...string) ([]byte, error) {
+		calls++
+		var runs []workflowRunAPIResponse
+		for i := range 2 * healthMinimumRuns {
+			runs = append(runs, workflowRunAPIResponse{ID: int64(calls*10 + i), Conclusion: "action_required"})
+		}
+		out, err := json.Marshal(map[string]any{"workflow_runs": runs})
+		require.NoError(t, err)
+		return out, nil
+	}
+	collection := &HealthCollection{Host: "github.com", APIRequests: healthMaximumAPIRequests - 2}
+	_, err := sampleHealthWorkflow(GitHubWorkflow{ID: 1, Name: "Example",
+		Path: ".github/workflows/example.lock.yml"}, "2026-10-08", "org/repo", false, collection)
+	require.ErrorIs(t, err, errHealthAPIBudget)
+	assert.Equal(t, 2, calls)
+	assert.True(t, collection.Partial)
+	assert.Equal(t, "request_budget", collection.Reason)
+}

pkg/cli/health_revisions.go

@@ -53,6 +53,9 @@ func healthRevisionStatuses(runs []WorkflowRun, collection *HealthCollection) []
 	byScope := make(map[string]int)
 	seen := make(map[int64]struct{})
 	for _, run := range ordered {
+		if isIgnoredHealthRun(run) {
+			continue
+		}
 		if _, duplicate := seen[run.DatabaseID]; duplicate {
 			continue
 		}