Merge pull request #217 from whywaita/fix/do-rescue

Fix to search repository
whywaita · Nov 6, 2024 · 180b31a · 180b31a
2 parents 7968300 + 0a186c5
commit 180b31a
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 11 deletions.
diff --git a/pkg/gh/run.go b/pkg/gh/run.go
@@ -28,13 +28,12 @@ func listRuns(ctx context.Context, client *github.Client, owner, repo string, op
 func ListRuns(owner, repo string) ([]*github.WorkflowRun, error) {
 	if cachedRs, expiration, found := responseCache.GetWithExpiration(getRunsCacheKey(owner, repo)); found {
 		if time.Until(expiration).Minutes() <= 1 {
-			go updateCache(context.Background(), owner, repo)
+			go updateCache(owner, repo)
 		}
-		logger.Logf(true, "found workflow runs (cache hit: expiration: %s) in %s/%s", expiration.Format("2006/01/02 15:04:05.000 -0700"), owner, repo)
 		return cachedRs.([]*github.WorkflowRun), nil
 	}
-	go updateCache(context.Background(), owner, repo)
-	return []*github.WorkflowRun{}, nil
+	go updateCache(owner, repo)
+	return []*github.WorkflowRun{}, nil // retry next time
 }
 
 func getRunsCacheKey(owner, repo string) string {
@@ -46,7 +45,8 @@ func ClearRunsCache(owner, repo string) {
 	responseCache.Delete(getRunsCacheKey(owner, repo))
 }
 
-func updateCache(ctx context.Context, owner, repo string) {
+func updateCache(owner, repo string) {
+	ctx := context.Background()
 	var opts = &github.ListWorkflowRunsOptions{
 		ListOptions: github.ListOptions{
 			Page:    0,
@@ -71,6 +71,6 @@ func updateCache(ctx context.Context, owner, repo string) {
 		return
 	}
 	storeRateLimit(getRateLimitKey(owner, repo), resp.Rate)
-	responseCache.Set(getRunsCacheKey(owner, repo), runs.WorkflowRuns, 15*time.Minute)
+	responseCache.Set(getRunsCacheKey(owner, repo), runs.WorkflowRuns, 3*time.Minute)
 	logger.Logf(true, "found %d workflow runs in %s/%s", len(runs.WorkflowRuns), owner, repo)
 }
diff --git a/pkg/metric/scrape_github.go b/pkg/metric/scrape_github.go
@@ -45,11 +45,11 @@ func (s ScraperGitHub) Scrape(ctx context.Context, ds datastore.Datastore, ch ch
 func scrapePendingRuns(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error {
 	gh.ActiveTargets.Range(func(key, value any) bool {
 		var pendings float64
-		scope := key.(string)
+		repoName := key.(string)
 		installationID := value.(int64)
-		target, err := ds.GetTargetByScope(ctx, scope)
+		target, err := datastore.SearchRepo(ctx, ds, repoName)
 		if err != nil {
-			logger.Logf(false, "failed to get target by scope (%s): %+v", scope, err)
+			logger.Logf(false, "failed to get target by scope (%s): %+v", repoName, err)
 			return true
 		}
 		owner, repo := target.OwnerRepo()
@@ -64,10 +64,14 @@ func scrapePendingRuns(ctx context.Context, ds datastore.Datastore, ch chan<- pr
 
 		for _, r := range runs {
 			if r.GetStatus() == "queued" || r.GetStatus() == "pending" {
-				if time.Since(r.CreatedAt.Time).Minutes() >= 30 {
+				oldMinutes := 30
+				sinceMinutes := time.Since(r.CreatedAt.Time).Minutes()
+				if sinceMinutes >= float64(oldMinutes) {
+					logger.Logf(false, "run %d is pending over %d minutes", r.GetID(), oldMinutes)
 					pendings++
 					gh.PendingRuns.Store(installationID, r)
 				}
+				logger.Logf(true, "run %d is pending, but not over %d minutes. So ignore (since: %f minutes)", r.GetID(), oldMinutes, sinceMinutes)
 			}
 		}
 		ch <- prometheus.MustNewConstMetric(githubPendingRunsDesc, prometheus.GaugeValue, pendings, target.UUID.String(), target.Scope)

diff --git a/pkg/starter/starter.go b/pkg/starter/starter.go
@@ -443,7 +443,7 @@ func (s *Starter) reRunWorkflow(ctx context.Context) {
 					domain = "https://github.com"
 				}
 
-				logger.Logf(false, "receive webhook repository: %s/%s", domain, repoName)
+				logger.Logf(false, "rescue pending job: (repo: %s/%s, runID: %d)", domain, repoName, run.GetID())
 				target, err := datastore.SearchRepo(ctx, s.ds, repoName)
 				if err != nil {
 					logger.Logf(false, "failed to search registered target: %+v", err)