karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3949 from whitewindmills/CronFederatedHPA_metrics 

feat: add cron federated hpa metrics
This commit is contained in:
karmada-bot 2023-08-23 10:11:02 +08:00 committed by GitHub
parent 19fbab565a 9e4caa393c
commit 273cc9a4dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 43 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/controllers/cronfederatedhpa/cronfederatedhpa_controller.go
View File

@ -15,6 +15,7 @@ package cronfederatedhpa
import (
"context"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
@ -28,6 +29,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller"
autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
"github.com/karmada-io/karmada/pkg/metrics"
"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
"github.com/karmada-io/karmada/pkg/util/helper"
)
@ -70,6 +72,10 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim
return controllerruntime.Result{}, nil
}
var err error
startTime := time.Now()
defer metrics.ObserveProcessCronFederatedHPALatency(err, startTime)
origRuleSets := sets.New[string]()
for _, history := range cronFHPA.Status.ExecutionHistories {
origRuleSets.Insert(history.RuleName)
@ -84,7 +90,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim
newRuleSets := sets.New[string]()
for _, rule := range cronFHPA.Spec.Rules {
if err := c.processCronRule(cronFHPA, rule); err != nil {
if err = c.processCronRule(cronFHPA, rule); err != nil {
return controllerruntime.Result{Requeue: true}, err
}
newRuleSets.Insert(rule.Name)
@ -96,7 +102,7 @@ func (c *CronFHPAController) Reconcile(ctx context.Context, req controllerruntim
continue
}
c.CronHandler.StopRuleExecutor(req.NamespacedName.String(), name)
if err := c.removeCronFHPAHistory(cronFHPA, name); err != nil {
if err = c.removeCronFHPAHistory(cronFHPA, name); err != nil {
return controllerruntime.Result{Requeue: true}, err
}
}

3
pkg/controllers/cronfederatedhpa/cronfederatedhpa_job.go
View File

@ -34,6 +34,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
"github.com/karmada-io/karmada/pkg/metrics"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/helper"
)
@ -86,7 +87,9 @@ func RunCronFederatedHPARule(c *CronFederatedHPAJob) {
}
var scaleErr error
start := time.Now()
defer func() {
metrics.ObserveProcessCronFederatedHPARuleLatency(scaleErr, start)
if scaleErr != nil {
c.eventRecorder.Event(cronFHPA, corev1.EventTypeWarning, "ScaleFailed", scaleErr.Error())
err = c.addFailedExecutionHistory(cronFHPA, scaleErr.Error())

38
pkg/metrics/resource.go
View File

@ -9,12 +9,14 @@ import (
)
const (
resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds"
resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds"
policyApplyAttemptsMetricsName = "policy_apply_attempts_total"
syncWorkDurationMetricsName = "binding_sync_work_duration_seconds"
syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds"
policyPreemptionMetricsName = "policy_preemption_total"
resourceMatchPolicyDurationMetricsName = "resource_match_policy_duration_seconds"
resourceApplyPolicyDurationMetricsName = "resource_apply_policy_duration_seconds"
policyApplyAttemptsMetricsName = "policy_apply_attempts_total"
syncWorkDurationMetricsName = "binding_sync_work_duration_seconds"
syncWorkloadDurationMetricsName = "work_sync_workload_duration_seconds"
policyPreemptionMetricsName = "policy_preemption_total"
cronFederatedHPADurationMetricsName = "cron_federated_hpa_process_duration_seconds"
cronFederatedHPARuleDurationMetricsName = "cron_federated_hpa_rule_process_duration_seconds"
)
var (
@ -51,6 +53,18 @@ var (
Name: policyPreemptionMetricsName,
Help: "Number of preemption for the resource template. By the result, 'error' means a resource template failed to be preempted by other propagation policies. Otherwise 'success'.",
}, []string{"result"})
cronFederatedHPADurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: cronFederatedHPADurationMetricsName,
Help: "Duration in seconds to process a cron federated HPA. By the result, 'error' means a cron federated HPA failed to be processed. Otherwise 'success'.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 12),
}, []string{"result"})
cronFederatedHPARuleDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: cronFederatedHPARuleDurationMetricsName,
Help: "Duration in seconds to process a cron federated HPA rule. By the result, 'error' means a cron federated HPA rule failed to be processed. Otherwise 'success'.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 12),
}, []string{"result"})
)
// ObserveFindMatchedPolicyLatency records the duration for the resource finding a matched policy.
@ -79,6 +93,16 @@ func CountPolicyPreemption(err error) {
policyPreemptionCounter.WithLabelValues(utilmetrics.GetResultByError(err)).Inc()
}
// ObserveProcessCronFederatedHPALatency records the duration to process a cron federated HPA.
func ObserveProcessCronFederatedHPALatency(err error, start time.Time) {
cronFederatedHPADurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start))
}
// ObserveProcessCronFederatedHPARuleLatency records the duration to process a cron federated HPA rule.
func ObserveProcessCronFederatedHPARuleLatency(err error, start time.Time) {
cronFederatedHPARuleDurationHistogram.WithLabelValues(utilmetrics.GetResultByError(err)).Observe(utilmetrics.DurationInSeconds(start))
}
// ResourceCollectors returns the collectors about resources.
func ResourceCollectors() []prometheus.Collector {
return []prometheus.Collector{
@ -88,6 +112,8 @@ func ResourceCollectors() []prometheus.Collector {
syncWorkDurationHistogram,
syncWorkloadDurationHistogram,
policyPreemptionCounter,
cronFederatedHPADurationHistogram,
cronFederatedHPARuleDurationHistogram,
}
}