Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[extension/cgroupruntime] Be aware of ECS task and CPU limits #36920

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion extension/cgroupruntimeextension/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

## Overview

The OpenTelemetry Cgroup Auto-Config Extension is designed to optimize Go runtime performance in containerized environments by automatically configuring GOMAXPROCS and GOMEMLIMIT based on the Linux cgroup filesystem. This extension leverages [automaxprocs](https://github.com/uber-go/automaxprocs) and [automemlimit](https://github.com/KimMachineGun/automemlimit) packages to dynamically adjust Go runtime variables, ensuring efficient resource usage aligned with container limits.
The OpenTelemetry Cgroup Auto-Config Extension is designed to optimize Go runtime performance in containerized environments by automatically configuring GOMAXPROCS and GOMEMLIMIT based on the Linux cgroup filesystem. This extension leverages [automaxprocs](https://github.com/uber-go/automaxprocs) or [gomaxecs](https://github.com/rdforte/gomaxecs) for AWS ECS Tasks and [automemlimit](https://github.com/KimMachineGun/automemlimit) packages to dynamically adjust Go runtime variables, ensuring efficient resource usage aligned with container limits.

## Configuration

Expand Down
15 changes: 12 additions & 3 deletions extension/cgroupruntimeextension/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"runtime/debug"

"github.com/KimMachineGun/automemlimit/memlimit"
gomaxecs "github.com/rdforte/gomaxecs/maxprocs"
"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/extension"
"go.uber.org/automaxprocs/maxprocs"
Expand Down Expand Up @@ -42,9 +43,17 @@ func createExtension(_ context.Context, set extension.Settings, cfg component.Co
cgroupConfig := cfg.(*Config)
return newCgroupRuntime(cgroupConfig, set.Logger,
func() (undoFunc, error) {
undo, err := maxprocs.Set(maxprocs.Logger(func(str string, params ...any) {
set.Logger.Debug(fmt.Sprintf(str, params))
}))
var undo func()
var err error
if gomaxecs.IsECS() {
undo, err = gomaxecs.Set(gomaxecs.WithLogger(func(str string, params ...any) {
set.Logger.Debug(fmt.Sprintf(str, params))
}))
} else {
undo, err = maxprocs.Set(maxprocs.Logger(func(str string, params ...any) {
set.Logger.Debug(fmt.Sprintf(str, params))
}))
}
return undoFunc(undo), err
},
func(ratio float64) (undoFunc, error) {
Expand Down
3 changes: 2 additions & 1 deletion extension/cgroupruntimeextension/go.mod
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
module github.com/open-telemetry/opentelemetry-collector-contrib/extension/cgroupruntimeextension

go 1.22.0
go 1.22.4

require (
github.com/KimMachineGun/automemlimit v0.6.1
github.com/containerd/cgroups/v3 v3.0.2
github.com/rdforte/gomaxecs v1.1.0
github.com/stretchr/testify v1.10.0
go.opentelemetry.io/collector/component v0.116.0
go.opentelemetry.io/collector/component/componenttest v0.116.0
Expand Down
2 changes: 2 additions & 0 deletions extension/cgroupruntimeextension/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 44 additions & 0 deletions extension/cgroupruntimeextension/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ package cgroupruntimeextension // import "github.com/open-telemetry/opentelemetr

import (
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
Expand Down Expand Up @@ -63,6 +66,21 @@ func cgroupMaxCpu(filename string) (quota int64, period uint64, err error) {
return quota, period, err
}

func startMockECSServer() *httptest.Server {
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response := map[string]interface{}{
"DockerID": "container-id",
"Limits": map[string]interface{}{
"CPU": 2.0,
},
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
})

return httptest.NewServer(handler)
}

func TestCgroupV2SudoIntegration(t *testing.T) {
checkCgroupSystem(t)
pointerInt64 := func(val int64) *int64 {
Expand All @@ -81,6 +99,7 @@ func TestCgroupV2SudoIntegration(t *testing.T) {
config *Config
expectedGoMaxProcs int
expectedGoMemLimit int64
setECSMetadataURI bool
}{
{
name: "90% the max cgroup memory and 12 GOMAXPROCS",
Expand Down Expand Up @@ -144,6 +163,24 @@ func TestCgroupV2SudoIntegration(t *testing.T) {
// 134217728 * 0.1
expectedGoMemLimit: 13421772,
},
{
name: "running on AWS ECS with 90% of max cgroup memory and 2 GOMAXPROCS",
cgroupCpuQuota: pointerInt64(-1),
cgroupCpuPeriod: 8000,
cgroupMaxMemory: 134217728, // 128 MB
config: &Config{
GoMaxProcs: GoMaxProcsConfig{
Enabled: true,
},
GoMemLimit: GoMemLimitConfig{
Enabled: true,
Ratio: 0.9,
},
},
expectedGoMaxProcs: 22,
expectedGoMemLimit: 120795955, // 134217728 * 0.9
setECSMetadataURI: true,
},
}

cgroupPath, err := cgroup2.PidGroupPath(os.Getpid())
Expand Down Expand Up @@ -220,6 +257,13 @@ func TestCgroupV2SudoIntegration(t *testing.T) {
})
require.NoError(t, err)

if test.setECSMetadataURI {
server := startMockECSServer()
defer server.Close()
os.Setenv("ECS_CONTAINER_METADATA_URI_V4", server.URL)
defer os.Unsetenv("ECS_CONTAINER_METADATA_URI_V4")
}

factory := NewFactory()
ctx := context.Background()
extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), test.config)
Expand Down