Merge pull request #6608 from dokku/k3s-resources
Fix issue with setting k3s resource values and lower the initial default values
This commit is contained in:
5 files changed
+163
-54
No files matched your search
@@ -361,20 +361,12 @@ App logs for the `logs` command are fetched by Dokku from running containers via
|
|||||||
|
|
||||||
### Supported Resource Management Properties
|
### Supported Resource Management Properties
|
||||||
|
|
||||||
The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported:
|
The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_:
|
||||||
|
|
||||||
#### Resource Limits
|
|
||||||
|
|
||||||
> [!NOTE]
|
|
||||||
> Cron tasks retrieve resource limits based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
|
|
||||||
|
|
||||||
- cpu: is specified in number of CPUs a process can access.
|
- cpu: is specified in number of CPUs a process can access.
|
||||||
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
|
- memory: should be specified with a suffix of `b` (bytes), `Ki` (kilobytes), `Mi` (megabytes), `Gi` (gigabytes). Default unit is `Mi` (megabytes).
|
||||||
|
|
||||||
#### Resource Reservations
|
If unspecified for any task, the default reservation will be `.1` CPU and `128Mi` RAM, with no limit set for either CPU or RAM. This is to avoid issues with overscheduling pods on a cluster. To avoid issues, set more specific values for at least resource reservations. If unbounded utilization is desired, set CPU and Memory to `0m` and `0Mi`, respectively.
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> Cron tasks retrieve resource reservations based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
|
> Cron tasks retrieve resource limits based on the computed cron task ID.
|
||||||
|
|
||||||
- cpu: is specified in number of CPUs a process can access.
|
|
||||||
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
|
|
||||||
@@ -125,7 +125,11 @@ func CallExecCommandWithContext(ctx context.Context, input ExecCommandInput) (Ex
|
|||||||
}
|
}
|
||||||
|
|
||||||
if os.Getenv("DOKKU_TRACE") == "1" {
|
if os.Getenv("DOKKU_TRACE") == "1" {
|
||||||
cmd.PrintCommand = true
|
argsSt := ""
|
||||||
|
if len(cmd.Args) > 0 {
|
||||||
|
argsSt = strings.Join(cmd.Args, " ")
|
||||||
|
}
|
||||||
|
LogWarn(fmt.Sprintf("exec: %s %s", cmd.Command, argsSt))
|
||||||
}
|
}
|
||||||
|
|
||||||
if input.Stdin != nil {
|
if input.Stdin != nil {
|
||||||
|
|||||||
+15
-1
@@ -2,7 +2,10 @@ package common
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PlugnTriggerInput is the input for CallPlugnTrigger
|
// PlugnTriggerInput is the input for CallPlugnTrigger
|
||||||
@@ -41,7 +44,7 @@ func CallPlugnTrigger(input PlugnTriggerInput) (ExecCommandResponse, error) {
|
|||||||
func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (ExecCommandResponse, error) {
|
func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (ExecCommandResponse, error) {
|
||||||
args := []string{"trigger", input.Trigger}
|
args := []string{"trigger", input.Trigger}
|
||||||
args = append(args, input.Args...)
|
args = append(args, input.Args...)
|
||||||
return CallExecCommandWithContext(ctx, ExecCommandInput{
|
result, err := CallExecCommandWithContext(ctx, ExecCommandInput{
|
||||||
Command: "plugn",
|
Command: "plugn",
|
||||||
Args: args,
|
Args: args,
|
||||||
DisableStdioBuffer: input.DisableStdioBuffer,
|
DisableStdioBuffer: input.DisableStdioBuffer,
|
||||||
@@ -51,4 +54,15 @@ func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (
|
|||||||
StreamStdout: input.StreamStdout,
|
StreamStdout: input.StreamStdout,
|
||||||
StreamStderr: input.StreamStderr,
|
StreamStderr: input.StreamStderr,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if os.Getenv("DOKKU_TRACE") == "1" {
|
||||||
|
for _, line := range strings.Split(result.Stderr, "\n") {
|
||||||
|
LogDebug(fmt.Sprintf("plugn trigger %s stderr: %s", input.Trigger, line))
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(result.Stdout, "\n") {
|
||||||
|
LogDebug(fmt.Sprintf("plugn trigger %s stdout: %s", input.Trigger, line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, err
|
||||||
}
|
}
|
||||||
@@ -824,55 +824,86 @@ func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32
|
|||||||
|
|
||||||
func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
|
func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
|
||||||
processResources := ProcessResourcesMap{
|
processResources := ProcessResourcesMap{
|
||||||
Limits: ProcessResources{
|
Limits: ProcessResources{},
|
||||||
CPU: "1000m",
|
|
||||||
Memory: "512Mi",
|
|
||||||
},
|
|
||||||
Requests: ProcessResources{
|
Requests: ProcessResources{
|
||||||
CPU: "1000m",
|
CPU: "100m",
|
||||||
Memory: "512Mi",
|
Memory: "128Mi",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...)
|
|
||||||
if err != nil && cpuLimit != "" && cpuLimit != "0" {
|
emptyValues := map[string]bool{
|
||||||
_, err := resource.ParseQuantity(cpuLimit)
|
"": true,
|
||||||
|
"0": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
|
||||||
|
Trigger: "resource-get-property",
|
||||||
|
Args: []string{appName, processType, "limit", "cpu"},
|
||||||
|
})
|
||||||
|
if err == nil && !emptyValues[result.StdoutContents()] {
|
||||||
|
quantity, err := resource.ParseQuantity(result.StdoutContents())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
|
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
|
||||||
}
|
}
|
||||||
processResources.Limits.CPU = cpuLimit
|
if quantity.MilliValue() != 0 {
|
||||||
|
processResources.Limits.CPU = quantity.String()
|
||||||
|
} else {
|
||||||
|
processResources.Limits.CPU = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
|
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
|
||||||
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
|
if err == nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
|
||||||
_, err := resource.ParseQuantity(nvidiaGpuLimit)
|
_, err := resource.ParseQuantity(nvidiaGpuLimit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
|
return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
|
||||||
}
|
}
|
||||||
processResources.Limits.NvidiaGPU = nvidiaGpuLimit
|
processResources.Limits.NvidiaGPU = nvidiaGpuLimit
|
||||||
}
|
}
|
||||||
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...)
|
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
||||||
if err != nil && memoryLimit != "" && memoryLimit != "0" {
|
Trigger: "resource-get-property",
|
||||||
_, err := resource.ParseQuantity(memoryLimit)
|
Args: []string{appName, processType, "limit", "memory"},
|
||||||
|
})
|
||||||
|
if err == nil && !emptyValues[result.StdoutContents()] {
|
||||||
|
quantity, err := parseMemoryQuantity(result.StdoutContents())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
|
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
|
||||||
}
|
}
|
||||||
processResources.Limits.Memory = memoryLimit
|
if quantity != "0Mi" {
|
||||||
|
processResources.Limits.Memory = quantity
|
||||||
|
} else {
|
||||||
|
processResources.Limits.Memory = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...)
|
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
||||||
if err != nil && cpuRequest != "" && cpuRequest != "0" {
|
Trigger: "resource-get-property",
|
||||||
_, err := resource.ParseQuantity(cpuRequest)
|
Args: []string{appName, processType, "reserve", "cpu"},
|
||||||
|
})
|
||||||
|
if err == nil && !emptyValues[result.StdoutContents()] {
|
||||||
|
quantity, err := resource.ParseQuantity(result.StdoutContents())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
|
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
|
||||||
}
|
}
|
||||||
processResources.Requests.CPU = cpuRequest
|
if quantity.MilliValue() != 0 {
|
||||||
|
processResources.Requests.CPU = quantity.String()
|
||||||
|
} else {
|
||||||
|
processResources.Requests.CPU = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...)
|
result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
|
||||||
if err != nil && memoryRequest != "" && memoryRequest != "0" {
|
Trigger: "resource-get-property",
|
||||||
_, err := resource.ParseQuantity(memoryRequest)
|
Args: []string{appName, processType, "reserve", "memory"},
|
||||||
|
})
|
||||||
|
if err == nil && !emptyValues[result.StdoutContents()] {
|
||||||
|
quantity, err := parseMemoryQuantity(result.StdoutContents())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
|
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
|
||||||
}
|
}
|
||||||
processResources.Requests.Memory = memoryRequest
|
if quantity != "0Mi" {
|
||||||
|
processResources.Requests.Memory = quantity
|
||||||
|
} else {
|
||||||
|
processResources.Requests.Memory = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return processResources, nil
|
return processResources, nil
|
||||||
@@ -1182,6 +1213,19 @@ func kubernetesNodeToNode(node v1.Node) Node {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseMemoryQuantity parses a string into a valid memory quantity
|
||||||
|
func parseMemoryQuantity(input string) (string, error) {
|
||||||
|
if _, err := strconv.ParseInt(input, 10, 64); err == nil {
|
||||||
|
input = fmt.Sprintf("%sMi", input)
|
||||||
|
}
|
||||||
|
quantity, err := resource.ParseQuantity(input)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return quantity.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
func uninstallHelperCommands(ctx context.Context) error {
|
func uninstallHelperCommands(ctx context.Context) error {
|
||||||
errs, _ := errgroup.WithContext(ctx)
|
errs, _ := errgroup.WithContext(ctx)
|
||||||
errs.Go(func() error {
|
errs.Go(func() error {
|
||||||
|
|||||||
@@ -70,22 +70,6 @@ uninstall_k3s() {
|
|||||||
assert_success
|
assert_success
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "(scheduler-k3s) install traefik" {
|
|
||||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
|
||||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
|
||||||
fi
|
|
||||||
|
|
||||||
install_k3s
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "(scheduler-k3s) install nginx" {
|
|
||||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
|
||||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
|
||||||
fi
|
|
||||||
|
|
||||||
INGRESS_CLASS=nginx install_k3s
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "(scheduler-k3s) install traefik with taint" {
|
@test "(scheduler-k3s) install traefik with taint" {
|
||||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
@@ -102,7 +86,7 @@ uninstall_k3s() {
|
|||||||
INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s
|
INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "(scheduler-k3s) deploy traefik" {
|
@test "(scheduler-k3s) deploy traefik [resource]" {
|
||||||
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
||||||
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
||||||
fi
|
fi
|
||||||
@@ -130,6 +114,77 @@ uninstall_k3s() {
|
|||||||
assert_success
|
assert_success
|
||||||
|
|
||||||
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
|
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
|
||||||
|
|
||||||
|
# include resource tests
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "100m"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "128Mi"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output ""
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output ""
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku resource:reserve $TEST_APP --memory 300 --cpu 0m --process-type web"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku resource:limit $TEST_APP --memory 512 --process-type web"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
run /bin/bash -c "sleep 20"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
|
||||||
|
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output ""
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "300Mi"
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output ""
|
||||||
|
|
||||||
|
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
|
||||||
|
echo "output: $output"
|
||||||
|
echo "status: $status"
|
||||||
|
assert_success
|
||||||
|
assert_output "512Mi"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "(scheduler-k3s) deploy nginx" {
|
@test "(scheduler-k3s) deploy nginx" {
|
||||||
|
|||||||
Reference in new issue
Block a user