Merge pull request #6608 from dokku/k3s-resources

Fix issue with setting k3s resource values and lower the initial default values
This commit is contained in:
Jose Diaz-Gonzalez authored and GitHub committed 2024-02-22 07:30:10 -05:00
commit 7d85e213be
5 files changed
+163 -54

No files matched your search

+4 -12
View File
@@ -361,20 +361,12 @@ App logs for the `logs` command are fetched by Dokku from running containers via
### Supported Resource Management Properties ### Supported Resource Management Properties
The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_. The following properties are supported: The `k3s` scheduler supports a minimal list of resource _limits_ and _reservations_:
#### Resource Limits
> [!NOTE]
> Cron tasks retrieve resource limits based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM.
- cpu: is specified in number of CPUs a process can access. - cpu: is specified in number of CPUs a process can access.
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes). - memory: should be specified with a suffix of `b` (bytes), `Ki` (kilobytes), `Mi` (megabytes), `Gi` (gigabytes). Default unit is `Mi` (megabytes).
#### Resource Reservations If unspecified for any task, the default reservation will be `.1` CPU and `128Mi` RAM, with no limit set for either CPU or RAM. This is to avoid issues with overscheduling pods on a cluster. To avoid issues, set more specific values for at least resource reservations. If unbounded utilization is desired, set CPU and Memory to `0m` and `0Mi`, respectively.
> [!NOTE] > [!NOTE]
> Cron tasks retrieve resource reservations based on the computed cron task ID. If unspecified, the default will be 1 CPU and 512m RAM. > Cron tasks retrieve resource limits based on the computed cron task ID.
- cpu: is specified in number of CPUs a process can access.
- memory: should be specified with a suffix of `b` (bytes), `k` (kilobytes), `m` (megabytes), `g` (gigabytes). Default unit is `m` (megabytes).
+5 -1
View File
@@ -125,7 +125,11 @@ func CallExecCommandWithContext(ctx context.Context, input ExecCommandInput) (Ex
} }
if os.Getenv("DOKKU_TRACE") == "1" { if os.Getenv("DOKKU_TRACE") == "1" {
cmd.PrintCommand = true argsSt := ""
if len(cmd.Args) > 0 {
argsSt = strings.Join(cmd.Args, " ")
}
LogWarn(fmt.Sprintf("exec: %s %s", cmd.Command, argsSt))
} }
if input.Stdin != nil { if input.Stdin != nil {
+15 -1
View File
@@ -2,7 +2,10 @@ package common
import ( import (
"context" "context"
"fmt"
"io" "io"
"os"
"strings"
) )
// PlugnTriggerInput is the input for CallPlugnTrigger // PlugnTriggerInput is the input for CallPlugnTrigger
@@ -41,7 +44,7 @@ func CallPlugnTrigger(input PlugnTriggerInput) (ExecCommandResponse, error) {
func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (ExecCommandResponse, error) { func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (ExecCommandResponse, error) {
args := []string{"trigger", input.Trigger} args := []string{"trigger", input.Trigger}
args = append(args, input.Args...) args = append(args, input.Args...)
return CallExecCommandWithContext(ctx, ExecCommandInput{ result, err := CallExecCommandWithContext(ctx, ExecCommandInput{
Command: "plugn", Command: "plugn",
Args: args, Args: args,
DisableStdioBuffer: input.DisableStdioBuffer, DisableStdioBuffer: input.DisableStdioBuffer,
@@ -51,4 +54,15 @@ func CallPlugnTriggerWithContext(ctx context.Context, input PlugnTriggerInput) (
StreamStdout: input.StreamStdout, StreamStdout: input.StreamStdout,
StreamStderr: input.StreamStderr, StreamStderr: input.StreamStderr,
}) })
if os.Getenv("DOKKU_TRACE") == "1" {
for _, line := range strings.Split(result.Stderr, "\n") {
LogDebug(fmt.Sprintf("plugn trigger %s stderr: %s", input.Trigger, line))
}
for _, line := range strings.Split(result.Stdout, "\n") {
LogDebug(fmt.Sprintf("plugn trigger %s stdout: %s", input.Trigger, line))
}
}
return result, err
} }
+67 -23
View File
@@ -824,55 +824,86 @@ func getProcessHealtchecks(healthchecks []appjson.Healthcheck, primaryPort int32
func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) { func getProcessResources(appName string, processType string) (ProcessResourcesMap, error) {
processResources := ProcessResourcesMap{ processResources := ProcessResourcesMap{
Limits: ProcessResources{ Limits: ProcessResources{},
CPU: "1000m",
Memory: "512Mi",
},
Requests: ProcessResources{ Requests: ProcessResources{
CPU: "1000m", CPU: "100m",
Memory: "512Mi", Memory: "128Mi",
}, },
} }
cpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "cpu"}...)
if err != nil && cpuLimit != "" && cpuLimit != "0" { emptyValues := map[string]bool{
_, err := resource.ParseQuantity(cpuLimit) "": true,
"0": true,
}
result, err := common.CallPlugnTrigger(common.PlugnTriggerInput{
Trigger: "resource-get-property",
Args: []string{appName, processType, "limit", "cpu"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := resource.ParseQuantity(result.StdoutContents())
if err != nil { if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err) return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu limit: %w", err)
} }
processResources.Limits.CPU = cpuLimit if quantity.MilliValue() != 0 {
processResources.Limits.CPU = quantity.String()
} else {
processResources.Limits.CPU = ""
}
} }
nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...) nvidiaGpuLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "nvidia-gpu"}...)
if err != nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" { if err == nil && nvidiaGpuLimit != "" && nvidiaGpuLimit != "0" {
_, err := resource.ParseQuantity(nvidiaGpuLimit) _, err := resource.ParseQuantity(nvidiaGpuLimit)
if err != nil { if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err) return ProcessResourcesMap{}, fmt.Errorf("Error parsing nvidia-gpu limit: %w", err)
} }
processResources.Limits.NvidiaGPU = nvidiaGpuLimit processResources.Limits.NvidiaGPU = nvidiaGpuLimit
} }
memoryLimit, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "limit", "memory"}...) result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
if err != nil && memoryLimit != "" && memoryLimit != "0" { Trigger: "resource-get-property",
_, err := resource.ParseQuantity(memoryLimit) Args: []string{appName, processType, "limit", "memory"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := parseMemoryQuantity(result.StdoutContents())
if err != nil { if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err) return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory limit: %w", err)
} }
processResources.Limits.Memory = memoryLimit if quantity != "0Mi" {
processResources.Limits.Memory = quantity
} else {
processResources.Limits.Memory = ""
}
} }
cpuRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "cpu"}...) result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
if err != nil && cpuRequest != "" && cpuRequest != "0" { Trigger: "resource-get-property",
_, err := resource.ParseQuantity(cpuRequest) Args: []string{appName, processType, "reserve", "cpu"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := resource.ParseQuantity(result.StdoutContents())
if err != nil { if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err) return ProcessResourcesMap{}, fmt.Errorf("Error parsing cpu request: %w", err)
} }
processResources.Requests.CPU = cpuRequest if quantity.MilliValue() != 0 {
processResources.Requests.CPU = quantity.String()
} else {
processResources.Requests.CPU = ""
}
} }
memoryRequest, err := common.PlugnTriggerOutputAsString("resource-get-property", []string{appName, processType, "reserve", "memory"}...) result, err = common.CallPlugnTrigger(common.PlugnTriggerInput{
if err != nil && memoryRequest != "" && memoryRequest != "0" { Trigger: "resource-get-property",
_, err := resource.ParseQuantity(memoryRequest) Args: []string{appName, processType, "reserve", "memory"},
})
if err == nil && !emptyValues[result.StdoutContents()] {
quantity, err := parseMemoryQuantity(result.StdoutContents())
if err != nil { if err != nil {
return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err) return ProcessResourcesMap{}, fmt.Errorf("Error parsing memory request: %w", err)
} }
processResources.Requests.Memory = memoryRequest if quantity != "0Mi" {
processResources.Requests.Memory = quantity
} else {
processResources.Requests.Memory = ""
}
} }
return processResources, nil return processResources, nil
@@ -1182,6 +1213,19 @@ func kubernetesNodeToNode(node v1.Node) Node {
} }
} }
// parseMemoryQuantity parses a string into a valid memory quantity
func parseMemoryQuantity(input string) (string, error) {
if _, err := strconv.ParseInt(input, 10, 64); err == nil {
input = fmt.Sprintf("%sMi", input)
}
quantity, err := resource.ParseQuantity(input)
if err != nil {
return "", err
}
return quantity.String(), nil
}
func uninstallHelperCommands(ctx context.Context) error { func uninstallHelperCommands(ctx context.Context) error {
errs, _ := errgroup.WithContext(ctx) errs, _ := errgroup.WithContext(ctx)
errs.Go(func() error { errs.Go(func() error {
+72 -17
View File
@@ -70,22 +70,6 @@ uninstall_k3s() {
assert_success assert_success
} }
@test "(scheduler-k3s) install traefik" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
install_k3s
}
@test "(scheduler-k3s) install nginx" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi
INGRESS_CLASS=nginx install_k3s
}
@test "(scheduler-k3s) install traefik with taint" { @test "(scheduler-k3s) install traefik with taint" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
@@ -102,7 +86,7 @@ uninstall_k3s() {
INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s INGRESS_CLASS=nginx TAINT_SCHEDULING=true install_k3s
} }
@test "(scheduler-k3s) deploy traefik" { @test "(scheduler-k3s) deploy traefik [resource]" {
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
fi fi
@@ -130,6 +114,77 @@ uninstall_k3s() {
assert_success assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server" assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
# include resource tests
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "100m"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "128Mi"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "dokku resource:reserve $TEST_APP --memory 300 --cpu 0m --process-type web"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku resource:limit $TEST_APP --memory 512 --process-type web"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "dokku ps:rebuild $TEST_APP"
echo "output: $output"
echo "status: $status"
assert_success
run /bin/bash -c "sleep 20"
echo "output: $output"
echo "status: $status"
assert_success
assert_http_localhost_response "http" "$TEST_APP.dokku.me" "80" "" "python/http.server"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.requests.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "300Mi"
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.cpu}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output ""
run /bin/bash -c "kubectl get pods -o=jsonpath='{.items[*]..resources.limits.memory}'"
echo "output: $output"
echo "status: $status"
assert_success
assert_output "512Mi"
} }
@test "(scheduler-k3s) deploy nginx" { @test "(scheduler-k3s) deploy nginx" {