44cd566178
Sysctls the kernel does not namespace, such as `vm.max_map_count`, cannot be set from a pod spec and previously had no answer beyond editing `/etc/sysctl.d` on each host by hand. `scheduler-k3s:node-sysctls:set` now applies them through a privileged daemonset, which reaches nodes joined later and reapplies after a reboot. Sysctls may be scoped to a node profile, with a profile scope inheriting the global values and overriding them on conflict so that every node is covered by exactly one daemonset. Clearing a sysctl stops dokku managing it but does not restore the previous value, which persists until the node reboots.
148 lines
4.2 KiB
Bash
148 lines
4.2 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
load test_helper
|
|
|
|
TEST_APP="rdmtestapp"
|
|
|
|
setup() {
|
|
uninstall_k3s || true
|
|
global_setup
|
|
dokku nginx:stop
|
|
export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
|
|
}
|
|
|
|
teardown() {
|
|
dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count || true
|
|
global_teardown
|
|
dokku nginx:start
|
|
uninstall_k3s || true
|
|
}
|
|
|
|
@test "(scheduler-k3s) docker-options sysctl" {
|
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
|
fi
|
|
|
|
INGRESS_CLASS=nginx install_k3s
|
|
|
|
run /bin/bash -c "dokku apps:create $TEST_APP"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run deploy_app python dokku@$DOKKU_DOMAIN:$TEST_APP
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "null"
|
|
|
|
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy '--sysctl net.ipv4.ip_unprivileged_port_start=1024'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "dokku ps:restart $TEST_APP"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls[0].name'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "net.ipv4.ip_unprivileged_port_start"
|
|
|
|
run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.securityContext.sysctls[0].value'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "1024"
|
|
|
|
run /bin/bash -c "dokku docker-options:remove $TEST_APP deploy '--sysctl net.ipv4.ip_unprivileged_port_start=1024'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "dokku docker-options:add $TEST_APP deploy '--sysctl vm.max_map_count=262144'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "dokku ps:restart $TEST_APP"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_failure
|
|
assert_output_contains "is not namespaced" -1
|
|
}
|
|
|
|
@test "(scheduler-k3s:node-sysctls) applies non-namespaced sysctls to nodes" {
|
|
if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then
|
|
skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"
|
|
fi
|
|
|
|
INGRESS_CLASS=nginx install_k3s
|
|
|
|
run /bin/bash -c "kubectl get daemonset -n kube-system dokku-node-sysctls-global"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_failure
|
|
|
|
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count 262144"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:report --format json | jq -r '.\"--global\".\"vm.max_map_count\"'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "262144"
|
|
|
|
run /bin/bash -c "kubectl rollout status daemonset -n kube-system dokku-node-sysctls-global --timeout=120s"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run /bin/bash -c "kubectl get daemonset -n kube-system dokku-node-sysctls-global -o json | jq -r '.status.desiredNumberScheduled'"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "$(kubectl get nodes --no-headers | wc -l | tr -d ' ')"
|
|
|
|
run /bin/bash -c "cat /proc/sys/vm/max_map_count"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
assert_output "262144"
|
|
|
|
run /bin/bash -c "dokku scheduler-k3s:node-sysctls:set --global vm.max_map_count"
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
|
|
run wait_for_daemonset_deletion dokku-node-sysctls-global
|
|
echo "output: $output"
|
|
echo "status: $status"
|
|
assert_success
|
|
}
|
|
|
|
wait_for_daemonset_deletion() {
|
|
declare desc="waits for a daemonset to be removed from the api server"
|
|
declare NAME="$1"
|
|
|
|
for _ in $(seq 1 30); do
|
|
if ! kubectl get daemonset -n kube-system "$NAME" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
echo "daemonset $NAME still exists after 60s"
|
|
return 1
|
|
}
|