diff --git a/tests.mk b/tests.mk
index 4d27156a7..b16a3fd90 100644
--- a/tests.mk
+++ b/tests.mk
@@ -55,7 +55,8 @@ endif
 	mkdir -p /root/.ssh
 	rm -f /root/.ssh/dokku_test_rsa*
 	echo -e  "y\n" | ssh-keygen -f /root/.ssh/dokku_test_rsa -t rsa -N ''
-	chmod 600 /root/.ssh/dokku_test_rsa*
+	chmod 600 /root/.ssh/dokku_test_rsa
+	chmod 644 /root/.ssh/dokku_test_rsa.pub
 
 	@echo "-----> Setting up ssh config..."
 ifneq ($(shell ls /root/.ssh/config >/dev/null 2>&1 ; echo $$?),0)