diff --git a/contrib/dokku_client.sh b/contrib/dokku_client.sh
index f1efe9e2e..32a3dde63 100755
--- a/contrib/dokku_client.sh
+++ b/contrib/dokku_client.sh
@@ -168,11 +168,18 @@ main() {
     remote:remove)
       shift 1
       git remote remove "$@"
-      exit "$?"
+      local exit_code="$?"
+      if [[ "$(git config dokku.remote)" == "$1" ]]; then
+        git config --unset dokku.remote
+      fi
+      exit "$exit_code"
       ;;
     remote:unset)
-      git config --unset dokku.remote
-      exit "$?"
+      if [[ -n "$(git config dokku.remote)" ]]; then
+        git config --unset dokku.remote
+        exit "$?"
+      fi
+      exit 0
       ;;
   esac
 
diff --git a/tests/unit/client.bats b/tests/unit/client.bats
index efdc1a70c..a31b2d77b 100644
--- a/tests/unit/client.bats
+++ b/tests/unit/client.bats
@@ -265,13 +265,12 @@ teardown() {
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output "dokku2"
 
   run /bin/bash -c "${BATS_TEST_DIRNAME}/../../contrib/dokku_client.sh remote:list"
   echo "output: $output"
   echo "status: $status"
   assert_success
-  assert_output "dokku\dokku2"
+  assert_output "$(printf "dokku\ndokku2")"
 
   run /bin/bash -c "${BATS_TEST_DIRNAME}/../../contrib/dokku_client.sh remote:remove dokku2"
   echo "output: $output"
@@ -289,6 +288,11 @@ teardown() {
   echo "status: $status"
   assert_success
 
+  run /bin/bash -c "${BATS_TEST_DIRNAME}/../../contrib/dokku_client.sh remote:unset"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
   run /bin/bash -c "${BATS_TEST_DIRNAME}/../../contrib/dokku_client.sh remote"
   echo "output: $output"
   echo "status: $status"