diff --git a/dokku b/dokku
index a32bd815c..98cb9679d 100755
--- a/dokku
+++ b/dokku
@@ -178,8 +178,12 @@ case "$1" in
         trap '' INT HUP
         sleep $WAIT
         for oldid in $oldids; do
-          docker stop $oldid &> /dev/null
-          docker kill $oldid &> /dev/null  # force a kill as docker seems to not send SIGKILL as the docs would indicate
+          # Attempt to stop, if that fails, then force a kill as docker seems
+          # to not send SIGKILL as the docs would indicate. If that fails, move
+          # on to the next.
+          docker stop $oldid \
+          || docker kill $oldid \
+          || :  # Always continue in case we have multiple to kill
         done
       ) & disown -a
       # Use trap since disown/nohup don't seem to keep child alive