From 96c737eae475eaa591b2e8ad84a50631a753b913 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:19:10 -0500 Subject: [PATCH 01/28] refactor: update digitalocean image building to match what Digitalocean does The scripts run/installed on the server are based on what was given to me by Digitalocean in 2019, so they may have diverged a bit since. --- .../files/etc/update-motd.d/99-one-click | 22 +++++ .../var/lib/cloud/scripts/per-once/001_setup | 25 +++++ .../lib/cloud/scripts/per-once/002_enable_ssh | 9 ++ .../images/digitalocean/in_parts/011-docker | 43 +++++++++ .../digitalocean/in_parts/011-ssh-message | 9 ++ .../digitalocean/in_parts/012-dokku-packages | 64 +++++++++++++ .../digitalocean/in_parts/012-grub-opts | 8 ++ .../digitalocean/in_parts/014-docker-dns | 6 ++ .../digitalocean/in_parts/014-ufw-rules | 12 +++ .../digitalocean/in_parts/099-application_tag | 20 ++++ .../images/digitalocean/in_parts/099-cleanup | 19 ++++ contrib/images/digitalocean/packer.pkr.hcl | 96 +++++++++++++++++++ contrib/packer.json | 32 ------- 13 files changed, 333 insertions(+), 32 deletions(-) create mode 100755 contrib/images/digitalocean/files/etc/update-motd.d/99-one-click create mode 100755 contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/001_setup create mode 100755 contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/002_enable_ssh create mode 100755 contrib/images/digitalocean/in_parts/011-docker create mode 100755 contrib/images/digitalocean/in_parts/011-ssh-message create mode 100755 contrib/images/digitalocean/in_parts/012-dokku-packages create mode 100755 contrib/images/digitalocean/in_parts/012-grub-opts create mode 100755 contrib/images/digitalocean/in_parts/014-docker-dns create mode 100755 contrib/images/digitalocean/in_parts/014-ufw-rules create mode 100755 contrib/images/digitalocean/in_parts/099-application_tag create mode 100755 contrib/images/digitalocean/in_parts/099-cleanup create mode 100644 contrib/images/digitalocean/packer.pkr.hcl delete mode 100644 contrib/packer.json diff --git a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click new file mode 100755 index 000000000..c9fc474ee --- /dev/null +++ b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Configured as part of the DigitalOcean 1-Click Image build process + +myip=$(hostname -I | awk '{print$1}') +cat < >(tee /var/log/one_click_setup.log) 2>&1 + +ufw --force enable +printf "dokku dokku/hostname string %s" hostname | debconf-set-selections + +export DEBIAN_FRONTEND=noninteractive +export LANG=C +export LC_ALL=C + +# Install Dokku.... +for count in {1..30}; do + dpkg -i /var/lib/digitalocean/debs/*deb || /bin/true + apt-get -f install || /bin/true + + version=$(dpkg-query --showformat='${Version}' -W dokku) + if [[ -n "$version" ]]; then + ran=0 + exit 0 + else + count=$(($count + 1)) + fi + sleep 5 +done diff --git a/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/002_enable_ssh b/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/002_enable_ssh new file mode 100755 index 000000000..48bd764a5 --- /dev/null +++ b/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/002_enable_ssh @@ -0,0 +1,9 @@ +#!/bin/bash +exec > >(tee /var/log/one_click_setup.log) 2>&1 + +# Remove the force command +sed -e '/Match user root/d' \ + -e '/.*ForceCommand.*droplet.*/d' \ + -i /etc/ssh/sshd_config + +systemctl restart ssh diff --git a/contrib/images/digitalocean/in_parts/011-docker b/contrib/images/digitalocean/in_parts/011-docker new file mode 100755 index 000000000..65d2955e3 --- /dev/null +++ b/contrib/images/digitalocean/in_parts/011-docker @@ -0,0 +1,43 @@ +#!/bin/bash +set -eo pipefail + +export DEBIAN_FRONTEND=noninteractive + +pkgs=(apt-transport-https + ca-certificates + curl + jq + linux-image-extra-virtual + software-properties-common +) + +echo '--> Updating apt repositories' +apt-get -qqy update + +echo "--> Installing basic dependencies: ${pkgs[*]}" +apt-get -qqy install "${pkgs[@]}" + +echo '--> Setting up docker apt repository' +curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + +cat >/etc/apt/sources.list.d/docker.list < Installing docker-ce' +apt-get -y update +apt-get -y install docker-ce docker-compose-plugin + +echo '--> Enabling docker-ce' +systemctl enable docker +systemctl start docker + +echo '--> Enabling docker live-restore' +if [[ ! -f /etc/docker/daemon.json ]]; then + echo "{}" >/etc/docker/daemon.json +fi + +config="$(jq '. + {"live-restore": true}' /etc/docker/daemon.json)" +echo "$config" >/etc/docker/daemon.json +systemctl reload docker +docker info | grep -i live diff --git a/contrib/images/digitalocean/in_parts/011-ssh-message b/contrib/images/digitalocean/in_parts/011-ssh-message new file mode 100755 index 000000000..da7a0b5df --- /dev/null +++ b/contrib/images/digitalocean/in_parts/011-ssh-message @@ -0,0 +1,9 @@ +#!/bin/bash +set -eo pipefail + +echo '--> Prevent login until after setup is complete' +# Be a bit of a dork about login in +cat >>/etc/ssh/sshd_config < Install core dependencies for dokku' +apt-get -y install "${pkgs[@]}" + +echo '--> Setting up dokku apt repository' +curl -fsSL https://packagecloud.io/dokku/dokku/gpgkey -o /etc/apt/keyrings/dokku.asc + +cat >/etc/apt/sources.list.d/dokku.list < Install dependencies for dokku from new repository' +apt-get -y clean +apt-get -y update +apt-get -y install "${dokku_pkgs[@]}" + +echo '--> Pre-cache dokku bits that are must run on boot' +apt-get -y clean +apt-get -y install --download-only "dokku=${DOKKU_VERSION}" dokku-event-listener herokuish + +mkdir -p /var/lib/digitalocean/debs +cp -au /var/cache/apt/archives/*deb /var/lib/digitalocean/debs diff --git a/contrib/images/digitalocean/in_parts/012-grub-opts b/contrib/images/digitalocean/in_parts/012-grub-opts new file mode 100755 index 000000000..72b471c77 --- /dev/null +++ b/contrib/images/digitalocean/in_parts/012-grub-opts @@ -0,0 +1,8 @@ +#!/bin/bash +set -eo pipefail + +echo '--> Update grub boot loader' +sed -e 's|GRUB_CMDLINE_LINUX="|GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1|g' \ + -i /etc/default/grub + +update-grub diff --git a/contrib/images/digitalocean/in_parts/014-docker-dns b/contrib/images/digitalocean/in_parts/014-docker-dns new file mode 100755 index 000000000..acd43f6ba --- /dev/null +++ b/contrib/images/digitalocean/in_parts/014-docker-dns @@ -0,0 +1,6 @@ +#!/bin/bash +set -eo pipefail + +echo '--> Enable Google DNS for Docker' +sed -e 's|#DOCKER_OPTS|DOCKER_OPTS|g' \ + -i /etc/default/docker diff --git a/contrib/images/digitalocean/in_parts/014-ufw-rules b/contrib/images/digitalocean/in_parts/014-ufw-rules new file mode 100755 index 000000000..5665ea827 --- /dev/null +++ b/contrib/images/digitalocean/in_parts/014-ufw-rules @@ -0,0 +1,12 @@ +#!/bin/bash +set -eo pipefail + +echo '--> Enable and configure UFW' +sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \ + -i /etc/default/ufw + +ufw allow ssh +ufw allow http +ufw allow https +ufw allow 2375/tcp +ufw allow 2376/tcp diff --git a/contrib/images/digitalocean/in_parts/099-application_tag b/contrib/images/digitalocean/in_parts/099-application_tag new file mode 100755 index 000000000..9bd9bff61 --- /dev/null +++ b/contrib/images/digitalocean/in_parts/099-application_tag @@ -0,0 +1,20 @@ +#!/bin/bash +set -eo pipefail + +echo '--> Write the application info' +application_version="$DOKKU_VERSION" +build_date="$(date -I)" +distro_arch="$(uname -m)" +distro_codename="$(lsb_release -sc)" +distro_release="$(lsb_release -sr)" +distro="$(lsb_release -si)" + +cat >/var/lib/digitalocean/application.info < Cleaning up' +rm -rf /tmp/* /var/tmp/* +history -c +cat /dev/null > /root/.bash_history +unset HISTFILE +apt-get -y autoremove +apt-get -y autoclean +rm -rf /var/lib/cloud/instances/* + +echo '--> Removing keys' +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* + +echo '--> Clearing log files' +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl new file mode 100644 index 000000000..21bb0f957 --- /dev/null +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -0,0 +1,96 @@ +packer { + required_plugins { + digitalocean = { + version = ">= 1.0.4" + source = "github.com/digitalocean/digitalocean" + } + } +} + +variable "source_image" { + type = string + default = "ubuntu-22-04-x64" +} + +variable "dokku_version" { + type = string +} + +source "digitalocean" "ubuntu" { + image = var.source_image + region = "nyc3" + size = "s-2vcpu-2gb" + ssh_username = "root" + snapshot_name = format("dokku-%s-%s", var.dokku_version, formatdate("YYYYMMDDhhmm", timestamp())) +} + +build { + name = "dokku" + sources = [ + "source.digitalocean.ubuntu" + ] + + provisioner "shell" { + inline = [ + "echo '--> Waiting until cloud-init is complete'", + "/usr/bin/cloud-init status --wait", + ] + } + + provisioner "file" { + source = "${path.root}/files/etc/update-motd.d/99-one-click" + destination = "/etc/update-motd.d/99-one-click" + } + + provisioner "file" { + source = "${path.root}/files/var/lib/cloud/scripts/per-once/001_setup" + destination = "/var/lib/cloud/scripts/per-once/001_setup" + } + + provisioner "file" { + source = "${path.root}/files/var/lib/cloud/scripts/per-once/002_enable_ssh" + destination = "/var/lib/cloud/scripts/per-once/002_enable_ssh" + } + + provisioner "shell" { + script = "${path.root}/in_parts/011-docker" + } + + provisioner "shell" { + script = "${path.root}/in_parts/011-ssh-message" + } + + provisioner "shell" { + environment_vars = [ + "DOKKU_VERSION=${var.dokku_version}" + ] + script = "${path.root}/in_parts/012-dokku-packages" + } + + provisioner "shell" { + script = "${path.root}/in_parts/012-grub-opts" + } + + provisioner "shell" { + script = "${path.root}/in_parts/014-docker-dns" + } + + provisioner "shell" { + script = "${path.root}/in_parts/014-ufw-rules" + } + + provisioner "shell" { + environment_vars = [ + "DOKKU_VERSION=${var.dokku_version}" + ] + script = "${path.root}/in_parts/099-application_tag" + } + + provisioner "shell" { + script = "${path.root}/in_parts/099-cleanup" + } +} + +post-processor "manifest" { + output = "digitalocean-manifest.json" +} diff --git a/contrib/packer.json b/contrib/packer.json deleted file mode 100644 index 96a60e5b2..000000000 --- a/contrib/packer.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "builders": [ - { - "type": "digitalocean", - "region": "nyc3", - "image": "ubuntu-14-04-x64" - } - ], - "provisioners": [ - { - "type": "file", - "source": "deb.mk", - "destination": "/tmp/Makefile" - }, - { - "type": "shell", - "inline": [ - "echo '--> Waiting 30 seconds for core services to be available'", - "sleep 30", - - "echo '--> Updating apt repositories'", - "sudo apt-get update -qq >/dev/null", - - "echo '--> Installing make requirement'", - "sudo apt-get -qq -y --no-install-recommends install build-essential", - - "cd /tmp && make install-from-deb", - "rm /root/.ssh/authorized_keys" - ] - } - ] -} From 6c099c4ad0614c3f0a8119f35f838825fd146882 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:19:42 -0500 Subject: [PATCH 02/28] tests: lint the packer config in CI --- .github/workflows/lint.yml | 15 +++++++++++++++ package.mk | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e4ac33938..4d287b5d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,6 +34,21 @@ jobs: config: '.github/linters/.markdown-lint.yml' args: './README.md' + packer: + name: packer + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup packer + uses: hashicorp/setup-packer@main + with: + version: latest + - name: Run packer init + run: "make image/init/digitalocean" + - name: Run `packer validate` + run: "make image/validate/digitalocean" + shellcheck: name: shellcheck runs-on: ubuntu-20.04 diff --git a/package.mk b/package.mk index dbfa1eaac..dbd31493b 100644 --- a/package.mk +++ b/package.mk @@ -1,3 +1,7 @@ +ifndef PKR_VAR_dokku_version + PKR_VAR_dokku_version = $(shell grep Version debian/control | cut -d' ' -f2) +endif + /tmp/build-dokku/var/lib/dokku/GIT_REV: mkdir -p /tmp/build-dokku mkdir -p /tmp/build-dokku/usr/share/bash-completion/completions @@ -31,3 +35,10 @@ ifdef DOKKU_GIT_REV else git rev-parse HEAD > /tmp/build-dokku/var/lib/dokku/GIT_REV endif +.PHONY: image/init/digitalocean +image/init/digitalocean: + packer init contrib/images/digitalocean/packer.pkr.hcl + +.PHONY: image/validate/digitalocean +image/validate/digitalocean: + packer validate -var 'dokku_version=${PKR_VAR_dokku_version}' contrib/images/digitalocean/packer.pkr.hcl From ae1b695fc53b3b5ac2cd02202ff5fe44343a18df Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:21:36 -0500 Subject: [PATCH 03/28] feat: build digitalocean images in ci on release While these don't (yet?) get submitted to the Digitalocean marketplace, it should bring us most of the way of making that possible. The next step would be to somehow get notifications of the new image snapshot sent... Somewhere and have the lovely DO folks test/release it for us. --- .github/workflows/release.yml | 33 +++++++++++++++++++++++++++++++++ package.mk | 5 +++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index edc508e81..a4a211334 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,6 +72,39 @@ jobs: name: build path: build + build-digitalocean: + name: build-digitalocean + runs-on: ubuntu-20.04 + needs: release + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup packer + uses: hashicorp/setup-packer@main + with: + version: latest + + - name: Run packer init + run: "make image/init/digitalocean" + + - name: Run `packer validate` + env: + PKR_VAR_dokku_version: ${{ needs.release.outputs.version }} + run: "make image/validate/digitalocean" + + - name: Bump Digitalocean Image + env: + DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} + PKR_VAR_dokku_version: ${{ needs.release.outputs.version }} + run: "make image/build/digitalocean" + + - name: upload packages + uses: actions/upload-artifact@v3 + with: + name: digitalocean-manifest.json + path: digitalocean-manifest.json + bump-azure: name: bump-azure runs-on: ubuntu-20.04 diff --git a/package.mk b/package.mk index dbd31493b..8a92a3393 100644 --- a/package.mk +++ b/package.mk @@ -35,6 +35,11 @@ ifdef DOKKU_GIT_REV else git rev-parse HEAD > /tmp/build-dokku/var/lib/dokku/GIT_REV endif + +.PHONY: image/build/digitalocean +image/build/digitalocean: + packer build -var 'dokku_version=${PKR_VAR_dokku_version}' contrib/images/digitalocean/packer.pkr.hcl + .PHONY: image/init/digitalocean image/init/digitalocean: packer init contrib/images/digitalocean/packer.pkr.hcl From adf98b53f55821ddde9109a4c55097251ee2b325 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:22:44 -0500 Subject: [PATCH 04/28] chore: run shfmt --- contrib/images/digitalocean/in_parts/099-cleanup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index f09f6fb82..afb21bdfa 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -4,7 +4,7 @@ set -eo pipefail echo '--> Cleaning up' rm -rf /tmp/* /var/tmp/* history -c -cat /dev/null > /root/.bash_history +cat /dev/null >/root/.bash_history unset HISTFILE apt-get -y autoremove apt-get -y autoclean @@ -16,4 +16,5 @@ rm -f /root/.ssh/authorized_keys /etc/ssh/*key* echo '--> Clearing log files' find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? -cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp +cat /dev/null >/var/log/lastlog +cat /dev/null >/var/log/wtmp From ec642788d328db0aa1dd6c492aa4e500d5403dda Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:25:40 -0500 Subject: [PATCH 05/28] fix: correct SC2004 --- .../digitalocean/files/var/lib/cloud/scripts/per-once/001_setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/001_setup b/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/001_setup index 9da800a0f..b343d77dc 100755 --- a/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/001_setup +++ b/contrib/images/digitalocean/files/var/lib/cloud/scripts/per-once/001_setup @@ -19,7 +19,7 @@ for count in {1..30}; do ran=0 exit 0 else - count=$(($count + 1)) + count=$((count + 1)) fi sleep 5 done From ba56474f1578c7909fc715de164248fe8447774b Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:26:29 -0500 Subject: [PATCH 06/28] fix: correct position of post-provisioner block --- contrib/images/digitalocean/packer.pkr.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 21bb0f957..3005c25e9 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -89,8 +89,8 @@ build { provisioner "shell" { script = "${path.root}/in_parts/099-cleanup" } -} -post-processor "manifest" { - output = "digitalocean-manifest.json" + post-processor "manifest" { + output = "digitalocean-manifest.json" + } } From 9ebe7087419260a42c7b11d8122540e3cb76913c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:31:41 -0500 Subject: [PATCH 07/28] fix: run packer validate with access to the DO token Without this, it fails when initializing the builder. --- .github/workflows/lint.yml | 2 ++ .github/workflows/release.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4d287b5d9..2ed0a039b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -47,6 +47,8 @@ jobs: - name: Run packer init run: "make image/init/digitalocean" - name: Run `packer validate` + env: + DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} run: "make image/validate/digitalocean" shellcheck: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4a211334..564183793 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,6 +90,7 @@ jobs: - name: Run `packer validate` env: + DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} PKR_VAR_dokku_version: ${{ needs.release.outputs.version }} run: "make image/validate/digitalocean" From 98d6a3b53e30cdb3aa7131befd8a43fff95636ca Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:32:30 -0500 Subject: [PATCH 08/28] fix: run build on smallest image This will allow anyone to use the resulting image, and not just those with at least 2GB of ram. --- contrib/images/digitalocean/packer.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 3005c25e9..71b564c6b 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -19,7 +19,7 @@ variable "dokku_version" { source "digitalocean" "ubuntu" { image = var.source_image region = "nyc3" - size = "s-2vcpu-2gb" + size = "s-1vcpu-512mb-10gb" ssh_username = "root" snapshot_name = format("dokku-%s-%s", var.dokku_version, formatdate("YYYYMMDDhhmm", timestamp())) } From 358fb72baf75391e00c1ed5f60cfc37449a263c8 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:33:26 -0500 Subject: [PATCH 09/28] fix: run build in nyc1 NYC1 is the only DC where the smallest size droplet is available. --- contrib/images/digitalocean/packer.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 71b564c6b..9321a3fb4 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -18,7 +18,7 @@ variable "dokku_version" { source "digitalocean" "ubuntu" { image = var.source_image - region = "nyc3" + region = "nyc1" size = "s-1vcpu-512mb-10gb" ssh_username = "root" snapshot_name = format("dokku-%s-%s", var.dokku_version, formatdate("YYYYMMDDhhmm", timestamp())) From afe042de46a6fc7cbe698e323853d3ad5e625b4d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:46:39 -0500 Subject: [PATCH 10/28] docs: cleanup motd - standardize text with the sample marketplace app - remove reference to web server (its unavailable in newer versions) --- .../images/digitalocean/files/etc/update-motd.d/99-one-click | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click index c9fc474ee..865183476 100755 --- a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click +++ b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click @@ -6,14 +6,13 @@ myip=$(hostname -I | awk '{print$1}') cat < Date: Thu, 16 Feb 2023 14:47:52 -0500 Subject: [PATCH 11/28] refactor: cleanup ufw rules - use nginx ful rules - limit ssh by default - force enable ufw - drop access to 2375 and 2376 (this allows passwordless docker access...) --- .../digitalocean/files/etc/update-motd.d/99-one-click | 3 +-- contrib/images/digitalocean/in_parts/014-ufw-rules | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click index 865183476..394b4e9ee 100755 --- a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click +++ b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click @@ -8,8 +8,7 @@ cat < Enable and configure UFW' sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \ -i /etc/default/ufw -ufw allow ssh -ufw allow http -ufw allow https -ufw allow 2375/tcp -ufw allow 2376/tcp +ufw limit ssh +ufw allow 'Nginx Full' + +ufw --force enable From ba06bac1b979548315eee369213fa489b34d8f45 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:48:43 -0500 Subject: [PATCH 12/28] feat: enable exit on error --- contrib/images/digitalocean/in_parts/011-docker | 1 + contrib/images/digitalocean/in_parts/011-ssh-message | 1 + contrib/images/digitalocean/in_parts/012-dokku-packages | 1 + contrib/images/digitalocean/in_parts/012-grub-opts | 1 + contrib/images/digitalocean/in_parts/014-docker-dns | 1 + contrib/images/digitalocean/in_parts/014-ufw-rules | 1 + contrib/images/digitalocean/in_parts/099-application_tag | 1 + contrib/images/digitalocean/in_parts/099-cleanup | 1 + 8 files changed, 8 insertions(+) diff --git a/contrib/images/digitalocean/in_parts/011-docker b/contrib/images/digitalocean/in_parts/011-docker index 65d2955e3..31942d492 100755 --- a/contrib/images/digitalocean/in_parts/011-docker +++ b/contrib/images/digitalocean/in_parts/011-docker @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit export DEBIAN_FRONTEND=noninteractive diff --git a/contrib/images/digitalocean/in_parts/011-ssh-message b/contrib/images/digitalocean/in_parts/011-ssh-message index da7a0b5df..83db94286 100755 --- a/contrib/images/digitalocean/in_parts/011-ssh-message +++ b/contrib/images/digitalocean/in_parts/011-ssh-message @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Prevent login until after setup is complete' # Be a bit of a dork about login in diff --git a/contrib/images/digitalocean/in_parts/012-dokku-packages b/contrib/images/digitalocean/in_parts/012-dokku-packages index 251c6df9f..3f2834e08 100755 --- a/contrib/images/digitalocean/in_parts/012-dokku-packages +++ b/contrib/images/digitalocean/in_parts/012-dokku-packages @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit export DEBIAN_FRONTEND=noninteractive diff --git a/contrib/images/digitalocean/in_parts/012-grub-opts b/contrib/images/digitalocean/in_parts/012-grub-opts index 72b471c77..fbb043cb5 100755 --- a/contrib/images/digitalocean/in_parts/012-grub-opts +++ b/contrib/images/digitalocean/in_parts/012-grub-opts @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Update grub boot loader' sed -e 's|GRUB_CMDLINE_LINUX="|GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1|g' \ diff --git a/contrib/images/digitalocean/in_parts/014-docker-dns b/contrib/images/digitalocean/in_parts/014-docker-dns index acd43f6ba..e3b509edc 100755 --- a/contrib/images/digitalocean/in_parts/014-docker-dns +++ b/contrib/images/digitalocean/in_parts/014-docker-dns @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Enable Google DNS for Docker' sed -e 's|#DOCKER_OPTS|DOCKER_OPTS|g' \ diff --git a/contrib/images/digitalocean/in_parts/014-ufw-rules b/contrib/images/digitalocean/in_parts/014-ufw-rules index 0488f0d1e..957728b28 100755 --- a/contrib/images/digitalocean/in_parts/014-ufw-rules +++ b/contrib/images/digitalocean/in_parts/014-ufw-rules @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Enable and configure UFW' sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \ diff --git a/contrib/images/digitalocean/in_parts/099-application_tag b/contrib/images/digitalocean/in_parts/099-application_tag index 9bd9bff61..97a1f1617 100755 --- a/contrib/images/digitalocean/in_parts/099-application_tag +++ b/contrib/images/digitalocean/in_parts/099-application_tag @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Write the application info' application_version="$DOKKU_VERSION" diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index afb21bdfa..8c449f928 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -1,5 +1,6 @@ #!/bin/bash set -eo pipefail +set -o errexit echo '--> Cleaning up' rm -rf /tmp/* /var/tmp/* From 197bd0c86452dd1ba5dfa11db76dfd618c784e0c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:49:13 -0500 Subject: [PATCH 13/28] feat: remove the default nginx site in built images This is confusing for users and shouldn't really ever exist. --- contrib/images/digitalocean/in_parts/012-dokku-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/images/digitalocean/in_parts/012-dokku-packages b/contrib/images/digitalocean/in_parts/012-dokku-packages index 3f2834e08..768c44924 100755 --- a/contrib/images/digitalocean/in_parts/012-dokku-packages +++ b/contrib/images/digitalocean/in_parts/012-dokku-packages @@ -56,6 +56,7 @@ echo '--> Install dependencies for dokku from new repository' apt-get -y clean apt-get -y update apt-get -y install "${dokku_pkgs[@]}" +rm -f /etc/nginx/sites-enabled/default echo '--> Pre-cache dokku bits that are must run on boot' apt-get -y clean From 4ce5ba6a75437238e3f2393c00e5c6c8c5b320ed Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:50:07 -0500 Subject: [PATCH 14/28] refactor: unify cleanup script with the one used by the DO marketplace --- .../images/digitalocean/in_parts/099-cleanup | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index 8c449f928..b6dfe7d09 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -2,20 +2,47 @@ set -eo pipefail set -o errexit -echo '--> Cleaning up' +echo '--> Cleaning up tmp dir' +# Ensure /tmp exists and has the proper permissions before +# checking for security updates +# https://github.com/digitalocean/marketplace-partners/issues/94 +if [[ ! -d /tmp ]]; then + mkdir /tmp +fi +chmod 1777 /tmp rm -rf /tmp/* /var/tmp/* + +echo '--> Cleaning up apt' +export DEBIAN_FRONTEND=noninteractive +apt-get -y update +apt-get -o Dpkg::Options::="--force-confold" upgrade -q -y --force-yes +apt-get -y autoremove +apt-get -y autoclean + +echo '--> Clearing history' history -c cat /dev/null >/root/.bash_history unset HISTFILE -apt-get -y autoremove -apt-get -y autoclean -rm -rf /var/lib/cloud/instances/* - -echo '--> Removing keys' -rm -f /root/.ssh/authorized_keys /etc/ssh/*key* echo '--> Clearing log files' find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? cat /dev/null >/var/log/lastlog cat /dev/null >/var/log/wtmp +rm -rf /var/lib/cloud/instances/* +rm -rf /tmp/* /var/tmp/* + +echo '--> Removing keys' +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +echo '--> Securely erase the unused portion of the filesystem' +printf "\n\033[0;32mWriting zeros to the remaining disk space to securely +erase the unused portion of the file system. +Depending on your disk size this may take several minutes. +The secure erase will complete successfully when you see:\033[0m + dd: writing to '/zerofile': No space left on device\n +Beginning secure erase now\n" + +dd if=/dev/zero of=/zerofile bs=4096 || rm /zerofile From 360940f68ea504932c9fce141d95f7b452a11850 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 14:53:58 -0500 Subject: [PATCH 15/28] fix: use https urls --- .../images/digitalocean/files/etc/update-motd.d/99-one-click | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click index 394b4e9ee..41291fc9e 100755 --- a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click +++ b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click @@ -11,9 +11,9 @@ To keep this Droplet secure, the UFW firewall is enabled. All ports are BLOCKED except 22 (SSH), 80 (HTTP), 443 (HTTPS). In a web browser, you can view: - * The Dokku One-Click Quickstart guide: http://do.co/dokku1804#start + * The Dokku One-Click Quickstart guide: https://do.co/dokku1804#start -For help and more information, visit http://do.co/dokku1804 +For help and more information, visit https://do.co/dokku1804 ******************************************************************************** To delete this message of the day: rm -rf $(readlink -f ${0}) From c24e0d38880294fcd695a52c4ae0e80663fecb14 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:00:37 -0500 Subject: [PATCH 16/28] feat: ensure we have a custom landing page for folks to view when going to their droplet ip/vhost --- .../etc/nginx/sites-available/digitalocean | 95 ++++++++++ .../files/etc/update-motd.d/99-one-click | 1 + .../files/var/www/html/index.html | 169 ++++++++++++++++++ contrib/images/digitalocean/packer.pkr.hcl | 10 ++ 4 files changed, 275 insertions(+) create mode 100644 contrib/images/digitalocean/files/etc/nginx/sites-available/digitalocean create mode 100644 contrib/images/digitalocean/files/var/www/html/index.html diff --git a/contrib/images/digitalocean/files/etc/nginx/sites-available/digitalocean b/contrib/images/digitalocean/files/etc/nginx/sites-available/digitalocean new file mode 100644 index 000000000..9e9b34ce9 --- /dev/null +++ b/contrib/images/digitalocean/files/etc/nginx/sites-available/digitalocean @@ -0,0 +1,95 @@ +# You may add here your +# server { +# ... +# } +# statements for each of your virtual hosts to this file + +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# http://wiki.nginx.org/Pitfalls +# http://wiki.nginx.org/QuickStart +# http://wiki.nginx.org/Configuration +# +# Generally, you will want to move this file somewhere, and start with a clean +# file but keep this around for reference. Or just disable in sites-enabled. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +server { + listen 80 default_server; + listen [::]:80 default_server ipv6only=on; + + root /var/www/html; + index index.php index.html index.htm; + + # Make site accessible from http://localhost/ + server_name localhost; + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + # Uncomment to enable naxsi on this location + # include /etc/nginx/naxsi.rules + } + + error_page 404 /404.html; + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location ~ \.php$ { + include snippets/fastcgi-php.conf; + fastcgi_pass unix:/run/php/php7.4-fpm.sock; + } + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} +} + + +# another virtual host using mix of IP-, name-, and port-based configuration +# +#server { +# listen 8000; +# listen somename:8080; +# server_name somename alias another.alias; +# root html; +# index index.html index.htm; +# +# location / { +# try_files $uri $uri/ =404; +# } +#} + + +# HTTPS server +# +#server { +# listen 443; +# server_name localhost; +# +# root html; +# index index.html index.htm; +# +# ssl on; +# ssl_certificate cert.pem; +# ssl_certificate_key cert.key; +# +# ssl_session_timeout 5m; +# +# ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2; +# ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES"; +# ssl_prefer_server_ciphers on; +# +# location / { +# try_files $uri $uri/ =404; +# } +#} diff --git a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click index 41291fc9e..c4c8f37dd 100755 --- a/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click +++ b/contrib/images/digitalocean/files/etc/update-motd.d/99-one-click @@ -17,4 +17,5 @@ For help and more information, visit https://do.co/dokku1804 ******************************************************************************** To delete this message of the day: rm -rf $(readlink -f ${0}) +To remove the default web landing page: rm -rf /etc/nginx/sites-available/digitalocean EOF diff --git a/contrib/images/digitalocean/files/var/www/html/index.html b/contrib/images/digitalocean/files/var/www/html/index.html new file mode 100644 index 000000000..989a2af6a --- /dev/null +++ b/contrib/images/digitalocean/files/var/www/html/index.html @@ -0,0 +1,169 @@ + + + + + + + +
+ +
© 2023 DigitalOcean, LLC. All rights reserved.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Please log into your Droplet with SSH to configure the Dokku installation.

+

See the Dokku 1-Click Quickstart guide for detailed assistance.

+ Quickstart Guide +
+ + + diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 9321a3fb4..94169f3d5 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -42,6 +42,16 @@ build { destination = "/etc/update-motd.d/99-one-click" } + provisioner "file" { + source = "${path.root}/files/etc/nginx/sites-available/digitalocean" + destination = "/etc/nginx/sites-available/digitalocean" + } + + provisioner "file" { + source = "${path.root}/files/var/www/html/index.html" + destination = "/var/www/html/index.html" + } + provisioner "file" { source = "${path.root}/files/var/lib/cloud/scripts/per-once/001_setup" destination = "/var/lib/cloud/scripts/per-once/001_setup" From d40ee363483d1e3136d906cd508c36fd9942e12a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:07:03 -0500 Subject: [PATCH 17/28] feat: update all base packages --- contrib/images/digitalocean/in_parts/011-docker | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/images/digitalocean/in_parts/011-docker b/contrib/images/digitalocean/in_parts/011-docker index 31942d492..6bb7c6b66 100755 --- a/contrib/images/digitalocean/in_parts/011-docker +++ b/contrib/images/digitalocean/in_parts/011-docker @@ -15,6 +15,9 @@ pkgs=(apt-transport-https echo '--> Updating apt repositories' apt-get -qqy update +echo '--> Updating all packages' +apt-get -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' full-upgrade + echo "--> Installing basic dependencies: ${pkgs[*]}" apt-get -qqy install "${pkgs[@]}" From 11743afbf0b570bd272093be4d7c6f4cdf11f89c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:07:25 -0500 Subject: [PATCH 18/28] chore: drop ability to set a custom base image We'll only ever test against the latest LTS. --- contrib/images/digitalocean/packer.pkr.hcl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 94169f3d5..bf8c487e6 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -7,17 +7,12 @@ packer { } } -variable "source_image" { - type = string - default = "ubuntu-22-04-x64" -} - variable "dokku_version" { type = string } source "digitalocean" "ubuntu" { - image = var.source_image + image = "ubuntu-22-04-x64" region = "nyc1" size = "s-1vcpu-512mb-10gb" ssh_username = "root" From b2fbefb6d3bbb3098a5e871a83ce5c7db3f66919 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:07:58 -0500 Subject: [PATCH 19/28] refactor: remove use of format function --- contrib/images/digitalocean/packer.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index bf8c487e6..8aaf94fb3 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -16,7 +16,7 @@ source "digitalocean" "ubuntu" { region = "nyc1" size = "s-1vcpu-512mb-10gb" ssh_username = "root" - snapshot_name = format("dokku-%s-%s", var.dokku_version, formatdate("YYYYMMDDhhmm", timestamp())) + snapshot_name = "dokku-${var.dokku_version}-snapshot-{{timestamp}}" } build { From 1eda16d1865198118f370e9d84cf61cb0de2a3ee Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:08:49 -0500 Subject: [PATCH 20/28] refactor: simplify provisioning --- contrib/images/digitalocean/packer.pkr.hcl | 72 ++++++---------------- 1 file changed, 20 insertions(+), 52 deletions(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 8aaf94fb3..87e495b81 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -33,66 +33,34 @@ build { } provisioner "file" { - source = "${path.root}/files/etc/update-motd.d/99-one-click" - destination = "/etc/update-motd.d/99-one-click" + source = "${path.root}/files/etc/" + destination = "/etc/" } provisioner "file" { - source = "${path.root}/files/etc/nginx/sites-available/digitalocean" - destination = "/etc/nginx/sites-available/digitalocean" - } - - provisioner "file" { - source = "${path.root}/files/var/www/html/index.html" - destination = "/var/www/html/index.html" - } - - provisioner "file" { - source = "${path.root}/files/var/lib/cloud/scripts/per-once/001_setup" - destination = "/var/lib/cloud/scripts/per-once/001_setup" - } - - provisioner "file" { - source = "${path.root}/files/var/lib/cloud/scripts/per-once/002_enable_ssh" - destination = "/var/lib/cloud/scripts/per-once/002_enable_ssh" + source = "${path.root}/files/var/" + destination = "/var/" } provisioner "shell" { - script = "${path.root}/in_parts/011-docker" - } - - provisioner "shell" { - script = "${path.root}/in_parts/011-ssh-message" - } - - provisioner "shell" { - environment_vars = [ + "environment_vars": [ + "application_name={{user `application_name`}}", + "application_version={{user `application_version`}}", "DOKKU_VERSION=${var.dokku_version}" + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8" + ], + scripts = [ + "${path.root}/in_parts/011-docker", + "${path.root}/in_parts/011-ssh-message", + "${path.root}/in_parts/012-dokku-packages", + "${path.root}/in_parts/012-grub-opts", + "${path.root}/in_parts/014-docker-dns", + "${path.root}/in_parts/014-ufw-rules", + "${path.root}/in_parts/099-application_tag", ] - script = "${path.root}/in_parts/012-dokku-packages" - } - - provisioner "shell" { - script = "${path.root}/in_parts/012-grub-opts" - } - - provisioner "shell" { - script = "${path.root}/in_parts/014-docker-dns" - } - - provisioner "shell" { - script = "${path.root}/in_parts/014-ufw-rules" - } - - provisioner "shell" { - environment_vars = [ - "DOKKU_VERSION=${var.dokku_version}" - ] - script = "${path.root}/in_parts/099-application_tag" - } - - provisioner "shell" { - script = "${path.root}/in_parts/099-cleanup" } post-processor "manifest" { From 55c3b73f35e0f42c3c51ab42a0cc4280627ee826 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:09:02 -0500 Subject: [PATCH 21/28] feat: run image check at the end of the build --- .../digitalocean/in_parts/100-image-check | 627 ++++++++++++++++++ contrib/images/digitalocean/packer.pkr.hcl | 1 + 2 files changed, 628 insertions(+) create mode 100755 contrib/images/digitalocean/in_parts/100-image-check diff --git a/contrib/images/digitalocean/in_parts/100-image-check b/contrib/images/digitalocean/in_parts/100-image-check new file mode 100755 index 000000000..e1cf432e5 --- /dev/null +++ b/contrib/images/digitalocean/in_parts/100-image-check @@ -0,0 +1,627 @@ +#!/bin/bash + +# DigitalOcean Marketplace Image Validation Tool +# © 2021-2022 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +VERSION="v. 1.8.1" +RUNDATE=$(date) + +# Script should be run with SUDO +if [ "$EUID" -ne 0 ]; then + echo "[Error] - This script must be run with sudo or as the root user." + exit 1 +fi + +STATUS=0 +PASS=0 +WARN=0 +FAIL=0 + +# $1 == command to check for +# returns: 0 == true, 1 == false +cmdExists() { + if command -v "$1" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +function getDistro { + if [ -f /etc/os-release ]; then + # freedesktop.org and systemd + # shellcheck disable=SC1091 + . /etc/os-release + OS=$NAME + VER=$VERSION_ID + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + OS=$(lsb_release -si) + VER=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + # shellcheck disable=SC1091 + . /etc/lsb-release + OS=$DISTRIB_ID + VER=$DISTRIB_RELEASE + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + VER=$(cat /etc/debian_version) + elif [ -f /etc/SuSe-release ]; then + # Older SuSE/etc. + : + elif [ -f /etc/redhat-release ]; then + # Older Red Hat, CentOS, etc. + VER=$(cut -d" " -f3 ", also works for BSD, etc. + OS=$(uname -s) + VER=$(uname -r) + fi +} +function loadPasswords { + SHADOW=$(cat /etc/shadow) +} + +function checkAgent { + # Check for the presence of the DO directory in the filesystem + if [ -d /opt/digitalocean ]; then + echo -en "\e[41m[FAIL]\e[0m DigitalOcean directory detected.\n" + ((FAIL++)) + STATUS=2 + if [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then + echo "To uninstall the agent: 'sudo yum remove droplet-agent'" + echo "To remove the DO directory: 'find /opt/digitalocean/ -type d -empty -delete'" + elif [[ $OS == "Ubuntu" ]] || [[ $OS == "Debian" ]]; then + echo "To uninstall the agent and remove the DO directory: 'sudo apt-get purge droplet-agent'" + fi + else + echo -en "\e[32m[PASS]\e[0m DigitalOcean Monitoring agent was not found\n" + ((PASS++)) + fi +} + +function checkLogs { + cp_ignore="/var/log/cpanel-install.log" + echo -en "\nChecking for log files in /var/log\n\n" + # Check if there are log archives or log files that have not been recently cleared. + for f in /var/log/*-????????; do + [[ -e $f ]] || break + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found; Contents:\n" + cat "${f}" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + done + for f in /var/log/*.[0-9]; do + [[ -e $f ]] || break + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found; Contents:\n" + cat "${f}" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + done + for f in /var/log/*.log; do + [[ -e $f ]] || break + if [[ "${f}" = '/var/log/lfd.log' && "$(grep -E -v '/var/log/messages has been reset| Watching /var/log/messages' "${f}" | wc -c)" -gt 50 ]]; then + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found; Contents:\n" + cat "${f}" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [[ "${f}" != '/var/log/lfd.log' && "$(wc -c <"${f}")" -gt 50 ]]; then + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found; Contents:\n" + cat "${f}" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + done +} +function checkTMP { + # Check the /tmp directory to ensure it is empty. Warn on any files found. + return 1 +} +function checkRoot { + user="root" + uhome="/root" + for usr in $SHADOW; do + IFS=':' read -r -a u <<<"$usr" + if [[ "${u[0]}" == "${user}" ]]; then + if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then + echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account.\n" + ((FAIL++)) + STATUS=2 + fi + fi + done + if [ -d ${uhome}/ ]; then + if [ -d ${uhome}/.ssh/ ]; then + if ls ${uhome}/.ssh/* >/dev/null 2>&1; then + for key in "${uhome}"/.ssh/*; do + if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then + + if [ "$(wc -c <"${key}")" -gt 50 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" + akey=$(cat "${key}") + echo "File Contents:" + echo "$akey" + echo "--------------" + ((FAIL++)) + STATUS=2 + fi + elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then + if [ "$(wc -c <"${key}")" -gt 0 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" + akey=$(cat "${key}") + echo "File Contents:" + echo "$akey" + echo "--------------" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory at \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + else + if [ "$(wc -c <"${key}")" -gt 50 ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a populated known_hosts file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + done + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" + fi + if [ -f /root/.bash_history ]; then + + BH_S=$(wc -c = 1000 && $1 != "nobody" {print $1}' /dev/null 2>&1; then + for key in "${uhome}"/.ssh/*; do + if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then + if [ "$(wc -c <"${key}")" -gt 50 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" + akey=$(cat "${key}") + echo "File Contents:" + echo "$akey" + echo "--------------" + ((FAIL++)) + STATUS=2 + fi + elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then + if [ "$(wc -c <"${key}")" -gt 0 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" + akey=$(cat "${key}") + echo "File Contents:" + echo "$akey" + echo "--------------" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" + # shellcheck disable=SC2030 + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then + + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory named \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + + else + if [ "$(wc -c <"${key}")" -gt 50 ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a known_hosts file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + + done + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have a directory in /home\n" + fi + + # Check for an uncleared .bash_history for this user + if [ -f "${uhome}/.bash_history" ]; then + BH_S=$(wc -c <"${uhome}/.bash_history") + + if [[ $BH_S -lt 200 ]]; then + echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m ${user}'s Bash History should be cleared to prevent sensitive information from leaking\n" + ((FAIL++)) + STATUS=2 + + fi + echo -en "\n\n" + fi + fi + done +} +function checkFirewall { + + if [[ $OS == "Ubuntu" ]]; then + fw="ufw" + ufwa=$(ufw status | head -1 | sed -e "s/^Status:\ //") + if [[ $ufwa == "active" ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + # shellcheck disable=SC2031 + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + # shellcheck disable=SC2031 + ((WARN++)) + fi + elif [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then + if [ -f /usr/lib/systemd/system/csf.service ]; then + fw="csf" + if [[ $(systemctl status $fw >/dev/null 2>&1) ]]; then + + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + elif cmdExists "firewall-cmd"; then + if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + fw="firewalld" + if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + fi + elif [[ "$OS" =~ Debian.* ]]; then + # user could be using a number of different services for managing their firewall + # we will check some of the most common + if cmdExists 'ufw'; then + fw="ufw" + ufwa=$(ufw status | head -1 | sed -e "s/^Status:\ //") + if [[ $ufwa == "active" ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + elif cmdExists "firewall-cmd"; then + fw="firewalld" + if [[ $(systemctl is-active --quiet $fw) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + # user could be using vanilla iptables, check if kernel module is loaded + fw="iptables" + if lsmod | grep -q '^ip_tables' 2>/dev/null; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + fi + fi + +} +function checkUpdates { + if [[ $OS == "Ubuntu" ]] || [[ "$OS" =~ Debian.* ]]; then + # Ensure /tmp exists and has the proper permissions before + # checking for security updates + # https://github.com/digitalocean/marketplace-partners/issues/94 + if [[ ! -d /tmp ]]; then + mkdir /tmp + fi + chmod 1777 /tmp + + echo -en "\nUpdating apt package database to check for security updates, this may take a minute...\n\n" + apt-get -y update >/dev/null + + uc=$(apt-get --just-print upgrade | grep -i "security" -c) + if [[ $uc -gt 0 ]]; then + update_count=$((uc / 2)) + else + update_count=0 + fi + + if [[ $update_count -gt 0 ]]; then + echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" + echo -en + echo -en "Here is a list of the security updates that are not installed:\n" + sleep 2 + apt-get --just-print upgrade | grep -i security | awk '{print $2}' | awk '!seen[$0]++' + echo -en + # shellcheck disable=SC2031 + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n\n" + ((PASS++)) + fi + elif [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then + echo -en "\nChecking for available security updates, this may take a minute...\n\n" + + update_count=$(yum check-update --security --quiet | wc -l) + if [[ $update_count -gt 0 ]]; then + echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n" + ((PASS++)) + fi + else + echo "Error encountered" + exit 1 + fi + + return 1 +} +function checkCloudInit { + + if hash cloud-init 2>/dev/null; then + CI="\e[32m[PASS]\e[0m Cloud-init is installed.\n" + ((PASS++)) + else + CI="\e[41m[FAIL]\e[0m No valid verison of cloud-init was found.\n" + ((FAIL++)) + STATUS=2 + fi + return 1 +} + +function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +clear +echo "DigitalOcean Marketplace Image Validation Tool ${VERSION}" +echo "Executed on: ${RUNDATE}" +echo "Checking local system for Marketplace compatibility..." + +getDistro + +echo -en "\n\e[1mDistribution:\e[0m ${OS}\n" +echo -en "\e[1mVersion:\e[0m ${VER}\n\n" + +ost=0 +osv=0 + +if [[ $OS == "Ubuntu" ]]; then + ost=1 + if [[ $VER == "22.10" ]] || [[ $VER == "22.04" ]] || [[ $VER == "20.04" ]] || [[ $VER == "18.04" ]] || [[ $VER == "16.04" ]]; then + osv=1 + fi + +elif [[ "$OS" =~ Debian.* ]]; then + ost=1 + case "$VER" in + 9) + osv=1 + ;; + 10) + osv=1 + ;; + 11) + osv=1 + ;; + *) + osv=2 + ;; + esac + +elif [[ $OS == "CentOS Linux" ]]; then + ost=1 + if [[ $VER == "8" ]]; then + osv=1 + elif [[ $VER == "7" ]]; then + osv=1 + elif [[ $VER == "6" ]]; then + osv=1 + else + osv=2 + fi +elif [[ $OS == "CentOS Stream" ]]; then + ost=1 + if [[ $VER == "8" ]]; then + osv=1 + else + osv=2 + fi +elif [[ $OS == "Rocky Linux" ]]; then + ost=1 + if [[ $VER =~ 8\. ]]; then + osv=1 + else + osv=2 + fi +elif [[ $OS == "AlmaLinux" ]]; then + ost=1 + if [[ "$VERSION" =~ 8.* ]] || [[ "$VERSION" =~ 9.* ]]; then + osv=1 + else + osv=2 + fi +else + ost=0 +fi + +if [[ $ost == 1 ]]; then + echo -en "\e[32m[PASS]\e[0m Supported Operating System Detected: ${OS}\n" + ((PASS++)) +else + echo -en "\e[41m[FAIL]\e[0m ${OS} is not a supported Operating System\n" + ((FAIL++)) + STATUS=2 +fi + +if [[ $osv == 1 ]]; then + echo -en "\e[32m[PASS]\e[0m Supported Release Detected: ${VER}\n" + ((PASS++)) +elif [[ $ost == 1 ]]; then + echo -en "\e[41m[FAIL]\e[0m ${OS} ${VER} is not a supported Operating System Version\n" + ((FAIL++)) + STATUS=2 +else + echo "Exiting..." + exit 1 +fi + +checkCloudInit + +echo -en "${CI}" + +checkFirewall + +echo -en "${FW_VER}" + +checkUpdates + +loadPasswords + +checkLogs + +echo -en "\n\nChecking all user-created accounts...\n" +checkUsers + +echo -en "\n\nChecking the root account...\n" +checkRoot + +checkAgent + +# Summary +echo -en "\n\n---------------------------------------------------------------------------------------------------\n" + +if [[ $STATUS == 0 ]]; then + echo -en "Scan Complete.\n\e[32mAll Tests Passed!\e[0m\n" +elif [[ $STATUS == 1 ]]; then + echo -en "Scan Complete. \n\e[93mSome non-critical tests failed. Please review these items.\e[0m\e[0m\n" +else + echo -en "Scan Complete. \n\e[41mOne or more tests failed. Please review these items and re-test.\e[0m\n" +fi +echo "---------------------------------------------------------------------------------------------------" +echo -en "\e[1m${PASS} Tests PASSED\e[0m\n" +echo -en "\e[1m${WARN} WARNINGS\e[0m\n" +echo -en "\e[1m${FAIL} Tests FAILED\e[0m\n" +echo -en "---------------------------------------------------------------------------------------------------\n" + +if [[ $STATUS == 0 ]]; then + echo -en "We did not detect any issues with this image. Please be sure to manually ensure that all software installed on the base system is functional, secure and properly configured (or facilities for configuration on first-boot have been created).\n\n" + exit 0 +elif [[ $STATUS == 1 ]]; then + echo -en "Please review all [WARN] items above and ensure they are intended or resolved. If you do not have a specific requirement, we recommend resolving these items before image submission\n\n" + exit 0 +else + echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the DigitalOcean Marketplace.\n\n" + exit 1 +fi diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 87e495b81..8fa6342b4 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -60,6 +60,7 @@ build { "${path.root}/in_parts/014-docker-dns", "${path.root}/in_parts/014-ufw-rules", "${path.root}/in_parts/099-application_tag", + "${path.root}/in_parts/100-image-check", ] } From f1152cd6b79a2baf60e8b082ea1c536ea7caca2c Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:10:49 -0500 Subject: [PATCH 22/28] fix: validate the packer image --- contrib/images/digitalocean/packer.pkr.hcl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 8fa6342b4..57006a0bf 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -43,15 +43,14 @@ build { } provisioner "shell" { - "environment_vars": [ - "application_name={{user `application_name`}}", - "application_version={{user `application_version`}}", - "DOKKU_VERSION=${var.dokku_version}" + environment_vars = [ + "DOKKU_VERSION=${var.dokku_version}", "DEBIAN_FRONTEND=noninteractive", "LC_ALL=C", "LANG=en_US.UTF-8", - "LC_CTYPE=en_US.UTF-8" - ], + "LC_CTYPE=en_US.UTF-8", + ] + scripts = [ "${path.root}/in_parts/011-docker", "${path.root}/in_parts/011-ssh-message", From 1eada6a5e8ed5e304ff245228472b629d2d0b2a0 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:32:55 -0500 Subject: [PATCH 23/28] fix: purge log files and droplet-agent This should fix a few of the warnings/errors shown during the image check process. --- contrib/images/digitalocean/in_parts/099-cleanup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index b6dfe7d09..c46a9f468 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -18,6 +18,7 @@ apt-get -y update apt-get -o Dpkg::Options::="--force-confold" upgrade -q -y --force-yes apt-get -y autoremove apt-get -y autoclean +apt-get -y purge droplet-agent echo '--> Clearing history' history -c @@ -27,6 +28,7 @@ unset HISTFILE echo '--> Clearing log files' find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +rm /var/log/alternatives.log /var/log/auth.log /var/log/cloud-init-output.log /var/log/cloud-init.log /var/log/dpkg.log /var/log/kern.log /var/log/ubuntu-advantage-timer.log /var/log/ufw.log /var/log/unattended-upgrades cat /dev/null >/var/log/lastlog cat /dev/null >/var/log/wtmp rm -rf /var/lib/cloud/instances/* @@ -37,6 +39,9 @@ rm -f /root/.ssh/authorized_keys /etc/ssh/*key* touch /etc/ssh/revoked_keys chmod 600 /etc/ssh/revoked_keys +echo '--> DO Directory' +ls -lah /opt/digitalocean + echo '--> Securely erase the unused portion of the filesystem' printf "\n\033[0;32mWriting zeros to the remaining disk space to securely erase the unused portion of the file system. From 7b1a2fa908766f70d8e60929f691cbb8b1811b77 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:49:02 -0500 Subject: [PATCH 24/28] fix: add missing cleanup step --- contrib/images/digitalocean/packer.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/images/digitalocean/packer.pkr.hcl b/contrib/images/digitalocean/packer.pkr.hcl index 57006a0bf..c0219a05a 100644 --- a/contrib/images/digitalocean/packer.pkr.hcl +++ b/contrib/images/digitalocean/packer.pkr.hcl @@ -59,6 +59,7 @@ build { "${path.root}/in_parts/014-docker-dns", "${path.root}/in_parts/014-ufw-rules", "${path.root}/in_parts/099-application_tag", + "${path.root}/in_parts/099-cleanup", "${path.root}/in_parts/100-image-check", ] } From 47fddf18993dde80266ca56f860419090ee17431 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:49:16 -0500 Subject: [PATCH 25/28] chore: remove extra cleanup line Logs are already zero'd out --- contrib/images/digitalocean/in_parts/099-cleanup | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index c46a9f468..25f253628 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -28,7 +28,6 @@ unset HISTFILE echo '--> Clearing log files' find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? -rm /var/log/alternatives.log /var/log/auth.log /var/log/cloud-init-output.log /var/log/cloud-init.log /var/log/dpkg.log /var/log/kern.log /var/log/ubuntu-advantage-timer.log /var/log/ufw.log /var/log/unattended-upgrades cat /dev/null >/var/log/lastlog cat /dev/null >/var/log/wtmp rm -rf /var/lib/cloud/instances/* From 635f9c1bc5e98558954a923fcbd56b1e8af0ee1a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 15:49:37 -0500 Subject: [PATCH 26/28] chore: remove unused function --- contrib/images/digitalocean/in_parts/100-image-check | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/images/digitalocean/in_parts/100-image-check b/contrib/images/digitalocean/in_parts/100-image-check index e1cf432e5..bd7178132 100755 --- a/contrib/images/digitalocean/in_parts/100-image-check +++ b/contrib/images/digitalocean/in_parts/100-image-check @@ -482,8 +482,6 @@ function checkCloudInit { return 1 } -function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } - clear echo "DigitalOcean Marketplace Image Validation Tool ${VERSION}" echo "Executed on: ${RUNDATE}" From 26a536bc9b4aaffe2139b4fd74b437fe25dd552a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 16:01:59 -0500 Subject: [PATCH 27/28] feat: log the output of the do directory --- contrib/images/digitalocean/in_parts/099-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/images/digitalocean/in_parts/099-cleanup b/contrib/images/digitalocean/in_parts/099-cleanup index 25f253628..6464914d6 100755 --- a/contrib/images/digitalocean/in_parts/099-cleanup +++ b/contrib/images/digitalocean/in_parts/099-cleanup @@ -38,8 +38,8 @@ rm -f /root/.ssh/authorized_keys /etc/ssh/*key* touch /etc/ssh/revoked_keys chmod 600 /etc/ssh/revoked_keys -echo '--> DO Directory' -ls -lah /opt/digitalocean +echo '--> DO Directory contents' +ls -lah /opt/digitalocean || true echo '--> Securely erase the unused portion of the filesystem' printf "\n\033[0;32mWriting zeros to the remaining disk space to securely From 4b8ccfbf3476b7c49f72b6b67c469f5ee5c43893 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 16 Feb 2023 16:22:32 -0500 Subject: [PATCH 28/28] fix: implement checkTMP --- .../images/digitalocean/in_parts/100-image-check | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/images/digitalocean/in_parts/100-image-check b/contrib/images/digitalocean/in_parts/100-image-check index bd7178132..a14bb87c1 100755 --- a/contrib/images/digitalocean/in_parts/100-image-check +++ b/contrib/images/digitalocean/in_parts/100-image-check @@ -136,7 +136,15 @@ function checkLogs { } function checkTMP { # Check the /tmp directory to ensure it is empty. Warn on any files found. - return 1 + if [[ -n "$(ls -A /tmp)" ]]; then + echo -en "\e[93m[WARN]\e[0m /tmp directory is not empty; Contents\n" + ls -A /tmp + ((WARN++)) + return 1 + fi + + echo -en "\e[32m[PASS]\e[0m /tmp directory is empty\n" + return 0 } function checkRoot { user="root" @@ -595,6 +603,9 @@ checkUsers echo -en "\n\nChecking the root account...\n" checkRoot +echo -en "\n\nChecking the /tmp directory...\n" +checkTMP + checkAgent # Summary