diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 65717429..9eca5648 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -17,7 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DOCKER_IMAGE: smallstep/step-ca
-      DOCKER_IMAGE_HSM: smallstep/step-ca-hsm
     outputs:
       version: ${{ steps.extract-tag.outputs.VERSION }}
       is_prerelease: ${{ steps.is_prerelease.outputs.IS_PRERELEASE }}
@@ -38,12 +37,12 @@ jobs:
           VERSION=${GITHUB_REF#refs/tags/v}
           echo "VERSION=${VERSION}" >> ${GITHUB_OUTPUT}
           echo "DOCKER_TAGS=${{ env.DOCKER_IMAGE }}:${VERSION}" >> ${GITHUB_ENV}
-          echo "DOCKER_TAGS_HSM=${{ env.DOCKER_IMAGE_HSM }}:${VERSION}" >> ${GITHUB_ENV}
+          echo "DOCKER_TAGS_HSM=${{ env.DOCKER_IMAGE }}:${VERSION}-hsm" >> ${GITHUB_ENV}
       - name: Add Latest Tag
         if: steps.is_prerelease.outputs.IS_PRERELEASE == 'false'
         run: |
           echo "DOCKER_TAGS=${{ env.DOCKER_TAGS }},${{ env.DOCKER_IMAGE }}:latest" >> ${GITHUB_ENV}
-          echo "DOCKER_TAGS_HSM=${{ env.DOCKER_TAGS_HSM }},${{ env.DOCKER_IMAGE_HSM }}:latest" >> ${GITHUB_ENV}
+          echo "DOCKER_TAGS_HSM=${{ env.DOCKER_TAGS_HSM }},${{ env.DOCKER_IMAGE }}:hsm" >> ${GITHUB_ENV}
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 00fa9c0e..553d831f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -3,19 +3,17 @@ FROM golang:alpine AS builder
 WORKDIR /src
 COPY . .
 
-RUN apk add --no-cache curl git make
-RUN make V=1 download
+RUN apk add --no-cache curl git make libcap
 RUN make V=1 bin/step-ca
+RUN setcap CAP_NET_BIND_SERVICE=+eip bin/step-ca
 
-FROM smallstep/step-kms-plugin-cloud:latest AS kms
+FROM smallstep/step-kms-plugin:cloud AS kms
 
 FROM smallstep/step-cli:latest
 
 COPY --from=builder /src/bin/step-ca /usr/local/bin/step-ca
 COPY --from=kms /usr/local/bin/step-kms-plugin /usr/local/bin/step-kms-plugin
 
-USER root
-RUN apk add --no-cache libcap && setcap CAP_NET_BIND_SERVICE=+eip /usr/local/bin/step-ca
 USER step
 
 ENV CONFIGPATH="/home/step/config/ca.json"
diff --git a/docker/Dockerfile.hsm b/docker/Dockerfile.hsm
index 05c1ef82..8ae1e7c7 100644
--- a/docker/Dockerfile.hsm
+++ b/docker/Dockerfile.hsm
@@ -1,23 +1,24 @@
-FROM golang:alpine AS builder
+FROM golang AS builder
 
 WORKDIR /src
 COPY . .
 
-RUN apk add --no-cache curl git make
-RUN apk add --no-cache gcc musl-dev pkgconf pcsc-lite-dev
-RUN make V=1 download
-RUN make V=1 GOFLAGS="" build
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends \
+        gcc pkgconf libpcsclite-dev libcap2-bin
+RUN make V=1 GOFLAGS="" bin/step-ca
+RUN setcap CAP_NET_BIND_SERVICE=+eip bin/step-ca
 
-FROM smallstep/step-kms-plugin:latest AS kms
+FROM smallstep/step-kms-plugin:bullseye AS kms
 
-FROM smallstep/step-cli:latest
+FROM smallstep/step-cli:bullseye
 
 COPY --from=builder /src/bin/step-ca /usr/local/bin/step-ca
 COPY --from=kms /usr/local/bin/step-kms-plugin /usr/local/bin/step-kms-plugin
 
 USER root
-RUN apk add --no-cache libcap && setcap CAP_NET_BIND_SERVICE=+eip /usr/local/bin/step-ca
-RUN apk add --no-cache pcsc-lite pcsc-lite-libs
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends pcscd libpcsclite1
 RUN mkdir -p /run/pcscd
 RUN chown step:step /run/pcscd
 USER step