diff --git a/.editorconfig b/.editorconfig index 7a5dc5155b..e9ae35946a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,7 +20,7 @@ indent_size = 4 [{release.json,upstream.json}] indent_style = tab -[scripts/{clean-manifests,plan-version,release,sync-cdn,undo-release}.sh] +[scripts/{clean-manifests,plan-version,release,size,sync-cdn,undo-release}.sh] indent_style = tab binary_next_line = true switch_case_indent = true diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 0de0334bcc..bee3f3ffcb 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -26,6 +26,7 @@ workflows: '.github/workflows/pr-quality.yaml': - 'peakoss/anti-slop@v0.3.0' '.github/workflows/pull-request.yaml': + - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - 'docker/login-action@v4.7.0' '.github/workflows/release.yaml': diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5e79fe2c4e..10c0af64e3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -81,6 +81,7 @@ jobs: run: | { echo "epoch=$(git log -1 --format=%ct)" + echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" echo "sha=$(git rev-parse HEAD)" } >> "$GITHUB_OUTPUT" @@ -96,11 +97,14 @@ jobs: file: docker/production/Dockerfile platforms: ${{ matrix.platform }} tags: coollabsio/coolify:test - outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true + outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true build-args: | COOLIFY_VERSION=${{ needs.plan.outputs.coolify }} SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} + CREATED=${{ steps.args.outputs.created }} GIT_SHA=${{ steps.args.outputs.sha }} + provenance: mode=max + sbom: true - name: Upload the image uses: actions/upload-artifact@v7.0.2 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f0dbd9a07c..61d20419d0 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -85,4 +85,7 @@ jobs: for index in "$RUNNER_TEMP"/images/*/index.json; do sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") done - docker buildx imagetools create --tag ghcr.io/coollabsio/coolify:main "${sources[@]}" + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" + docker buildx imagetools create "${annotations[@]}" --tag ghcr.io/coollabsio/coolify:main "${sources[@]}" diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index e21e1269f1..688b843a01 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -100,10 +100,13 @@ jobs: for index in "$RUNNER_TEMP"/images/*/index.json; do sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" for repository in ghcr.io/coollabsio/coolify docker.io/coollabsio/coolify; do tags=(--tag "$repository:${TAG#v}") [ "$LINE" != latest ] || tags+=(--tag "$repository:nightly") - docker buildx imagetools create "${tags[@]}" "${sources[@]}" + docker buildx imagetools create "${annotations[@]}" "${tags[@]}" "${sources[@]}" done cdn: diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 42a585f23d..1755e7d97c 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -81,4 +81,42 @@ jobs: for index in "$RUNNER_TEMP"/images/*/index.json; do sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") done - docker buildx imagetools create --tag "ghcr.io/coollabsio/coolify:$TAG" "${sources[@]}" + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" + docker buildx imagetools create "${annotations[@]}" --tag "ghcr.io/coollabsio/coolify:$TAG" "${sources[@]}" + + size: + name: Compare size with latest + needs: build + permissions: + contents: read + pull-requests: write # comment the size report on the pull request + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Download the built images + uses: actions/download-artifact@v8.0.2 + with: + pattern: image-* + path: ${{ runner.temp }}/images + + - name: Compare the size with the latest release + env: + IMAGE_NAME: pr-${{ github.event.pull_request.number }} + OUTPUT_FILE: ${{ runner.temp }}/size.md + run: | + scripts/size.sh "$RUNNER_TEMP/images" coollabsio/coolify:latest + cat "$OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment the size on the pull request + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: gh pr comment "$PULL_REQUEST" --body-file "$RUNNER_TEMP/size.md" --edit-last --create-if-none diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ab98ab102a..2f7819b8f5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -105,8 +105,11 @@ jobs: for index in "$RUNNER_TEMP"/images/*/index.json; do sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" for repository in ghcr.io/coollabsio/coolify docker.io/coollabsio/coolify; do - docker buildx imagetools create --tag "$repository:${TAG#v}" "${sources[@]}" + docker buildx imagetools create "${annotations[@]}" --tag "$repository:${TAG#v}" "${sources[@]}" done - name: Create the GitHub release diff --git a/CHANGELOG.md b/CHANGELOG.md index e8eb867ebf..a5280b75af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,16 +22,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- Attach an SBOM attestation to every image, which lists every package of the image +- Copy the labels of the image to the annotations of its index, ghcr.io reads the description and the license of an image of several platforms from there - Choose the update channel of an instance on the update page: stable, preview with betas and release candidates, nightly with a warning that breaking changes are possible, or a custom image such as `pr-1234` - Keep the updates of the old channel after a switch to a more stable channel, until the stable release of the installed version is out - Publish prereleases from `next` and the release branches, tagged like `4.6.0-beta.1`, and the moving `preview` tag - Publish a nightly version twice a day, tagged like `4.6.0-alpha.0.20261026.1`, with the moving `nightly` tag and the nightly channel of the CDN, from `next` or from the release branch while one is in progress - Build `main` twice a day when it changed and move the `main` tag on ghcr.io to it, versioned like `4.5.0.20261028.1`, to test fixes before their patch release - Publish the image of every pull request from a branch of this repository as `ghcr.io/coollabsio/coolify:pr-`, so a change can be tried before it is merged -- Add OCI labels with the version and the commit to the image +- Add OCI labels to the image, such as its version, commit, creation date, license and base image, instead of the labels it took over from its serversideup/php base image ### Changed +- Compress the image layers with zstd instead of gzip, it compresses better and compresses and decompresses faster, pulling needs Docker Engine 23.0 or later and Coolify requires 24.0 - Move the helper image to [coollabsio/coolify-helper](https://github.com/coollabsio/coolify-helper), which releases it - Replace the release process, `RELEASE.md` describes it - Keep the Coolify versions in `release.json`, `stable.coolify` for `main` and the maintenance branches and `preview.coolify` for `next` and the release branches @@ -76,6 +79,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Maintenance +- Add `scripts/size.sh` to measure the compressed and the on-disk size of every platform +- Comment the size change of every pull request compared with the latest release - Add `scripts/clean-manifests.sh` and a workflow that delete the unused manifests from ghcr.io and docker.io, such as the leftovers of the old workflows and the images of closed pull requests, and keep every release, every moving tag, the nightlies of the last 14 days and the images of open pull requests - Give every job a name, a timeout, a fixed runner image and a concurrency group - Bump `actions/stale` to v11, GitHub Actions removed the Node.js 20 it ran on diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index ce77ae4ea4..2e7bf28fda 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -116,6 +116,28 @@ RUN mkdir -p /etc/nginx/conf.d && \ COPY --from=minio-client /usr/bin/mc /usr/bin/mc RUN chmod +x /usr/bin/mc +ARG SERVERSIDEUP_PHP_VERSION +ARG COOLIFY_VERSION +ARG CREATED +ARG GIT_SHA + +LABEL org.opencontainers.image.title="Coolify Development" \ + org.opencontainers.image.description="The development image of Coolify, it runs the checkout that Docker Compose mounts into it." \ + org.opencontainers.image.version="${COOLIFY_VERSION}" \ + org.opencontainers.image.created="${CREATED}" \ + org.opencontainers.image.revision="${GIT_SHA}" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.authors="coollabsio" \ + org.opencontainers.image.vendor="coollabsio" \ + org.opencontainers.image.url="https://coolify.io" \ + org.opencontainers.image.source="https://github.com/coollabsio/coolify" \ + org.opencontainers.image.documentation="https://github.com/coollabsio/coolify/blob/main/CONTRIBUTING.md" \ + org.opencontainers.image.base.name="docker.io/serversideup/php:${SERVERSIDEUP_PHP_VERSION}" \ + io.coolify.nginx.version="${NGINX_VERSION}" \ + io.coolify.cloudflared.version="${CLOUDFLARED_VERSION}" \ + io.coolify.postgresql-client.version="${POSTGRES_VERSION}" \ + io.coolify.php-uv.version="${PHP_UV_VERSION}" + # Stay as root for s6 init so bind-mounted workspaces work even when the host # tree is root-owned (CI, Jean, rootful Docker). PHP-FPM/nginx still run as # www-data via their service configs. Do NOT set USER www-data here — that diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 018dca1d18..fcd20d01e0 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -17,6 +17,12 @@ ARG PHP_UV_VERSION=0.3.0 ARG USER_ID=9999 ARG GROUP_ID=9999 +# Set by the build workflow +ARG COOLIFY_VERSION +ARG SOURCE_DATE_EPOCH +ARG CREATED +ARG GIT_SHA + # ================================================================= # Stage 1: Composer dependencies # ================================================================= @@ -220,10 +226,28 @@ RUN mkdir -p /etc/nginx/conf.d && \ COPY --from=minio-client /usr/bin/mc /usr/bin/mc RUN chmod +x /usr/bin/mc +ARG SERVERSIDEUP_PHP_VERSION +ARG NODE_VERSION +ARG CREATED ARG GIT_SHA -LABEL org.opencontainers.image.version="${COOLIFY_VERSION}" \ - org.opencontainers.image.revision="${GIT_SHA}" +LABEL org.opencontainers.image.title="Coolify" \ + org.opencontainers.image.description="An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers." \ + org.opencontainers.image.version="${COOLIFY_VERSION}" \ + org.opencontainers.image.created="${CREATED}" \ + org.opencontainers.image.revision="${GIT_SHA}" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.authors="coollabsio" \ + org.opencontainers.image.vendor="coollabsio" \ + org.opencontainers.image.url="https://coolify.io" \ + org.opencontainers.image.source="https://github.com/coollabsio/coolify" \ + org.opencontainers.image.documentation="https://coolify.io/docs" \ + org.opencontainers.image.base.name="docker.io/serversideup/php:${SERVERSIDEUP_PHP_VERSION}" \ + io.coolify.nginx.version="${NGINX_VERSION}" \ + io.coolify.node.version="${NODE_VERSION}" \ + io.coolify.cloudflared.version="${CLOUDFLARED_VERSION}" \ + io.coolify.postgresql-client.version="${POSTGRES_VERSION}" \ + io.coolify.php-uv.version="${PHP_UV_VERSION}" # Switch to non-root user USER www-data diff --git a/scripts/size.sh b/scripts/size.sh new file mode 100755 index 0000000000..b37ee987d2 --- /dev/null +++ b/scripts/size.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +IMAGE="${1:-}" +BASELINE="${2:-}" +PLATFORM="${PLATFORM:-}" +IMAGE_NAME="${IMAGE_NAME:-}" +OUTPUT_FILE="${OUTPUT_FILE:-}" +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^[^-][^[:space:]]*( [^-][^[:space:]]*)?$ ]] || die "'$OPTION' is not an option, expected an image and an optional baseline" + [ -z "$OUTPUT_FILE" ] || [ -n "$BASELINE" ] || die "OUTPUT_FILE is set without a baseline, expected an image and a baseline to compare" + measure_image image "$IMAGE" + [ -z "$BASELINE" ] || measure_image baseline "$BASELINE" + [ -z "$BASELINE" ] || compare_with_baseline + [ -z "$OUTPUT_FILE" ] || write_report +} + +measure_image() { + heading "Sizes of $2" + mkdir "$WORK_DIR/$1" + if [ -d "$2" ]; then + read_layouts "$1" "$2" + else + read_repository "$1" "$2" + fi + measure_manifests "$1" "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1.tsv" ] || die "$2 has no image for ${PLATFORM:-a platform}" + sort -o "$WORK_DIR/$1.tsv" "$WORK_DIR/$1.tsv" +} + +read_layouts() { + layout="$2" + find "$2" -name index.json -exec jq -s '{manifests: map(.manifests[])}' {} + > "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1/index.json" ] || die "$2 holds no OCI layout, expected an index.json in it" +} + +read_repository() { + local name="${2#docker.io/}" reference + layout="" + repository="${name%%[:@]*}" + reference="${name:${#repository}+1}" + [[ "$repository" == */* ]] || repository="library/$repository" + read_object manifests "${reference:-latest}" > "$WORK_DIR/$1/index.json" 2> /dev/null || die "$2 is not on Docker Hub or not public" +} + +measure_manifests() { + local digest + if jq -e 'has("manifests")' "$2" > /dev/null; then + for digest in $(jq -r '.manifests[] | select(.platform.os != "unknown") | .digest' "$2"); do + read_object manifests "$digest" > "$WORK_DIR/$1/$digest" + measure_manifests "$1" "$WORK_DIR/$1/$digest" + done + else + measure_platform "$1" "$2" + fi +} + +measure_platform() { + local platform compressed unpacked=0 media_type digest blocks + read_object blobs "$(jq -r .config.digest "$2")" > "$2.config" + platform="$(jq -r '[.os, .architecture, .variant // empty] | join("/")' "$2.config")" + [ -z "$PLATFORM" ] || [ "$platform" = "$PLATFORM" ] || return 0 + compressed=$(($(wc -c < "$2") + $(jq '.config.size + ([.layers[].size] | add // 0)' "$2"))) + while read -r media_type digest; do + blocks="$(read_object blobs "$digest" | decompress "$media_type" | count_blocks)" + unpacked=$((unpacked + blocks)) + done < <(jq -r '.layers[] | "\(.mediaType) \(.digest)"' "$2") + printf '%s\t%s\t%s\t%s\n' "$platform" "$compressed" $((compressed + unpacked)) \ + "$(jq -r '.config.Labels["org.opencontainers.image.version"] // ""' "$2.config")" >> "$WORK_DIR/$1.tsv" + ok "$BOLD$platform$RESET is $(format_size "$compressed") compressed and $(format_size $((compressed + unpacked))) on disk, $(format_size "$unpacked") of it unpacked" +} + +decompress() { + case "$1" in + *+zstd) zstd -dc ;; + *+gzip | *.tar.gzip) gzip -dc ;; + *.tar) cat ;; + *) die "a layer is $1, expected a tar archive, plain or compressed with gzip or zstd" ;; + esac +} + +count_blocks() { + tar -tvf - | awk ' + BEGIN { blocks = 1 } + /^-/ { blocks += int((($2 ~ /\// ? $3 : $5) + 4095) / 4096) } + /^d/ { blocks++ } + /^l/ && length(substr($0, index($0, " -> ") + 4)) >= 60 { blocks++ } + END { print blocks * 4096 }' +} + +compare_with_baseline() { + local platform compressed on_disk base_compressed base_on_disk compressed_change on_disk_change + heading "Changes from $BASELINE" + while IFS=$'\t' read -r platform compressed on_disk base_compressed base_on_disk; do + compressed_change="$(format_change "$compressed" "$base_compressed")" + on_disk_change="$(format_change "$on_disk" "$base_on_disk")" + if [ "$compressed" = - ]; then + warn "$BOLD$platform$RESET is gone, $IMAGE has no image for it" + elif [ "$base_compressed" = - ]; then + ok "$BOLD$platform$RESET is new, $BASELINE has no image for it" + else + ok "$BOLD$platform$RESET changes by $compressed_change compressed and $on_disk_change on disk" + fi + printf '| %s | %s | %s | %s | %s | %s | %s |\n' "$platform" "$(format_size "$compressed")" "$(format_size "$base_compressed")" "$compressed_change" \ + "$(format_size "$on_disk")" "$(format_size "$base_on_disk")" "$on_disk_change" >> "$WORK_DIR/rows.md" + done < <(join -t $'\t' -a 1 -a 2 -e - -o 0,1.2,1.3,2.2,2.3 "$WORK_DIR/image.tsv" "$WORK_DIR/baseline.tsv") +} + +format_change() { + awk -v size="$1" -v base="$2" 'BEGIN { + if (size == "-" || base == "-") { print "-"; exit } + delta = size - base + if (delta == 0) { print "±0"; exit } + if (delta <= -1000000 || delta >= 1000000) amount = sprintf("%+.2f MB", delta / 1000000) + else if (delta <= -1000 || delta >= 1000) amount = sprintf("%+.1f kB", delta / 1000) + else amount = sprintf("%+d B", delta) + printf "%s (%+.1f %%)\n", amount, delta * 100 / base + }' +} + +write_report() { + local image_name baseline_name + image_name="$(short_name image "$IMAGE")" + baseline_name="$(short_name baseline "$BASELINE")" + if [ "$image_name" = "$baseline_name" ]; then + image_name="\`$IMAGE\`" + baseline_name="\`$BASELINE\`" + fi + { + printf '## Image size\n\n' + printf '%s compared with %s%s.\n\n' "$image_name" "$baseline_name" "$(format_version baseline "$BASELINE")" + printf '| Platform | Compressed %s | Compressed %s | Change | On disk %s | On disk %s | Change |\n' "$image_name" "$baseline_name" "$image_name" "$baseline_name" + printf '| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n' + cat "$WORK_DIR/rows.md" + printf "\nCompressed is what \`docker pull\` downloads, the CONTENT SIZE of the platform in \`docker image ls --tree\`. " + printf "On disk adds the unpacked files in 4 KiB blocks of ext4, the DISK USAGE that Docker Desktop shows, as the containerd image store keeps both. " + printf "Neither counts the SBOM and provenance attestations.\n" + } > "$OUTPUT_FILE" + ok "$OUTPUT_FILE holds the Markdown report" +} + +short_name() { + if [ "$1" = image ] && [ -n "$IMAGE_NAME" ]; then + printf "\`%s\`" "$IMAGE_NAME" + elif [ -d "$2" ]; then + printf 'the build' + else + printf "\`%s\`" "${2##*:}" + fi +} + +format_version() { + local version + [ ! -d "$2" ] || return 0 + version="$(head -n 1 "$WORK_DIR/$1.tsv" | cut -f 4)" + [ -z "$version" ] || printf " (\`%s\`)" "$version" +} + +read_object() { + local file + if [ -z "$layout" ]; then + fetch -H "Authorization: Bearer $(fetch "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$repository:pull" | jq -r .token)" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/$repository/$1/$2" + else + file="$(find "$layout" -path "*/blobs/${2/://}" -print -quit)" + [ -n "$file" ] || die "$layout has no blob $2" + cat "$file" + fi +} + +fetch() { + curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-all-errors -L "$@" +} + +format_size() { + awk -v bytes="$1" 'BEGIN { + if (bytes == "-") print "-" + else if (bytes >= 1000000) printf "%.2f MB\n", bytes / 1000000 + else if (bytes >= 1000) printf "%.1f kB\n", bytes / 1000 + else printf "%d B\n", bytes + }' +} + +main