From 75f0ad7459a7047d6589ce96466fcd8c50d3bb31 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:22:39 +0100 Subject: [PATCH] feat(scripts): use the undocumented docker.io endpoint only for untagged manifests - read the docker.io tags from the documented tags API - check that docker.io still has a manifest before deleting it, docker.io also answers 202 to the deletion of a missing manifest - count every manifest in --delete --- scripts/clean-manifests.sh | 127 ++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index b43061ec3b..ec659b5464 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -35,9 +35,9 @@ main() { GHCR_TOKEN="$(gh auth token)" || die "gh is not signed in with a token that has read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" read_ghcr_versions - read_dockerhub_manifests + read_dockerhub_tags read_dockerhub_manifests_deleted_on_ghcr - read_dockerhub_images + read_dockerhub_untagged_manifests select_kept_tags read_manifests select_kept_manifests @@ -57,23 +57,34 @@ read_ghcr_versions() { ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged" } -# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice -read_dockerhub_manifests() { - local hub base url page key +read_dockerhub_tags() { + local hub url page hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" - base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100" - url="$base" + url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100" + : > "$WORK_DIR/hub-tags.tsv" + : > "$WORK_DIR/hub-images.tsv" + : > "$WORK_DIR/dockerhub-images.tsv" while [ -n "$url" ]; do - page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the manifests of docker.io/$IMAGE from the Docker Hub API" - jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" \ - >> "$WORK_DIR/dockerhub-manifests.tsv" - key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")" - url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}" + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the tags of docker.io/$IMAGE from the Docker Hub API" + jq -r '.results[] | [.name, (.digest // ""), (.media_type // ""), (.tag_last_pushed // .last_updated // ""), (.full_size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-tags.tsv" + jq -r '.results[] | .name as $tag | .images[]? | select(.digest) | [$tag, .digest, ([.os, .architecture] | join("/")), (.last_pushed // ""), (.size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-images.tsv" + url="$(jq -r '.next // empty' <<< "$page")" done - awk -F '\t' -v OFS='\t' '{ n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv" - grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return the tags of every manifest" - awk -F '\t' '$4 != "" { found = 1; exit } END { exit !found }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists no push dates, expected them from the Docker Hub API" - ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub-manifests.tsv") manifests, $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) of them untagged" + awk -F '\t' '$2 == "" || $3 == "" { print $1 }' "$WORK_DIR/hub-tags.tsv" > "$WORK_DIR/old-tags.txt" + request_manifests docker.io "$WORK_DIR/old-tags.txt" -f --head -o /dev/null -w '{}\t%header{docker-content-digest}\t%header{content-type}\n' > "$WORK_DIR/old-tags.tsv" \ + || die "could not read the digests of the old tags of docker.io/$IMAGE, the Docker Hub API lists them without one" + awk -F '\t' -v OFS='\t' -v images="$WORK_DIR/dockerhub-images.tsv" ' + FILENAME == ARGV[1] { old_digest[$1] = $2; old_type[$1] = $3; next } + FILENAME == ARGV[2] { if ($2 == "") $2 = old_digest[$1]; if ($3 == "") $3 = old_type[$1]; root[$1] = $2; tags[$2] = ($2 in tags) ? tags[$2] "," $1 : $1; type[$2] = $3; if ($4 > pushed[$2]) pushed[$2] = $4; next } + $2 == root[$1] { platform[$2] = $3; size[$2] = $5; next } + !((root[$1], $2) in pair) { pair[root[$1], $2]; print root[$1], $2 > images } + !($2 in tags) && !seen[$2]++ { print $2, "", "", $4, $5, $3 } + END { for (d in tags) print d, type[d], tags[d], pushed[d], (type[d] ~ /index|list/ ? 0 : size[d] + 0), platform[d] }' \ + "$WORK_DIR/old-tags.tsv" "$WORK_DIR/hub-tags.tsv" "$WORK_DIR/hub-images.tsv" > "$WORK_DIR/dockerhub-manifests.tsv" + awk -F '\t' -v OFS='\t' '$3 != "" { n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv" + grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return every tag" + awk -F '\t' '$1 !~ /^sha256:/ || $4 == "" { exit 1 }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists manifests without a digest or push date, expected both from the Docker Hub API" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(awk -F '\t' '$3 != ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests, which reference $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) more" } read_dockerhub_manifests_deleted_on_ghcr() { @@ -85,20 +96,23 @@ read_dockerhub_manifests_deleted_on_ghcr() { ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/deleted-on-ghcr.tsv" | count) of the $(count < "$WORK_DIR/ghcr-deleted.tsv") versions that ghcr.io deleted last and no tag references" } -read_dockerhub_images() { - local hub url page +# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice. It only adds the untagged manifests that no tag references +read_dockerhub_untagged_manifests() { + local hub base url page key hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" - url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100" + base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100" + url="$base" + : > "$WORK_DIR/listing.tsv" while [ -n "$url" ]; do - if ! page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")"; then - warn "the Docker Hub API refused a page of the tags of docker.io/$IMAGE, the images of an index that only docker.io has are deleted in a later run" - : > "$WORK_DIR/dockerhub-images.tsv" - return - fi - jq -r '.results[] | .digest as $index | .images[]? | select($index and .digest and .digest != $index) | [$index, .digest] | @tsv' <<< "$page" >> "$WORK_DIR/dockerhub-images.tsv" - url="$(jq -r '.next // empty' <<< "$page")" + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" \ + && jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" >> "$WORK_DIR/listing.tsv" \ + || { warn "the undocumented manifests listing of the Docker Hub API failed, the untagged manifests that no tag references wait for a later run"; return; } + key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")" + url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}" done - ok "read the images of the $(cut -f 1 "$WORK_DIR/dockerhub-images.tsv" | sort -u | count) indexes of the tags of docker.io/$IMAGE from the Docker Hub API" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { known[$1]; next } !($1 in known) && $3 == "" { print $1, $2, "", $4, $5, $6 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/listing.tsv" > "$WORK_DIR/untagged.tsv" + cat "$WORK_DIR/untagged.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "the undocumented manifests listing adds $(count < "$WORK_DIR/untagged.tsv") untagged manifests that no tag references" } select_kept_tags() { @@ -120,25 +134,18 @@ select_kept_tags() { } read_manifests() { - local digest tag hub mkdir "$WORK_DIR/manifests" request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ || die "could not read the manifests of ghcr.io/$IMAGE" - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ - "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" - awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" - request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept manifests that only docker.io has" - hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" - while IFS=$'\t' read -r digest tag; do - curl -sSf --retry 3 -H "Authorization: Bearer $hub" "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ - || die "could not read the images of $tag from the Docker Hub API" - done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && $2 ~ /oci\.image\.index/ && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept indexes that only docker.io has" manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ - >> "$WORK_DIR/children.tsv" + > "$WORK_DIR/children.tsv" manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ > "$WORK_DIR/kinds.tsv" - ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and $(count < "$WORK_DIR/dockerhub-roots.tsv") kept manifests that only docker.io has, $(count < "$WORK_DIR/dockerhub-indexes.txt") of them with a GET that counts as a pull" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and the $(count < "$WORK_DIR/dockerhub-indexes.txt") kept indexes that only docker.io has, each with a GET that counts as a pull" } select_kept_manifests() { @@ -166,15 +173,14 @@ write_list() { FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } FILENAME == ARGV[6] { on_docker_hub[$1] - if (!($1 in kept) && !($1 in kind) && $4 < cutoff && !seen[$1]++) + if (!($1 in kept) && !($1 in kind) && $4 != "" && $4 < cutoff && !seen[$1]++) print "no", "yes", ($2 ~ /index|list/ ? "index" : ($6 == "unknown/unknown" ? "attestation" : "image")), ($6 == "unknown/unknown" ? "" : $6), $3, $4, $5, "", $1 next } !($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", (($1 in on_docker_hub) ? "yes" : "no"), kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-active.tsv" \ | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/sorted.tsv" - : > "$WORK_DIR/grouped.txt" - awk -F '\t' -v grouped="$WORK_DIR/grouped.txt" 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) { print list[i] > grouped; emit(list[i]) } } + awk -F '\t' 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) emit(list[i]) } FILENAME == ARGV[1] || FILENAME == ARGV[2] { if (!(($1, $2) in pair)) { pair[$1, $2]; children[$1] = children[$1] " " $2 } next } { order[++rows] = $9; line[$9] = $0; type[$9] = $3 } END { @@ -189,13 +195,14 @@ write_list() { } delete_manifests() { - local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status + local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status gone=0 if [ -z "$LIMIT" ]; then - info "a dry run deletes nothing, --delete with a count deletes that many from the top of the list" + info "a dry run deletes nothing, --delete with a count deletes that many manifests from the top of the list" return fi heading "Deleting" - write_table /dev/null "$DELETED_FILE" + head -n 1 "$TO_DELETE_FILE" > "$DELETED_FILE" + : > "$WORK_DIR/deleted.tsv" while IFS='|' read -r on_ghcr on_docker_hub type tags version digest rest; do [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest" name="$digest${tags:+ ($tags)}" @@ -208,20 +215,24 @@ delete_manifests() { status="$(delete_dockerhub_manifest "$digest")" if [[ "$status" == 2?? ]]; then deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + elif [ "$status" = 404 ]; then + info "docker.io had already deleted the $type $BOLD$name$RESET together with its index" + gone=$((gone + 1)) fi fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" - write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" | tee -a "$WORK_DIR/deleted.tsv" >> "$DELETED_FILE" fi case "$status" in "" | 2?? | 404) ;; 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(awk -F '\t' -v limit="$LIMIT" 'FILENAME == ARGV[1] { grouped[$1]; next } FNR > limit && !($9 in grouped) { exit } { print }' "$WORK_DIR/grouped.txt" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }') + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }') + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 " (" $3 "), they free " $7 : "deleted nothing") }' "$DELETED_FILE")" + [ "$gone" = 0 ] || info "docker.io had already deleted $gone of the listed manifests together with their index, it deletes the children of an index that nothing else references" info "$DELETED_FILE lists what this run deleted and where, ghcr.io can restore them for 30 days, docker.io deletes them permanently" } @@ -251,9 +262,7 @@ request_manifests() { split -l 1000 "$list" "$WORK_DIR/chunk-" for chunk in "$WORK_DIR"/chunk-*; do token="$(registry_token "$registry" pull)" - xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ - "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 rm "$chunk" done } @@ -264,9 +273,9 @@ manifest_fields() { kept_manifests() { awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } - FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } - FILENAME == ARGV[2] { tag[$1]; next } - { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" + FILENAME == ARGV[1] || FILENAME == ARGV[2] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[3] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/kept-tags.txt" "$1" } write_table() { @@ -302,11 +311,15 @@ delete_ghcr_version() { sleep 1 } +# docker.io answers 202 to the deletion of any digest, so a HEAD tells first whether the manifest is still there delete_dockerhub_manifest() { - local status - while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ - "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + local token status + token="$(registry_token docker.io pull,push,delete)" + status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' --head -H "Authorization: Bearer $token" -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" + [ "$status" = 200 ] || { echo "$status"; return; } + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do sleep 60 + token="$(registry_token docker.io pull,push,delete)" done echo "$status" }