feat(scripts): use the undocumented docker.io endpoint only for untagged manifests

- read the docker.io tags from the documented tags API
- check that docker.io still has a manifest before deleting it, docker.io also answers 202 to the deletion of a missing manifest
- count every manifest in --delete
This commit is contained in:
peaklabs-dev
2026-11-03 17:41:33 +01:00
parent ef88ea07da
commit 75f0ad7459
+70 -57
View File
@@ -35,9 +35,9 @@ main() {
GHCR_TOKEN="$(gh auth token)" || die "gh is not signed in with a token that has read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages"
heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE"
read_ghcr_versions
read_dockerhub_manifests
read_dockerhub_tags
read_dockerhub_manifests_deleted_on_ghcr
read_dockerhub_images
read_dockerhub_untagged_manifests
select_kept_tags
read_manifests
select_kept_manifests
@@ -57,23 +57,34 @@ read_ghcr_versions() {
ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged"
}
# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice
read_dockerhub_manifests() {
local hub base url page key
read_dockerhub_tags() {
local hub url page
hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME"
base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100"
url="$base"
url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100"
: > "$WORK_DIR/hub-tags.tsv"
: > "$WORK_DIR/hub-images.tsv"
: > "$WORK_DIR/dockerhub-images.tsv"
while [ -n "$url" ]; do
page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the manifests of docker.io/$IMAGE from the Docker Hub API"
jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" \
>> "$WORK_DIR/dockerhub-manifests.tsv"
key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")"
url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}"
page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the tags of docker.io/$IMAGE from the Docker Hub API"
jq -r '.results[] | [.name, (.digest // ""), (.media_type // ""), (.tag_last_pushed // .last_updated // ""), (.full_size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-tags.tsv"
jq -r '.results[] | .name as $tag | .images[]? | select(.digest) | [$tag, .digest, ([.os, .architecture] | join("/")), (.last_pushed // ""), (.size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-images.tsv"
url="$(jq -r '.next // empty' <<< "$page")"
done
awk -F '\t' -v OFS='\t' '{ n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv"
grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return the tags of every manifest"
awk -F '\t' '$4 != "" { found = 1; exit } END { exit !found }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists no push dates, expected them from the Docker Hub API"
ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub-manifests.tsv") manifests, $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) of them untagged"
awk -F '\t' '$2 == "" || $3 == "" { print $1 }' "$WORK_DIR/hub-tags.tsv" > "$WORK_DIR/old-tags.txt"
request_manifests docker.io "$WORK_DIR/old-tags.txt" -f --head -o /dev/null -w '{}\t%header{docker-content-digest}\t%header{content-type}\n' > "$WORK_DIR/old-tags.tsv" \
|| die "could not read the digests of the old tags of docker.io/$IMAGE, the Docker Hub API lists them without one"
awk -F '\t' -v OFS='\t' -v images="$WORK_DIR/dockerhub-images.tsv" '
FILENAME == ARGV[1] { old_digest[$1] = $2; old_type[$1] = $3; next }
FILENAME == ARGV[2] { if ($2 == "") $2 = old_digest[$1]; if ($3 == "") $3 = old_type[$1]; root[$1] = $2; tags[$2] = ($2 in tags) ? tags[$2] "," $1 : $1; type[$2] = $3; if ($4 > pushed[$2]) pushed[$2] = $4; next }
$2 == root[$1] { platform[$2] = $3; size[$2] = $5; next }
!((root[$1], $2) in pair) { pair[root[$1], $2]; print root[$1], $2 > images }
!($2 in tags) && !seen[$2]++ { print $2, "", "", $4, $5, $3 }
END { for (d in tags) print d, type[d], tags[d], pushed[d], (type[d] ~ /index|list/ ? 0 : size[d] + 0), platform[d] }' \
"$WORK_DIR/old-tags.tsv" "$WORK_DIR/hub-tags.tsv" "$WORK_DIR/hub-images.tsv" > "$WORK_DIR/dockerhub-manifests.tsv"
awk -F '\t' -v OFS='\t' '$3 != "" { n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv"
grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return every tag"
awk -F '\t' '$1 !~ /^sha256:/ || $4 == "" { exit 1 }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists manifests without a digest or push date, expected both from the Docker Hub API"
ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(awk -F '\t' '$3 != ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests, which reference $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) more"
}
read_dockerhub_manifests_deleted_on_ghcr() {
@@ -85,20 +96,23 @@ read_dockerhub_manifests_deleted_on_ghcr() {
ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/deleted-on-ghcr.tsv" | count) of the $(count < "$WORK_DIR/ghcr-deleted.tsv") versions that ghcr.io deleted last and no tag references"
}
read_dockerhub_images() {
local hub url page
# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice. It only adds the untagged manifests that no tag references
read_dockerhub_untagged_manifests() {
local hub base url page key
hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME"
url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100"
base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100"
url="$base"
: > "$WORK_DIR/listing.tsv"
while [ -n "$url" ]; do
if ! page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")"; then
warn "the Docker Hub API refused a page of the tags of docker.io/$IMAGE, the images of an index that only docker.io has are deleted in a later run"
: > "$WORK_DIR/dockerhub-images.tsv"
return
fi
jq -r '.results[] | .digest as $index | .images[]? | select($index and .digest and .digest != $index) | [$index, .digest] | @tsv' <<< "$page" >> "$WORK_DIR/dockerhub-images.tsv"
url="$(jq -r '.next // empty' <<< "$page")"
page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" \
&& jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" >> "$WORK_DIR/listing.tsv" \
|| { warn "the undocumented manifests listing of the Docker Hub API failed, the untagged manifests that no tag references wait for a later run"; return; }
key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")"
url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}"
done
ok "read the images of the $(cut -f 1 "$WORK_DIR/dockerhub-images.tsv" | sort -u | count) indexes of the tags of docker.io/$IMAGE from the Docker Hub API"
awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { known[$1]; next } !($1 in known) && $3 == "" { print $1, $2, "", $4, $5, $6 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/listing.tsv" > "$WORK_DIR/untagged.tsv"
cat "$WORK_DIR/untagged.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv"
ok "the undocumented manifests listing adds $(count < "$WORK_DIR/untagged.tsv") untagged manifests that no tag references"
}
select_kept_tags() {
@@ -120,25 +134,18 @@ select_kept_tags() {
}
read_manifests() {
local digest tag hub
mkdir "$WORK_DIR/manifests"
request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \
|| die "could not read the manifests of ghcr.io/$IMAGE"
awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \
"$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv"
awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt"
request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept manifests that only docker.io has"
hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME"
while IFS=$'\t' read -r digest tag; do
curl -sSf --retry 3 -H "Authorization: Bearer $hub" "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \
|| die "could not read the images of $tag from the Docker Hub API"
done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv"
awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && $2 ~ /oci\.image\.index/ && !seen[$1]++ { print $1 }' \
"$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-indexes.txt"
request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept indexes that only docker.io has"
manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \
>> "$WORK_DIR/children.tsv"
> "$WORK_DIR/children.tsv"
manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv"
manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \
> "$WORK_DIR/kinds.tsv"
ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and $(count < "$WORK_DIR/dockerhub-roots.tsv") kept manifests that only docker.io has, $(count < "$WORK_DIR/dockerhub-indexes.txt") of them with a GET that counts as a pull"
ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and the $(count < "$WORK_DIR/dockerhub-indexes.txt") kept indexes that only docker.io has, each with a GET that counts as a pull"
}
select_kept_manifests() {
@@ -166,15 +173,14 @@ write_list() {
FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next }
FILENAME == ARGV[6] {
on_docker_hub[$1]
if (!($1 in kept) && !($1 in kind) && $4 < cutoff && !seen[$1]++)
if (!($1 in kept) && !($1 in kind) && $4 != "" && $4 < cutoff && !seen[$1]++)
print "no", "yes", ($2 ~ /index|list/ ? "index" : ($6 == "unknown/unknown" ? "attestation" : "image")), ($6 == "unknown/unknown" ? "" : $6), $3, $4, $5, "", $1
next
}
!($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", (($1 in on_docker_hub) ? "yes" : "no"), kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \
"$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-active.tsv" \
| sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/sorted.tsv"
: > "$WORK_DIR/grouped.txt"
awk -F '\t' -v grouped="$WORK_DIR/grouped.txt" 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) { print list[i] > grouped; emit(list[i]) } }
awk -F '\t' 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) emit(list[i]) }
FILENAME == ARGV[1] || FILENAME == ARGV[2] { if (!(($1, $2) in pair)) { pair[$1, $2]; children[$1] = children[$1] " " $2 } next }
{ order[++rows] = $9; line[$9] = $0; type[$9] = $3 }
END {
@@ -189,13 +195,14 @@ write_list() {
}
delete_manifests() {
local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status
local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status gone=0
if [ -z "$LIMIT" ]; then
info "a dry run deletes nothing, --delete with a count deletes that many from the top of the list"
info "a dry run deletes nothing, --delete with a count deletes that many manifests from the top of the list"
return
fi
heading "Deleting"
write_table /dev/null "$DELETED_FILE"
head -n 1 "$TO_DELETE_FILE" > "$DELETED_FILE"
: > "$WORK_DIR/deleted.tsv"
while IFS='|' read -r on_ghcr on_docker_hub type tags version digest rest; do
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest"
name="$digest${tags:+ ($tags)}"
@@ -208,20 +215,24 @@ delete_manifests() {
status="$(delete_dockerhub_manifest "$digest")"
if [[ "$status" == 2?? ]]; then
deleted="${deleted:+$deleted and }docker.io" docker_hub=yes
elif [ "$status" = 404 ]; then
info "docker.io had already deleted the $type $BOLD$name$RESET together with its index"
gone=$((gone + 1))
fi
fi
if [ -n "$deleted" ]; then
ok "deleted the $type $BOLD$name$RESET from $deleted"
printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv"
write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE"
printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" | tee -a "$WORK_DIR/deleted.tsv" >> "$DELETED_FILE"
fi
case "$status" in
"" | 2?? | 404) ;;
403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;;
*) die "could not delete the $type $name from docker.io, it answered $status" ;;
esac
done < <(awk -F '\t' -v limit="$LIMIT" 'FILENAME == ARGV[1] { grouped[$1]; next } FNR > limit && !($9 in grouped) { exit } { print }' "$WORK_DIR/grouped.txt" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }')
done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }')
write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE"
ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 " (" $3 "), they free " $7 : "deleted nothing") }' "$DELETED_FILE")"
[ "$gone" = 0 ] || info "docker.io had already deleted $gone of the listed manifests together with their index, it deletes the children of an index that nothing else references"
info "$DELETED_FILE lists what this run deleted and where, ghcr.io can restore them for 30 days, docker.io deletes them permanently"
}
@@ -251,9 +262,7 @@ request_manifests() {
split -l 1000 "$list" "$WORK_DIR/chunk-"
for chunk in "$WORK_DIR"/chunk-*; do
token="$(registry_token "$registry" pull)"
xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \
"$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1
xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1
rm "$chunk"
done
}
@@ -264,9 +273,9 @@ manifest_fields() {
kept_manifests() {
awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) }
FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next }
FILENAME == ARGV[2] { tag[$1]; next }
{ n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1"
FILENAME == ARGV[1] || FILENAME == ARGV[2] { children[$1] = children[$1] " " $2; next }
FILENAME == ARGV[3] { tag[$1]; next }
{ n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/kept-tags.txt" "$1"
}
write_table() {
@@ -302,11 +311,15 @@ delete_ghcr_version() {
sleep 1
}
# docker.io answers 202 to the deletion of any digest, so a HEAD tells first whether the manifest is still there
delete_dockerhub_manifest() {
local status
while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \
"https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do
local token status
token="$(registry_token docker.io pull,push,delete)"
status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' --head -H "Authorization: Bearer $token" -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")"
[ "$status" = 200 ] || { echo "$status"; return; }
while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do
sleep 60
token="$(registry_token docker.io pull,push,delete)"
done
echo "$status"
}