fix(offline-repo): read dpkg-deb output in both of its shapes
dpkg-deb prints a BARE VALUE when one control field is requested and "<Field>: <value>" when several are. Every reader was written for one shape and broke on the other, so no pool could be read and no pin could be matched against real files: - build-index.sh: kept the label, so "Package: curl" failed the name check and the pool came out empty -- the build refused with "no .deb files"; - add-packages.sh (read_deb): rejected every genuine .deb as an invalid name; (scan_existing): produced empty keys, so a version conflict went undetected and the same package was copied twice; - build-offline-repo.sh: compared a labelled value with a "package=version" pin, which can never match. A first attempt with `sed 's/^Field: //'` was wrong in the other direction: sed -v prints nothing when the pattern does not match, which emptied the single-field reads. The label is now dropped only when it is present (awk: if the first word ends with a colon), which leaves a bare value and a version like "1:2.3-4" untouched. Also fixed in build-index.sh: the foreign-file guard did not list repo-manifest.yaml, the record this script writes into the bundle itself, so every second build stopped on its own previous output. Accepting it is what makes a rebuild possible at all. Tests - The dpkg-deb stub now reproduces both shapes, the single- and the multi-field one. It emitted only one, which is exactly why this class stayed invisible while the suite was green. - The rebuild case mirrors the production layout (record inside the output directory) and is proven by mutation: removing repo-manifest.yaml from the guard's allow-list turns it red.
This commit is contained in:
parent
2116882d82
commit
d2672eb776
5 changed files with 73 additions and 30 deletions
|
|
@ -147,12 +147,14 @@ inspect_deb() {
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
# Only the value of each field belongs in these variables: dpkg-deb prints
|
# Only the value of each field belongs in these variables: dpkg-deb prints
|
||||||
# "<Field>: <value>", and keeping the label made every real package fail the
|
# "<Field>: <value>" when several fields are asked for, and a bare value when
|
||||||
# name check ("Package: curl" is not a package name). The test suite stubs
|
# one is. Strip a leading "<word>: " only if it is present: always stripping
|
||||||
# dpkg-deb, so this only shows up against real .deb files.
|
# emptied the single-field reads, never stripping left "Package: curl" as a
|
||||||
DEB_PKG="$(printf '%s\n' "$raw" | sed -n 's/^Package: //p')"
|
# package name. A version cannot start with a letter-and-colon ("1:2.3" is
|
||||||
DEB_VER="$(printf '%s\n' "$raw" | sed -n 's/^Version: //p')"
|
# fine), so the pattern is safe.
|
||||||
DEB_ARCH="$(printf '%s\n' "$raw" | sed -n 's/^Architecture: //p')"
|
DEB_PKG="$(printf '%s\n' "$raw" | sed -n '1p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
|
DEB_VER="$(printf '%s\n' "$raw" | sed -n '2p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
|
DEB_ARCH="$(printf '%s\n' "$raw" | sed -n '3p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
|
|
||||||
if [[ ! "$DEB_PKG" =~ $PACKAGE_NAME_RE ]]; then
|
if [[ ! "$DEB_PKG" =~ $PACKAGE_NAME_RE ]]; then
|
||||||
printf 'HATA: paket adı geçersiz: "%s" (%s)\n' "$DEB_PKG" "$file" >&2
|
printf 'HATA: paket adı geçersiz: "%s" (%s)\n' "$DEB_PKG" "$file" >&2
|
||||||
|
|
@ -179,12 +181,12 @@ scan_existing() {
|
||||||
[[ -d "$PACKAGES_DIR" ]] || return 0
|
[[ -d "$PACKAGES_DIR" ]] || return 0
|
||||||
while IFS= read -r -d '' deb; do
|
while IFS= read -r -d '' deb; do
|
||||||
sha="$(file_sha256 "$deb")"
|
sha="$(file_sha256 "$deb")"
|
||||||
# Strip the "<Field>: " label dpkg-deb prints; the map keys are package and
|
# A single requested field comes back bare ("curl"), so strip a label only
|
||||||
# version values, and a labelled key never matches the incoming candidate
|
# when dpkg-deb actually printed one. Always stripping emptied these keys,
|
||||||
# (so a version conflict went undetected and the same package was copied
|
# which made a labelled/empty entry never match the incoming candidate: a
|
||||||
# twice).
|
# version conflict went undetected and the same package was copied twice.
|
||||||
pkg="$(dpkg-deb -f "$deb" Package 2>/dev/null | sed -n 's/^Package: //p' || true)"
|
pkg="$(dpkg-deb -f "$deb" Package 2>/dev/null | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }' || true)"
|
||||||
ver="$(dpkg-deb -f "$deb" Version 2>/dev/null | sed -n 's/^Version: //p' || true)"
|
ver="$(dpkg-deb -f "$deb" Version 2>/dev/null | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }' || true)"
|
||||||
if [[ -z "$pkg" || -z "$ver" ]]; then
|
if [[ -z "$pkg" || -z "$ver" ]]; then
|
||||||
printf 'UYARI: packages/ içindeki okunamayan dosya yok sayıldı: %s\n' "${deb##*/}" >&2
|
printf 'UYARI: packages/ içindeki okunamayan dosya yok sayıldı: %s\n' "${deb##*/}" >&2
|
||||||
continue
|
continue
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,10 @@ LOCK_HEADER="$(printf 'package\tversion\tarchitecture\tsha256\tsource')"
|
||||||
UNPINNED='UNPINNED'
|
UNPINNED='UNPINNED'
|
||||||
SCHEMA_VERSION=1
|
SCHEMA_VERSION=1
|
||||||
# Entries an output directory may hold without being treated as foreign.
|
# Entries an output directory may hold without being treated as foreign.
|
||||||
ALLOWED_ENTRIES='pool Packages Packages.gz packages.lock.tsv SHA256SUMS .gitkeep'
|
# repo-manifest.yaml belongs here: this script writes it into the bundle after
|
||||||
|
# the index, so a rebuild would otherwise refuse its own previous output as a
|
||||||
|
# stranger and stop ("bu depoya ait olmayan dosya") on every second run.
|
||||||
|
ALLOWED_ENTRIES='pool Packages Packages.gz packages.lock.tsv SHA256SUMS repo-manifest.yaml .gitkeep'
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
|
|
@ -289,9 +292,15 @@ while IFS= read -r -d '' deb; do
|
||||||
# check and the pool came out empty -- a stub dpkg-deb in the test suite hides
|
# check and the pool came out empty -- a stub dpkg-deb in the test suite hides
|
||||||
# this, so it only surfaces against real .deb files.
|
# this, so it only surfaces against real .deb files.
|
||||||
raw="$(dpkg-deb -f "$deb" Package Version Architecture 2>/dev/null || true)"
|
raw="$(dpkg-deb -f "$deb" Package Version Architecture 2>/dev/null || true)"
|
||||||
package="$(printf '%s\n' "$raw" | sed -n 's/^Package: //p')"
|
# dpkg-deb prints a bare value for a single requested field but
|
||||||
version="$(printf '%s\n' "$raw" | sed -n 's/^Version: //p')"
|
# "<Field>: <value>" for several. Both shapes have to survive the read: the
|
||||||
arch="$(printf '%s\n' "$raw" | sed -n 's/^Architecture: //p')"
|
# awk expression drops the label only when the first word ends with a colon,
|
||||||
|
# and sed -n 's/.../p' (used here first) printed nothing at all when the
|
||||||
|
# pattern did not match, which emptied every row. A version may contain a
|
||||||
|
# colon ("1:2.3") but never starts with a word-and-colon, so this is safe.
|
||||||
|
package="$(printf '%s\n' "$raw" | sed -n '1p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
|
version="$(printf '%s\n' "$raw" | sed -n '2p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
|
arch="$(printf '%s\n' "$raw" | sed -n '3p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
if [[ ! "$package" =~ $PACKAGE_NAME_RE || -z "$version" ]]; then
|
if [[ ! "$package" =~ $PACKAGE_NAME_RE || -z "$version" ]]; then
|
||||||
POOL_ERRORS+=("$base: geçerli bir Debian paketi okunamadı; add-packages.sh ile ekleyin")
|
POOL_ERRORS+=("$base: geçerli bir Debian paketi okunamadı; add-packages.sh ile ekleyin")
|
||||||
continue
|
continue
|
||||||
|
|
|
||||||
|
|
@ -249,9 +249,10 @@ for list in "$SCRIPT_DIR/manifests/base-packages.txt" "$SCRIPT_DIR/manifests/rem
|
||||||
pins=$((pins + 1))
|
pins=$((pins + 1))
|
||||||
found=0
|
found=0
|
||||||
for deb in "${debs[@]}"; do
|
for deb in "${debs[@]}"; do
|
||||||
# dpkg-deb prints "<Field>: <value>", so the label has to be stripped
|
# A single requested field comes back bare, so strip a label only when
|
||||||
# before the value can be compared with a "package=version" pin.
|
# dpkg-deb printed one; always stripping emptied both halves and no pin
|
||||||
deb_pin="$(dpkg-deb -f "$deb" Package | sed -n 's/^Package: //p')=$(dpkg-deb -f "$deb" Version | sed -n 's/^Version: //p')"
|
# could ever match.
|
||||||
|
deb_pin="$(dpkg-deb -f "$deb" Package | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')=$(dpkg-deb -f "$deb" Version | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||||
[[ "$deb_pin" == "$pin" ]] && found=1 && break
|
[[ "$deb_pin" == "$pin" ]] && found=1 && break
|
||||||
done
|
done
|
||||||
[[ "$found" -eq 1 ]] || { printf 'Pinned package not supplied locally: %s\n' "$pin" >&2; exit 1; }
|
[[ "$found" -eq 1 ]] || { printf 'Pinned package not supplied locally: %s\n' "$pin" >&2; exit 1; }
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ repository:
|
||||||
state: 'BUILT'
|
state: 'BUILT'
|
||||||
ubuntu_release: '26.04'
|
ubuntu_release: '26.04'
|
||||||
architecture: 'amd64'
|
architecture: 'amd64'
|
||||||
generated_at: '2026-09-16T18:35:49Z'
|
generated_at: '2026-09-16T18:55:57Z'
|
||||||
generated_by: 'provisioning/offline-repo/build-index.sh'
|
generated_by: 'provisioning/offline-repo/build-index.sh'
|
||||||
package_count: 1321
|
package_count: 1321
|
||||||
total_bytes: 927393560
|
total_bytes: 927393560
|
||||||
|
|
|
||||||
|
|
@ -426,10 +426,11 @@ cat > "$STUB_BIN/dpkg-deb" <<'STUB'
|
||||||
# Test stub: real dpkg-deb reads the control member; the identity is read from
|
# Test stub: real dpkg-deb reads the control member; the identity is read from
|
||||||
# the <package>_<version>_<arch>.deb file name instead.
|
# the <package>_<version>_<arch>.deb file name instead.
|
||||||
#
|
#
|
||||||
# The output carries the "<Field>: <value>" labels exactly as dpkg-deb prints
|
# The output shape mirrors the real tool exactly, which matters more than it
|
||||||
# them. Emitting bare values made every consumer look correct while the real
|
# looks: dpkg-deb prints a BARE VALUE when one field is requested and
|
||||||
# tool was mis-parsed -- the pool came out empty against genuine .deb files
|
# "<Field>: <value>" when several are, and callers exist in both styles. A stub
|
||||||
# because the callers kept the label and "Package: curl" is not a package name.
|
# that always emitted one shape let a parser break the other pass every test --
|
||||||
|
# the real pool came out empty while this suite stayed green.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
[[ "${1:-}" == '-f' ]] || exit 1
|
[[ "${1:-}" == '-f' ]] || exit 1
|
||||||
file="${2:-}"
|
file="${2:-}"
|
||||||
|
|
@ -442,14 +443,28 @@ esac
|
||||||
name="${base%.deb}"; arch="${name##*_}"; rest="${name%_*}"
|
name="${base%.deb}"; arch="${name##*_}"; rest="${name%_*}"
|
||||||
version="${rest##*_}"; package="${rest%_*}"
|
version="${rest##*_}"; package="${rest%_*}"
|
||||||
[[ -n "$package" && -n "$version" && -n "$arch" ]] || exit 1
|
[[ -n "$package" && -n "$version" && -n "$arch" ]] || exit 1
|
||||||
for field in "$@"; do
|
|
||||||
|
# dpkg-deb prints a BARE VALUE for one requested field and "<Field>: <value>"
|
||||||
|
# for several. Both shapes must be reproduced: a stub that always emitted one
|
||||||
|
# of them let a reader that only handled the other pass every run, and the real
|
||||||
|
# pool came out empty while this suite stayed green.
|
||||||
|
if [[ $# -eq 1 ]]; then
|
||||||
|
case "$1" in
|
||||||
|
Package) printf '%s\n' "$package" ;;
|
||||||
|
Version) printf '%s\n' "$version" ;;
|
||||||
|
Architecture) printf '%s\n' "$arch" ;;
|
||||||
|
*) printf '\n' ;;
|
||||||
|
esac
|
||||||
|
else
|
||||||
|
for field in "$@"; do
|
||||||
case "$field" in
|
case "$field" in
|
||||||
Package) printf 'Package: %s\n' "$package" ;;
|
Package) printf 'Package: %s\n' "$package" ;;
|
||||||
Version) printf 'Version: %s\n' "$version" ;;
|
Version) printf 'Version: %s\n' "$version" ;;
|
||||||
Architecture) printf 'Architecture: %s\n' "$arch" ;;
|
Architecture) printf 'Architecture: %s\n' "$arch" ;;
|
||||||
*) printf '\n' ;;
|
*) printf '\n' ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
fi
|
||||||
STUB
|
STUB
|
||||||
cat > "$STUB_BIN/apt-ftparchive" <<'STUB'
|
cat > "$STUB_BIN/apt-ftparchive" <<'STUB'
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
@ -625,6 +640,22 @@ bundle_after="$(find "$OUT" -type f -exec sha256sum {} + | sort)"
|
||||||
record_after="$(sha256sum "$TMP/repo-manifest.yaml" | awk '{print $1}')"
|
record_after="$(sha256sum "$TMP/repo-manifest.yaml" | awk '{print $1}')"
|
||||||
[[ "$rc" -eq 0 && "$bundle_before" == "$bundle_after" ]] && pass 'rebuilding an unchanged pool leaves the bundle byte-identical' || fail 'rebuilding an unchanged pool changed the bundle'
|
[[ "$rc" -eq 0 && "$bundle_before" == "$bundle_after" ]] && pass 'rebuilding an unchanged pool leaves the bundle byte-identical' || fail 'rebuilding an unchanged pool changed the bundle'
|
||||||
grep -q 'DEĞİŞİKLİK YOK' <<<"$out" && pass 'build-index.sh reports the unchanged bundle instead of rewriting it' || fail 'build-index.sh must report an unchanged bundle'
|
grep -q 'DEĞİŞİKLİK YOK' <<<"$out" && pass 'build-index.sh reports the unchanged bundle instead of rewriting it' || fail 'build-index.sh must report an unchanged bundle'
|
||||||
|
# The guard must accept this script's own output as its own. In production the
|
||||||
|
# record lives INSIDE the output directory (--manifest-out built/repo-manifest.yaml),
|
||||||
|
# and the earlier fixtures wrote it outside, so a guard that rejected
|
||||||
|
# repo-manifest.yaml as a foreign file passed every run while a real rebuild
|
||||||
|
# stopped with "bu depoya ait olmayan dosya". Mirror the real layout.
|
||||||
|
REBUILD_OUT="$TMP/rebuild-out"
|
||||||
|
mkdir -p "$REBUILD_OUT"
|
||||||
|
bash "$BUILD_INDEX" --packages "$PKG_OUT" --output "$REBUILD_OUT" --manifest-dir "$TMP/m" >/dev/null 2>&1
|
||||||
|
[[ -f "$REBUILD_OUT/repo-manifest.yaml" ]] || fail 'the record was not written inside the output directory'
|
||||||
|
rc_again=0; out_again="$(bash "$BUILD_INDEX" --packages "$PKG_OUT" --output "$REBUILD_OUT" --manifest-dir "$TMP/m" 2>&1)" || rc_again=$?
|
||||||
|
if [[ "$rc_again" -eq 0 ]] && ! grep -q 'bu depoya ait olmayan dosya' <<<"$out_again"; then
|
||||||
|
pass 'rebuilding the same output directory is accepted (its own record is not foreign)'
|
||||||
|
else
|
||||||
|
fail "a second build over its own output failed (exit $rc_again)"
|
||||||
|
printf '%s\n' "$out_again" >&2
|
||||||
|
fi
|
||||||
[[ "$record_before" == "$record_after" ]] && pass 'an unchanged rebuild leaves the repository record untouched' || fail 'an unchanged rebuild must not restamp the repository record'
|
[[ "$record_before" == "$record_after" ]] && pass 'an unchanged rebuild leaves the repository record untouched' || fail 'an unchanged rebuild must not restamp the repository record'
|
||||||
git_status_note="$(git -C "$REPO_ROOT" status --porcelain -- "provisioning/offline-repo/built" 2>/dev/null || true)"
|
git_status_note="$(git -C "$REPO_ROOT" status --porcelain -- "provisioning/offline-repo/built" 2>/dev/null || true)"
|
||||||
[[ "$git_status_note" == "$built_state_before" ]] \
|
[[ "$git_status_note" == "$built_state_before" ]] \
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue