fix(offline-repo): read dpkg-deb output in both of its shapes
dpkg-deb prints a BARE VALUE when one control field is requested and "<Field>: <value>" when several are. Every reader was written for one shape and broke on the other, so no pool could be read and no pin could be matched against real files: - build-index.sh: kept the label, so "Package: curl" failed the name check and the pool came out empty -- the build refused with "no .deb files"; - add-packages.sh (read_deb): rejected every genuine .deb as an invalid name; (scan_existing): produced empty keys, so a version conflict went undetected and the same package was copied twice; - build-offline-repo.sh: compared a labelled value with a "package=version" pin, which can never match. A first attempt with `sed 's/^Field: //'` was wrong in the other direction: sed -v prints nothing when the pattern does not match, which emptied the single-field reads. The label is now dropped only when it is present (awk: if the first word ends with a colon), which leaves a bare value and a version like "1:2.3-4" untouched. Also fixed in build-index.sh: the foreign-file guard did not list repo-manifest.yaml, the record this script writes into the bundle itself, so every second build stopped on its own previous output. Accepting it is what makes a rebuild possible at all. Tests - The dpkg-deb stub now reproduces both shapes, the single- and the multi-field one. It emitted only one, which is exactly why this class stayed invisible while the suite was green. - The rebuild case mirrors the production layout (record inside the output directory) and is proven by mutation: removing repo-manifest.yaml from the guard's allow-list turns it red.
This commit is contained in:
emirhanmamak-blueforce
parent
2116882d82
commit
d2672eb776
5 changed files with 73 additions and 30 deletions
|
|
@ -147,12 +147,14 @@ inspect_deb() {
|
|||
return 1
|
||||
fi
|
||||
# Only the value of each field belongs in these variables: dpkg-deb prints
|
||||
# "<Field>: <value>", and keeping the label made every real package fail the
|
||||
# name check ("Package: curl" is not a package name). The test suite stubs
|
||||
# dpkg-deb, so this only shows up against real .deb files.
|
||||
DEB_PKG="$(printf '%s\n' "$raw" | sed -n 's/^Package: //p')"
|
||||
DEB_VER="$(printf '%s\n' "$raw" | sed -n 's/^Version: //p')"
|
||||
DEB_ARCH="$(printf '%s\n' "$raw" | sed -n 's/^Architecture: //p')"
|
||||
# "<Field>: <value>" when several fields are asked for, and a bare value when
|
||||
# one is. Strip a leading "<word>: " only if it is present: always stripping
|
||||
# emptied the single-field reads, never stripping left "Package: curl" as a
|
||||
# package name. A version cannot start with a letter-and-colon ("1:2.3" is
|
||||
# fine), so the pattern is safe.
|
||||
DEB_PKG="$(printf '%s\n' "$raw" | sed -n '1p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
DEB_VER="$(printf '%s\n' "$raw" | sed -n '2p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
DEB_ARCH="$(printf '%s\n' "$raw" | sed -n '3p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
|
||||
if [[ ! "$DEB_PKG" =~ $PACKAGE_NAME_RE ]]; then
|
||||
printf 'HATA: paket adı geçersiz: "%s" (%s)\n' "$DEB_PKG" "$file" >&2
|
||||
|
|
@ -179,12 +181,12 @@ scan_existing() {
|
|||
[[ -d "$PACKAGES_DIR" ]] || return 0
|
||||
while IFS= read -r -d '' deb; do
|
||||
sha="$(file_sha256 "$deb")"
|
||||
# Strip the "<Field>: " label dpkg-deb prints; the map keys are package and
|
||||
# version values, and a labelled key never matches the incoming candidate
|
||||
# (so a version conflict went undetected and the same package was copied
|
||||
# twice).
|
||||
pkg="$(dpkg-deb -f "$deb" Package 2>/dev/null | sed -n 's/^Package: //p' || true)"
|
||||
ver="$(dpkg-deb -f "$deb" Version 2>/dev/null | sed -n 's/^Version: //p' || true)"
|
||||
# A single requested field comes back bare ("curl"), so strip a label only
|
||||
# when dpkg-deb actually printed one. Always stripping emptied these keys,
|
||||
# which made a labelled/empty entry never match the incoming candidate: a
|
||||
# version conflict went undetected and the same package was copied twice.
|
||||
pkg="$(dpkg-deb -f "$deb" Package 2>/dev/null | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }' || true)"
|
||||
ver="$(dpkg-deb -f "$deb" Version 2>/dev/null | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }' || true)"
|
||||
if [[ -z "$pkg" || -z "$ver" ]]; then
|
||||
printf 'UYARI: packages/ içindeki okunamayan dosya yok sayıldı: %s\n' "${deb##*/}" >&2
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -52,7 +52,10 @@ LOCK_HEADER="$(printf 'package\tversion\tarchitecture\tsha256\tsource')"
|
|||
UNPINNED='UNPINNED'
|
||||
SCHEMA_VERSION=1
|
||||
# Entries an output directory may hold without being treated as foreign.
|
||||
ALLOWED_ENTRIES='pool Packages Packages.gz packages.lock.tsv SHA256SUMS .gitkeep'
|
||||
# repo-manifest.yaml belongs here: this script writes it into the bundle after
|
||||
# the index, so a rebuild would otherwise refuse its own previous output as a
|
||||
# stranger and stop ("bu depoya ait olmayan dosya") on every second run.
|
||||
ALLOWED_ENTRIES='pool Packages Packages.gz packages.lock.tsv SHA256SUMS repo-manifest.yaml .gitkeep'
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
|
|
@ -289,9 +292,15 @@ while IFS= read -r -d '' deb; do
|
|||
# check and the pool came out empty -- a stub dpkg-deb in the test suite hides
|
||||
# this, so it only surfaces against real .deb files.
|
||||
raw="$(dpkg-deb -f "$deb" Package Version Architecture 2>/dev/null || true)"
|
||||
package="$(printf '%s\n' "$raw" | sed -n 's/^Package: //p')"
|
||||
version="$(printf '%s\n' "$raw" | sed -n 's/^Version: //p')"
|
||||
arch="$(printf '%s\n' "$raw" | sed -n 's/^Architecture: //p')"
|
||||
# dpkg-deb prints a bare value for a single requested field but
|
||||
# "<Field>: <value>" for several. Both shapes have to survive the read: the
|
||||
# awk expression drops the label only when the first word ends with a colon,
|
||||
# and sed -n 's/.../p' (used here first) printed nothing at all when the
|
||||
# pattern did not match, which emptied every row. A version may contain a
|
||||
# colon ("1:2.3") but never starts with a word-and-colon, so this is safe.
|
||||
package="$(printf '%s\n' "$raw" | sed -n '1p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
version="$(printf '%s\n' "$raw" | sed -n '2p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
arch="$(printf '%s\n' "$raw" | sed -n '3p' | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
if [[ ! "$package" =~ $PACKAGE_NAME_RE || -z "$version" ]]; then
|
||||
POOL_ERRORS+=("$base: geçerli bir Debian paketi okunamadı; add-packages.sh ile ekleyin")
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -249,9 +249,10 @@ for list in "$SCRIPT_DIR/manifests/base-packages.txt" "$SCRIPT_DIR/manifests/rem
|
|||
pins=$((pins + 1))
|
||||
found=0
|
||||
for deb in "${debs[@]}"; do
|
||||
# dpkg-deb prints "<Field>: <value>", so the label has to be stripped
|
||||
# before the value can be compared with a "package=version" pin.
|
||||
deb_pin="$(dpkg-deb -f "$deb" Package | sed -n 's/^Package: //p')=$(dpkg-deb -f "$deb" Version | sed -n 's/^Version: //p')"
|
||||
# A single requested field comes back bare, so strip a label only when
|
||||
# dpkg-deb printed one; always stripping emptied both halves and no pin
|
||||
# could ever match.
|
||||
deb_pin="$(dpkg-deb -f "$deb" Package | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')=$(dpkg-deb -f "$deb" Version | awk '{ if ($1 ~ /:$/) { $1=""; sub(/^ /, "") } print }')"
|
||||
[[ "$deb_pin" == "$pin" ]] && found=1 && break
|
||||
done
|
||||
[[ "$found" -eq 1 ]] || { printf 'Pinned package not supplied locally: %s\n' "$pin" >&2; exit 1; }
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ repository:
|
|||
state: 'BUILT'
|
||||
ubuntu_release: '26.04'
|
||||
architecture: 'amd64'
|
||||
generated_at: '2026-09-16T18:35:49Z'
|
||||
generated_at: '2026-09-16T18:55:57Z'
|
||||
generated_by: 'provisioning/offline-repo/build-index.sh'
|
||||
package_count: 1321
|
||||
total_bytes: 927393560
|
||||
|
|
|
|||
|
|
@ -426,10 +426,11 @@ cat > "$STUB_BIN/dpkg-deb" <<'STUB'
|
|||
# Test stub: real dpkg-deb reads the control member; the identity is read from
|
||||
# the <package>_<version>_<arch>.deb file name instead.
|
||||
#
|
||||
# The output carries the "<Field>: <value>" labels exactly as dpkg-deb prints
|
||||
# them. Emitting bare values made every consumer look correct while the real
|
||||
# tool was mis-parsed -- the pool came out empty against genuine .deb files
|
||||
# because the callers kept the label and "Package: curl" is not a package name.
|
||||
# The output shape mirrors the real tool exactly, which matters more than it
|
||||
# looks: dpkg-deb prints a BARE VALUE when one field is requested and
|
||||
# "<Field>: <value>" when several are, and callers exist in both styles. A stub
|
||||
# that always emitted one shape let a parser break the other pass every test --
|
||||
# the real pool came out empty while this suite stayed green.
|
||||
set -euo pipefail
|
||||
[[ "${1:-}" == '-f' ]] || exit 1
|
||||
file="${2:-}"
|
||||
|
|
@ -442,6 +443,19 @@ esac
|
|||
name="${base%.deb}"; arch="${name##*_}"; rest="${name%_*}"
|
||||
version="${rest##*_}"; package="${rest%_*}"
|
||||
[[ -n "$package" && -n "$version" && -n "$arch" ]] || exit 1
|
||||
|
||||
# dpkg-deb prints a BARE VALUE for one requested field and "<Field>: <value>"
|
||||
# for several. Both shapes must be reproduced: a stub that always emitted one
|
||||
# of them let a reader that only handled the other pass every run, and the real
|
||||
# pool came out empty while this suite stayed green.
|
||||
if [[ $# -eq 1 ]]; then
|
||||
case "$1" in
|
||||
Package) printf '%s\n' "$package" ;;
|
||||
Version) printf '%s\n' "$version" ;;
|
||||
Architecture) printf '%s\n' "$arch" ;;
|
||||
*) printf '\n' ;;
|
||||
esac
|
||||
else
|
||||
for field in "$@"; do
|
||||
case "$field" in
|
||||
Package) printf 'Package: %s\n' "$package" ;;
|
||||
|
|
@ -450,6 +464,7 @@ for field in "$@"; do
|
|||
*) printf '\n' ;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
STUB
|
||||
cat > "$STUB_BIN/apt-ftparchive" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
|
|
@ -625,6 +640,22 @@ bundle_after="$(find "$OUT" -type f -exec sha256sum {} + | sort)"
|
|||
record_after="$(sha256sum "$TMP/repo-manifest.yaml" | awk '{print $1}')"
|
||||
[[ "$rc" -eq 0 && "$bundle_before" == "$bundle_after" ]] && pass 'rebuilding an unchanged pool leaves the bundle byte-identical' || fail 'rebuilding an unchanged pool changed the bundle'
|
||||
grep -q 'DEĞİŞİKLİK YOK' <<<"$out" && pass 'build-index.sh reports the unchanged bundle instead of rewriting it' || fail 'build-index.sh must report an unchanged bundle'
|
||||
# The guard must accept this script's own output as its own. In production the
|
||||
# record lives INSIDE the output directory (--manifest-out built/repo-manifest.yaml),
|
||||
# and the earlier fixtures wrote it outside, so a guard that rejected
|
||||
# repo-manifest.yaml as a foreign file passed every run while a real rebuild
|
||||
# stopped with "bu depoya ait olmayan dosya". Mirror the real layout.
|
||||
REBUILD_OUT="$TMP/rebuild-out"
|
||||
mkdir -p "$REBUILD_OUT"
|
||||
bash "$BUILD_INDEX" --packages "$PKG_OUT" --output "$REBUILD_OUT" --manifest-dir "$TMP/m" >/dev/null 2>&1
|
||||
[[ -f "$REBUILD_OUT/repo-manifest.yaml" ]] || fail 'the record was not written inside the output directory'
|
||||
rc_again=0; out_again="$(bash "$BUILD_INDEX" --packages "$PKG_OUT" --output "$REBUILD_OUT" --manifest-dir "$TMP/m" 2>&1)" || rc_again=$?
|
||||
if [[ "$rc_again" -eq 0 ]] && ! grep -q 'bu depoya ait olmayan dosya' <<<"$out_again"; then
|
||||
pass 'rebuilding the same output directory is accepted (its own record is not foreign)'
|
||||
else
|
||||
fail "a second build over its own output failed (exit $rc_again)"
|
||||
printf '%s\n' "$out_again" >&2
|
||||
fi
|
||||
[[ "$record_before" == "$record_after" ]] && pass 'an unchanged rebuild leaves the repository record untouched' || fail 'an unchanged rebuild must not restamp the repository record'
|
||||
git_status_note="$(git -C "$REPO_ROOT" status --porcelain -- "provisioning/offline-repo/built" 2>/dev/null || true)"
|
||||
[[ "$git_status_note" == "$built_state_before" ]] \
|
||||
|
|
|
|||
Loading…
Reference in a new issue