Attachment 22353 Details for Bug 60117 – gear-ours-merge-audit.sh: audit gear repos for empty ours-merges (first-parent, periods, tar:/diff: rules)

gear-ours-merge-audit.sh: audit gear repos for empty ours-merges (first-parent, periods, tar:/diff: rules)

gear-ours-merge-audit.sh (text/plain), 12.65 KB, created by Vitaly Lipatov on 2026-10-07 00:18:43 MSK

(hide)

Description:

Filename:

MIME Type:

Creator: Vitaly Lipatov

Created: 2026-10-07 00:18:43 MSK

Size: 12.65 KB

patch

obsolete

>#!/bin/bash
># gear-ours-merge-audit.sh â find gear repos poisoned by `git merge -s ours` (ALT bug 60117)
>#
># rpmgs from etersoft-build-utils 3.3.1-3.3.2 (d223e6c..bd83b59, 2026-02-07..2026-06-29)
># and again 2e21b42..af5e7ff (2026-07-29..2026-08-21) merged upstream tags with
># `git merge -s ours` for .gear/tags-based packages: the upstream tree was dropped,
># but the tag counts as merged, so later merges only bring increments.
># Dangerous when the tarball is built from the branch tree (`tar: .`), not from the tag.
>#
># Usage:
>#   gear-ours-merge-audit.sh [-j N] [-w WORKDIR] [-k] [-s DATE] NAME...
>#   gear-ours-merge-audit.sh [-j N] ... -f FILE      # names from file ('-' = stdin)
>#   gear-ours-merge-audit.sh ... < names.txt          # names from stdin
>#
>#   -j N        parallel jobs (default 1; keep <= 4 to spare git.altlinux.org)
>#   -w WORKDIR  work dir (default ${TMPDIR:-/tmp}/ours-audit); clones in WORKDIR/repos/NAME.git,
>#               per-package results in WORKDIR/results/NAME.txt
>#   -k          keep clones of unaffected repos (default: remove them, keep affected)
>#   -s DATE     shallow-clone since DATE (default 2025-06-01; deepened automatically
>#               when a shallow boundary commit is inside the periods or a merge parent /
>#               merge-base is cut off); '-s full' for full clone
>#
># Detector (per merge commit m of our branch, git log --merges --first-parent):
>#   tree(m) == tree(m^1), tree(m^1) != tree(m^2), and merge-base(m^1,m^2)..m^2 changed
>#   something  => the merge discarded the other side (empty ours-merge).
>#   Only merges dated inside the rpmgs ours periods count as 'affected';
>#   empty merges outside them are reported separately as 'out of period'.
>#
># Archive source (from .gear/rules on HEAD):
>#   tag  â tar*/zip: REF:path (e.g. v@version@:.)    tree â tar*/zip: . or dir (branch tree)
>#   none â no tar/zip rule (committed tarball etc.)
>#   '+diff' suffix â a `diff:` rule (e.g. diff: @version@:. . name=upstream.patch) applies
>#   the tag->branch-tree difference on top of the archive, so lost upstream changes are
>#   reverted in the built package.
># RISK = affected and (archive from tree or a diff: rule present).
># For RISK: git diff --shortstat between the upstream tag commit of the current version
># (.gear/tags/list -> .gear/tags/<id> -> 'object <commit>', or refs/tags) and HEAD,
># excluding .gear, with up to 10 changed files; plus the sisyphus builds
># (gb-sisyphus-task* tags) made after the first ours-merge.
>#
># Output: one line per package
>#   NAME | affected | archive | RISK | diff-stat | builds-after
># followed by indented 'ours:' lines (hash date subject; '(out of period)' marked)
># and 'file:' lines for RISK packages.
>#
># Read-only: nothing is pushed or modified upstream.
>
>set -euo pipefail
>
>JOBS=1
>WORKDIR="${TMPDIR:-/tmp}/ours-audit"
>KEEP=0
>SINCE=2025-06-01
>LISTFILE=
>ONE=
>
># rpmgs ours periods (inclusive)
>P1_FROM=2026-02-07 P1_TO=2026-06-29
>P2_FROM=2026-07-29 P2_TO=2026-08-21
>
>usage() { sed -n '2,/^$/{s/^# \{0,1\}//;p}' "$0"; exit "${1:-0}"; }
>
>while getopts "j:w:ks:f:1:h" opt; do
>    case "$opt" in
>        j) JOBS=$OPTARG ;;
>        w) WORKDIR=$OPTARG ;;
>        k) KEEP=1 ;;
>        s) SINCE=$OPTARG ;;
>        f) LISTFILE=$OPTARG ;;
>        1) ONE=$OPTARG ;;
>        h) usage 0 ;;
>        *) usage 1 ;;
>    esac
>done
>shift $((OPTIND - 1))
>
>mkdir -p "$WORKDIR/repos" "$WORKDIR/results"
>
>in_period()
>{
>    local d=$1
>    [[ ! "$d" < "$P1_FROM" && ! "$d" > "$P1_TO" ]] && return 0
>    [[ ! "$d" < "$P2_FROM" && ! "$d" > "$P2_TO" ]] && return 0
>    return 1
>}
>
>repo_url()
>{
>    local name=$1
>    echo "git://git.altlinux.org/gears/${name:0:1}/$name.git"
>}
>
># classify archive source from rules text on stdin
>archive_source()
>{
>    local rules src=none line path has_diff=0
>    rules=$(cat)
>    while read -r line; do
>        case "$line" in
>            tar*:*|zip*:*)
>                path=$(printf '%s\n' "$line" | sed -E 's/^[a-z0-9.]+:[[:space:]]*//; s/[[:space:]].*//')
>                if [[ "$path" == *:* ]]; then
>                    [ "$src" = none ] && src=tag
>                else
>                    src=tree
>                fi
>                ;;
>            diff*:*) has_diff=1 ;;
>        esac
>    done <<< "$rules"
>    [ "$has_diff" = 1 ] && src=$src+diff
>    echo "$src"
>}
>
># print spec Version on HEAD with trivial %define/%global expansion
>spec_version()
>{
>    local git=$1 spec text ver name val i
>    spec=$(git -C "$git" cat-file -p HEAD:.gear/rules 2>/dev/null | sed -nE 's/^spec:[[:space:]]*([^[:space:]]+).*/\1/p' | sed -n 1p)
>    [ -n "$spec" ] || spec=$(git -C "$git" ls-tree --name-only HEAD | sed -n '/\.spec$/p' | sed -n 1p)
>    [ -n "$spec" ] || return 0
>    text=$(git -C "$git" cat-file -p "HEAD:$spec" 2>/dev/null) || return 0
>    ver=$(printf '%s\n' "$text" | sed -nE 's/^Version:[[:space:]]*([^[:space:]]+).*/\1/p' | sed -n 1p)
>    for i in 1 2 3; do
>        [[ "$ver" == *%* ]] || break
>        name=$(printf '%s' "$ver" | sed -nE 's/.*%\{?([A-Za-z_][A-Za-z0-9_]*)\}?.*/\1/p')
>        [ -n "$name" ] || break
>        val=$(printf '%s\n' "$text" | sed -nE "s/^%(define|global)[[:space:]]+$name[[:space:]]+([^[:space:]]+).*/\2/p" | sed -n 1p)
>        [ -n "$val" ] || break
>        ver=$(printf '%s' "$ver" | sed -E "s/%\{?$name\}?/$val/")
>    done
>    echo "$ver"
>}
>
># .gear/tags/<tag object id> -> 'object <commit>'; lightweight tags list the commit itself
>gear_tag_commit()
>{
>    if git -C "$1" cat-file -e "HEAD:.gear/tags/$2" 2>/dev/null; then
>        git -C "$1" cat-file -p "HEAD:.gear/tags/$2" | sed -n 's/^object //p' | sed -n 1p
>    elif [ "$(git -C "$1" cat-file -t "$2" 2>/dev/null)" = commit ]; then
>        echo "$2"
>    fi
>}
>
># find upstream tag commit for VERSION: .gear/tags/list, then refs/tags
>upstream_tag_commit()
>{
>    local git=$1 ver=$2 rulesref=$3 sha t id
>    [ -n "$ver" ] || return 0
>    rulesref=${rulesref//@version@/$ver}
>    if [ -n "$rulesref" ]; then
>        id=$(git -C "$git" cat-file -p HEAD:.gear/tags/list 2>/dev/null | awk -v t="$rulesref" '$2 == t {print $1; exit}')
>        [ -n "$id" ] && sha=$(gear_tag_commit "$git" "$id") && [ -n "$sha" ] && { echo "$sha $rulesref"; return 0; }
>    fi
>    read -r id t < <(git -C "$git" cat-file -p HEAD:.gear/tags/list 2>/dev/null |
>          awk -v v="$ver" '{ t=$2; sub(/^[^0-9]*/, "", t); if (t == v) { print $1, $2; exit } }') || true
>    [ -n "${id:-}" ] && sha=$(gear_tag_commit "$git" "$id") && [ -n "$sha" ] && { echo "$sha $t"; return 0; }
>    for t in "v$ver" "$ver" "release-$ver" "$rulesref"; do
>        [ -n "$t" ] || continue
>        sha=$(git -C "$git" rev-parse -q --verify "refs/tags/$t^{commit}" 2>/dev/null) && { echo "$sha $t"; return 0; }
>    done
>    return 0
>}
>
># replace a shallow clone with a full one (fetch --unshallow is unreliable
># when the shallow file lists objects that were not sent)
>deepen()
>{
>    local git=$1 url=$2
>    [ -f "$git/shallow" ] || return 1
>    rm -rf "$git.full"
>    git clone --bare -q "$url" "$git.full" >/dev/null 2>&1 || { rm -rf "$git.full"; return 1; }
>    rm -rf "$git"
>    mv "$git.full" "$git"
>}
>
>audit_one()
>{
>    local name=$1 git url out affected=0 lines="" m d p1 p2 base src risk=no
>    local stat=- builds=- first="" ref="" ver="" tc
>    git="$WORKDIR/repos/$name.git"
>    out="$WORKDIR/results/$name.txt"
>    url=$(repo_url "$name")
>
>    if [ ! -d "$git" ]; then
>        local copt=(--bare -q)
>        [ "$SINCE" = full ] || copt+=(--shallow-since="$SINCE")
>        if ! git clone "${copt[@]}" "$url" "$git" >/dev/null 2>&1; then
>            rm -rf "$git"
>            # shallow-since fails when nothing is newer than SINCE: retry with depth 1 history
>            if [ "$SINCE" = full ] || ! git clone --bare -q --depth 50 "$url" "$git" >/dev/null 2>&1; then
>                rm -rf "$git"
>                echo "$name | no-gear | - | - | - | -" | tee "$out"
>                return 0
>            fi
>        fi
>    fi
>    # shallow-since drops refs with no commits after SINCE: retry with a full clone
>    if ! git -C "$git" rev-parse -q --verify HEAD >/dev/null && [ -f "$git/shallow" ]; then
>        deepen "$git" "$url" || true
>    fi
>    if ! git -C "$git" rev-parse -q --verify HEAD >/dev/null; then
>        echo "$name | no-head (gears HEAD branch missing, built from srpm) | - | - | - | -" | tee "$out"
>        return 0
>    fi
>
>    # a shallow boundary commit loses all its parents (a merge there becomes invisible):
>    # deepen when a boundary merge commit is newer than the first period start
>    if [ -f "$git/shallow" ]; then
>        local sb
>        while read -r sb; do
>            # commit objects keep their parent lines even at the shallow boundary
>            [ "$(git -C "$git" cat-file -p "$sb" 2>/dev/null | sed '/^$/q' | grep -c '^parent ')" -ge 2 ] || continue
>            if [[ ! "$(git -C "$git" log -1 --format=%cd --date=short "$sb" 2>/dev/null)" < "$P1_FROM" ]]; then
>                deepen "$git" "$url" || true
>                break
>            fi
>        done < "$git/shallow"
>    fi
>
>    local outper=0
>    for pass in 1 2; do
>        affected=0 outper=0 lines="" first="" local_cut=0
>        while read -r m d; do
>            if ! git -C "$git" cat-file -e "$m^1" 2>/dev/null || ! git -C "$git" cat-file -e "$m^2" 2>/dev/null; then
>                local_cut=1; break
>            fi
>            git -C "$git" diff --quiet "$m^1" "$m" 2>/dev/null || continue
>            git -C "$git" diff --quiet "$m^1" "$m^2" 2>/dev/null && continue
>            base=$(git -C "$git" merge-base "$m^1" "$m^2" 2>/dev/null) || base=""
>            if [ -z "$base" ]; then
>                [ -f "$git/shallow" ] && { local_cut=1; break; }
>                # unrelated histories: other side brought its whole tree
>            elif git -C "$git" diff --quiet "$base" "$m^2" 2>/dev/null; then
>                continue
>            fi
>            if in_period "$d"; then
>                affected=$((affected + 1))
>                lines+="    ours: $(git -C "$git" log -1 --format='%h %cd %s' --date=short "$m")"$'\n'
>                first=$d
>            else
>                outper=$((outper + 1))
>                lines+="    ours: $(git -C "$git" log -1 --format='%h %cd %s' --date=short "$m") (out of period)"$'\n'
>            fi
>        done < <(git -C "$git" log --merges --first-parent --format='%H %cd' --date=short HEAD)
>        [ "$local_cut" = 1 ] && [ "$pass" = 1 ] && deepen "$git" "$url" && continue
>        break
>    done
>
>    local rules
>    rules=$(git -C "$git" cat-file -p HEAD:.gear/rules 2>/dev/null || true)
>    src=$(printf '%s\n' "$rules" | archive_source)
>    ref=$(printf '%s\n' "$rules" | sed -nE 's/^(tar|zip)[a-z0-9.]*:[[:space:]]*([^[:space:]:]+):.*/\2/p' | sed -n 1p)
>
>    local files=""
>    if [ "$affected" -gt 0 ] && [[ "$src" == tree* || "$src" == *+diff ]]; then
>        risk=yes
>        ver=$(spec_version "$git")
>        tc=$(upstream_tag_commit "$git" "$ver" "$ref")
>        if [ -n "$tc" ] && git -C "$git" cat-file -e "${tc%% *}^{commit}" 2>/dev/null; then
>            stat=$(git -C "$git" diff --shortstat "${tc%% *}" HEAD -- . ':!.gear' | sed 's/^ *//')
>            [ -n "$stat" ] || stat="identical"
>            files=$(git -C "$git" diff --name-status "${tc%% *}" HEAD -- . ':!.gear' | sed -n '1,10s/^/    file: /p')
>            stat="vs ${tc#* }: $stat"
>        else
>            stat="no upstream tag for '${ver:-?}'"
>        fi
>        # sisyphus builds after the first (oldest) ours-merge
>        builds=$(git -C "$git" for-each-ref 'refs/tags/gb-sisyphus-task*' \
>                     --format='%(creatordate:short) %(refname:short)' |
>                 awk -v f="$first" '$1 >= f' | sort | while read -r bd bt; do
>                     printf '%s(%s)\n' "$(git -C "$git" log -1 --format=%s "$bt" | cut -c1-30)" "$bd"
>                 done | awk -F'(' '!seen[$1]++' | tr '\n' ' ')
>        builds=${builds:-none}
>    fi
>
>    {
>        if [ "$affected" -gt 0 ]; then
>            echo "$name | yes $affected$([ "$outper" = 0 ] || echo " (+$outper out of period)") | $src | $risk | $stat | $builds"
>        elif [ "$outper" -gt 0 ]; then
>            echo "$name | out-of-period $outper | $src | no | - | -"
>        else
>            echo "$name | no | $src | no | - | -"
>        fi
>        printf '%s' "$lines"
>        [ -z "$files" ] || printf '%s\n' "$files"
>    } | tee "$out"
>
>    if [ "$affected" = 0 ] && [ "$outper" = 0 ] && [ "$KEEP" = 0 ]; then
>        rm -rf "$git"
>    fi
>}
>
>if [ -n "$ONE" ]; then
>    audit_one "$ONE"
>    exit 0
>fi
>
>names=("$@")
>if [ -n "$LISTFILE" ]; then
>    mapfile -t -O "${#names[@]}" names < <(if [ "$LISTFILE" = - ]; then cat; else cat "$LISTFILE"; fi)
>elif [ ${#names[@]} -eq 0 ] && [ ! -t 0 ]; then
>    mapfile -t names
>fi
>[ ${#names[@]} -gt 0 ] || usage 1
>
>echo "NAME | affected | archive | RISK | diff-stat | builds-after"
>printf '%s\n' "${names[@]}" | grep -v '^[[:space:]]*$#\|$$' |
>    xargs -r -P "$JOBS" -I{} "$0" -w "$WORKDIR" -s "$SINCE" $([ "$KEEP" = 1 ] && echo -k) -1 {}

Actions: View

Attachments on bug 60117: 22353