Attachment #1090 for bug #7889

View | Details | Raw Unified | Return to bug 7889
Collapse All | Expand All




# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA.
#

. /etc/sisyphus/config
. /etc/sisyphus/functions

TEST=
component=classic
force_yes=
dereference=1

show_help()
{
	cat <<EOF
Usage: $PROG [options]

Valid options are:
  -p, --prefix=DIR          path to repository location;
  -c, --component=NAME      repository component to use;
  -a, --architectures=LIST  architectures to handle;
  -f, --force-yes           force yes answer;
  -P, --no-dereference      do not dereference symbolic links;
  -d, --dry-run             dry run;
  -h, --help                show this text and exit.
EOF
	exit
}

show_usage()
{
	[ -z "$*" ] || Info "$*"
	echo "Try \`$PROG --help' for more information." >&2
	exit 1
}

process_dups()
{
	local rep arch
	rep="$1" && shift || Fatal "first argument required"
	arch="$1" && shift || Fatal "second argument required"

	mkdir -p -m700 -- "$WORKDIR/$rep"
	echo -n "Processing $rep: "
	find "$PREFIX/$rep" -name "*\.$arch\.rpm" -execdir rpmrdups \{\} \+ > "$WORKDIR/$rep/dups" &&
	    echo "done" || Fatal "failed"
}

sources_index() {
    echo -n "Generating src-bin list: "
    
    local arch paths
    for arch in $ARCHITECTURES; do
	paths="$paths $PREFIX/$NEW_FILE_PREFIX/$arch/RPMS"
    done

    [ -n "$paths" ] || Fatal "No binary paths found"

    find $paths -mindepth 1 -maxdepth 1 -type f -name '*.rpm' \
	-execdir rpmquery -p --qf '%{SOURCERPM}\t%{ARCH}\n' -- \{\} \+ |
        sort -u > "$WORKDIR/src-bin" &&
	echo "done" || { echo "failed"; return 1; }
}

correct_dups() {
	local rep="$1" && shift || Fatal "first argument required"
	local n f
	>"$WORKDIR/$rep/needremove"
	cut -d\  -f2- -- "$WORKDIR/$rep/dups" | tr \  \\n |
	while read f; do
	    [ -e "$f" ] || continue
	    n="$(grep -c -m2 --mmap -F "${f##*/}" "$WORKDIR/src-bin" ||:)"
    	    [ "$n" -gt 1 ] || { printf '%s\n' "$f" >> "$WORKDIR/$rep/needremove"; continue; }
	    printf %s\\n "Package retained: ${f##*/}"
	done
}

confirm_cleanup() {
    local REPLY
    while :; do
	echo "Really purge files listed above? (yes/no)"
	read
	if [ "$REPLY" = no ]; then
	    echo Cancelled!
	    return 1
	fi
	if [ "$REPLY" = yes ]; then
	    break
	fi
    done
    return 0
}

repos_cleanup() {
    local type="$1" && shift || Fatal "first argument required"
    [ "$#" -gt 0 ] || return 1
    local rep list_cmd

    for rep in $*; do
	[ -s "$WORKDIR/$rep/dups" ] || continue

	echo "Duplicated files found in \"$rep\" repository:"
	cat -- "$WORKDIR/$rep/dups"
	echo

	if [ "$type" = "bin" ]; then
	    cut -d\  -f2- -- "$WORKDIR/$rep/dups" > "$WORKDIR/$rep/needremove"
	elif [ "$type" = "src" ]; then
	    correct_dups "$rep"
	fi

	cd "$PREFIX/$rep"
	if [ -z "$force_yes" ]; then
	    cat -- "$WORKDIR/$rep/needremove" |
		xargs -r ls -Llt --
	    confirm_cleanup || continue
	fi

	if [ -n "$dereference" ]; then
	    cat -- "$WORKDIR/$rep/needremove" |
		xargs -r realpath |
		xargs -r $TEST rm -v --
	else
	    cat -- "$WORKDIR/$rep/needremove" |
		xargs -r $TEST rm -v --
	fi
    done
}

WORKDIR=
exit_handler()
{
	local rc=$?
	trap - EXIT
	[ -z "$WORKDIR" ] || rm -rf -- "$WORKDIR"
	exit $rc
}

TEMP=`getopt -n $PROG -o a:,c:,f,h,p:,P,d -l architectures:,apt-config:,component:,force-yes,help,no-dereference,prefix:,dry-run -- "$@"` || show_usage
eval set -- "$TEMP"

while :; do
	case "$1" in
		-a|--architectures) shift
			ARCHITECTURES="$1"
			;;
		-c|--component) shift

			;;
		-p|--prefix) shift
			PREFIX="$(readlink -ev "$1")"
			;;
		-f|--force-yes) force_yes=1
			;;
		-P|--no-dereference) dereference=
			;;
		-d|--dry-run) TEST=echo
			;;
		-h|--help) show_help
			;;
		--) shift; break
			;;
		*) Fatal "unrecognized option: $1"
			;;
	esac
	shift

[ -n "$ARCHITECTURES" ] || Fatal 'Invalid ARCHITECTURES.'
[ -n "$component" ] || Fatal 'Invalid component.'

trap exit_handler HUP INT QUIT TERM EXIT
WORKDIR="$(mktemp -dt "$PROG.XXXXXXXXXX")"

echo 'Making dups in repositories...'

reps_bin=
reps_src=
	local rep arch
	rep="$1" && shift
	arch="$1" && shift

	mkdir -p -m700 -- "$WORKDIR/$rep"
	cd "$PREFIX/$rep"
	echo -n "Processing $rep: "
	>"$WORKDIR/$rep/dups" || Fatal "$rep: failed"
	for n in `ls -1 |grep ".$arch.rpm\$" |cut -c1 |LC_COLLATE=C sort -u`; do
		printf %s "$n"
		rpmrdups "$n"*."$arch".rpm >>"$WORKDIR/$rep/dups" ||
			Fatal "$rep/$n: failed"
	done
	echo
}

reps=
for arch in $ARCHITECTURES; do
    reps_bin="$reps_bin $arch/RPMS.$component"
    reps_src="$reps_src $arch/SRPMS.$component"
	process_dups "$arch/SRPMS.$component" "src"
done

    process_dups "$arch/RPMS.$component" "$arch"
    process_dups "$arch/SRPMS.$component" "src"
		echo "Duplicated files found in \"$rep\" repository:"
		cat -- "$WORKDIR/$rep/dups"
		echo

		cd "$PREFIX/$rep"
		if [ -z "$force_yes" ]; then
			cut -d\  -f2- -- "$WORKDIR/$rep/dups" |
				xargs -r ls -Llt --
			while :; do
				echo "Really purge files listed above? (yes/no)"
				read
				if [ "$REPLY" = no ]; then
					echo Cancelled!
					continue 2;
				fi
				if [ "$REPLY" = yes ]; then
					break;
				fi
			done
		fi
		if [ -n "$dereference" ]; then
			cut -d\  -f2- -- "$WORKDIR/$rep/dups" |
				xargs -r realpath |
				xargs -r rm -v --
		else
			cut -d\  -f2- -- "$WORKDIR/$rep/dups" |
				xargs -r rm -v --		
		fi
	fi
done

repos_cleanup bin $reps_bin
sources_index
repos_cleanup src $reps_src

Return to bug 7889

Lines 21-73 Link Here

(-)sisyphus-0.9.1/bin/sisyphus_cleanup_dups.cleanupdups (-55 / +108 lines)
21	# You should have received a copy of the GNU General Public License	21	# You should have received a copy of the GNU General Public License
22	# along with this program; if not, write to the Free Software	22	# along with this program; if not, write to the Free Software
23	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.	23	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
24	#	24	#
25		25
26	. /etc/sisyphus/config	26	. /etc/sisyphus/config
27	. /etc/sisyphus/functions	27	. /etc/sisyphus/functions
28		28
		29	TEST=
29	component=classic	30	component=classic
30	force_yes=	31	force_yes=
31	dereference=1	32	dereference=1
32		33
33	show_help()	34	show_help()
34	{	35	{
35	cat <<EOF	36	cat <<EOF
36	Usage: $PROG [options]	37	Usage: $PROG [options]
37		38
38	Valid options are:	39	Valid options are:
39	-p, --prefix=DIR path to repository location;	40	-p, --prefix=DIR path to repository location;
40	-c, --component=NAME repository component to use;	41	-c, --component=NAME repository component to use;
41	-a, --architectures=LIST architectures to handle;	42	-a, --architectures=LIST architectures to handle;
42	-f, --force-yes force yes answer;	43	-f, --force-yes force yes answer;
43	-P, --no-dereference do not dereference symbolic links;	44	-P, --no-dereference do not dereference symbolic links;
		45	-d, --dry-run dry run;
44	-h, --help show this text and exit.	46	-h, --help show this text and exit.
45	EOF	47	EOF
46	exit	48	exit
47	}	49	}
48		50
49	show_usage()	51	show_usage()
50	{	52	{
51	[ -z "$" ] \|\| Info "$"	53	[ -z "$" ] \|\| Info "$"
52	echo "Try \`$PROG --help' for more information." >&2	54	echo "Try \`$PROG --help' for more information." >&2
53	exit 1	55	exit 1
54	}	56	}
55		57
		58	process_dups()
		59	{
		60	local rep arch
		61	rep="$1" && shift \|\| Fatal "first argument required"
		62	arch="$1" && shift \|\| Fatal "second argument required"
		63
		64	mkdir -p -m700 -- "$WORKDIR/$rep"
		65	echo -n "Processing $rep: "
		66	find "$PREFIX/$rep" -name "*\.$arch\.rpm" -execdir rpmrdups \{\} \+ > "$WORKDIR/$rep/dups" &&
		67	echo "done" \|\| Fatal "failed"
		68	}
		69
		70	sources_index() {
		71	echo -n "Generating src-bin list: "
		72
		73	local arch paths
		74	for arch in $ARCHITECTURES; do
		75	paths="$paths $PREFIX/$NEW_FILE_PREFIX/$arch/RPMS"
		76	done
		77
		78	[ -n "$paths" ] \|\| Fatal "No binary paths found"
		79
		80	find $paths -mindepth 1 -maxdepth 1 -type f -name '*.rpm' \
		81	-execdir rpmquery -p --qf '%{SOURCERPM}\t%{ARCH}\n' -- \{\} \+ \|
		82	sort -u > "$WORKDIR/src-bin" &&
		83	echo "done" \|\| { echo "failed"; return 1; }
		84	}
		85
		86	correct_dups() {
		87	local rep="$1" && shift \|\| Fatal "first argument required"
		88	local n f
		89	>"$WORKDIR/$rep/needremove"
		90	cut -d\ -f2- -- "$WORKDIR/$rep/dups" \| tr \ \\n \|
		91	while read f; do
		92	[ -e "$f" ] \|\| continue
		93	n="$(grep -c -m2 --mmap -F "${f##*/}" "$WORKDIR/src-bin" \|\|:)"
		94	[ "$n" -gt 1 ] \|\| { printf '%s\n' "$f" >> "$WORKDIR/$rep/needremove"; continue; }
		95	printf %s\\n "Package retained: ${f##*/}"
		96	done
		97	}
		98
		99	confirm_cleanup() {
		100	local REPLY
		101	while :; do
		102	echo "Really purge files listed above? (yes/no)"
		103	read
		104	if [ "$REPLY" = no ]; then
		105	echo Cancelled!
		106	return 1
		107	fi
		108	if [ "$REPLY" = yes ]; then
		109	break
		110	fi
		111	done
		112	return 0
		113	}
		114
		115	repos_cleanup() {
		116	local type="$1" && shift \|\| Fatal "first argument required"
		117	[ "$#" -gt 0 ] \|\| return 1
		118	local rep list_cmd
		119
		120	for rep in $*; do
		121	[ -s "$WORKDIR/$rep/dups" ] \|\| continue
122
123	echo "Duplicated files found in \"$rep\" repository:"
124	cat -- "$WORKDIR/$rep/dups"
125	echo
126
127	if [ "$type" = "bin" ]; then
128	cut -d\ -f2- -- "$WORKDIR/$rep/dups" > "$WORKDIR/$rep/needremove"
129	elif [ "$type" = "src" ]; then
130	correct_dups "$rep"
131	fi
132
133	cd "$PREFIX/$rep"
134	if [ -z "$force_yes" ]; then
135	cat -- "$WORKDIR/$rep/needremove" \|
136	xargs -r ls -Llt --
137	confirm_cleanup \|\| continue
138	fi
139
140	if [ -n "$dereference" ]; then
141	cat -- "$WORKDIR/$rep/needremove" \|
142	xargs -r realpath \|
143	xargs -r $TEST rm -v --
144	else
145	cat -- "$WORKDIR/$rep/needremove" \|
146	xargs -r $TEST rm -v --
147	fi
148	done
149	}
150
56	WORKDIR=	151	WORKDIR=
57	exit_handler()	152	exit_handler()
58	{	153	{
59	local rc=$?	154	local rc=$?
60	trap - EXIT	155	trap - EXIT
61	[ -z "$WORKDIR" ] \|\| rm -rf -- "$WORKDIR"	156	[ -z "$WORKDIR" ] \|\| rm -rf -- "$WORKDIR"
62	exit $rc	157	exit $rc
63	}	158	}
64		159
65	TEMP=`getopt -n $PROG -o a:,c:,f,h,p:,P -l architectures:,apt-config:,component:,force-yes,help,no-dereference,prefix: -- "$@"` \|\| show_usage	160	TEMP=`getopt -n $PROG -o a:,c:,f,h,p:,P,d -l architectures:,apt-config:,component:,force-yes,help,no-dereference,prefix:,dry-run -- "$@"` \|\| show_usage
66	eval set -- "$TEMP"	161	eval set -- "$TEMP"
67		162
68	while :; do	163	while :; do
69	case "$1" in	164	case "$1" in
70	-a\|--architectures) shift	165	-a\|--architectures) shift
71	ARCHITECTURES="$1"	166	ARCHITECTURES="$1"
72	;;	167	;;
73	-c\|--component) shift	168	-c\|--component) shift
Lines 75-90 while :; do Link Here
75	;;	170	;;
76	-p\|--prefix) shift	171	-p\|--prefix) shift
77	PREFIX="$(readlink -ev "$1")"	172	PREFIX="$(readlink -ev "$1")"
78	;;	173	;;
79	-f\|--force-yes) force_yes=1	174	-f\|--force-yes) force_yes=1
80	;;	175	;;
81	-P\|--no-dereference) dereference=	176	-P\|--no-dereference) dereference=
82	;;	177	;;
		178	-d\|--dry-run) TEST=echo
		179	;;
83	-h\|--help) show_help	180	-h\|--help) show_help
84	;;	181	;;
85	--) shift; break	182	--) shift; break
86	;;	183	;;
87	*) Fatal "unrecognized option: $1"	184	*) Fatal "unrecognized option: $1"
88	;;	185	;;
89	esac	186	esac
90	shift	187	shift
Lines 93-157 done Link Here
93	[ -n "$ARCHITECTURES" ] \|\| Fatal 'Invalid ARCHITECTURES.'	190	[ -n "$ARCHITECTURES" ] \|\| Fatal 'Invalid ARCHITECTURES.'
94	[ -n "$component" ] \|\| Fatal 'Invalid component.'	191	[ -n "$component" ] \|\| Fatal 'Invalid component.'
95		192
96	trap exit_handler HUP INT QUIT TERM EXIT	193	trap exit_handler HUP INT QUIT TERM EXIT
97	WORKDIR="$(mktemp -dt "$PROG.XXXXXXXXXX")"	194	WORKDIR="$(mktemp -dt "$PROG.XXXXXXXXXX")"
98		195
99	echo 'Making dups in repositories...'	196	echo 'Making dups in repositories...'
100		197
101	process_dups()	198	reps_bin=
102	{	199	reps_src=
103	local rep arch
104	rep="$1" && shift
105	arch="$1" && shift
106
107	mkdir -p -m700 -- "$WORKDIR/$rep"
108	cd "$PREFIX/$rep"
109	echo -n "Processing $rep: "
110	>"$WORKDIR/$rep/dups" \|\| Fatal "$rep: failed"
111	for n in `ls -1 \|grep ".$arch.rpm\$" \|cut -c1 \|LC_COLLATE=C sort -u`; do
112	printf %s "$n"
113	rpmrdups "$n"*."$arch".rpm >>"$WORKDIR/$rep/dups" \|\|
114	Fatal "$rep/$n: failed"
115	done
116	echo
117	}
118
119	reps=
120	for arch in $ARCHITECTURES; do	200	for arch in $ARCHITECTURES; do
121	reps="$reps $arch/RPMS.$component $arch/SRPMS.$component"	201	reps_bin="$reps_bin $arch/RPMS.$component"
122	process_dups "$arch/RPMS.$component" "$arch"	202	reps_src="$reps_src $arch/SRPMS.$component"
123	process_dups "$arch/SRPMS.$component" "src"
124	done
125		203
126	for rep in $reps; do	204	process_dups "$arch/RPMS.$component" "$arch"
127	if [ -s "$WORKDIR/$rep/dups" ]; then	205	process_dups "$arch/SRPMS.$component" "src"
128	echo "Duplicated files found in \"$rep\" repository:"
129	cat -- "$WORKDIR/$rep/dups"
130	echo
131
132	cd "$PREFIX/$rep"
133	if [ -z "$force_yes" ]; then
134	cut -d\ -f2- -- "$WORKDIR/$rep/dups" \|
135	xargs -r ls -Llt --
136	while :; do
137	echo "Really purge files listed above? (yes/no)"
138	read
139	if [ "$REPLY" = no ]; then
140	echo Cancelled!
141	continue 2;
142	fi
143	if [ "$REPLY" = yes ]; then
144	break;
145	fi
146	done
147	fi
148	if [ -n "$dereference" ]; then
149	cut -d\ -f2- -- "$WORKDIR/$rep/dups" \|
150	xargs -r realpath \|
151	xargs -r rm -v --
152	else
153	cut -d\ -f2- -- "$WORKDIR/$rep/dups" \|
154	xargs -r rm -v --
155	fi
156	fi
157	done	206	done
		207
		208	repos_cleanup bin $reps_bin
		209	sources_index
		210	repos_cleanup src $reps_src