#!/bin/bash

# This script requires curl, fedora-release-rawhide, perl-XML-Rules, rpm-build and yum-utils

SOURCE_REPO=rawhide-source
BINARY_REPO=rawhide
BINARY_ARCHLIST="$(uname -i),$(uname -m),noarch"
DATA_DIR=$HOME/lib/rawhide-repo-data
TIMESTAMP_FILE=$DATA_DIR/rawhide-metadata-timestamp
EXPIRE_MINUTES=60
PROXY_CACHE=http://yum.intra.city-fan.org:3128/
QUICK=no

# Places to look for perl modules to rebuild
# These need to be the values for Rawhide (host arch), not the host distribution
# Do not add trailing slashes to these directory names
PERL_VENDORLIB=/usr/share/perl5/vendor_perl
PERL_VENDORARCH=/usr/lib64/perl5/vendor_perl
PERL_PRIVLIB=/usr/share/perl5
PERL_ARCHLIB=/usr/lib64/perl5

# Filter specified keywords from file
filter-from () {
	file=$1
	shift
	if [ ! -f "$file" ]; then
		touch $file
	fi
	while [ -n "$1" ]
	do
		grep -v "^$1 " ${file} > ${file}.tmp
		mv ${file}.tmp ${file}
		shift
	done
}

# Get primary.xml for a repo
get-primary-xml () {
	repo="$1"
	outfile="$2"
	# Get yum to give us the baseurl for the repo
	baseurl=$(
		yum --quiet \
		--disablerepo=\* \
		--enablerepo=$repo \
		repolist -v |
		awk '/^Repo-baseurl : / { print $3 }' |
		sed -e 's|/$||'
	)
	# Retrieve the repo's repomd.xml
	repomd=$(mktemp)
	curl --silent --output "$repomd" "$baseurl/repodata/repomd.xml"
	# Get the name of the repo's primary.xml file
	primaryfile=$(
		perl -se '
			use IO::File;
			use XML::Rules;
			my $repomd = new IO::File($xml_repomd);
			my @rules = (
				data => sub {
					print $_[1]->{location}->{href} if $_[1]->{type} eq "primary";
					return;
				},
			);
			my $parser = XML::Rules->new(rules => \@rules);
			$parser->parse($repomd);' -- -xml_repomd="$repomd"
	)
	rm -f "$repomd"
	case "$primaryfile" in
	*.gz)	decompress="gzip -dc";;
	*.xz)	decompress="xz -dc";;
	*.zst)	decompress="zstd -dc";;
	*)	decompress="cat";;
	esac
	timestamp=$(
		curl --silent --head "$baseurl/$primaryfile" |
		awk '/^Last-Modified:/ { print substr($0, 16) }'
	)
	curl --silent "$baseurl/$primaryfile" | $decompress > "$outfile"
	touch --date="$timestamp" "$outfile"
}

# Check options; the only one expected is "--quick"
case $# in
0)	;;
1)	if [ "x$1" = "x--quick" ]; then
		QUICK=yes
	else
		echo "rawhide-perl-build-order: unrecognized option: $1" 1>&2
		exit 1
	fi
	;;
*)	echo "rawhide-perl-build-order: usage: rawhide-perl-build-order [--quick]" 1>&2
	exit 1
	;;
esac

# Create data directory and required subdirs if they don't already exist
mkdir -p $DATA_DIR
mkdir -p $DATA_DIR/git
mkdir -p $DATA_DIR/src.data
mkdir -p $DATA_DIR/dep.data

# Use a caching proxy if possible
if [ -n "$PROXY_CACHE" ]; then
	export http_proxy="$PROXY_CACHE"
	export ftp_proxy="$PROXY_CACHE"
fi

# Skip the lengthy metadata extraction if requested, and use data from previous runs
if [ "$QUICK" != "yes" ]; then

# Check to see if we've updated the metadata recently
[ ! -f $TIMESTAMP_FILE ] && touch --date=2000-01-01 $TIMESTAMP_FILE
if [ -n "$(find $TIMESTAMP_FILE -mmin +$EXPIRE_MINUTES)" ]; then
	# Data not updated recently, so refresh

	# Generate local metadata caches
	echo "================================================================="
	echo "Generate local metadata caches"
	echo "================================================================="
	yum	--quiet \
		--disablerepo=\* \
		--enablerepo=$SOURCE_REPO \
		--enablerepo=$BINARY_REPO \
		makecache

	# Get the list of packages in the default buildroot, to be treated as leaf packages
	# Can't use cache here as "yum makecache" doesn't pull in group info
	echo "================================================================="
	echo "Get full list of packages in the default buildroot"
	echo "================================================================="
	BUILDROOT_TOP_LIST=$(
		repoquery \
			--disablerepo=\* \
			--enablerepo=$BINARY_REPO \
			--group --grouppkgs=all --list buildsys-build
	)
	# Expand out to the full buildroot package list
	python -c '
from __future__ import print_function
import sys
import yum

yb = yum.YumBase()
yb.preconf.debuglevel = 0
yb.setCacheDir()
yb.conf.cache = True
binary_repo = "'$BINARY_REPO'"

yb.repos.disableRepo("*")
yb.repos.enableRepo(binary_repo)

deps_done = set()
names_done = set()

def BestProviderPackage(dep):
    return yb.bestPackagesFromList(yb.returnPackagesByDep(dep), single_name = True)

def RecursiveRequires(dep):
    if dep in deps_done:
        return
    deps_done.add(dep)
    for prov in BestProviderPackage(dep):
        names_done.add(prov.name)
        for dep in prov.returnPrco(prcotype = "requires", printable = True):
            RecursiveRequires(dep)

for pkgname in sys.argv[1:]:
    RecursiveRequires(pkgname)

for dep in sorted(names_done, key = str.lower):
    print(dep)
'	$BUILDROOT_TOP_LIST > $DATA_DIR/buildsys-build-package-list

	# Get primary.xml for repos
	#get-primary-xml $SOURCE_REPO $DATA_DIR/source-primary.xml
	get-primary-xml $BINARY_REPO $DATA_DIR/binary-primary.xml

	# Touch the timestamp file so we don't repeat this process for a while
	touch $TIMESTAMP_FILE

fi

# Get list of binary packages requiring perl
echo "================================================================="
echo "Get lists of Binary RPMs of interest"
echo "================================================================="
# Strategy here is to split the perl ecosystem into two groups:
# * Those that *provide* modules in the standard perl directory locations,
#   which should have perl(:MODULE_COMPAT_*) dependencies and need
#   rebuilding
# * Those that *use* modules from the standard perl directory locations,
#   but don't actually provide any themselves; I don't think these need
#   rebuilding really and they shouldn't have perl(:MODULE_COMPAT_*)
#   dependencies, but they may be needed to build packages from the first
#   group and so their requirements have an influence on the build order.
#   If it's felt that they need rebuilding anyway, it could be done after
#   the first group has been bootstrapped.

# TODO: write a separate script looking for directory ownership issues
# in the perl module packages, i.e. modules that place files under
# ${PERL_VENDORLIB} etc. but in directories not owned by the main perl
# package itself. Here is not the place for that check.
#
# TODO: perhaps that script could also check that perl module providers
# include a perl(:MODULE_COMPAT_*) dependency, and perl module consumers
# do not?
#
# Find the providers
#
# First, get a list of all perl module files/dirs in all packages in the repo
repoquery --cache \
	--disablerepo=\* \
	--enablerepo=$BINARY_REPO \
	--archlist $BINARY_ARCHLIST \
	--qf '%{NAME}' \
	--list '*' |
	grep -E '^('${PERL_VENDORLIB}'|'${PERL_VENDORARCH}'|'${PERL_PRIVLIB}'|'${PERL_ARCHLIB}')/' \
	> $DATA_DIR/perl-full-filelist
# Then find out which packages own those files/dirs
cat $DATA_DIR/perl-full-filelist |
xargs repoquery --cache \
	--disablerepo=\* \
	--enablerepo=$BINARY_REPO \
	--archlist $BINARY_ARCHLIST \
	--qf '%{NAME}' \
	--whatprovides |
	sort -u > $DATA_DIR/rpms-with-perl-modules

# Find the consumers; there are about 800 packages in this group,
# including for example autoconf, automake, glibc-utils
repoquery --cache \
	--disablerepo=\* \
	--enablerepo=$BINARY_REPO \
	--archlist $BINARY_ARCHLIST \
	--qf '%{NAME}' \
	--whatrequires $(cat $DATA_DIR/rpms-with-perl-modules) |
	grep -Fvxf $DATA_DIR/rpms-with-perl-modules |
	sort -u > $DATA_DIR/rpms-using-perl

echo "================================================================="
echo "Determine which Binary RPMs are built from each SRPM"
echo "================================================================="
# Process the primary metadata and map RPMs to SRPMs
#
# Whilst doing this, we note the list of dual-lived perl modules built from the
# main perl package, as we don't need to build those before we can use the ones
# from the main perl package
#
# We also identify the SRPMs that need rebuilding, i.e. those that generate a
# binary RPM that includes a perl module
perl -se '
	use IO::File;
	use XML::Rules;

	my %rpm_to_build = ();
	my %srpm_to_build = ();
	open(PERL_MODULE_PACKAGES, "<", "$xxx_data_dir/rpms-with-perl-modules");
	while (my $line = <PERL_MODULE_PACKAGES>) {
		chomp(my $rpm = $line);
		$rpm_to_build{$rpm} = 1;
	}
	close(PERL_MODULE_PACKAGES);

	my $primaryxml = new IO::File("$xxx_data_dir/$xml_primary");
	my %srpmdata = ();
	chomp(my $platform = `uname -i`);
	chomp(my $machine = `uname -m`);
	my @rules = (
		_default => content,
		format => sub {
			my $srpm = $_[1]->{"rpm:sourcerpm"};
			$srpm =~ s/-[^-]*-[^-]*$//;
			return "srpm" => $srpm;
		},
		package => sub {
			my $arch = $_[1]->{arch};
			return unless $arch eq $platform or $arch eq $machine or $arch eq "noarch";
			my $rpm = $_[1]->{name};
			my $srpm = $_[1]->{srpm};
			$srpm_to_build{$srpm} = 1 if $rpm_to_build{$rpm} == 1;
			@srpmdata{$srpm} = () unless exists $srpmdata{$srpm};
			push @{ $srpmdata{$srpm} }, $rpm;
			return;
		},
	);
	my $parser = XML::Rules->new(rules => \@rules);
	$parser->parse($primaryxml);

	open(DUAL_LIVED, ">", "$xxx_data_dir/perl-dual-lived-packages");
	foreach $rpm (sort @{ $srpmdata{"perl"} }) {
		print DUAL_LIVED "$rpm\n";
	}
	close(DUAL_LIVED);

	open(PERL_MODULE_SRPMS, ">", "$xxx_data_dir/srpms-with-perl-modules");
	open(RPMS_BEING_REBUILT, ">", "$xxx_data_dir/rpms-being-rebuilt");
	foreach $srpm (sort keys %srpmdata) {
		# Assume perl itself is already built
		print PERL_MODULE_SRPMS "$srpm\n" if $srpm_to_build{$srpm} == 1 and $srpm ne "perl";
		my $fh;
		my $datafile = "$xxx_data_dir/src.data/$srpm".".src.data";
		open($fh, ">", $datafile)
			or die "Cannot create $datafile";
		print $fh "Package: $srpm\n";
		foreach $rpm (sort @{ $srpmdata{$srpm} }) {
			print $fh "RPM: $rpm\n";
			print RPMS_BEING_REBUILT "$rpm\n" if $srpm_to_build{$srpm} == 1;
		}
	}
	close(PERL_MODULE_SRPMS);
	close(RPMS_BEING_REBUILT);
	' -- -xxx_data_dir=$DATA_DIR -xml_primary=binary-primary.xml

# We now have a list of rpms that will be rebuilt, some of which (e.g. lots
# of -tests subpackages that still exist) will be in the list of packages
# using perl that don't provide perl modules themselves; packages to be
# rebuilt can be removed from that list.
grep -Fvxf $DATA_DIR/rpms-being-rebuilt $DATA_DIR/rpms-using-perl \
	> $DATA_DIR/rpms-using-perl-and-not-being-rebuilt

# Get list of build dependencies for each SRPM
echo "================================================================="
echo "Get list of build dependencies for each SRPM to be rebuilt"
echo "================================================================="
# Create a small dummy file to use for sources and patches
dd if=/dev/zero of=$DATA_DIR/dummy-file bs=128 count=1 &> /dev/null
for srpm in $(cat $DATA_DIR/srpms-with-perl-modules)
do
	echo "Processing $srpm"
	# Create a minimal git clone of the package, or update existing one
	if [ ! -d "$DATA_DIR/git/${srpm}.git" ]; then
		cd $DATA_DIR/git
		git clone --depth=1 --mirror git://pkgs.fedoraproject.org/$srpm
		cd - &> /dev/null
	else
		cd "$DATA_DIR/git/${srpm}.git"
		git fetch
		cd - &> /dev/null
	fi
	# Retrieve the spec file for the package, with sources and patches
	# renamed to dummy-file, so we can create a dummy SRPM from which we
	# can then extract build dependencies, with or without %{perl_bootstrap}
	cd "$DATA_DIR/git/${srpm}.git"
	# See if the repo has changed since we last worked out the buildreqs, and if so, skip this package
	LAST_COMMIT=$(git rev-parse HEAD)
	STORED_COMMIT=$(awk '/^Git-Head: / { print $2 }' "$DATA_DIR/src.data/${srpm}.src.data")
	if [ "$LAST_COMMIT" = "$STORED_COMMIT" ]; then
		cd - &> /dev/null
		continue
	fi
	# Otherwise, make an SRPM and grok the buildreqs
	filter-from "$DATA_DIR/src.data/${srpm}.src.data" Git-Head:
	echo "Git-Head: $LAST_COMMIT" >> "$DATA_DIR/src.data/${srpm}.src.data"
	git show "HEAD:${srpm}.spec" |
	sed	-e 's|^\(Source[0-9]*\).*|\1: dummy-file|i' \
		-e 's|^\(Patch[0-9]*\).*|\1: dummy-file|i' \
		-e 's|^\(Version\).*|\1: 0|i' \
		-e 's|^\(Release\).*|\1: 0|i' > "$DATA_DIR/${srpm}.spec"
	cd - &> /dev/null
	# Grok regular buildreqs
	filter-from "$DATA_DIR/src.data/${srpm}.src.data" BR:
	rpmbuild \
		--define "_sourcedir $DATA_DIR" \
		--define "_specdir $DATA_DIR" \
		--define "_builddir $DATA_DIR" \
		--define "_srcrpmdir $DATA_DIR" \
		--define "_rpmdir $DATA_DIR" \
		-bs "$DATA_DIR/${srpm}.spec" >/dev/null 2>"$DATA_DIR/${srpm}.warnings"
	grep -v "listed twice" "$DATA_DIR/${srpm}.warnings" 1>&2
	rm "$DATA_DIR/${srpm}.warnings"
	rpm -qp --requires "$DATA_DIR/${srpm}-0-0.src.rpm" |
	grep -v '^rpmlib[(]' |
	sort |
	awk '{ print "BR: " $0 }' >> $DATA_DIR/src.data/${srpm}.src.data
	rm "$DATA_DIR/${srpm}-0-0.src.rpm"
	# Grok boot buildreqs
	filter-from "$DATA_DIR/src.data/${srpm}.src.data" BootBR:
	rpmbuild $EXTRA_DEFINE \
		--define "_sourcedir $DATA_DIR" \
		--define "_specdir $DATA_DIR" \
		--define "_builddir $DATA_DIR" \
		--define "_srcrpmdir $DATA_DIR" \
		--define "_rpmdir $DATA_DIR" \
		--define "perl_bootstrap 1" \
		-bs "$DATA_DIR/${srpm}.spec" >/dev/null 2>"$DATA_DIR/${srpm}.warnings"
	grep -v "listed twice" "$DATA_DIR/${srpm}.warnings" 1>&2
	rm "$DATA_DIR/${srpm}.warnings"
	rpm -qp --requires "$DATA_DIR/${srpm}-0-0.src.rpm" |
	grep -v '^rpmlib[(]' |
	sort |
	awk '{ print "BootBR: " $0 }' >> "$DATA_DIR/src.data/${srpm}.src.data"
	rm "$DATA_DIR/${srpm}-0-0.src.rpm"
	rm "$DATA_DIR/${srpm}.spec"
	# Work out which, if any, buildreqs are not required when bootstrapping
	filter-from "$DATA_DIR/src.data/${srpm}.src.data" BootX:
	awk '/^BR: / { print substr($0, 5) }' "$DATA_DIR/src.data/${srpm}.src.data" \
		> "$DATA_DIR/${srpm}.BR"
	awk '/^BootBR: / { print substr($0, 9) }' "$DATA_DIR/src.data/${srpm}.src.data" \
		> "$DATA_DIR/${srpm}.BootBR"
	grep -Fvxf "$DATA_DIR/${srpm}.BootBR" "$DATA_DIR/${srpm}.BR" |
	awk '{ print "BootX: " $0 }' >> "$DATA_DIR/src.data/${srpm}.src.data"
	rm "$DATA_DIR/${srpm}.BR" "$DATA_DIR/${srpm}.BootBR"
done

# In quick mode, we resume here
fi

# Resolve buildreqs to a list of directly-required packages
echo "================================================================="
echo "Resolve build dependencies to packages for packages being rebuilt"
echo "================================================================="
python -c '
from __future__ import print_function
import os
import sys
import yum
from string import index

yb = yum.YumBase()
yb.preconf.debuglevel = 0
yb.setCacheDir()
yb.conf.cache = True
binary_repo = "'$BINARY_REPO'"
buildsys_build_pkg_file = "buildsys-build-package-list"
dual_lived_package_file = "perl-dual-lived-packages"
data_dir = "'$DATA_DIR'"
platform = "'`uname -i`'"
machine = "'`uname -m`'"

yb.repos.disableRepo("*")
yb.repos.enableRepo(binary_repo)

buildsys_pkgset = set()
infile = open(os.path.join(data_dir, buildsys_build_pkg_file), "r")
for line in infile:
    buildsys_pkgset.add(line.strip())
infile.close()

dual_lived_pkgset = set()
infile = open(os.path.join(data_dir, dual_lived_package_file), "r")
for line in infile:
    dual_lived_pkgset.add(line.strip())
infile.close()

# First of all, read the override file to pick up deps that we need to ignore
# A BootX entry removes a build requirement when bootstrapping
# A BootDepX entry removes a runtime requirement of a bootstrap-built package
# We should keep track of any packages with BootDepX entries and rebuild all
# packages that pull in that package post-bootstrap
bootx = {}
bootxdep = {}
overridefile_name = os.path.join(data_dir, "override.data")
overridefile = open(overridefile_name, "r")
def target(s):
    openp = index(s, "(")
    closep = index(s, ")")
    return s[openp+1:closep]
def exclusion(s):
    spacep = index(s, ": ")
    return s[spacep+1:].strip()
for line in overridefile:
    if line.startswith("BootX("):
        srpm = target(line)
        if srpm not in bootx:
            bootx[srpm] = set()
        bootx[srpm].add(exclusion(line))
    if line.startswith("BootDepX("):
        rpm = target(line)
        if rpm not in bootxdep:
            bootxdep[rpm] = set()
        bootxdep[rpm].add(exclusion(line))
overridefile.close()

def BestProviderPackage(dep):
    providers = []
    buildsys_providers = []
    for pkg in yb.returnPackagesByDep(dep):
        if pkg.arch != "noarch" and pkg.arch != platform and pkg.arch != machine:
            continue
        if pkg.name in buildsys_pkgset | dual_lived_pkgset:
            buildsys_providers.append(pkg)
        else:
            providers.append(pkg)
    if len(buildsys_providers) > 0:
        return yb.bestPackagesFromList(buildsys_providers, single_name = True)
    else:
        return yb.bestPackagesFromList(providers, single_name = True)

# Now find out which dependency rpms are excluded by each BootDepX entry in the override file
excluded_depset = {}
for rpm in bootxdep:
    excluded_depset[rpm] = set()
    for dep in bootxdep[rpm]:
        for pkg in BestProviderPackage(dep):
            excluded_depset[rpm].add(pkg.name)

# Now process each srpm in turn, working out its build dependency tree
for srpm in sys.argv[1:]:
    print("Processing %s" % srpm)
    infile_name = os.path.join(data_dir, "src.data", srpm+".src.data")
    outfile_name = os.path.join(data_dir, "src.data", srpm+".src.data.tmp")
    infile = open(infile_name, "r")
    outfile = open(outfile_name, "w")
    depset = set()
    bootdepset = set()
    binary_rpmset = set()
    for line in infile:
        if line.startswith("BR: "):
            depset.add(line[4:].strip())
        if line.startswith("BootBR: "):
            bootdepset.add(line[8:].strip())
        if line.startswith("RPM: "):
            binary_rpmset.add(line[5:].strip())
        if line.startswith("PBR: "):
            continue
        if line.startswith("BootPBR: "):
            continue
        if line.startswith("BootBRdep: "):
            continue
        if line.startswith("BootPBRdep: "):
            continue
        if line.startswith("BootXPdep("):
            continue
        if line.startswith("BootXdepRebuild: "):
            continue
        print(line.rstrip("\r\n"), file = outfile)
    infile.close()
    # Apply any manual BootX dependency overrides
    if srpm in bootx:
        bootdepset -= bootx[srpm]

    provider_names = set()
    provider_deps = set()
    boot_provider_names = set()
    boot_provider_deps = set()
    excluded_deps = set()
    deps_done = set()
    names_done = set()
    buildsys_dep = set()
    for dep in depset:
        for prov in BestProviderPackage(dep):
            provider_names.add(prov.name)
            if dep in bootdepset:
                boot_provider_names.add(prov.name)
                names_done.add(prov.name)
            # Do not follow deps for buildsys packages that should always be installable
            if prov.name not in buildsys_pkgset | dual_lived_pkgset:
                runtime_deps = set(prov.returnPrco(prcotype = "requires", printable = True))
                provider_deps |= runtime_deps
                if dep in bootdepset:
                    # Skip excluded runtime deps for this package, if any
                    if prov.name in bootxdep:
                        runtime_deps -= bootxdep[prov.name]
                        excluded_deps |= excluded_depset[prov.name]
                    boot_provider_deps |= runtime_deps

    for prov in sorted(provider_names, key = str.lower):
        print("PBR: %s" % prov, file = outfile)
    for prov in sorted(boot_provider_names, key = str.lower):
        print("BootPBR: %s" % prov, file = outfile)
    for rpm in sorted(excluded_depset, key = str.lower):
        if rpm in binary_rpmset:
            for dep in sorted(excluded_depset[rpm], key = str.lower):
                print("BootXPdep(%s): %s" % (rpm, dep), file = outfile)

    def RecursiveRequires(dep):
        global excluded_deps
        global deps_done
        global names_done
        global buildsys_dep
        if dep in deps_done:
            return
        deps_done.add(dep)
        for prov in BestProviderPackage(dep):
            if prov.name in buildsys_pkgset:
                buildsys_dep.add(dep)
                continue
            names_done.add(prov.name)
            for dep in prov.returnPrco(prcotype = "requires", printable = True):
                # Skip excluded runtime deps for this package, if any
                if prov.name in bootxdep and dep in bootxdep[prov.name]:
                    excluded_deps |= excluded_depset[prov.name]
                    continue
                RecursiveRequires(dep)

    for dep in boot_provider_deps:
        RecursiveRequires(dep)
    
    bootbrdep_set = deps_done - buildsys_dep
    bootpbrdep_set = names_done - boot_provider_names

    for dep in sorted(bootbrdep_set, key = str.lower):
        print("BootBRdep: %s" % dep, file = outfile)
    for prov in sorted(bootpbrdep_set, key = str.lower):
        print("BootPBRdep: %s" % prov, file = outfile)
    for pkg in sorted(excluded_deps - names_done, key = str.lower):
        print("BootXdepRebuild: %s" % pkg, file = outfile)

    outfile.close()
    os.rename(outfile_name, infile_name)
' $(cat $DATA_DIR/srpms-with-perl-modules)

# Determine the dependencies of the perl-consuming packages
echo "================================================================="
echo "Resolve dependencies to packages for perl consumer packages"
echo "================================================================="
python -c '
from __future__ import print_function
import os
import sys
import yum
from string import index

yb = yum.YumBase()
yb.preconf.debuglevel = 0
yb.setCacheDir()
yb.conf.cache = True
binary_repo = "'$BINARY_REPO'"
buildsys_build_pkg_file = "buildsys-build-package-list"
dual_lived_package_file = "perl-dual-lived-packages"
perl_consumer_pkg_file = "rpms-using-perl-and-not-being-rebuilt"
data_dir = "'$DATA_DIR'"
platform = "'`uname -i`'"
machine = "'`uname -m`'"

yb.repos.disableRepo("*")
yb.repos.enableRepo(binary_repo)

buildsys_pkgset = set()
infile = open(os.path.join(data_dir, buildsys_build_pkg_file), "r")
for line in infile:
    buildsys_pkgset.add(line.strip())
infile.close()

dual_lived_pkgset = set()
infile = open(os.path.join(data_dir, dual_lived_package_file), "r")
for line in infile:
    dual_lived_pkgset.add(line.strip())
infile.close()

perl_consumer_pkgset = set()
infile = open(os.path.join(data_dir, perl_consumer_pkg_file), "r")
for line in infile:
    perl_consumer_pkgset.add(line.strip())
infile.close()

# process each rpm in turn, working out its dependency tree
for rpm in sorted(perl_consumer_pkgset):
    print("Processing %s" % rpm)
    outfile_name = os.path.join(data_dir, "dep.data", rpm+".dep.data")
    outfile = open(outfile_name, "w")

    def BestProviderPackage(dep):
        providers = []
        buildsys_providers = []
        for pkg in yb.returnPackagesByDep(dep):
            if pkg.arch != "noarch" and pkg.arch != platform and pkg.arch != machine:
                continue
            if pkg.name in buildsys_pkgset | dual_lived_pkgset:
                buildsys_providers.append(pkg)
            else:
                providers.append(pkg)
        if len(buildsys_providers) > 0:
            return yb.bestPackagesFromList(buildsys_providers, single_name = True)
        else:
            return yb.bestPackagesFromList(providers, single_name = True)

    deps_done = set()
    names_done = set()
    buildsys_dep = set()

    def RecursiveRequires(dep):
        if dep in deps_done:
            return
        deps_done.add(dep)
        for prov in BestProviderPackage(dep):
            if prov.name in buildsys_pkgset | dual_lived_pkgset:
                continue
            names_done.add(prov.name)
            for dep in prov.returnPrco(prcotype = "requires", printable = True):
                RecursiveRequires(dep)

    RecursiveRequires(rpm)

    for dep in sorted(names_done, key = str.lower):
        # List all packages providing runtime dependencies, other than the package itself
        # since it will be installable if all other dependencies are satisfied
        if dep != rpm:
            print(dep, file = outfile)

    outfile.close()
'

# Run the boot order routine
echo "================================================================="
echo "Calculate build order"
echo "================================================================="
python -c '
from __future__ import print_function
import os
import sys
from string import index

data_dir = "'$DATA_DIR'"
buildroot_package_file = os.path.join(data_dir, "buildsys-build-package-list")
dual_lived_package_file = os.path.join(data_dir, "perl-dual-lived-packages")
perl_consumer_pkg_file = os.path.join(data_dir, "rpms-using-perl-and-not-being-rebuilt")

rpmlist = {}
buildreqs = {}
builddeps = {}
sourcepkg = {}
needs_rebuild_for_retest = {}
needs_rebuild_for_deps = {}
boot_build_missing_deps = {}
rpm_to_build = set()
srpm_to_build = set()
consumer_deps = {}
boottree = set()
fails_to_build_from_source = set()
manual_build = set()

perl_consumer_pkgset = set()
infile = open(os.path.join(data_dir, perl_consumer_pkg_file), "r")
for line in infile:
    consumer_pkg = line.strip()
    perl_consumer_pkgset.add(consumer_pkg)
    consumer_deps[consumer_pkg] = set()
    depfile_name = os.path.join(data_dir, "dep.data", consumer_pkg+".dep.data")
    depfile = open(depfile_name, "r")
    for dep in depfile:
         consumer_deps[consumer_pkg].add(dep.strip())
    depfile.close()
infile.close()
consumer_packages_remaining = perl_consumer_pkgset

overridefile_name = os.path.join(data_dir, "override.data")
overridefile = open(overridefile_name, "r")
def target(s):
    openp = index(s, "(")
    closep = index(s, ")")
    return s[openp+1:closep]
for line in overridefile:
    if line.startswith("BootTree("):
        srpm = target(line)
        boottree.add(srpm)
    if line.startswith("FTBFS("):
        srpm = target(line)
        fails_to_build_from_source.add(srpm)
    if line.startswith("Manual("):
        srpm = target(line)
        manual_build.add(srpm)
overridefile.close()

for srpm in sys.argv[1:]:
    infile_name = os.path.join(data_dir, "src.data", srpm+".src.data")
    try:
        infile = open(infile_name, "r")
        srpm_to_build.add(srpm)
        rpmlist[srpm] = []
        buildreqs[srpm] = []
        builddeps[srpm] = []
        boot_build_missing_deps[srpm] = set()
        for line in infile:
            if line.startswith("RPM: "):
                rpm = line[5:].strip()
                rpmlist[srpm].append(rpm)
                rpm_to_build.add(rpm)
                sourcepkg[rpm] = srpm
                continue
            if line.startswith("BootPBR: "):
                buildreqs[srpm].append(line[9:].strip())
                continue
            if line.startswith("BootPBRdep: "):
                builddeps[srpm].append(line[12:].strip())
                continue
            if line.startswith("BootX: "):
                needs_rebuild_for_retest[srpm] = True
                continue
            if line.startswith("BootXPdep("):
                needs_rebuild_for_deps[srpm] = True
                continue
            if line.startswith("BootXdepRebuild: "):
                boot_build_missing_deps[srpm].add(line[17:].strip())
                continue
        infile.close()
    except IOError:
        pass
# RPMs in the default buildroot or from the base perl package can be assumed
# to be available for building with, even if we are going to build our own version
for f in [ buildroot_package_file, dual_lived_package_file ]:
    infile = open(f, "r")
    for line in infile:
        pkg = line.strip()
        rpm_to_build.discard(pkg)
    infile.close()

# Some packages may be built manually at the start of the process
if len(manual_build) > 0:
    print("Pass 0 (manual build)")
    srpms_built_this_pass = set()
    rpms_built_this_pass = set()
    for srpm in sorted(manual_build):
        print("Build: %s (%s)" % (srpm, " ".join(rpmlist[srpm])))
        srpms_built_this_pass.add(srpm)
        for rpm in rpmlist[srpm]:
            rpms_built_this_pass.add(rpm)
    srpm_to_build = srpm_to_build.difference(srpms_built_this_pass)
    rpm_to_build = rpm_to_build.difference(rpms_built_this_pass)

# Evaluate the build order
passnum = 0
while True:
    passnum += 1
    print("Pass %s" % passnum)
    print("Packages remaining: %s" % len(srpm_to_build))
    print("Consumer packages unavailable: %s" % len(consumer_packages_remaining))
    consumer_packages_available_this_pass = set()
    for rpm in sorted(consumer_packages_remaining):
        for dep in consumer_deps[rpm]:
            if dep in rpm_to_build:
                break
        else:
            print("Available: %s" % rpm)
            consumer_packages_available_this_pass.add(rpm)
            consumer_packages_remaining.remove(rpm)
    if len(consumer_packages_available_this_pass) > 0 or len(consumer_packages_remaining) > 0:
        print("Consumer packages made available this pass: %s" % len(consumer_packages_available_this_pass))
    srpms_built_this_pass = set()
    rpms_built_this_pass = set()
    for srpm in sorted(srpm_to_build):
        if srpm in fails_to_build_from_source:
            continue
        for dep in buildreqs[srpm] + builddeps[srpm]:
            if dep in rpm_to_build | consumer_packages_remaining:
                break
        else:
            print("Build: %s (%s)" % (srpm, " ".join(rpmlist[srpm])))
            srpms_built_this_pass.add(srpm)
            for rpm in rpmlist[srpm]:
                rpms_built_this_pass.add(rpm)
    if len(srpms_built_this_pass) > 0:
        srpm_to_build = srpm_to_build.difference(srpms_built_this_pass)
        rpm_to_build = rpm_to_build.difference(rpms_built_this_pass)
        print("Packages built: %s" % len(srpms_built_this_pass))
        if len(srpm_to_build) == 0:
            print("Packages remaining: 0")
            break
    else:
        break

# Now list the remaining packages, if any, and their unsatisfied buildreqs
cycle_count = {}
if len(srpm_to_build) > 0:

    unsatisfied_deps_this_package = 0

    # Identify dependency cycle and increment count for each instance
    # Sort in reverse order to avoid indirect deps being the start/endpoint, which looks a bit wierd
    def register_cycle(stack, looper):
        startpoint = stack.index(looper)
        loop_list = stack[startpoint:]
        # Normalize the cycle
        startpoint = loop_list.index(sorted(loop_list, reverse = True)[0])
        loop_list = loop_list[startpoint:] + loop_list[:startpoint] + [ loop_list[startpoint] ]
        cycle_name = "->".join(loop_list)
        if cycle_name in cycle_count:
            cycle_count[cycle_name] += 1
        else:
            cycle_count[cycle_name] = 1
        return cycle_name

    # Generate a tree view of the unsatisfied dependencies of a package
    # Print the tree view in verbose mode, but always work out the dependency cycles
    depstack = []
    def deptree(package, level = 0, indirect = False, verbose = False, consumer = False):
        global unsatisfied_deps_this_package
        if verbose:
            print("%s%s" % ("  " * level, package), end = "")
            if package in sourcepkg and package != sourcepkg[package]:
                print(" (from %s)" % sourcepkg[package], end = "")
            if consumer:
                print(" [consumer package]", end = "")
            if indirect:
                print(" [indirect dependency]", end = "")
        if not package in rpm_to_build | consumer_packages_remaining:
            if verbose:
                print(" [available]")
            return
        unsatisfied_deps_this_package += 1
        if unsatisfied_deps_this_package > 500:
            if verbose:
                print(" [Aborting deptree due to reaching depth limit]")
            return
        if consumer:
            # Highlight consumer packages in cycles by putting them in angle brackets
            package_str = "<" + package + ">"
        else:
            # For provider packages, map binary packages back to their SRPMs
            package = sourcepkg[package]
            package_str = package
        if package in fails_to_build_from_source:
            if verbose:
                print(" [FTBFS]")
            return
        if package != "[*]" and package_str in depstack:
            cycle = register_cycle(depstack, package_str)
            if verbose:
                print(" [circular dep: %s]" % cycle)
            return
        depstack.append(package_str)
        if verbose:
            print("")
        # Providers and consumers have different trees to follow
        if not consumer:
            # Follow provider package dep tree, looking at direct dependencies first
            all_brs_available = True
            for dep in buildreqs[package]:
                if dep in rpm_to_build | consumer_packages_remaining:
                    all_brs_available = False
                deptree(dep, level + 1, verbose = verbose, consumer = dep in consumer_packages_remaining)
            # If all direct dependencies are available, look at the indirect ones
            if all_brs_available:
                for dep in builddeps[package]:
                    # Push "[*]" to the dep stack to flag an indirect dependency for the cycle checker
                    depstack.append("[*]")
                    deptree(dep, level = level + 1, indirect = True, verbose = verbose, consumer = dep in consumer_packages_remaining)
                    depstack.pop()
        else:
            # Follow consumer package dep tree
            for dep in consumer_deps[package]:
                deptree(dep, level + 1, verbose = verbose, consumer = dep in consumer_packages_remaining)
        depstack.pop()
        if verbose and level == 0:
            print("")

    print("================")
    print("Unbuilt packages")
    print("================")
    for srpm in sorted(srpm_to_build):
        print("%s needs" % srpm, end = "")
        for dep in sorted(buildreqs[srpm]):
            if dep in rpm_to_build | consumer_packages_remaining:
                print(" %s" % dep, end = "")
        for dep in sorted(builddeps[srpm]):
            if dep in rpm_to_build:
                print(" [%s]" % dep, end = "")
        if srpm in fails_to_build_from_source:
            print(" [FTBFS]", end = "")
        print("")
        unsatisfied_deps_this_package = 0
        deptree(srpm, verbose = (srpm in boottree))
    if len(consumer_packages_remaining) > 0:
        print("=============================")
        print("Unavailable consumer packages")
        print("=============================")
        for rpm in sorted(consumer_packages_remaining):
            print("%s needs" % rpm, end = "")
            for dep in sorted(consumer_deps[rpm]):
                if dep in rpm_to_build | consumer_packages_remaining:
                    print(" %s" % dep, end = "")
            print("")
    print("=================")
    print("Dependency Cycles")
    print("=================")
    for cycle in sorted(cycle_count, key = cycle_count.get, reverse = True):
        print("%s %s" % ("{0:10}".format(cycle_count[cycle]), cycle))
else:
    print("====================================================================")
    print("Packages to rebuild post-bootstrap, with %{perl_bootstrap} undefined")
    print("====================================================================")
    if len(needs_rebuild_for_deps) > 0:
        print("=================================================")
        print("Rebuild packages to add full runtime dependencies")
        print("=================================================")
        for srpm in sorted(needs_rebuild_for_deps, key = str.lower):
            print("Rebuild: %s" % srpm)
    if len(needs_rebuild_for_retest) > 0:
        print("======================================================")
        print("Rebuild packages with full buildreq set for re-testing")
        print("======================================================")
        for srpm in sorted(needs_rebuild_for_retest, key = str.lower):
            print("Rebuild: %s" % srpm)
    if len(needs_rebuild_for_deps) > 0:
        print("=======================================================")
        print("Rebuild packages with complete buildroot for re-testing")
        print("=======================================================")
        for srpm in sorted(boot_build_missing_deps, key = str.lower):
            if len(boot_build_missing_deps[srpm]) == 0:
                continue
            if srpm in needs_rebuild_for_retest:
                # Rebuild already done in previous pass
                continue
            print("Rebuild: %s (missing %s)" % (srpm, ", ".join(boot_build_missing_deps[srpm])))
' $(cat $DATA_DIR/srpms-with-perl-modules) > $DATA_DIR/buildorder