#!/bin/bash

# This script requires fedora-release-rawhide and perl-XML-Rules

LOCAL_SOURCE_REPO=city-fan.org-rawhide-source
LOCAL_BINARY_REPO=city-fan.org-rawhide
BASE_SOURCE_REPO=rawhide-source
BASE_BINARY_REPO=rawhide
BINARY_ARCHLIST="$(uname -i),$(uname -m),noarch"
DATA_DIR=$HOME/lib/repo-data
TEMP_BUILD_DIR=$HOME/lib/repo-data/BUILD

# Create voluminous debug output for dependency tracing if requested
if [ "$1" = "-d" ]; then
	DEBUG_MODE=True
else
	DEBUG_MODE=False
fi

# Filter specified keywords from file
filter-from () {
	file=$1
	shift
	if [ ! -f "$file" ]; then
		touch $file
	fi
	while [ -n "$1" ]
	do
		grep -v "^$1 " ${file} > ${file}.tmp
		mv ${file}.tmp ${file}
		shift
	done
}

echo "================================================================="
echo "Refresh repository metadata"
echo "================================================================="
cfo-build-data-update

# Get list of local SRPMs
echo "================================================================="
echo "Get list of Source RPMs of interest"
echo "================================================================="
SRPM_LIST=$(
	dnf repoquery \
		--quiet \
		--disablerepo=\* \
		--enablerepo=$LOCAL_SOURCE_REPO \
		--all --arch=src --qf '%{NAME}'
)

# Get list of binary RPMs
echo "================================================================="
echo "Get list of Binary RPMs of interest"
echo "================================================================="
RPM_LIST=$(
	dnf repoquery \
		--quiet \
		--disablerepo=\* \
		--enablerepo=$LOCAL_BINARY_REPO \
		--arch $BINARY_ARCHLIST \
		--all --qf '%{NAME}'
)

# Get list of build dependencies for each SRPM
echo "================================================================="
echo "Get list of build dependencies for each SRPM"
echo "================================================================="
mkdir -p ${TEMP_BUILD_DIR}/{SPECS,SRPMS,SOURCES,BUILD,RPMS}
for pkg in ${SRPM_LIST}
do
	echo "Processing $pkg"
	PKG_DATA_FILE="${DATA_DIR}/${pkg}.src.data"
	[ ! -r ${PKG_DATA_FILE} ] && echo "Package: ${pkg}" > ${PKG_DATA_FILE}
	dnf download \
		--quiet \
		--disablerepo=\* \
		--enablerepo=$LOCAL_BINARY_REPO \
		--destdir ${TEMP_BUILD_DIR} \
		--source \
		--best \
		${pkg}
	CURRENT_SRPM=$(echo ${TEMP_BUILD_DIR}/${pkg}-*.src.rpm)
	if [ ! -r ${CURRENT_SRPM} ]; then
		# Perhaps the SRPM does not produce a binary package of the same name;
		# try again with one of the binary package names (may need priming)
		binpkg=$(grep '^RPM: ' ${PKG_DATA_FILE} | awk '{ print substr($0, 6); exit }')
		if [ -z "${binpkg}" ]; then
			echo "cfo-build-order: SRPM not found for ${pkg}, skipping" 1>&2
			continue
		fi
		dnf download \
			--quiet \
			--disablerepo=\* \
			--enablerepo=$LOCAL_BINARY_REPO \
			--destdir ${TEMP_BUILD_DIR} \
			--source \
			--best \
			${binpkg}
		CURRENT_SRPM=$(echo ${TEMP_BUILD_DIR}/${pkg}-*.src.rpm)
		if [ -r ${CURRENT_SRPM} ]; then
			echo "No binary ${pkg} package, used source for ${binpkg} instead"
		else
			echo "cfo-build-order: SRPM not found for ${pkg} or ${binpkg}, skipping" 1>&2
			continue
		fi
	fi
	# Grok regular build requirements
	rpmbuild \
		--define "_sourcedir ${TEMP_BUILD_DIR}/SOURCES" \
		--define "_specdir ${TEMP_BUILD_DIR}/SPECS" \
		--define "_builddir ${TEMP_BUILD_DIR}/BUILD" \
		--define "_srcrpmdir ${TEMP_BUILD_DIR}/SRPMS" \
		--define "_rpmdir ${TEMP_BUILD_DIR}/RPMS" \
		--rmsource --rmspec --nodeps \
		-rs ${CURRENT_SRPM} >/dev/null 2>${TEMP_BUILD_DIR}/${pkg}.warnings
	grep -v '^[+]' ${TEMP_BUILD_DIR}/${pkg}.warnings |
	grep -Fv " does not exist - using root"
	REBUILT_SRPM=$(echo ${TEMP_BUILD_DIR}/SRPMS/${pkg}-*.src.rpm)
	filter-from ${PKG_DATA_FILE} BR:
	rpm -qp --requires ${REBUILT_SRPM} |
	grep -v '^rpmlib[(]' |
	sort |
	awk '{ print "BR: " $0 }' >> ${PKG_DATA_FILE}
	rm -f ${REBUILT_SRPM} ${TEMP_BUILD_DIR}/${pkg}.warnings
	# Grok boot buildreqs
	rpmbuild \
		--define "_sourcedir ${TEMP_BUILD_DIR}/SOURCES" \
		--define "_specdir ${TEMP_BUILD_DIR}/SPECS" \
		--define "_builddir ${TEMP_BUILD_DIR}/BUILD" \
		--define "_srcrpmdir ${TEMP_BUILD_DIR}/SRPMS" \
		--define "_rpmdir ${TEMP_BUILD_DIR}/RPMS" \
		--rmsource --rmspec --nodeps \
		--define "perl_bootstrap 1" \
		-rs ${CURRENT_SRPM} >/dev/null 2>${TEMP_BUILD_DIR}/${pkg}.warnings
	grep -v '^[+]' ${TEMP_BUILD_DIR}/${pkg}.warnings |
	grep -Fv " does not exist - using root"
	filter-from ${PKG_DATA_FILE} BootBR:
	rpm -qp --requires ${REBUILT_SRPM} |
	grep -v '^rpmlib[(]' |
	sort |
	awk '{ print "BootBR: " $0 }' >> ${PKG_DATA_FILE}
	rm -f ${CURRENT_SRPM} ${REBUILT_SRPM} ${TEMP_BUILD_DIR}/${pkg}.warnings
	# Work out which, if any, buildreqs are not required when bootstrapping
	filter-from ${PKG_DATA_FILE} BootX:
	awk '/^BR: / { print substr($0, 5) }' ${PKG_DATA_FILE} \
		> "$DATA_DIR/${pkg}.BR"
	awk '/^BootBR: / { print substr($0, 9) }' ${PKG_DATA_FILE} \
		> "$DATA_DIR/${pkg}.BootBR"
	grep -Fvxf "$DATA_DIR/${pkg}.BootBR" "$DATA_DIR/${pkg}.BR" |
	awk '{ print "BootX: " $0 }' >> ${PKG_DATA_FILE}
	rm "$DATA_DIR/${pkg}.BR" "$DATA_DIR/${pkg}.BootBR"
done

# Resolve these to a list of directly-required packages
echo "================================================================="
echo "Resolve build dependencies to packages"
echo "================================================================="
python3 -c '
import os
import sys
import dnf

local_binary_repo = "'$LOCAL_BINARY_REPO'"
base_binary_repo = "'$BASE_BINARY_REPO'"
data_dir = "'$DATA_DIR'"
arch = "'`arch`'"

with dnf.Base() as base:
    base.read_all_repos()
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching(local_binary_repo)
    repos.enable()
    repos = base.repos.get_matching(base_binary_repo)
    repos.enable()
    base.fill_sack(load_system_repo=False)
    q = base.sack.query()
    available = base.sack.query().available().filter()
    for srpm in sys.argv[1:]:
        print("Processing %s" % srpm)
        infile_name = os.path.join(data_dir, srpm+".src.data")
        outfile_name = os.path.join(data_dir, srpm+".src.data.tmp")
        infile = open(infile_name, "r")
        outfile = open(outfile_name, "w")
        deplist = []
        for line in infile:
            if line.startswith("PBR: "):
                continue
            if line.startswith("WARNING: unsatisfied build dependency: "):
                continue
            if line.startswith("WARNING: multiple providers for dependency: "):
                continue
            if line.startswith("BR: "):
                deplist.append(line[4:].strip())
            print("%s" % line.rstrip("\r\n"), file = outfile)
        infile.close()
        providers = []
        for dep in deplist:
            local_providers = []
            base_providers = []
            if dep.startswith("/"):
                pkgs = available.filter(file=dep)
                if len(pkgs) < 1:
                    pkgs = available.filter(provides=dep)
            else:
                pkgs = available.filter(provides=dep)
            for pkg in pkgs:
                if pkg.arch not in [ arch, "noarch" ]:
                    continue
                if pkg.repo.id == local_binary_repo:
                    local_providers.append(pkg)
                else:
                    base_providers.append(pkg)
            best_providers = local_providers if len(local_providers) > 0 else base_providers[0:1]
            if len(best_providers) < 1:
                print("WARNING: unsatisfied build dependency: %s" % dep, file = outfile)
            if len(best_providers) > 1:
                print("WARNING: multiple providers for dependency: %s" % dep, file = outfile)
            for pkg in best_providers:
                providers.append(pkg.name)
        for prov in sorted(set(providers), key = str.lower):
            print("PBR: %s" % prov, file = outfile)
        outfile.close()
        os.rename(outfile_name, infile_name)
' $SRPM_LIST

# Find the packages that each SRPM directly buildrequires when bootstrapping,
# by excluding BR: entries that also have matching BootX: entries (added manually)
echo "================================================================="
echo "Get list of bootstrap build dependencies for each SRPM"
echo "================================================================="
for srpm in $SRPM_LIST
do
	# Clear existing DEBUG data
	filter-from $DATA_DIR/${srpm}.src.data DEBUG:
done
python3 -c '
import os
import sys
import dnf

local_binary_repo = "'$LOCAL_BINARY_REPO'"
base_binary_repo = "'$BASE_BINARY_REPO'"
buildsys_build_pkg_file = "buildsys-build-package-list"
dual_lived_package_file = "perl-dual-lived-packages"
data_dir = "'$DATA_DIR'"
platform = "'`uname -i`'"
machine = "'`uname -m`'"
debug = '$DEBUG_MODE'

with dnf.Base() as base:
    base.read_all_repos()
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching(local_binary_repo)
    repos.enable()
    repos = base.repos.get_matching(base_binary_repo)
    repos.enable()
    base.fill_sack(load_system_repo=False)
    q = base.sack.query()
    available = base.sack.query().available().filter()

    buildsys_pkgset = set()
    infile = open(os.path.join(data_dir, buildsys_build_pkg_file), "r")
    for line in infile:
        buildsys_pkgset.add(line.strip())
    infile.close()

    dual_lived_pkgset = set()
    infile = open(os.path.join(data_dir, dual_lived_package_file), "r")
    for line in infile:
        dual_lived_pkgset.add(line.strip())
    infile.close()

    for srpm in sys.argv[1:]:
        print("Processing %s" % srpm)
        infile_name = os.path.join(data_dir, srpm+".src.data")
        outfile_name = os.path.join(data_dir, srpm+".src.data.tmp")
        infile = open(infile_name, "r")
        outfile = open(outfile_name, "w")
        depset = set()
        bootxset = set()
        for line in infile:
            if line.startswith("BootBR: "):
                continue
            if line.startswith("BootPBR: "):
                continue
            if line.startswith("BootBRdep: "):
                continue
            if line.startswith("BootPBRdep: "):
                continue
            if line.startswith("BR: "):
                depset.add(line[4:].strip())
            if line.startswith("BootX: "):
                bootxset.add(line[7:].strip())
            print("%s" % line.rstrip("\r\n"), file = outfile)
        infile.close()
        depset = depset.difference(bootxset)

        def BestProviderPackage(dep):
            local_providers = []
            base_providers = []
            buildsys_providers = []
            if dep.startswith("/"):
                pkgs = available.filter(file=dep)
                if len(pkgs) < 1:
                    pkgs = available.filter(provides=dep)
            else:
                pkgs = available.filter(provides=dep)
            for pkg in pkgs:
                if pkg.arch not in [ machine, platform, "noarch" ]:
                    if debug:
                        print("DEBUG: provider for %s: %s-%s-%s (%s) [%s] IGNORED - wrong arch" %
                               (dep, pkg.name, pkg.version, pkg.release, pkg.arch, pkg.repo.id), file = outfile)
                    continue
                if pkg.repo.id == base_binary_repo and pkg.name in buildsys_pkgset | dual_lived_pkgset:
                    if debug:
                        print("DEBUG: provider for %s: %s-%s-%s (%s) [%s] marked as buildsys provider" %
                               (dep, pkg.name, pkg.version, pkg.release, pkg.arch, pkg.repo.id), file = outfile)
                    buildsys_providers.append(pkg)
                elif pkg.repo.id == local_binary_repo:
                    if debug:
                        print("DEBUG: provider for %s: %s-%s-%s (%s) [%s] marked as local provider" %
                               (dep, pkg.name, pkg.version, pkg.release, pkg.arch, pkg.repo.id), file = outfile)
                    local_providers.append(pkg)
                else:
                    if debug:
                        print("DEBUG: provider for %s: %s-%s-%s (%s) [%s] marked as base provider" %
                               (dep, pkg.name, pkg.version, pkg.release, pkg.arch, pkg.repo.id), file = outfile)
                    base_providers.append(pkg)
            if len(buildsys_providers) > 0:
                best_providers = buildsys_providers[0:1]
            elif len(local_providers) > 0:
                best_providers = local_providers[0:1]
            else:
                best_providers = base_providers[0:1]
            if len(best_providers) < 1:
                print("WARNING: unsatisfied build dependency: %s" % dep, file = outfile)
            else:
                if debug:
                    bp = best_providers[0]
                    print("DEBUG: BEST PROVIDER for %s: %s-%s-%s (%s) [%s]" %
                           (dep, bp.name, bp.version, bp.release, bp.arch, bp.repo.id), file = outfile)
            return best_providers

        provider_names = []
        provider_pkgs = []
        provider_deps = set()
        for dep in depset:
            for prov in BestProviderPackage(dep):
                provider_pkgs.append(prov)
                provider_names.append(prov.name)
                # Do not follow deps for buildsys packages that should always be installable
                if prov.repo.id != base_binary_repo or prov.name not in buildsys_pkgset | dual_lived_pkgset:
                    for req in sorted(prov.requires, key=str):
                        if debug:
                            print("DEBUG: Adding %s as a dependency of %s-%s-%s (%s) [%s]" %
                                   (str(req), prov.name, prov.version, prov.release, prov.arch, prov.repo.id), file = outfile)
                        provider_deps.add(str(req))
        provider_names = list(set(provider_names))

        for prov in sorted(provider_names, key = str.lower):
            print("BootPBR: %s" % prov, file = outfile)

        deps_done = set()
        names_done = set()
        buildsys_dep = set()

        def RecursiveRequires(dep, prefix=""):
            if dep in deps_done:
                return
            deps_done.add(dep)
            if debug:
                print("DEBUG: looking for dependencies of %s%s" % (prefix, dep), file = outfile)
            for prov in BestProviderPackage(dep):
                if prov.name in buildsys_pkgset:
                    buildsys_dep.add(dep)
                    continue
                if prov.repoid != local_binary_repo:
                    # Pulling in base package, do not descend further
                    buildsys_dep.add(dep)
                    continue
                names_done.add(prov.name)
                prefix = prefix + dep + "→"
                for dep in sorted(prov.requires, key=str):
                    RecursiveRequires(str(dep), prefix=prefix)

        for dep in provider_deps:
            RecursiveRequires(dep)

        bootbrdep_set = deps_done.difference(buildsys_dep)
        bootpbrdep_set = names_done.difference(set(provider_names))

        for dep in sorted(bootbrdep_set, key = str.lower):
            print("BootBRdep: %s" % dep, file = outfile)
        for prov in sorted(bootpbrdep_set, key = str.lower):
            print("BootPBRdep: %s" % prov, file = outfile)

        outfile.close()
        os.rename(outfile_name, infile_name)
' $SRPM_LIST

# Find the SRPM that each binary package is built from
echo "================================================================="
echo "Find the SRPM that each binary package is built from"
echo "================================================================="
for srpm in $SRPM_LIST
do
	# Clear existing binary RPM data
	filter-from $DATA_DIR/${srpm}.src.data RPM:
done
# Process the local binary primary metadata and map RPMs to SRPMs
perl -se '
	use IO::File;
	use XML::Rules;

	my $primaryxml = new IO::File("$xml_datadir/$xml_primary");
	my %srpmdata = ();
	chomp(my $platform = `uname -i`);
	chomp(my $machine = `uname -m`);
	my @rules = (
		_default => content,
		format => sub {
			my $srpm = $_[1]->{"rpm:sourcerpm"};
			$srpm =~ s/-[^-]*-[^-]*$//;
			return "srpm" => $srpm;
		},
		package => sub {
			my $arch = $_[1]->{arch};
			return unless $arch eq $platform or $arch eq $machine or $arch eq "noarch";
			my $rpm = $_[1]->{name};
			my $srpm = $_[1]->{srpm};
			#print "$rpm appears to have been built from $srpm\n";
			@srpmdata{$srpm} = () unless exists $srpmdata{$srpm};
			push @{ $srpmdata{$srpm} }, $rpm;
			return;
		},
	);
	my $parser = XML::Rules->new(rules => \@rules);
	$parser->parse($primaryxml);

	foreach $srpm (sort keys %srpmdata) {
		my $fh;
		my $datafile = "$xml_datadir/$srpm.src.data";
		open($fh, ">>", $datafile)
			or die "Cannot create $datafile";
		foreach $rpm (sort @{ $srpmdata{$srpm} }) {
			print $fh "RPM: $rpm\n";
		}
	}' -- -xml_datadir=$DATA_DIR -xml_primary=local-binary-primary.xml
# Remove any stale data files for packages removed from the repo that are no longer
# building binary RPMs in the development repo
for f in $(grep --files-without-match '^RPM: ' $DATA_DIR/*.src.data)
do
	echo "Removing stale/source-only data file: " $(basename $f)
	rm $f
done

# Run the boot order routine
echo "================================================================="
echo "Calculate build order"
echo "================================================================="
python3 -c '
import os
import sys

data_dir = "'$DATA_DIR'"
buildroot_package_file = os.path.join(data_dir, "buildsys-build-package-list")
dual_lived_package_file = os.path.join(data_dir, "perl-dual-lived-packages")

rpmlist = {}
buildreqs = {}
builddeps = {}
sourcepkg = {}
needs_rebuild = {}
rpm_to_build = set()
srpm_to_build = set()

for srpm in sys.argv[1:]:
    infile_name = os.path.join(data_dir, srpm+".src.data")
    try:
        infile = open(infile_name, "r")
        srpm_to_build.add(srpm)
        rpmlist[srpm] = []
        buildreqs[srpm] = []
        builddeps[srpm] = []
        for line in infile:
            if line.startswith("RPM: "):
                rpm = line[5:].strip()
                rpmlist[srpm].append(rpm)
                rpm_to_build.add(rpm)
                sourcepkg[rpm] = srpm
                continue
            if line.startswith("BootPBR: "):
                buildreqs[srpm].append(line[9:].strip())
                continue
            if line.startswith("BootPBRdep: "):
                builddeps[srpm].append(line[12:].strip())
                continue
            if line.startswith("BootX: "):
                needs_rebuild[srpm] = True
                continue
        infile.close()
    except IOError:
        pass
# RPMs in the default buildroot or from the base perl package can be assumed
# to be available for building with, even if we are going to build our own version
for f in [ buildroot_package_file, dual_lived_package_file ]:
    infile = open(f, "r")
    for line in infile:
        pkg = line.strip()
        rpm_to_build.discard(pkg)
    infile.close()

# Evaluate the build order
passnum = 0
while True:
    passnum += 1
    print("Pass %s" % passnum)
    print("Packages remaining: %s" % len(srpm_to_build))
    srpms_built_this_pass = set()
    rpms_built_this_pass = set()
    for srpm in sorted(srpm_to_build):
        for dep in buildreqs[srpm] + builddeps[srpm]:
            if dep in rpm_to_build:
                break
        else:
            print("Build: %s (" % srpm, end = "")
            print("%s" % " ".join(rpmlist[srpm]), end = "")
            print(")")
            srpms_built_this_pass.add(srpm)
            for rpm in rpmlist[srpm]:
                rpms_built_this_pass.add(rpm)
    if len(srpms_built_this_pass) > 0:
        srpm_to_build = srpm_to_build.difference(srpms_built_this_pass)
        rpm_to_build = rpm_to_build.difference(rpms_built_this_pass)
        print("Packages built: %s" % len(srpms_built_this_pass))
        if len(srpm_to_build) == 0:
            print("Packages remaining: 0")
            break
    else:
        break

# Now list the remaining packages, if any, and their unsatisfied buildreqs
cycle_count = {}
if len(srpm_to_build) > 0:

    # Identify dependency cycle and increment count for each instance
    def register_cycle(stack, looper):
        startpoint = stack.index(looper)
        loop_list = stack[startpoint:]
        # Normalize the cycle
        startpoint = loop_list.index(sorted(loop_list)[0])
        loop_list = loop_list[startpoint:] + loop_list[:startpoint] + [ loop_list[startpoint] ]
        cycle_name = "→".join(loop_list)
        if cycle_name in cycle_count:
            cycle_count[cycle_name] += 1
            if cycle_count[cycle_name] > 999999:
                print("=====================================================")
                print("Processing aborted due to excessive dependency cycles")
                print("Please resolve these, then try again")
                print("=====================================================")
                for cycle in sorted(cycle_count, key = cycle_count.get, reverse = True):
                    print("%s %s" % ("{0:7}".format(cycle_count[cycle]), cycle))
                raise SystemExit
        else:
            cycle_count[cycle_name] = 1
        return cycle_name

    # Generate a tree view of the unsatisfied dependencies of a package
    # Print the tree view in verbose mode, but always work out the dependency cycles
    depstack = []
    def deptree(package, level = 0, indirect = False, verbose = False):
        if verbose:
            print("%s%s" % ("  " * level, package), end = "")
        if verbose and package in sourcepkg and package != sourcepkg[package]:
            print(" (from %s)" % sourcepkg[package],  end = "")
        if verbose and indirect:
            print(" [indirect dependency]", end = "")
        if not package in rpm_to_build:
            if verbose:
                print(" [available]")
            return
        if package != sourcepkg[package]:
            package = sourcepkg[package]
        if package in depstack:
            cycle = register_cycle(depstack, package)
            if verbose:
                print(" [circular dep: %s]" % cycle)
            return
        depstack.append(package)
        if verbose:
            print("")
        all_brs_available = True
        for dep in buildreqs[package]:
            if dep in rpm_to_build:
                all_brs_available = False
            deptree(dep, level + 1, verbose = verbose)
        if all_brs_available:
            for dep in builddeps[package]:
                deptree(dep, level = level + 1, indirect = True, verbose = verbose)
        depstack.pop()
        if verbose and level == 0:
            print("")

    print("================")
    print("Unbuilt packages")
    print("================")
    for srpm in sorted(srpm_to_build):
        print("%s needs" % srpm, end = "")
        for dep in sorted(buildreqs[srpm]):
            if dep in rpm_to_build:
                print(" %s" % dep, end = "")
        for dep in sorted(builddeps[srpm]):
            if dep in rpm_to_build:
                print(" [%s]" % dep, end = "")
        print("")
        deptree(srpm, verbose = False)
    print("=================")
    print("Dependency Cycles")
    print("=================")
    for cycle in sorted(cycle_count, key = cycle_count.get, reverse = True):
        print("%s %s" % ("{0:6}".format(cycle_count[cycle]), cycle))
else:
    print("==================================")
    print("Packages to rebuild post-bootstrap")
    print("==================================")
    for srpm in sorted(needs_rebuild):
        print("Rebuild: %s" % srpm)
' $SRPM_LIST > $DATA_DIR/buildorder