From fc2df4ccb5f4054ce0b63259e5a3e7c833e048fc Mon Sep 17 00:00:00 2001 From: Alexander Lakhin Date: Thu, 13 Feb 2014 12:03:20 +0400 Subject: [PATCH] Rework broken dependencies visualization --- analyze-repodb.py | 176 +++++++++++++++++++++++++--------------------- prepare-repodb.py | 2 +- 2 files changed, 96 insertions(+), 82 deletions(-) diff --git a/analyze-repodb.py b/analyze-repodb.py index c57080e..4d5ab1d 100755 --- a/analyze-repodb.py +++ b/analyze-repodb.py @@ -22,16 +22,37 @@ def parseargs(): opts = parser.parse_args() return opts -def detect_broken_dependencies(dbc): - def print_broken_packages(): +def detect_broken_dependencies(dbc, dot_output): + def print_broken_packages(repo_packages_broken, build_arch, current_repodir, reponames, all_broken): + def build_dep_chains(pkg_id, current_repodir, all_broken, build_arch, reponames): + package_title = all_broken[pkg_id]['nvra'] + dep_chains = [] + if all_broken[pkg_id]['repo'] != current_repodir: + package_title += ' {' + reponames[all_broken[pkg_id]['repo']] + '}' + else: + deps = all_broken[pkg_id]['deps'] + if deps is not None: + for dep_id in deps: + if deps[dep_id]['build_arch'] == build_arch: + chains = build_dep_chains(dep_id, current_repodir, + all_broken, build_arch, + reponames) + for chain in chains: + dep_chains.append(chain) + if len(dep_chains) == 0: + dep_chains.append([]) + for dep_chain in dep_chains: + dep_chain.insert(0, package_title) + return dep_chains + for rpb_name in sorted(repo_packages_broken.keys()): rpb_id = repo_packages_broken[rpb_name] - dep_chain = [] - dep_id = all_broken[rpb_id]['depid'] - while dep_id != 0: - dep_chain.append('%s (%d)' % (all_broken[dep_id]['nvra'], all_broken[dep_id]['repo'])) - dep_id = all_broken[dep_id]['depid'] - print '\t' + rpb_name + ' => '+ (' => '.join(dep_chain)) + alternatives = False + dep_chains = build_dep_chains(rpb_id, current_repodir, all_broken, build_arch, reponames) + if len(dep_chains) > 1: + print '\t\tMultiple dependencies (for %s):' % rpb_name + for dep_chain in dep_chains: + print '\t' + (' => '.join(dep_chain)) print 'Total: %d' % len(repo_packages_broken) print '' @@ -55,14 +76,15 @@ SELECT packages.id, nvra, repodir_id, repodirs.name, bp_reqs = [] pre_repodir_id = -1 pre_bp_id = -1 - pre_bp_nvra = -1 pre_cnt = 0 for bp in broken_level0: (bp_id, bp_nvra, bp_repodir_id, bp_repodir_name, bp_reqname, bp_reqarch) = \ (bp[0], bp[1], bp[2], bp[3], bp[4], bp[5]) broken[bp_id] = bp_nvra if pre_bp_id != bp_id and pre_bp_id != -1: - print '\t%s (%s)' % (pre_bp_nvra, ', '.join(bp_reqs)) + all_broken[pre_bp_id]['brokenreqs'] = bp_reqs + print '\t%s (%s)' % (all_broken[pre_bp_id]['nvra'], + ', '.join(bp_reqs)) pre_cnt += 1 bp_reqs = [] if bp_reqarch is not None: @@ -76,12 +98,16 @@ SELECT packages.id, nvra, repodir_id, repodirs.name, pre_repodir_id = bp_repodir_id pre_cnt = 0 if bp_id not in all_broken: - all_broken[bp_id] = {'repo': bp_repodir_id, 'nvra': bp_nvra, 'reqname': bp_reqname, 'depid': 0} + all_broken[bp_id] = {'nvra': bp_nvra, + 'repo': bp_repodir_id, + 'brokenreqs': [], + 'deps': None} pre_bp_id = bp_id - pre_bp_nvra = bp_nvra if pre_bp_id != -1: - print '\t%s (%s)' % (pre_bp_nvra, ','.join(bp_reqs)) + all_broken[pre_bp_id]['brokenreqs'] = bp_reqs + print '\t%s (%s)' % (all_broken[pre_bp_id]['nvra'], + ', '.join(bp_reqs)) print 'Total: %d' % pre_cnt all_broken_cnt = -1 @@ -101,15 +127,23 @@ SELECT packages.id, nvra, repodir_id, repodirs.name, packages.id NOT IN (%(pids)s) ORDER BY repodir_id, nvra""" % {'pids': pids}).fetchall() for packb in packages_broken_recurs: - all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1], - 'reqname': packb[4], 'build_arch': packb[5], - 'depid': packb[6]} - broken_recursive.append(packb[0]) + pkg_id = packb[0] + if pkg_id not in all_broken: + all_broken[pkg_id] = {'nvra': packb[1], 'repo': packb[2], + 'deps': {}} + dep_pkg_id = packb[6] + deps = all_broken[pkg_id]['deps'] + if dep_pkg_id not in deps: + deps[dep_pkg_id] = {'build_arch': packb[5], 'req_names': []} + deps[dep_pkg_id]['req_names'].append(packb[4]) + broken_recursive.append(pkg_id) + + all_repodirs = dbc.execute(""" +SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall() + reponames = {repodir[0]: repodir[1] for repodir in all_repodirs} if broken_recursive: print 'Recursive broken dependencies:' - all_repodirs = dbc.execute(""" -SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall() for rd in all_repodirs: (rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2]) if rd_sources == '.': @@ -118,16 +152,51 @@ SELECT DISTINCT build_arch FROM rpm_requires WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?) """, [rd_id]).fetchall() for arch_rec in archs: - repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive \ - if all_broken[id]['repo'] == rd_id and all_broken[id]['build_arch'] == arch_rec[0]} + arch = arch_rec[0] + repo_packages_broken = {} + for pkg_id in broken_recursive: + package = all_broken[pkg_id] + if package['repo'] == rd_id: + for dep in package['deps']: + if package['deps'][dep]['build_arch'] == arch: + repo_packages_broken[package['nvra']] = pkg_id if repo_packages_broken: - print '%d) %s (%s)' % (rd_id, rd_name, arch_rec[0]) - print_broken_packages() + print '%d) %s (%s)' % (rd_id, rd_name, arch) + print_broken_packages(repo_packages_broken, arch, + rd_id, reponames, all_broken) else: - repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive if all_broken[id]['repo'] == rd_id} + repo_packages_broken = {all_broken[id]['nvra']: id + for id in broken_recursive + if all_broken[id]['repo'] == rd_id} if repo_packages_broken: print '%d) %s' % (rd_id, rd_name) - print_broken_packages() + print_broken_packages(repo_packages_broken, None, + rd_id, reponames, all_broken) + if dot_output: + for rd in all_repodirs: + (rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2]) + dot_file = None + for pkg_id in all_broken: + package = all_broken[pkg_id] + if package['repo'] != rd_id: + continue + if not dot_file: + dot_file = open('broken-repo-%d.dot' % rd_id, 'w') + OutputGraphHead(dot_file, rd_name) + if package['deps'] is None: + dot_file.write('"%s" [color="red"];\n' % package['nvra']) + else: + deps = package['deps'] + for dep_id in deps: + dep_package_title = all_broken[dep_id]['nvra'] + if all_broken[dep_id]['repo'] != rd_id: + dep_package_title += ' {' + \ + reponames[all_broken[dep_id]['repo']] + '}' + dot_file.write('"%s" -> "%s" [color="blue"];\n' % + (package['nvra'], dep_package_title)) + + if dot_file: + OutputGraphTail(dot_file) def OutputGraphHead(file_output, dg_name): @@ -144,59 +213,6 @@ def OutputGraphTail(file_output): """ file_output.write('}\n') -def render_dot_graphs(dbc): - repodirs = dbc.execute(""" -SELECT id, name, sources, path FROM repodirs ORDER BY id -""").fetchall() - for repodir in repodirs: - (rd_id, rd_name) = (repodir[0], repodir[1]) - - dot_file = open('repo-%d.dot' % rd_id, 'w') - packages_processed = {} - low_level_pkgs = dbc.execute(""" -SELECT packages.id, packages.nvra FROM packages - WHERE repodir_id = ? AND - NOT EXISTS (SELECT 1 FROM package_depend_res, packages dp - WHERE package_id = packages.id AND - dp.id = dep_package_id AND dp.repodir_id = ?) - ORDER BY packages.id""", [rd_id, rd_id]).fetchall() - for pkg_rec in low_level_pkgs: - packages_processed[pkg_rec[0]] = pkg_rec[1] - - OutputGraphHead(dot_file, rd_name) - pkg_linked = {} - level = 0 - curr_level_pkgs = [pkg_rec[0] for pkg_rec in low_level_pkgs] - while len(curr_level_pkgs) > 0: - in_curr_pkgs = ','.join([str(pkg_id) - for pkg_id in curr_level_pkgs]) - depend_pkgs = dbc.execute(""" -SELECT DISTINCT packages.id, packages.nvra, package_depend_res.dep_package_id - FROM package_depend_res, packages - WHERE repodir_id = ? AND package_depend_res.dep_package_id IN (%s) - AND package_depend_res.package_id = packages.id - ORDER BY packages.id""" % in_curr_pkgs, [rd_id]).fetchall() - next_level_pkgs = [] - for pkg_rec in depend_pkgs: - if level == 0: - pkg_linked[pkg_rec[2]] = True - if pkg_rec[0] not in packages_processed: - packages_processed[pkg_rec[0]] = pkg_rec[1] - next_level_pkgs.append(pkg_rec[0]) - if pkg_rec[0] != pkg_rec[2]: - dot_file.write('"%s" -> "%s" [color="0.66 1 0.66"];\n' % - (packages_processed[pkg_rec[0]], - packages_processed[pkg_rec[2]])) - if level == 0: - for ll_rec in low_level_pkgs: - if ll_rec[0] not in pkg_linked: - dot_file.write('"%s" [color="0.66 0.66 1"];\n' % - packages_processed[ll_rec[0]]) - curr_level_pkgs = next_level_pkgs - level += 1 - - OutputGraphTail(dot_file) - def detect_loops(dbc): header = '===\n' \ 'Loopbacks:' @@ -506,9 +522,7 @@ def main(args): conn = sqlite3.connect(DB) dbc = conn.cursor() - detect_broken_dependencies(dbc) - if options.dot_graphs: - render_dot_graphs(dbc) + detect_broken_dependencies(dbc, options.dot_graphs) #detect_loops(dbc) detect_lost_sources(dbc) analyze_partitioning(dbc) diff --git a/prepare-repodb.py b/prepare-repodb.py index 2a1dd7c..d67fd26 100755 --- a/prepare-repodb.py +++ b/prepare-repodb.py @@ -166,7 +166,7 @@ SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id = ? target_obj_id = tofile[0] new_target_path = tofile[1] if not target_obj_id: - # Just two level of dependency recursion - TODO: Full depth recursion? + # Just two levels of dependency recursion - TODO: Full depth recursion? tofile = dbc.execute(""" SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN ( SELECT dep_package_id FROM package_depend_res WHERE package_id = ?