Rework broken dependencies visualization

analyze-repodb.py

@@ -22,16 +22,37 @@ def parseargs():
     opts = parser.parse_args()
     return opts
 
-def detect_broken_dependencies(dbc):
+def detect_broken_dependencies(dbc, dot_output):
-    def print_broken_packages():
+    def print_broken_packages(repo_packages_broken, build_arch, current_repodir, reponames, all_broken):
+        def build_dep_chains(pkg_id, current_repodir, all_broken, build_arch, reponames):
+            package_title = all_broken[pkg_id]['nvra']
+            dep_chains = []
+            if all_broken[pkg_id]['repo'] != current_repodir:
+                package_title += ' {' + reponames[all_broken[pkg_id]['repo']] + '}'
+            else:
+                deps = all_broken[pkg_id]['deps']
+                if deps is not None:
+                    for dep_id in deps:
+                        if deps[dep_id]['build_arch'] == build_arch:
+                            chains = build_dep_chains(dep_id, current_repodir,
+                                                      all_broken, build_arch,
+                                                      reponames)
+                            for chain in chains:
+                                dep_chains.append(chain)
+            if len(dep_chains) == 0:
+                dep_chains.append([])
+            for dep_chain in dep_chains:
+                dep_chain.insert(0, package_title)
+            return dep_chains
+
         for rpb_name in sorted(repo_packages_broken.keys()):
             rpb_id = repo_packages_broken[rpb_name]
-            dep_chain = []
+            alternatives = False
-            dep_id = all_broken[rpb_id]['depid']
+            dep_chains = build_dep_chains(rpb_id, current_repodir, all_broken, build_arch, reponames)
-            while dep_id != 0:
+            if len(dep_chains) > 1:
-                dep_chain.append('%s (%d)' % (all_broken[dep_id]['nvra'], all_broken[dep_id]['repo']))
+                print '\t\tMultiple dependencies (for %s):' % rpb_name
-                dep_id = all_broken[dep_id]['depid']
+            for dep_chain in dep_chains:
-            print '\t' + rpb_name + ' => '+ (' => '.join(dep_chain))
+                print '\t' + (' => '.join(dep_chain))
         print 'Total: %d' % len(repo_packages_broken)
         print ''
 
@@ -55,14 +76,15 @@ SELECT packages.id, nvra, repodir_id, repodirs.name,
         bp_reqs = []
         pre_repodir_id = -1
         pre_bp_id = -1
-        pre_bp_nvra = -1
         pre_cnt = 0
         for bp in broken_level0:
             (bp_id, bp_nvra, bp_repodir_id, bp_repodir_name, bp_reqname, bp_reqarch) = \
             (bp[0], bp[1],   bp[2],         bp[3],           bp[4],      bp[5])
             broken[bp_id] = bp_nvra
             if pre_bp_id != bp_id and pre_bp_id != -1:
-                print '\t%s (%s)' % (pre_bp_nvra, ', '.join(bp_reqs))
+                all_broken[pre_bp_id]['brokenreqs'] = bp_reqs
+                print '\t%s (%s)' % (all_broken[pre_bp_id]['nvra'],
+                                     ', '.join(bp_reqs))
                 pre_cnt += 1
                 bp_reqs = []
             if bp_reqarch is not None:
@@ -76,12 +98,16 @@ SELECT packages.id, nvra, repodir_id, repodirs.name,
                 pre_repodir_id = bp_repodir_id
                 pre_cnt = 0
             if bp_id not in all_broken:
-                all_broken[bp_id] = {'repo': bp_repodir_id, 'nvra': bp_nvra, 'reqname': bp_reqname, 'depid': 0}
+                all_broken[bp_id] = {'nvra': bp_nvra,
+                                     'repo': bp_repodir_id,
+                                     'brokenreqs': [],
+                                     'deps': None}
             pre_bp_id = bp_id
-            pre_bp_nvra = bp_nvra
 
         if pre_bp_id != -1:
-            print '\t%s (%s)' % (pre_bp_nvra, ','.join(bp_reqs))
+            all_broken[pre_bp_id]['brokenreqs'] = bp_reqs
+            print '\t%s (%s)' % (all_broken[pre_bp_id]['nvra'],
+                                    ', '.join(bp_reqs))
             print 'Total: %d' % pre_cnt
 
     all_broken_cnt = -1
@@ -101,15 +127,23 @@ SELECT packages.id, nvra, repodir_id, repodirs.name,
         packages.id NOT IN (%(pids)s)
   ORDER BY repodir_id, nvra""" % {'pids': pids}).fetchall()
         for packb in packages_broken_recurs:
-            all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1],
+            pkg_id = packb[0]
-                                    'reqname': packb[4], 'build_arch': packb[5],
+            if pkg_id not in all_broken:
-                                    'depid': packb[6]}
+                all_broken[pkg_id] = {'nvra': packb[1], 'repo': packb[2],
-            broken_recursive.append(packb[0])
+                                      'deps': {}}
+            dep_pkg_id = packb[6]
+            deps = all_broken[pkg_id]['deps']
+            if dep_pkg_id not in deps:
+                deps[dep_pkg_id] = {'build_arch': packb[5], 'req_names': []}
+            deps[dep_pkg_id]['req_names'].append(packb[4])
+            broken_recursive.append(pkg_id)
+
+    all_repodirs = dbc.execute("""
+SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall()
+    reponames = {repodir[0]: repodir[1] for repodir in all_repodirs}
 
     if broken_recursive:
         print 'Recursive broken dependencies:'
-        all_repodirs = dbc.execute("""
-SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall()
         for rd in all_repodirs:
             (rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2])
             if rd_sources == '.':
@@ -118,16 +152,51 @@ SELECT DISTINCT build_arch FROM rpm_requires
   WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?)
 """, [rd_id]).fetchall()
                 for arch_rec in archs:
-                    repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive \
+                    arch = arch_rec[0]
-                        if all_broken[id]['repo'] == rd_id and all_broken[id]['build_arch'] == arch_rec[0]}
+                    repo_packages_broken = {}
+                    for pkg_id in broken_recursive:
+                        package = all_broken[pkg_id]
+                        if package['repo'] == rd_id:
+                            for dep in package['deps']:
+                                if package['deps'][dep]['build_arch'] == arch:
+                                    repo_packages_broken[package['nvra']] = pkg_id
                     if repo_packages_broken:
-                        print '%d) %s (%s)' % (rd_id, rd_name, arch_rec[0])
+                        print '%d) %s (%s)' % (rd_id, rd_name, arch)
-                        print_broken_packages()
+                        print_broken_packages(repo_packages_broken, arch,
+                                              rd_id, reponames, all_broken)
             else:
-                repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive if all_broken[id]['repo'] == rd_id}
+                repo_packages_broken = {all_broken[id]['nvra']: id
+                                        for id in broken_recursive
+                                          if all_broken[id]['repo'] == rd_id}
                 if repo_packages_broken:
                     print '%d) %s' % (rd_id, rd_name)
-                    print_broken_packages()
+                    print_broken_packages(repo_packages_broken, None,
+                                          rd_id, reponames, all_broken)
+    if dot_output:
+        for rd in all_repodirs:
+            (rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2])
+            dot_file = None
+            for pkg_id in all_broken:
+                package = all_broken[pkg_id]
+                if package['repo'] != rd_id:
+                    continue
+                if not dot_file:
+                    dot_file = open('broken-repo-%d.dot' % rd_id, 'w')
+                    OutputGraphHead(dot_file, rd_name)
+                if package['deps'] is None:
+                    dot_file.write('"%s" [color="red"];\n' % package['nvra'])
+                else:
+                    deps = package['deps']
+                    for dep_id in deps:
+                        dep_package_title = all_broken[dep_id]['nvra']
+                        if all_broken[dep_id]['repo'] != rd_id:
+                            dep_package_title += ' {' + \
+                              reponames[all_broken[dep_id]['repo']] + '}'
+                        dot_file.write('"%s" -> "%s" [color="blue"];\n' %
+                                       (package['nvra'], dep_package_title))
+
+            if dot_file:
+                OutputGraphTail(dot_file)
 
 
 def OutputGraphHead(file_output, dg_name):
@@ -144,59 +213,6 @@ def OutputGraphTail(file_output):
     """
     file_output.write('}\n')
 
-def render_dot_graphs(dbc):
-    repodirs = dbc.execute("""
-SELECT id, name, sources, path FROM repodirs ORDER BY id
-""").fetchall()
-    for repodir in repodirs:
-        (rd_id, rd_name) = (repodir[0], repodir[1])
-
-        dot_file = open('repo-%d.dot' % rd_id, 'w')
-        packages_processed = {}
-        low_level_pkgs = dbc.execute("""
-SELECT packages.id, packages.nvra FROM packages
-  WHERE repodir_id = ? AND
-   NOT EXISTS (SELECT 1 FROM package_depend_res, packages dp
-                WHERE package_id = packages.id AND
-                dp.id = dep_package_id AND dp.repodir_id = ?)
-  ORDER BY packages.id""", [rd_id, rd_id]).fetchall()
-        for pkg_rec in low_level_pkgs:
-            packages_processed[pkg_rec[0]] = pkg_rec[1]
-
-        OutputGraphHead(dot_file, rd_name)
-        pkg_linked = {}
-        level = 0
-        curr_level_pkgs = [pkg_rec[0] for pkg_rec in low_level_pkgs]
-        while len(curr_level_pkgs) > 0:
-            in_curr_pkgs = ','.join([str(pkg_id)
-                                    for pkg_id in curr_level_pkgs])
-            depend_pkgs = dbc.execute("""
-SELECT DISTINCT packages.id, packages.nvra, package_depend_res.dep_package_id
-  FROM package_depend_res, packages
-  WHERE repodir_id = ? AND package_depend_res.dep_package_id IN (%s)
-    AND package_depend_res.package_id = packages.id
-  ORDER BY packages.id""" % in_curr_pkgs, [rd_id]).fetchall()
-            next_level_pkgs = []
-            for pkg_rec in depend_pkgs:
-                if level == 0:
-                    pkg_linked[pkg_rec[2]] = True
-                if pkg_rec[0] not in packages_processed:
-                    packages_processed[pkg_rec[0]] = pkg_rec[1]
-                    next_level_pkgs.append(pkg_rec[0])
-                if pkg_rec[0] != pkg_rec[2]:
-                    dot_file.write('"%s" -> "%s" [color="0.66 1 0.66"];\n' %
-                                   (packages_processed[pkg_rec[0]],
-                                    packages_processed[pkg_rec[2]]))
-            if level == 0:
-                for ll_rec in low_level_pkgs:
-                    if ll_rec[0] not in pkg_linked:
-                        dot_file.write('"%s" [color="0.66 0.66 1"];\n' %
-                                    packages_processed[ll_rec[0]])
-            curr_level_pkgs = next_level_pkgs
-            level += 1
-
-        OutputGraphTail(dot_file)
-
 def detect_loops(dbc):
     header = '===\n' \
              'Loopbacks:'
@@ -506,9 +522,7 @@ def main(args):
 
     conn = sqlite3.connect(DB)
     dbc = conn.cursor()
-    detect_broken_dependencies(dbc)
+    detect_broken_dependencies(dbc, options.dot_graphs)
-    if options.dot_graphs:
-        render_dot_graphs(dbc)
     #detect_loops(dbc)
     detect_lost_sources(dbc)
     analyze_partitioning(dbc)

prepare-repodb.py

@@ -166,7 +166,7 @@ SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id = ?
                 target_obj_id = tofile[0]
                 new_target_path = tofile[1]
             if not target_obj_id:
-                # Just two level of dependency recursion - TODO: Full depth recursion?
+                # Just two levels of dependency recursion - TODO: Full depth recursion?
                 tofile = dbc.execute("""
 SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN (
   SELECT dep_package_id FROM package_depend_res WHERE package_id = ?