repo-analyzer/prepare-repodb.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
import gettext
import argparse
import sqlite3
import re
import rpm

RPMSENSE_LESS      = 0x02
RPMSENSE_GREATER   = 0x04
RPMSENSE_EQUAL     = 0x08
RPMSENSE_SENSEMASK = 0x0f
RPMSENSE_FIND_PROVIDES = 0x8000
RPMSENSE_MISSINGOK    = 0x80000
RPMSENSE_SCRIPT_POST   =  0x400
RPMSENSE_SCRIPT_PREUN  =  0x800
RPMSENSE_SCRIPT_POSTUN = 0x1000

DB = 'repo.db'

def version_ok(required_version, compare_flag, candidate_version):
    def sep_version(version):
        vrem = re.match(r'\A(.+)(\-[^\-\:]+)(\:[^\:]+|)\Z', version)
        if vrem:
            return (vrem.group(1), vrem.group(2), vrem.group(3))
        return (version, '', '')

    def simple_version(version):
        version = re.sub(r'[\-:]', '.', version)
        version = re.sub(r'[a-z]+', '.', version, flags=re.I)
        version = re.sub(r'\.\Z', '', version)
        return version

    def format_versions(ver1, ver2):
        #see urpm-repoclosure, formatVersions
        # v1 - provided
        # v2 - required
        (e1, e2) = (None, None)
        e1_m = re.match(r'\A([^\-\:]+)\:(.*)', ver1)
        if e1_m:
            (e1, ver1) = (e1_m.group(1), e1_m.group(2))
        e2_m = re.match(r'\A([^\-\:]+)\:(.*)', ver2)
        if e2_m:
            (e2, ver2) = (e2_m.group(1), e2_m.group(2))
        (ver1_m, ver1_r, ver1_rr) = sep_version(ver1)
        (ver2_m, ver2_r, ver2_rr) = sep_version(ver2)
        if not ver2_rr:
            ver1_rr = ''
        if not ver2_r:
            ver1_r = ''
        ver1 = ver1_m + ver1_r + ver1_rr
        ver2 = ver2_m + ver2_r + ver2_rr
        if e1_m and e2_m:
            ver1 = e1 + '.' + ver1
            ver2 = e2 + '.' + ver2
        return (simple_version(ver1), simple_version(ver2))

    def cmp_nums(num1, num2):
        # 00503
        # 12
        if num1 == num2:
            return 0
        lzeros1 = re.match(r'\A([0]+)([1-9].*)', num1)
        if lzeros1:
            (num1, num2) = (lzeros1.group(2), num2 + lzeros1.group(1))
        lzeros2 = re.match(r'\A([0]+)([1-9].*)', num2)
        if lzeros2:
            (num2, num1) = (lzeros2.group(2), num1 + lzeros2.group(1))
        diff = int(num1, 10) - int(num2, 10)
        return 0 if diff == 0 else \
               (1 if diff > 0 else -1)

    def cmp_versions(version1, version2):
        #see urpm-repoclosure, cmpVersions
        # 3.2.5-5:2011.0
        # NOTE: perl 5.00503 and 5.12
        (v1, v2) = format_versions(version1, version2)
        if v1 == v2:
            return 0
        v1parts = v1.split('.')
        v2parts = v2.split('.')
        for i in xrange(0, min(len(v1parts), len(v2parts))):
            (num1, num2)= (v1parts[i], v2parts[i])
            if (len(num1) > 0 and len(num2) == 0):
                return 1
            if (len(num1) == 0 and len(num2) > 0):
                return -1
            num_diff = cmp_nums(num1, num2)
            if num_diff != 0:
                return num_diff
        if len(v1parts) < len(v2parts):
            return -1
        if len(v1parts) > len(v2parts):
            return 1
        return 0

    def rpm_cmp_versions(version1, version2):
        def stringToVersion(verstring):
            # from rpmUtils
            if verstring in [None, '']:
                return (None, None, None)
            e1_m = re.match(r'\A([^\-\:]+)\:(.*)', verstring)
            epoch = None
            if e1_m:
                (epoch, verstring) = (e1_m.group(1), e1_m.group(2))
            j = verstring.find('-')
            if j != -1:
                if verstring[:j] == '':
                    version = None
                else:
                    version = verstring[:j]
                release = verstring[j + 1:]
            else:
                if verstring == '':
                    version = None
                else:
                    version = verstring
                release = None
            return (epoch, version, release)

        (e1, v1, r1) = stringToVersion(version1)
        (e2, v2, r2) = stringToVersion(version2)
        if e1 is None or e2 is None:
            e1 = '0'
            e2 = '0'
        result = rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
        return result

#    print '===', required_version, compare_flag, candidate_version
    if compare_flag == 0:
        return True
    if candidate_version == '*':
        return True
    #see urpm-repoclosure, checkDeps
    if compare_flag == RPMSENSE_EQUAL and \
       candidate_version == required_version:
        return True

    cmp_res = None
    try:
        cmp_res = cmp_versions(candidate_version, required_version)
    except ValueError as ex:
        print ('Error when comparing versions: "%s" and "%s"\n%s' %
               (candidate_version, required_version, str(ex)))
        return False
    rpm_cmp_res = rpm_cmp_versions(candidate_version, required_version)
    #if (cmp_res != rpm_cmp_res):
        #print >> sys.stderr, ('Invalid compare: "%s" vs "%s"! Results: rc: %d, rpm: %d.' %
                        #(candidate_version, required_version, cmp_res, rpm_cmp_res))
    if compare_flag == RPMSENSE_EQUAL:
        return cmp_res == 0
    elif compare_flag == RPMSENSE_LESS | RPMSENSE_EQUAL:
        return cmp_res <= 0
    elif compare_flag == RPMSENSE_GREATER | RPMSENSE_EQUAL:
        return cmp_res >= 0
    elif compare_flag == RPMSENSE_LESS:
        return cmp_res < 0
    elif compare_flag == RPMSENSE_GREATER:
        return cmp_res > 0
    return False


def process_repodir_dependencies(dbc, repodir_id, repodir_name, repodir_depends, dep_type):
    package_depends = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
       dep.id, dep.name, flags, dep.version
  FROM packages, rpm_%s dep
  WHERE repodir_id = ? AND dep.package_id = packages.id
  ORDER BY packages.name, dep.name
""" % dep_type, [repodir_id]).fetchall()
    search_repodirs = [repodir_id]
    search_repodirs.extend(repodir_depends)
    in_repodirs = ','.join(str(id) for id in search_repodirs)
    dependency_cache = {}
    for packdep in package_depends:
        (cpackage_id, package_nvra, dep_id,    dep_name,   dep_flags,  dep_version) = \
         (packdep[0],  packdep[2], packdep[3], packdep[4], packdep[5], packdep[6])
        dependency_uid = dep_name + '\x00' + str(dep_flags) + '\x00' + dep_version
        dep_res = dependency_cache.get(dependency_uid, None)
        if dep_res is None:
            dep_res = []
            depend_candidates = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
       prov.id, prov.name, flags, prov.version
  FROM packages, rpm_provides AS prov
  WHERE prov.package_id = packages.id AND repodir_id IN (%s) AND prov.name = ?
  ORDER by packages.name, packages.nvra
""" % in_repodirs, [dep_name]).fetchall()
            for dep_cand in depend_candidates:
                (pkg_id,      provides_id, provides_flags, provides_version) = \
                (dep_cand[0], dep_cand[3], dep_cand[5],    dep_cand[6])
                if provides_flags & RPMSENSE_SENSEMASK == 0:
                    if not provides_version:
                        provides_version = '*'
                    else:
                        raise Exception('Invalid provides version '
                                        '(flags = %d, version = %s)!' %
                                        (provides_flags, provides_version))
                if version_ok(dep_version, dep_flags & RPMSENSE_SENSEMASK,
                               provides_version):
                    dep_res.append({'prov_id': provides_id, 'pkg_id':  pkg_id})
        if len(dep_res) > 0:
            for res_rec in dep_res:
                dbc.execute("""
INSERT INTO package_%(dep)s_res(package_id, %(dep)s_id,
  provides_id, dep_package_id)
VALUES (?, ?, ?, ?)""" % {'dep': dep_type}, [cpackage_id, dep_id,
                          res_rec.get('prov_id'), res_rec.get('pkg_id')])
        dependency_cache[dependency_uid] = dep_res

def process_repodir_requires(dbc, repodir_id, repodir_name, repodir_depends, requires_build_arch):
    global n

    print 'Processing repo %d: %s (with depends: %s)' % (repodir_id, repodir_name, str(repodir_depends))
    package_requires = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
       req.id, req.name, flags, req.version
  FROM packages, rpm_requires req
  WHERE repodir_id = ? AND req.package_id = packages.id %s
  ORDER BY packages.name, req.name
""" % ((" AND build_arch = '%s'" % requires_build_arch)
       if requires_build_arch is not None else ""),
     [repodir_id]).fetchall()

    search_repodirs = [repodir_id]
    search_repodirs.extend(repodir_depends)
    in_repodirs = ','.join(str(id) for id in search_repodirs)
#        print 'package requires count: ', len(package_requires)
    broken_dep = 0

    for packreq in package_requires:
        (cpackage_id, package_nvra, requires_id, requires_name, requires_flags, requires_version) = \
         (packreq[0],  packreq[2],   packreq[3],  packreq[4],    packreq[5],     packreq[6])

        req_res = []
        if (re.match(r'\A(rpmlib|executable)\(.+\)\Z', requires_name)):
            # see if($N=~/\A(rpmlib|executable)\(.+\)\Z/) in urpm_repoclosure.pl
            req_res.append({})
        else:
            depend_candidates = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
       prov.id, prov.name, flags, prov.version
  FROM packages, rpm_provides AS prov
  WHERE prov.package_id = packages.id AND repodir_id IN (%s) AND prov.name = ?
  ORDER by packages.name, packages.nvra
""" % in_repodirs, [requires_name]).fetchall()
            preferred_version = None
            for dep_cand in depend_candidates:
                (pkg_id,      provides_id,
                    provides_flags, provides_version) = \
                (dep_cand[0], dep_cand[3],
                    dep_cand[5],    dep_cand[6])
                if provides_flags & RPMSENSE_SENSEMASK == 0:
                    if not provides_version:
                        provides_version = '*'
                    else:
                        raise Exception('Invalid provides version '
                                        '(flags = %d, version = %s)!' %
                                        (provides_flags, provides_version))
                if version_ok(requires_version,
                                requires_flags & RPMSENSE_SENSEMASK,
                                provides_version):
                    if pkg_id == cpackage_id:
                        # the same package is preferred over any other
                        req_res.append({'prov_id': provides_id,
                                        'pkg_id':  pkg_id})
                        preferred_version = None
                        break
                    better_version = preferred_version is None or \
                                        provides_version == '*'
                    if not better_version:
                        better_version = version_ok(provides_version,
                                                    RPMSENSE_GREATER,
                                                    preferred_version)
                    if better_version:
                        preferred_version = provides_version

            if preferred_version is not None:
                for dep_cand in depend_candidates:
                    (pkg_id,      provides_id, provides_version) = \
                    (dep_cand[0], dep_cand[3], dep_cand[6])
                    if provides_version == preferred_version or \
                        version_ok(provides_version, RPMSENSE_EQUAL,
                                    preferred_version):
                        req_res.append({'prov_id': provides_id,
                                        'pkg_id':  pkg_id})


            if len(req_res) == 0 and requires_name.startswith('/'): # file dependency
                if (requires_flags & (RPMSENSE_SCRIPT_POST |
                                        RPMSENSE_SCRIPT_PREUN |
                                        RPMSENSE_SCRIPT_POSTUN)) != 0:
                    int_files_cnt = dbc.execute("""
SELECT COUNT(1) FROM package_files WHERE package_id = ? AND path = ?
""", [cpackage_id, requires_name]).fetchone()
                    if int_files_cnt[0] > 0:
                        req_res.append({})
                else:
            #TODO: Check file dependencies (/usr/bin/python (required by ant-scripts-1.7.1-7.0.6.noarch), /usr/sbin/useradd (required by tomcat5-5.5.28-0.5.2.noarch))?
                    files_deps = dbc.execute("""
SELECT package_id FROM package_files
  WHERE path = ? AND
        package_id in (SELECT id FROM packages WHERE repodir_id IN (%s))
""" % in_repodirs, [requires_name]).fetchall()
                    for file_dep in files_deps:
                        req_res.append({'pkg_id': file_dep[0]})

            if len(req_res) == 0 and (requires_flags & RPMSENSE_MISSINGOK) != 0:
                req_res.append({})

        if len(req_res) > 0:
            for res_rec in req_res:
                dbc.execute("""
INSERT INTO package_requires_res(package_id, requires_id,
 provides_id, dep_package_id)
VALUES (?, ?, ?, ?)
""", [cpackage_id, requires_id, res_rec.get('prov_id'), res_rec.get('pkg_id')])
        else:
            print requires_name, ' ', requires_version, ' (required by %s)' % package_nvra, ' not found!!!'
            broken_dep += 1
        n = n + 1
        #print "n = ", n
#            if n == 60000:
#                break
    print 'broken_deps: ', broken_dep
    print ''

def extract_arch(arch_template, repo_name):
    arch_sign = '$arch'
    spos = arch_template.find(arch_sign)
    if spos >= 0:
        repo_prefix = arch_template[:spos]
        repo_postfix = arch_template[spos + len(arch_sign):]
        if repo_name.startswith(repo_prefix) and \
           repo_name.endswith(repo_postfix):
            return repo_name[len(repo_prefix) :
                             len(repo_name) - len(repo_postfix)]
    return None

def process_repodir_file_links(dbc, repodir_id, repodir_name, repodir_depends):
    package_files_links = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
       package_files.id AS object_id, package_files.path, package_files.link_to_path
  FROM packages, package_files
  WHERE repodir_id = ? AND package_files.package_id = packages.id AND
        link_to_path IS NOT NULL
  ORDER BY packages.name, link_to_path
""", [repodir_id]).fetchall()
    for file_link in package_files_links:
        pkg_id = file_link[0]
        pkg_nvra = file_link[2]
        object_id = file_link[3]
        target_obj_id = None
        target_path = os.path.normpath(file_link[5])
        target_paths = {}
        target_paths[target_path] = True
        while target_path != '':
            new_target_path = None
            tofile = dbc.execute("""
SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id = ?
""", [target_path, pkg_id]).fetchone()
            if tofile:
                target_obj_id = tofile[0]
                new_target_path = tofile[1]
            if not target_obj_id:
                # Just two levels of dependency recursion - TODO: Full depth recursion?
                tofile = dbc.execute("""
SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN (
  SELECT dep_package_id FROM package_requires_res WHERE package_id = ?
  UNION
  SELECT dep_package_id FROM package_requires_res WHERE package_id IN
   (SELECT dep_package_id FROM package_requires_res WHERE package_id = ?)
)
""", [target_path, pkg_id, pkg_id]).fetchone()
                if tofile:
                    target_obj_id = tofile[0]
                    new_target_path = tofile[1]
            if new_target_path is None:
                break
            target_path = os.path.normpath(new_target_path)
            if target_path in target_paths:
                print 'Link loop detected! %s: %s -> %s' % (pkg_nvra, file_link[5], target_path)
                target_obj_id = None
                break
            target_paths[target_path] = True

        if target_obj_id:
            dbc.execute("""
UPDATE package_files SET link_to_file_id = ? WHERE id = ?
""", [target_obj_id, object_id])
        else:
#            print 'target %s not found (%d: %s)' % (target_path, pkg_id, pkg_name)
            pass

def process_repodir_so_needed(dbc, repodir_id, repodir_name, repodir_depends):
    print 'Searching object files resolutions (1)...'
    dbc.execute("""
INSERT INTO so_needed_res(so_needed_id, dep_obj_file_id, res_type)
  SELECT so_needed.id, tpf.id, 1 FROM packages
   CROSS JOIN package_files spf CROSS JOIN so_needed CROSS JOIN rpm_requires
   CROSS JOIN package_requires_res req_res CROSS JOIN package_files tpf
    WHERE so_needed.obj_file_id = spf.id AND spf.package_id = packages.id AND
     packages.repodir_id = ? AND spf.package_id = rpm_requires.package_id AND
     (so_needed.name = rpm_requires.name OR
      so_needed.name || '()(64bit)' = rpm_requires.name) AND
     packages.id = req_res.package_id AND
     rpm_requires.id = req_res.requires_id AND
     req_res.dep_package_id = tpf.package_id AND
     so_needed.name = tpf.basename
""", [repodir_id])

    search_repodirs = [repodir_id]
    search_repodirs.extend(repodir_depends)
    in_repodirs = ','.join(str(id) for id in search_repodirs)

    objects_not_resolved1 = dbc.execute("""
SELECT packages.id AS package_id, packages.nvra,
       package_files.id AS object_id, package_files.basename AS object_name,
       so_needed.id AS so_needed_id, so_needed.name AS so_needed_name
  FROM packages CROSS JOIN package_files CROSS JOIN so_needed
  WHERE repodir_id = ? AND package_files.package_id = packages.id AND
        so_needed.obj_file_id = package_files.id AND
        NOT EXISTS (SELECT 1 FROM so_needed_res
                      WHERE so_needed_res.so_needed_id = so_needed.id)
  ORDER BY packages.nvra, package_files.basename, so_needed.name
""", [repodir_id]).fetchall()
    print 'Object files not resolved by rpm requires-provides: ', len(objects_not_resolved1)
    if objects_not_resolved1:
        print 'Searching object files resolutions (2)...'
        in_so_needed = ','.join(str(obj_rec[4]) for obj_rec in objects_not_resolved1)
        dbc.execute("""
INSERT INTO so_needed_res(so_needed_id, dep_obj_file_id, res_type)
  SELECT so_needed.id, tpf.id, 2 FROM packages, package_files tpf, so_needed
    WHERE packages.repodir_id IN (%s) AND packages.id = tpf.package_id AND
     so_needed.id IN (%s) AND tpf.basename = so_needed.name
""" % (in_repodirs, in_so_needed))

    objects_not_resolved2 = dbc.execute("""
SELECT packages.id AS package_id, packages.nvra,
       package_files.id AS object_id, package_files.basename AS object_name,
       so_needed.id AS so_needed_id, so_needed.name AS so_needed_name
  FROM packages, package_files, so_needed
  WHERE repodir_id = ? AND package_files.package_id = packages.id AND
        so_needed.obj_file_id = package_files.id AND
        NOT EXISTS (SELECT 1 FROM so_needed_res WHERE so_needed_res.so_needed_id = so_needed.id)
  ORDER BY packages.nvra, package_files.basename, so_needed.name
""", [repodir_id]).fetchall()
    print 'Object files not resolved: ', len(objects_not_resolved2)


def process_repodir_obj_symbols(dbc, repodir_id, repodir_name, repodir_depends):
    print 'Searching symbols resolutions (1)...'
# EXPLAIN QUERY PLAN
    dbc.execute("""
INSERT INTO obj_symbols_res(obj_sym_id, dep_obj_sym_id, res_type)
SELECT sos.id, tos.id, 1 FROM packages CROSS JOIN package_files spf CROSS JOIN obj_symbols sos CROSS JOIN
  so_needed CROSS JOIN so_needed_res CROSS JOIN package_files tpf CROSS JOIN obj_symbols tos
   WHERE packages.repodir_id = ? AND packages.id = spf.package_id AND spf.id = sos.obj_file_id AND
    sos.sym_type = 0 AND sos.obj_file_id = so_needed.obj_file_id AND so_needed.id = so_needed_res.so_needed_id AND
    so_needed_res.res_type = 1 AND so_needed_res.dep_obj_file_id = tpf.id AND
    (tos.obj_file_id = tpf.id OR tos.obj_file_id = tpf.link_to_file_id) AND
    tos.sym_type = 1 AND tos.name = sos.name
""", [repodir_id])
    print 'Searching symbols resolutions (2)...'
    dbc.execute("""
INSERT INTO obj_symbols_res(obj_sym_id, dep_obj_sym_id, res_type)
SELECT sos.id, tos.id, 2 FROM packages CROSS JOIN package_files CROSS JOIN obj_symbols sos CROSS JOIN
  so_needed CROSS JOIN so_needed_res CROSS JOIN obj_symbols tos
   WHERE packages.repodir_id = ? AND packages.id = package_files.package_id AND package_files.id = sos.obj_file_id AND
    sos.sym_type = 0 AND sos.obj_file_id = so_needed.obj_file_id AND so_needed.id = so_needed_res.so_needed_id AND
    so_needed_res.res_type = 2 AND so_needed_res.dep_obj_file_id = tos.obj_file_id AND
    tos.sym_type = 1 AND tos.name = sos.name
""", [repodir_id])
    print 'Searching symbols resolutions (3)...'
    search_repodirs = [repodir_id]
    search_repodirs.extend(repodir_depends)
    in_repodirs = ','.join(str(id) for id in search_repodirs)
    dbc.execute("""
INSERT INTO obj_symbols_res(obj_sym_id, dep_obj_sym_id, res_type)
SELECT sos.id, tos.id, 3 FROM packages CROSS JOIN package_files CROSS JOIN obj_symbols sos CROSS JOIN
 obj_symbols tos CROSS JOIN package_files tpf
  WHERE repodir_id = ? AND packages.id = package_files.package_id AND package_files.id = sos.obj_file_id AND
  sos.sym_type = 0 AND NOT EXISTS (SELECT 1 FROM obj_symbols_res WHERE obj_sym_id = sos.id) AND
  sos.name = tos.name AND tos.sym_type = 1 AND tos.obj_file_id = tpf.id AND
  tpf.package_id IN (SELECT id FROM packages WHERE repodir_id IN (%s))
""" % in_repodirs, [repodir_id])


def process_repodir(dbc, repo_id, repo_name, repo_sources, depend_repodir_list, repodirs_processed, dep_arch):
    all_depends_ready = True
    repodir_depends = []

    in_repodirs = ','.join(str(id) for id in repodirs_processed)
    for dr_name in depend_repodir_list:
        repodir_depend_found = dbc.execute("""
SELECT id, name FROM repodirs WHERE id IN (%s) AND name = ?
""" % in_repodirs, [dr_name]).fetchall()
        if len(repodir_depend_found) == 0:
            all_depends_ready = False
            break
        else:
            for rdf in repodir_depend_found:
                repodir_depends.append(rdf[0])
    if not all_depends_ready:
        return False
    print repo_name, ' ', depend_repodir_list, ' ', dep_arch
    process_repodir_dependencies(dbc, repo_id, repo_name, repodir_depends, 'conflicts')
    process_repodir_dependencies(dbc, repo_id, repo_name, repodir_depends, 'obsoletes')
    process_repodir_requires(dbc, repo_id, repo_name, repodir_depends, dep_arch)
    process_repodir_file_links(dbc, repo_id, repo_name, repodir_depends)
    process_repodir_so_needed(dbc, repo_id, repo_name, repodir_depends)
    process_repodir_obj_symbols(dbc, repo_id, repo_name, repodir_depends)

    if repo_sources:
        print 'Searching source rpms...'
        dbc.execute("""
UPDATE packages SET sourcerpm_package =
  (SELECT id FROM packages ps
     WHERE repodir_id IN (SELECT id FROM repodirs WHERE name = ?) AND
      ps.nvra = substr(packages.sourcerpm, 1, length(packages.sourcerpm)-4)
  )
  WHERE repodir_id = ?  AND sourcerpm LIKE '%.rpm'
""", [repo_sources, repo_id])
    return True

def main(args):

    conn = sqlite3.connect(DB)
    dbc = conn.cursor()

    global n
    n = 0
    dbc.executescript("""
PRAGMA cache_size = -1048576;
DELETE FROM package_requires_res;
DELETE FROM package_conflicts_res;
DELETE FROM package_obsoletes_res;
DELETE FROM so_needed_res;
DELETE FROM obj_symbols_res;
UPDATE package_files SET link_to_file_id = NULL;
UPDATE packages SET sourcerpm_package = NULL;
ANALYZE;
""")
    repodirs_processed = []
    #Process binary rpms
    repodirs_processed_cnt = -1
    while repodirs_processed_cnt < len(repodirs_processed):
        in_repodirs = ','.join(str(id) for id in repodirs_processed)
        repodirs = dbc.execute("""
SELECT id, name, sources, path FROM repodirs WHERE sources <> '.' AND id NOT IN (%s)
""" % in_repodirs).fetchall()
        for repodir in repodirs:
            (repo_id, repo_name, repo_sources) = (repodir[0], repodir[1], repodir[2])
            depend_repodir_names = dbc.execute(
"""
SELECT depend_repodir_name FROM repodir_depends WHERE repodir_id = ?
""", [repo_id]).fetchall()
            depend_repodir_list = [drn[0] for drn in depend_repodir_names]
            if process_repodir(dbc, repo_id, repo_name, repo_sources, depend_repodir_list, repodirs_processed, None):
                repodirs_processed.append(repo_id)
        repodirs_processed_cnt = len(repodirs_processed)

    #Process SRPMS
    repodirs_processed_cnt = -1
    while repodirs_processed_cnt < len(repodirs_processed):
        repodirs = dbc.execute("""
SELECT id, name, sources, path FROM repodirs WHERE sources = '.'
""").fetchall()
        for repodir in repodirs:
            (repo_id, repo_name, repo_sources) = (repodir[0], repodir[1], repodir[2])
            src_build_archs = []
            depend_repodir_names = dbc.execute(
"""
SELECT depend_repodir_name FROM repodir_depends WHERE repodir_id = ?
""", [repo_id]).fetchall()
            for drn in depend_repodir_names:
                dr_name = drn[0]
                if '$arch' in dr_name:
                    depend_repodir_found = dbc.execute(
"""
SELECT id, name FROM repodirs WHERE name LIKE ?
""", [dr_name.replace('$arch', '%')]).fetchall()
                    if len(depend_repodir_found) == 0:
                        raise Exception('Dependancy repositories not found!')
                    for drf in depend_repodir_found:
                        arch = extract_arch(dr_name, drf[1])
                        if arch:
                            if arch == 'SRPMS':
                                continue
                            src_build_archs.append(arch)
                        else:
                            raise Exception('Source repository should depend on */$arch/* repo.')
            for arch in src_build_archs:
                depend_repodir_list = [drn[0].replace('$arch', arch)
                                        for drn in depend_repodir_names]
                if not process_repodir(dbc, repo_id, repo_name, None, depend_repodir_list, repodirs_processed, arch):
                    raise Exception('Couldn\'t process SRPMS repository!')
            repodirs_processed.append(repo_id)

        repodirs_processed_cnt = len(repodirs_processed)

    in_repodirs = ','.join(str(id) for id in repodirs_processed)
    repodirs_not_processed = dbc.execute("""
SELECT id, name, sources, path FROM repodirs rd WHERE id NOT IN (%s)
""" % in_repodirs).fetchall()
    if len(repodirs_not_processed) > 0:
        print 'Repodirs not processed due to dependencies:'
        for rdna in repodirs_not_processed:
            print rdna[1]
    dbc.execute("""
ANALYZE""")
    conn.commit()

if __name__ == "__main__":
    main(sys.argv)