Update centos.git.repolist.py to use pagure api

centos.git.repolist.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+#pylint: disable=line-too-long
 #
 #  License: GPLv3
 #
@@ -6,73 +7,112 @@
 #         Updates:
 #                  Pat Riehecky <riehecky@fnal.gov>
 #
-'''Get list of repos from gitblit RPC, to grab CentOS sources'''
+'''Get list of repos from pagure, to grab CentOS sources'''
 
-import optparse
-import requests
-import simplejson as json
+# for python3 compat
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import print_function
+
+import logging
 import sys
+import json
+import textwrap
+import time
 
-RPCURL = "https://git.centos.org/rpc/?req=LIST_REPOSITORIES"
+sys.setrecursionlimit(500)
 
-def read_args():
+try:
+    from argparse import ArgumentParser
+except ImportError:  # pragma: no cover
+    print("Please install argparse - rpm: python-argparse", file=sys.stderr)
+    raise
+
+try:
+    import requests
+except ImportError:  # pragma: no cover
+    print("Please install requests - rpm: python-requests", file=sys.stderr)
+    raise
+
+
+def setup_args():
     '''
-        read in the command line args and set things up
+        Setup the argparse object.
+
+        Make sure all fields have defaults so we could use this as an object
     '''
+    parser = ArgumentParser(description=textwrap.dedent(__doc__))
 
-    desc = '''Get list of git repositories from the GitBlit json RPC
-           '''
+    parser.add_argument('--debug', action='store_true', default=False,
+                        help='Print debugging information')
+    parser.add_argument('--hostname', default='git.centos.org',
+                        type=str, help='What host should we query?')
+    parser.add_argument('--apiver', default='0',
+                        type=str, help='What api version is the host?')
+    parser.add_argument('--namespace', default='rpms',
+                        type=str, help='What project namespace?')
+    parser.add_argument('--show-forks', action='store_true', default=False,
+                        help='Should we also show project forks?')
 
-    usage = "usage: %prog [options] "
-    parser = optparse.OptionParser(usage=usage, description=desc)
-    parser.add_option('-p', '--project', metavar="<PROJECTS>",
-                      help='''project path (default 'rpms', could be 'all', 'core-sig'...)''',
-                      default='rpms')
+    return parser
 
-    parser.add_option('-b', '--branch', metavar="<branch name>",
-                      help='Only list repos with this branch (default master)',
-                      default = 'master')
+def run_query(hostname, api, namespace, forks):
+    '''
+        Actually call the API version
+    '''
+    list_of_urls = []
+    if str(api) == '0':
+        query = 'https://{hostname}/api/0/projects?per_page=50&namespace={namespace}'.format(hostname=hostname, namespace=namespace)
+        if forks:
+            query = query + '&forks=1'
+        else:
+            query = query + '&forks=0'
 
-    parser.add_option('-u', '--url', metavar="<URL>",
-                      help='URL to check (default %s)' % (RPCURL),
-                      default = RPCURL)
+        fetch_prefix = 'https://{hostname}/'.format(hostname=hostname)
 
-    (options, args) = parser.parse_args()
-    return options
+        fetch_next_v0(query, fetch_prefix, list_of_urls)
+    else:
+        raise NotImplementedError("Unknown API version %s", api)
 
-def get_repo_list(url, branch, projectpath):
-    '''return a list of repo URLs'''
+    list_of_urls.sort()
+    return list_of_urls
+
+def fetch_next_v0(page, fetch_prefix, list_of_urls):
+    '''
+        Recursively fetch the page until we are done
+    '''
+    logging.debug('Trying to fetch %s', page)
     try:
-        req = requests.get(url)
+        req = requests.get(page)
     except requests.exceptions.RequestException as err_msg:
-        print err_msg
-        sys.exit(1)
+        print(err_msg, file=sys.stderr)
+        raise
 
-    if req.status_code != 200:
-        print "Unable to access gitblit api at " + url
-        sys.exit(1)
+    try:
+        message = json.loads(req.text)
+    except ValueError as err_msg:
+        print(page, file=sys.stderr)
+        print(req.text, file=sys.stderr)
+        print(err_msg, file=sys.stderr)
+        raise
 
+    for project in message['projects']:
+        list_of_urls.append(fetch_prefix + project['fullname'])
 
-    payload = req.text
-    repos = json.loads(payload)
-    branchname = 'refs/heads/' + branch
+    if 'next' in message['pagination']:
+        if message['pagination']['next']:
+            time.sleep(0.25) # Add a smallish delay to help with load
+            fetch_next_v0(message['pagination']['next'], fetch_prefix, list_of_urls)
 
-    for repo in repos.keys():
-        if projectpath != 'all':
-            if repos[repo]['projectPath'] != projectpath:
-                del repos[repo]
-                continue
-        if branchname not in repos[repo]['availableRefs']:
-            del repos[repo]
+if __name__ == '__main__':
 
-    return repos.keys()
+    PARSER = setup_args()
+    ARGS = PARSER.parse_args()
 
-def main():
-    '''Broken out so it can be inherited if someone wants'''
-    options = read_args()
-    repos = get_repo_list(url=options.url, branch=options.branch, projectpath=options.project)
-    if repos:
-        print '\n'.join(repos)
+    if ARGS.debug:
+        logging.basicConfig(level=logging.DEBUG)
 
-if __name__ == "__main__":
-    main()
+    URLS = run_query(ARGS.hostname, ARGS.apiver, ARGS.namespace, ARGS.show_forks)
+
+    for URL in URLS:
+        print(URL)