[PATCH] fdo#46538 Easyhack: Add support for more bug trackers to get-bugzilla-attachments-by-mimetype

Hello,

Here is a patch adding support for Novell, KDE (for KOffice/Calligra), GNOME (for Gnumeric), 
AbiSource
(for AbiWord) bugzillas and Launchpad to bin/get-bugzilla-attachments-by-mimetype script as 
required by easyhack#46538. However, there two glitches and workarounds:

1- As stated in the code, Novell bugzilla requires users to log in to bugzilla in
order to get full information about the bugs such as attachment bodies etc. Since
Novell bugzilla uses some proprietary login extensions it's hard to implement log in
mechanism and use it via xmlrpc API. python-bugzilla claims that it supports Novell
bugzilla login but I could not make it work. As a workaround, comments containing
"Created an attachment (id=xxxxxxx)" pattern are parsed and attachments are downloaded
using urllib. Attachments are filtered using mimetype info obtained from http headers.

2- For launchpad, launchpadlib python module is used. But currently launchpad api
does not support really advanced search like bugzilla. For example, it's not possible
to search for bugs having application/pdf attachments. So, again a workaround is used. Code
iterates over all the bugs of the *libreoffice* source package and check their attachments.

---
 bin/get-bugzilla-attachments-by-mimetype |  278 ++++++++++++++++++++----------
 1 file changed, 191 insertions(+), 87 deletions(-)

diff --git a/bin/get-bugzilla-attachments-by-mimetype b/bin/get-bugzilla-attachments-by-mimetype
index 0b18060..7316f0a 100755
--- a/bin/get-bugzilla-attachments-by-mimetype
+++ b/bin/get-bugzilla-attachments-by-mimetype
@@ -39,6 +39,7 @@
 import urllib
 import feedparser
 import base64
+import re
 import os, os.path
 import xmlrpclib
 from xml.dom import minidom
@@ -83,6 +84,42 @@ def get_from_bug_url_via_xml(url, mimetype, prefix, suffix):
                     break
             attachmentid += 1
 
+def get_novell_bug_via_xml(url, mimetype, prefix, suffix):
+    id = url.rsplit('=', 2)[1]
+    print "id is", prefix, id, suffix
+    if os.path.isfile(suffix + '/' + prefix + id + '-1.' + suffix):
+        print "assuming", id, "is up to date"
+    else:
+        print "parsing", id
+        sock = urlopen_retry(url+"&ctype=xml")
+        dom = minidom.parse(sock)
+        sock.close()
+        attachmentid=1
+        for comment in dom.getElementsByTagName('thetext'):
+            commentText = comment.firstChild.nodeValue
+            match = re.search(r".*Created an attachment \(id=([0-9]+)\)", commentText)
+            if not match:
+                continue
+
+            realAttachmentId = match.group(1)
+            handle = urlopen_retry(novellattach + realAttachmentId)
+            if not handle:
+                print "attachment %s is not accessible", realAttachmentId
+                continue 
+
+            remoteMime = handle.info().gettype()
+            if remoteMime != mimetype:
+                print "skipping"
+                continue
+
+            download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix
+            print 'downloading as', download
+            f = open(download, 'w')
+            f.write(handle.read())
+            f.close()
+            break
+            attachmentid += 1
+
 def get_through_rpc_query(rpcurl, showurl, mimetype, prefix, suffix):
     try:
         proxy = xmlrpclib.ServerProxy(rpcurl)
@@ -109,108 +146,164 @@ def get_through_rss_query_url(url, mimetype, prefix, suffix):
     except:
         pass
     d = feedparser.parse(url)
+
+    #Getting detailed bug information and downloading an attachment body is not possible without 
logging in to Novell bugzilla
+    #get_novell_bug_via_xml function is a workaround for that situation
+    get_bug_function = get_novell_bug_via_xml if prefix == "novell" else get_from_bug_url_via_xml
+
     for entry in d['entries']:
-        get_from_bug_url_via_xml(entry['id'], mimetype, prefix, suffix)
+        get_bug_function(entry['id'], mimetype, prefix, suffix)
 
 def get_through_rss_query(queryurl, mimetype, prefix, suffix):
     url = queryurl + 
'?query_format=advanced&field0-0-0=attachments.mimetype&type0-0-0=equals&value0-0-0=' + 
escape(mimetype) + '&ctype=rss'
     print 'url is', url
     get_through_rss_query_url(url, mimetype, prefix, suffix)
 
+def get_launchpad_bugs(prefix):
+    #launchpadlib python module is required to download launchpad attachments
+    from launchpadlib.launchpad import Launchpad
+
+    launchpad = Launchpad.login_anonymously("attachmentdownload", "production")
+    ubuntu = launchpad.distributions["ubuntu"]
+
+    #since searching bugs having attachments with specific mimetypes is not available in launchpad 
API
+    #we're iterating over all bugs of the libreoffice source package
+    libo = ubuntu.getSourcePackage(name="libreoffice")
+    libobugs = libo.getBugTasks()
+
+    for bugtask in libobugs:
+        bug = bugtask.bug
+        id = str(bug.id)
+        print "parsing ", id, "status:", bugtask.status, "title:", bug.title[:50]
+        attachmentid = 1
+        for attachment in bug.attachments:
+            handle = attachment.data.open()
+            if not handle.content_type in mimetypes:
+                #print "skipping"
+                continue
+
+            suffix = mimetypes[handle.content_type]
+            if not os.path.isdir(suffix):
+                try:
+                    os.mkdir(suffix)
+                except:
+                    pass
+
+            download = suffix + '/' + prefix + id + '-' + str(attachmentid) + '.' + suffix
+
+            if os.path.isfile(download):
+                print "assuming", id, "is up to date"
+                break
+
+            print 'mimetype is', handle.content_type, 'downloading as', download
+
+            f = open(download, "w")
+            f.write(handle.read())
+            f.close()
+            attachmentid += 1
 
 freedesktop = 'http://bugs.freedesktop.org/buglist.cgi'
+abisource = 'http://bugzilla.abisource.com/buglist.cgi' #added for abiword
+gnome = 'http://bugzilla.gnome.org/buglist.cgi' # added for gnumeric
+kde = 'http://bugs.kde.org/buglist.cgi' # added for koffice/calligra
 openoffice = 'https://issues.apache.org/ooo/buglist.cgi'
 redhatrpc = 'https://bugzilla.redhat.com/xmlrpc.cgi'
 redhatbug = 'https://bugzilla.redhat.com/show_bug.cgi?id='
-novell = 'https://bugzilla.novell.com/buglist.cgi'
 mozilla = 'https://bugzilla.mozilla.org/buglist.cgi'
 
-mimetypes = [
+#Novell Bugzilla requires users to log in in order to get details of the bugs such as attachment 
bodies etc.
+#As a dirty workaround, we parse comments containing "Created an attachment (id=xxxxxx)" and 
download attachments manually
+#python-bugzilla claims that it supports Novell bugzilla login but it's not working right now and 
novell bugzilla login
+#system is a nightmare
+novellattach = 'https://bugzilla.novell.com/attachment.cgi?id='
+novell = 'https://bugzilla.novell.com/buglist.cgi'
+
+mimetypes = {
 # ODF
-    ('application/vnd.oasis.opendocument.base', 'odb'),
-    ('application/vnd.oasis.opendocument.database', 'odb'),
-    ('application/vnd.oasis.opendocument.chart', 'odc'),
-    ('application/vnd.oasis.opendocument.chart-template', 'otc'),
-    ('application/vnd.oasis.opendocument.formula', 'odf'),
-    ('application/vnd.oasis.opendocument.formula-template', 'otf'),
-    ('application/vnd.oasis.opendocument.graphics', 'odg'),
-    ('application/vnd.oasis.opendocument.graphics-template', 'otg'),
-    ('application/vnd.oasis.opendocument.graphics-flat-xml', 'fodg'),
-    ('application/vnd.oasis.opendocument.presentation', 'odp'),
-    ('application/vnd.oasis.opendocument.presentation-template', 'otp'),
-    ('application/vnd.oasis.opendocument.presentation-flat-xml', 'fodp'),
-    ('application/vnd.oasis.opendocument.spreadsheet', 'ods'),
-    ('application/vnd.oasis.opendocument.spreadsheet-template', 'ots'),
-    ('application/vnd.oasis.opendocument.spreadsheet-flat-xml', 'fods'),
-    ('application/vnd.oasis.opendocument.text', 'odt'),
-    ('application/vnd.oasis.opendocument.text-flat-xml', 'fodt'),
-    ('application/vnd.oasis.opendocument.text-master', 'odm'),
-    ('application/vnd.oasis.opendocument.text-template', 'ott'),
-    ('application/vnd.oasis.opendocument.text-web', 'oth'),
+    'application/vnd.oasis.opendocument.base': 'odb',
+    'application/vnd.oasis.opendocument.database': 'odb',
+    'application/vnd.oasis.opendocument.chart': 'odc',
+    'application/vnd.oasis.opendocument.chart-template': 'otc',
+    'application/vnd.oasis.opendocument.formula': 'odf',
+    'application/vnd.oasis.opendocument.formula-template': 'otf',
+    'application/vnd.oasis.opendocument.graphics': 'odg',
+    'application/vnd.oasis.opendocument.graphics-template': 'otg',
+    'application/vnd.oasis.opendocument.graphics-flat-xml': 'fodg',
+    'application/vnd.oasis.opendocument.presentation': 'odp',
+    'application/vnd.oasis.opendocument.presentation-template': 'otp',
+    'application/vnd.oasis.opendocument.presentation-flat-xml': 'fodp',
+    'application/vnd.oasis.opendocument.spreadsheet': 'ods',
+    'application/vnd.oasis.opendocument.spreadsheet-template': 'ots',
+    'application/vnd.oasis.opendocument.spreadsheet-flat-xml': 'fods',
+    'application/vnd.oasis.opendocument.text': 'odt',
+    'application/vnd.oasis.opendocument.text-flat-xml': 'fodt',
+    'application/vnd.oasis.opendocument.text-master': 'odm',
+    'application/vnd.oasis.opendocument.text-template': 'ott',
+    'application/vnd.oasis.opendocument.text-web': 'oth',
 # OOo XML
-    ('application/vnd.sun.xml.base', 'odb'),
-    ('application/vnd.sun.xml.calc', 'sxc'),
-    ('application/vnd.sun.xml.calc.template', 'stc'),
-    ('application/vnd.sun.xml.chart', 'sxs'),
-    ('application/vnd.sun.xml.draw', 'sxd'),
-    ('application/vnd.sun.xml.draw.template', 'std'),
-    ('application/vnd.sun.xml.impress', 'sxi'),
-    ('application/vnd.sun.xml.impress.template', 'sti'),
-    ('application/vnd.sun.xml.math', 'sxm'),
-    ('application/vnd.sun.xml.writer', 'sxw'),
-    ('application/vnd.sun.xml.writer.global', 'sxg'),
-    ('application/vnd.sun.xml.writer.template', 'stw'),
-    ('application/vnd.sun.xml.writer.web', 'stw'),
+    'application/vnd.sun.xml.base': 'odb',
+    'application/vnd.sun.xml.calc': 'sxc',
+    'application/vnd.sun.xml.calc.template': 'stc',
+    'application/vnd.sun.xml.chart': 'sxs',
+    'application/vnd.sun.xml.draw': 'sxd',
+    'application/vnd.sun.xml.draw.template': 'std',
+    'application/vnd.sun.xml.impress': 'sxi',
+    'application/vnd.sun.xml.impress.template': 'sti',
+    'application/vnd.sun.xml.math': 'sxm',
+    'application/vnd.sun.xml.writer': 'sxw',
+    'application/vnd.sun.xml.writer.global': 'sxg',
+    'application/vnd.sun.xml.writer.template': 'stw',
+    'application/vnd.sun.xml.writer.web': 'stw',
 # MSO
-    ('application/rtf', 'rtf'),
-    ('text/rtf', 'rtf'),
-    ('application/msword', 'doc'),
-    ('application/vnd.ms-powerpoint', 'ppt'),
-    ('application/vnd.ms-excel', 'xls'),
-    ('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'xlsx'),
-    ('application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'xltx'),
-    ('application/vnd.openxmlformats-officedocument.presentationml.presentation', 'pptx'),
-    ('application/vnd.openxmlformats-officedocument.presentationml.template', 'ppotx'),
-    ('application/vnd.openxmlformats-officedocument.presentationml.slideshow', 'ppsx'),
-    ('application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'docx'),
-    ('application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'dotx'),
-    ('application/vnd.visio', 'vsd'),
+    'application/rtf': 'rtf',
+    'text/rtf': 'rtf',
+    'application/msword': 'doc',
+    'application/vnd.ms-powerpoint': 'ppt',
+    'application/vnd.ms-excel': 'xls',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.template': 'xltx',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
+    'application/vnd.openxmlformats-officedocument.presentationml.template': 'ppotx',
+    'application/vnd.openxmlformats-officedocument.presentationml.slideshow': 'ppsx',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.template': 'dotx',
+    'application/vnd.visio': 'vsd',
 # W3C
-    ('application/xhtml+xml', 'xhtml'),
-    ('application/mathml+xml', 'mml'),
-    ('text/html', 'html'),
-    ('application/docbook+xml', 'docbook'),
+    'application/xhtml+xml': 'xhtml',
+    'application/mathml+xml': 'mml',
+    'text/html': 'html',
+    'application/docbook+xml': 'docbook',
 # misc
-    ('text/spreadsheet', 'slk'),
-    ('application/pdf', 'pdf'),
-    ('application/vnd.corel-draw', 'cdr'),
-    ('application/vnd.lotus-wordpro', 'lwp'),
-    ('application/vnd.lotus-1-2-3', 'wks'),
-    ('application/vnd.wordperfect', 'wpd'),
-    ('application/vnd.ms-works', 'wps'),
-    ('application/x-hwp', 'hwp'),
-    ('application/x-aportisdoc', 'pdb'),
-    ('application/x-pocket-word', 'psw'),
-    ('application/x-t602', '602'),
+    'text/spreadsheet': 'slk',
+    'application/pdf': 'pdf',
+    'application/vnd.corel-draw': 'cdr',
+    'application/vnd.lotus-wordpro': 'lwp',
+    'application/vnd.lotus-1-2-3': 'wks',
+    'application/vnd.wordperfect': 'wpd',
+    'application/vnd.ms-works': 'wps',
+    'application/x-hwp': 'hwp',
+    'application/x-aportisdoc': 'pdb',
+    'application/x-pocket-word': 'psw',
+    'application/x-t602': '602',
 # binfilter
-    ('application/x-starcalc', 'sdc'),
-    ('application/vnd.stardivision.calc', 'sdc5'),
-    ('application/x-starchart', 'sds'),
-    ('application/vnd.stardivision.chart', 'sds5'),
-    ('application/x-stardraw', 'sdd_d'),
-    ('application/vnd.stardivision.draw', 'sda5'),
-    ('application/x-starimpress', 'sdd_i'),
-    ('application/vnd.stardivision.impress', 'sdd5'),
-    ('application/vnd.stardivision.impress-packed', 'sdp5'),
-    ('application/x-starmath', 'smf'),
-    ('application/vnd.stardivision.math', 'smf5'),
-    ('application/x-starwriter', 'sdw'),
-    ('application/vnd.stardivision.writer', 'sdw5'),
-    ('application/vnd.stardivision.writer-global', 'sgl5'),
+    'application/x-starcalc': 'sdc',
+    'application/vnd.stardivision.calc': 'sdc5',
+    'application/x-starchart': 'sds',
+    'application/vnd.stardivision.chart': 'sds5',
+    'application/x-stardraw': 'sdd_d',
+    'application/vnd.stardivision.draw': 'sda5',
+    'application/x-starimpress': 'sdd_i',
+    'application/vnd.stardivision.impress': 'sdd5',
+    'application/vnd.stardivision.impress-packed': 'sdp5',
+    'application/x-starmath': 'smf',
+    'application/vnd.stardivision.math': 'smf5',
+    'application/x-starwriter': 'sdw',
+    'application/vnd.stardivision.writer': 'sdw5',
+    'application/vnd.stardivision.writer-global': 'sgl5',
 # unusual image mimetypes
-    ('image/cgm', 'cgm'),
-    ('image/x-targa', 'tga'),
-]
+    'image/cgm': 'cgm',
+    'image/x-targa': 'tga',
+}
 
 # disabled for now, this would download gigs of pngs/jpegs...
 common_image_mimetypes = [
@@ -241,16 +334,27 @@ common_image_mimetypes = [
     ('image/png', 'png'),
 ]
 
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
+    get_through_rss_query(novell, mimetype, "novell", extension)
+
+for (mimetype,extension) in mimetypes.items():
+    get_through_rss_query(kde, mimetype, "kde", extension)
+
+for (mimetype,extension) in mimetypes.items():
+    get_through_rss_query(gnome, mimetype, "gnome", extension)
+
+for (mimetype,extension) in mimetypes.items():
+    get_through_rss_query(abisource, mimetype, "abi", extension)
+
+for (mimetype,extension) in mimetypes.items():
     get_through_rss_query(freedesktop, mimetype, "fdo", extension)
 
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
     get_through_rpc_query(redhatrpc, redhatbug, mimetype, "rhbz", extension)
 
-#to-do, get attachments some other way, not inline in xml
-#get_through_rss_query(novell, 'application/msword', "n", "doc")
-
-for (mimetype,extension) in mimetypes:
+for (mimetype,extension) in mimetypes.items():
     get_through_rss_query(openoffice, mimetype, "ooo", extension)
 
+get_launchpad_bugs("lp")
+
 # vim:set shiftwidth=4 softtabstop=4 expandtab:
-- 
1.7.10