fixed the gpg thing. WHEW. what a PITA.

also fleshed out some logging.
2019-12-19 14:04:34 -05:00
parent f25e6bee2a
commit b4c9caefbd
7 changed files with 348 additions and 41 deletions
--- a/aif/utils/sources.py
+++ b/aif/utils/sources.py
@@ -1,10 +1,12 @@
 import ftplib
 import io
+import logging
 import pathlib
 import re
 ##
 import requests
 import requests.auth
+from lxml import etree
 ##
 import aif.constants_fallback
 from . import gpg_handler
@@ -12,13 +14,26 @@ from . import hash_handler
 from . import parser


+_logger = logging.getLogger(__name__)
+
+
 class ChecksumFile(object):
    _bsd_re = re.compile(r'^(?P<fname>\(.*\))\s+=\s+(?P<cksum>.*)$')

    def __init__(self, checksum_xml, filetype):
        self.xml = checksum_xml
+        if self.xml is not None:
+            _logger.debug('checksum_xml: {0}'.format(etree.tostring(self.xml).decode('utf-8')))
+        else:
+            _logger.error('checksum_xml is required but not specified')
+            raise ValueError('checksum_xml is required')
        self.uri = self.xml.text.strip()
        self.filetype = filetype
+        if filetype:
+            _logger.debug('URI and filetype: {{{0}}}{1}'.format(self.uri, self.filetype))
+        else:
+            _logger.error('filetype is required but not specified')
+            raise ValueError('filetype is required')
        self.hashes = None
        downloader = getDLHandler(self.uri)  # Recursive objects for the win?
        dl = downloader(self.xml)
@@ -28,14 +43,14 @@ class ChecksumFile(object):
        self._convert()

    def _convert(self):
-        data = self.data
-        if not isinstance(data, str):
-            data = data.decode('utf-8')
-        data.strip()
+        if not isinstance(self.data, str):
+            self.data = self.data.decode('utf-8')
+        self.data.strip()
        self.hashes = {}
        if self.filetype not in ('gnu', 'bsd'):
+            _logger.error('Passed an invalid filetype: {0}'.format(self.filetype))
            raise ValueError('filetype attribute must be either "gnu" or "bsd"')
-        for line in data.splitlines():
+        for line in self.data.splitlines():
            if self.filetype == 'gnu':
                hashtype = None  # GNU style splits their hash types into separate files by default.
                h, fname = line.split(None, 1)
@@ -48,17 +63,29 @@ class ChecksumFile(object):
            if hashtype not in self.hashes:
                self.hashes[hashtype] = {}
            self.hashes[hashtype][fname] = h
+        _logger.debug('Generated hash set: {0}'.format(self.hashes))
        return(None)


 class Downloader(object):
    def __init__(self, netresource_xml, *args, **kwargs):
        self.xml = netresource_xml
+        _logger.info('Instantiated class {0}'.format(type(self).__name__))
+        if netresource_xml is not None:
+            _logger.debug('netresource_xml: {0}'.format(etree.tostring(self.xml).decode('utf-8')))
+        else:
+            _logger.error('netresource_xml is required but not specified')
+            raise ValueError('netresource_xml is required')
+        _logger.debug('args: {0}'.format(','.join(args)))
+        _logger.debug('kwargs: {0}'.format(kwargs))
        self.uri = parser.URI(self.xml.text.strip())
+        _logger.debug('Parsed URI: {0}'.format(self.uri))
        self.user = self.xml.attrib.get('user')
        if not self.user and self.uri.user:
            self.user = self.uri.user
        self.password = self.xml.attrib.get('password')
+        _logger.debug('Parsed user: {0}'.format(self.user))
+        _logger.debug('Parsed password: {0}'.format(self.password))
        if not self.password and self.uri.password:
            self.password = self.uri.password
        self.real_uri = ('{0}://'
@@ -68,6 +95,7 @@ class Downloader(object):
                                       (self.uri.base if self.uri.base else ''),
                                       (':{0}'.format(self.uri.port) if self.uri.port else ''),
                                       self.uri.path)
+        _logger.debug('Rebuilt URI: {0}'.format(self.real_uri))
        self.gpg = None
        self.checksum = None
        self.data = io.BytesIO()
@@ -77,11 +105,19 @@ class Downloader(object):
        return(None)

    def parseGpgVerify(self, results):
-        pass
+        pass  # TODO? Might not need to.

    def verify(self, verify_xml, *args, **kwargs):
        gpg_xml = verify_xml.find('gpg')
+        if gpg_xml is not None:
+            _logger.debug('gpg_xml: {0}'.format(etree.tostring(gpg_xml).decode('utf-8')))
+        else:
+            _logger.debug('No <gpg> in verify_xml')
        hash_xml = verify_xml.find('hash')
+        if hash_xml is not None:
+            _logger.debug('Hash XML: {0}'.format(etree.tostring(hash_xml).decode('utf-8')))
+        else:
+            _logger.debug('No <hash> in verify_xml')
        results = {}
        if gpg_xml is not None:
            results['gpg'] = self.verifyGPG(gpg_xml)
@@ -94,35 +130,72 @@ class Downloader(object):
        # We don't allow custom GPG homedirs since this is probably running from a LiveCD/USB/whatever anyways.
        # This means we can *always* instantiate the GPG handler from scratch.
        self.gpg = gpg_handler.GPG()
+        _logger.info('Established GPG session.')
+        _logger.debug('GPG home dir: {0}'.format(self.gpg.homedir))
+        _logger.debug('GPG primary key: {0}'.format(self.gpg.primary_key.fpr))
        keys_xml = gpg_xml.find('keys')
+        if keys_xml is not None:
+            _logger.debug('keys_xml: {0}'.format(etree.tostring(keys_xml).decode('utf-8')))
+        else:
+            _logger.error('No required <keys> in gpg_xml')
+            raise ValueError('<keys> is required in a GPG verification block')
        sigs_xml = gpg_xml.find('sigs')
-        fnargs = {'keyring_import': True}
+        if sigs_xml is not None:
+            _logger.debug('Keys XML: {0}'.format(etree.tostring(keys_xml).decode('utf-8')))
+        else:
+            _logger.error('No required <keys> in gpg_xml')
+            raise ValueError('<sigs> is required in a GPG verification block')
+        fnargs = {'strict': keys_xml.attrib.get('detect')}
+        if fnargs['strict']:  # We have to manually do this since it's in our parent's __init__
+            if fnargs['strict'].lower() in ('true', '1'):
+                fnargs['strict'] = True
+            else:
+                fnargs['strict'] = False
+        else:
+            fnargs['strict'] = False
        fnargs.update(kwargs)
        if keys_xml is not None:
            fnargs['keys'] = []
            for key_id_xml in keys_xml.findall('keyID'):
+                _logger.debug('Found <keyID>: {0}'.format(etree.tostring(key_id_xml).decode('utf-8')))
                if key_id_xml.text == 'auto':
-                    k = self.gpg.findKeyByID(aif.constants_fallback.ARCH_RELENG_KEY, **fnargs)
-                elif key_id_xml.text == 'detect':
-                    fnargs['strict'] = False
-                    continue
+                    _logger.debug('Key ID was set to "auto"; using {0}'.format(aif.constants_fallback.ARCH_RELENG_KEY))
+                    self.gpg.findKeyByID(aif.constants_fallback.ARCH_RELENG_KEY, source = 'remote',
+                                         keyring_import = True, **fnargs)
+                    k = self.gpg.findKeyByID(aif.constants_fallback.ARCH_RELENG_KEY, source = 'local', **fnargs)
                else:
-                    k = self.gpg.findKeyByID(key_id_xml.text.strip(), **fnargs)
+                    _logger.debug('Finding key: {0}'.format(key_id_xml.text.strip()))
+                    self.gpg.findKeyByID(key_id_xml.text.strip(), source = 'remote', keyring_import = True, **fnargs)
+                    k = self.gpg.findKeyByID(key_id_xml.text.strip(), source = 'local', **fnargs)
+                    if k:
+                        _logger.debug('Key {0} found'.format(k.fpr))
+                    else:
+                        _logger.error('Key {0} not found'.format(key_id_xml.text.strip()))
+                        raise RuntimeError('Could not find key ID specified')
                fnargs['keys'].append(k)
            for key_file_xml in keys_xml.findall('keyFile'):
+                _logger.debug('Found <keyFile>: {0}'.format(etree.tostring(key_file_xml).decode('utf-8')))
                downloader = getDLHandler(key_file_xml.text.strip())  # Recursive objects for the win?
                dl = downloader(key_file_xml)
                dl.get()
-                k = self.gpg.getKeyData(dl.data.read(), **fnargs)[0]
+                k = self.gpg.getKeyData(dl.data.read(), keyring_import = True, **fnargs)[0]
+                if k:
+                    fnargs['keys'].extend(k)
+                else:
+                    pass  # No keys found in key file. We log this in GPG.getKeyData() though.
                dl.data.seek(0, 0)
-                fnargs['keys'].extend(k)
+            if not fnargs['keys']:
+                _logger.debug('Found no keys in keys_xml')
+                raise ValueError('Could not find any keys')
        if sigs_xml is not None:
            for sig_text_xml in sigs_xml.findall('signature'):
+                _logger.debug('Found <signature>')
                sig = sig_text_xml.text.strip()
                sigchk = self.gpg.verifyData(self.data.read(), detached = sig, **fnargs)
                self.data.seek(0, 0)
                results.update(sigchk)
            for sig_file_xml in sigs_xml.findall('signatureFile'):
+                _logger.debug('Found <signatureFile>: {0}'.format(sig_file_xml.text.strip()))
                downloader = getDLHandler(sig_file_xml.text.strip())
                dl = downloader(sig_file_xml)
                dl.get()
@@ -131,6 +204,7 @@ class Downloader(object):
                self.data.seek(0, 0)
                results.update(sigchk)
        self.gpg.clean()
+        _logger.debug('Rendered results: {0}'.format(results))
        return(results)

    def verifyHash(self, hash_xml, *args, **kwargs):
@@ -144,6 +218,7 @@ class Downloader(object):
        self.data.seek(0, 0)
        if checksum_file_xml is not None:
            for cksum_xml in checksum_file_xml:
+                _logger.debug('Found <checksumFile>: {0}'.format(etree.tostring(cksum_xml).decode('utf-8')))
                htype = cksum_xml.attrib['hashType'].strip().lower()
                ftype = cksum_xml.attrib['fileType'].strip().lower()
                fname = cksum_xml.attrib.get('filePath',
@@ -154,14 +229,28 @@ class Downloader(object):
                elif ftype == 'bsd':
                    cksum = cksum_file.hashes[htype][fname]
                result = (cksum == checksums[htype])
+                if result:
+                    _logger.debug('Checksum type {0} matches ({1})'.format(htype, cksum))
+                else:
+                    _logger.warning(('Checksum type {0} mismatch: '
+                                     '{1} (data) vs. {2} (specified)').format(htype, checksums[htype], cksum))
                results.append(result)
        if checksum_xml is not None:
            for cksum_xml in checksum_xml:
+                _logger.debug('Found <checksum>: {0}'.format(etree.tostring(cksum_xml).decode('utf-8')))
                # Thankfully, this is a LOT easier.
                htype = cksum_xml.attrib['hashType'].strip().lower()
                result = (cksum_xml.text.strip().lower() == checksums[htype])
+                if result:
+                    _logger.debug('Checksum type {0} matches ({1})'.format(htype, checksums[htype]))
+                else:
+                    _logger.warning(('Checksum type {0} mismatch: '
+                                     '{1} (data) vs. {2} (specified)').format(htype,
+                                                                              checksums[htype],
+                                                                              cksum_xml.text.strip().lower()))
                results.append(result)
        result = all(results)
+        _logger.debug('Overall result of checksumming: {0}'.format(result))
        return(result)


@@ -176,6 +265,7 @@ class FSDownloader(Downloader):
        with open(self.uri.path, 'rb') as fh:
            self.data.write(fh.read())
        self.data.seek(0, 0)
+        _logger.info('Read in {0} bytes'.format(self.data.getbuffer().nbytes))
        return(None)


@@ -188,11 +278,15 @@ class FTPDownloader(Downloader):
            self.password = ''
        self.port = (self.uri.port if self.uri.port else 0)
        self._conn = None
+        _logger.debug('User: {0}'.format(self.user))
+        _logger.debug('Password: {0}'.format(self.password))
+        _logger.debug('Port: {0}'.format(self.port))

    def _connect(self):
        self._conn = ftplib.FTP()
        self._conn.connect(host = self.uri.base, port = self.port)
        self._conn.login(user = self.user, passwd = self.password)
+        _logger.info('Connected.')
        return(None)

    def get(self):
@@ -201,10 +295,12 @@ class FTPDownloader(Downloader):
        self._conn.retrbinary('RETR {0}'.format(self.uri.path), self.data.write)
        self.data.seek(0, 0)
        self._close()
+        _logger.info('Read in {0} bytes'.format(self.data.getbuffer().nbytes))
        return(None)

    def _close(self):
        self._conn.quit()
+        _logger.info('Closed connection')
        return(None)


@@ -217,6 +313,7 @@ class FTPSDownloader(FTPDownloader):
        self._conn.connect(host = self.uri.base, port = self.port)
        self._conn.login(user = self.user, passwd = self.password)
        self._conn.prot_p()
+        _logger.info('Connected.')
        return(None)


@@ -225,6 +322,7 @@ class HTTPDownloader(Downloader):
        super().__init__(netresource_xml, *args, **kwargs)
        self.auth = self.xml.attrib.get('authType', 'none').lower()
        if self.auth == 'none':
+            _logger.debug('No auth.')
            self.auth = None
            self.realm = None
            self.user = None
@@ -232,14 +330,17 @@ class HTTPDownloader(Downloader):
        else:
            if self.auth == 'basic':
                self.auth = requests.auth.HTTPBasicAuth(self.user, self.password)
+                _logger.info('HTTP basic auth configured.')
            elif self.auth == 'digest':
                self.auth = requests.auth.HTTPDigestAuth(self.user, self.password)
+                _logger.info('HTTP digest auth configured.')

    def get(self):
        self.data.seek(0, 0)
        req = requests.get(self.real_uri, auth = self.auth)
        self.data.write(req.content)
        self.data.seek(0, 0)
+        _logger.info('Read in {0} bytes'.format(self.data.getbuffer().nbytes))
        return(None)


@@ -254,5 +355,6 @@ def getDLHandler(uri):
    elif re.search(r'^ftps://', uri, re.IGNORECASE):
        return(FTPSDownloader)
    else:
+        _logger.error('Unable to detect which download handler to instantiate.')
        raise RuntimeError('Could not detect which download handler to use')
    return(None)