merge trunk at bzr190

author: Scott Moser <[email protected]> 2017-12-19 13:34:22 -0500
committer: Scott Moser <[email protected]> 2017-12-19 13:34:22 -0500
commit: e5c1d36b06042da3c5ca03e130373dbdeb11025d (patch)
tree: 106344dec33e99bc589d2fe71cf764ff1ae28b16
parent: c09def7c57d43ee0f5d3662c695c5d7bf752110f (diff)
parent: fce179949da27c3e82c26e223ed3132138426ed6 (diff)
14 files changed, 666 insertions, 110 deletions
diff --git a/curtin/block/__init__.py b/curtin/block/__init__.py
index 20bc4948..27ee9fbb 100644
--- a/curtin/block/__init__.py
+++ b/curtin/block/__init__.py
@@ -168,24 +168,31 @@ def get_pardevs_on_blockdevs(devs):
     return ret
 
 
-def get_root_device(dev):
+def get_root_device(dev, fpath="curtin"):
     """
-    Get root partition for specified device
+    Get root partition for specified device, based on presence of /curtin.
     """
     partitions = get_pardevs_on_blockdevs(dev)
     target = None
+    tmp_mount = tempfile.mkdtemp()
     for i in partitions:
         dev_path = partitions[i]['device_path']
-        tmp_mount = tempfile.mkdtemp()
+        mp = None
         try:
             util.do_mount(dev_path, tmp_mount)
-            curtin_dir = os.path.join(tmp_mount, 'curtin')
-            if os.path.isdir(curtin_dir) is False:
-                continue
-            target = dev_path
-            util.do_umount(tmp_mount)
+            mp = tmp_mount
+            curtin_dir = os.path.join(tmp_mount, fpath)
+            if os.path.isdir(curtin_dir):
+                target = dev_path
+                break
         except:
-            util.do_umount(tmp_mount)
+            pass
+        finally:
+            if mp:
+                util.do_umount(mp)
+
+    os.rmdir(tmp_mount)
+
     if target is None:
         raise ValueError("Could not find root device")
     return target
diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py
index 622c6525..e08baee3 100644
--- a/curtin/commands/block_meta.py
+++ b/curtin/commands/block_meta.py
@@ -69,7 +69,7 @@ def write_image_to_disk(source, dev):
                     ('wget "$1" --progress=dot:mega -O - |'
                      'tar -SxOzf - | dd of="$2"'),
                     '--', source, devnode])
-    util.subp(['partprobe'])
+    util.subp(['partprobe', devnode])
     util.subp(['udevadm', 'settle'])
     return block.get_root_device([devname, ])
 
@@ -189,14 +189,14 @@ def meta_simple(args):
     if bootpt['enabled'] and ptfmt in ("uefi", "prep"):
         raise ValueError("format=%s with boot partition not supported" % ptfmt)
 
-    if ptfmt == "prep":
-        rootdev = devnode + "2"
-
     if bootpt['enabled']:
         bootdev = devnode + "1"
         rootdev = devnode + "2"
     else:
-        rootdev = devnode + "1"
+        if ptfmt == "prep":
+            rootdev = devnode + "2"
+        else:
+            rootdev = devnode + "1"
         bootdev = None
 
     LOG.debug("rootdev=%s bootdev=%s fmt=%s bootpt=%s",
diff --git a/curtin/commands/curthooks.py b/curtin/commands/curthooks.py
index 9621e57b..3b2f0733 100644
--- a/curtin/commands/curthooks.py
+++ b/curtin/commands/curthooks.py
@@ -294,13 +294,14 @@ def setup_grub(cfg, target):
             blockdevs.add(blockdev)
 
         if platform.machine().startswith("ppc64"):
-            # ppc64 we want the PReP partitions on the installed block devices.
-            # the shnip here prints /dev/xxxN for each N that has 'prep' flags
+            # assume we want partitions that are 4100 (PReP). The snippet here
+            # just prints the partition number partitions of that type.
             shnip = textwrap.dedent("""
                 export LANG=C;
                 for d in "$@"; do
-                  parted --machine "$d" print |
-                    awk -F: "\$7 ~ /prep/ { print d \$1 }" d=$d; done
+                    sgdisk "$d" --print |
+                        awk "\$6 == prep { print d \$1 }" "d=$d" prep=4100
+                done
                 """)
             try:
                 out, err = util.subp(
@@ -308,10 +309,11 @@ def setup_grub(cfg, target):
                     capture=True)
                 instdevs = str(out).splitlines()
                 if not instdevs:
-                    LOG.warn("No PReP partitions found!")
+                    LOG.warn("No power grub target partitions found!")
+                    instdevs = None
             except util.ProcessExecutionError as e:
-                LOG.warn("Failed to find PReP partitions with parted: %s", e)
-                instdevs = ["none"]
+                LOG.warn("Failed to find power grub partitions: %s", e)
+                instdevs = None
         else:
             instdevs = list(blockdevs)
 
@@ -344,7 +346,7 @@ def setup_grub(cfg, target):
     if instdevs:
         instdevs = [block.get_dev_name_entry(i)[1] for i in instdevs]
     else:
-        instdevs = []
+        instdevs = ["none"]
     LOG.debug("installing grub to %s [replace_default=%s]",
               instdevs, replace_default)
     with util.ChrootableTarget(target):
diff --git a/curtin/commands/install.py b/curtin/commands/install.py
index b8fb7a14..ae501454 100644
--- a/curtin/commands/install.py
+++ b/curtin/commands/install.py
@@ -21,15 +21,21 @@ import os
 import re
 import shlex
 import shutil
+import subprocess
+import sys
 import tempfile
 
 from curtin import config
-from curtin.log import LOG
 from curtin import util
-
+from curtin.log import LOG
+from curtin.reporter import (
+    INSTALL_LOG,
+    load_reporter,
+    clear_install_log,
+    writeline_install_log,
+    )
 from . import populate_one_subcmd
 
-
 CONFIG_BUILTIN = {
     'sources': {},
     'stages': ['early', 'partitioning', 'network', 'extract', 'curthooks',
@@ -88,10 +94,27 @@ class WorkingDir(object):
 
 
 class Stage(object):
+
     def __init__(self, name, commands, env):
         self.name = name
         self.commands = commands
         self.env = env
+        self.install_log = self.open_install_log()
+
+    def open_install_log(self):
+        """Open the install log."""
+        try:
+            return open(INSTALL_LOG, 'a')
+        except IOError:
+            return None
+
+    def write(self, data):
+        """Write data to stdout and to the install_log."""
+        sys.stdout.write(data)
+        sys.stdout.flush()
+        if self.install_log is not None:
+            self.install_log.write(data)
+            self.install_log.flush()
 
     def run(self):
         for cmdname in sorted(self.commands.keys()):
@@ -101,17 +124,37 @@ class Stage(object):
             shell = not isinstance(cmd, list)
             with util.LogTimer(LOG.debug, cmdname):
                 try:
-                    util.subp(cmd, shell=shell, env=self.env)
-                except util.ProcessExecutionError:
+                    sp = subprocess.Popen(
+                        cmd,
+                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                        env=self.env, shell=shell)
+                except OSError as e:
+                    LOG.warn("%s command failed", cmdname)
+                    raise util.ProcessExecutionError(cmd=cmd, reason=e)
+
+                output = ""
+                while True:
+                    data = sp.stdout.read(1)
+                    if data == '' and sp.poll() is not None:
+                        break
+                    self.write(data)
+                    output += data
+
+                rc = sp.returncode
+                if rc != 0:
                     LOG.warn("%s command failed", cmdname)
-                    raise
+                    raise util.ProcessExecutionError(
+                        stdout=output, stderr="",
+                        exit_code=rc, cmd=cmd)
 
 
 def apply_power_state(pstate):
-    # power_state:
-    #  delay: 5
-    #  mode: poweroff
-    #  message: Bye Bye
+    """
+    power_state:
+     delay: 5
+     mode: poweroff
+     message: Bye Bye
+    """
     cmd = load_power_state(pstate)
     if not cmd:
         return
@@ -129,7 +172,7 @@ def apply_power_state(pstate):
 
 
 def load_power_state(pstate):
-    # returns a command to reboot the system if power_state should.
+    """Returns a command to reboot the system if power_state should."""
     if pstate is None:
         return None
 
@@ -163,9 +206,11 @@ def load_power_state(pstate):
 
 
 def apply_kexec(kexec, target):
-    # load kexec kernel from target dir, similar to /etc/init.d/kexec-load
-    # kexec:
-    #  mode: on
+    """
+    load kexec kernel from target dir, similar to /etc/init.d/kexec-load
+    kexec:
+     mode: on
+    """
     grubcfg = "boot/grub/grub.cfg"
     target_grubcfg = os.path.join(target, grubcfg)
 
@@ -250,13 +295,17 @@ def cmd_install(args):
     if cfg.get('http_proxy'):
         os.environ['http_proxy'] = cfg['http_proxy']
 
+    # Load MAAS Reporter
+    clear_install_log()
+    maas_reporter = load_reporter(cfg)
+
     try:
         dd_images = util.get_dd_images(cfg.get('sources', {}))
         if len(dd_images) > 1:
             raise ValueError("You may not use more then one disk image")
+
         workingd = WorkingDir(cfg)
         LOG.debug(workingd.env())
-
         env = os.environ.copy()
         env.update(workingd.env())
 
@@ -270,6 +319,13 @@ def cmd_install(args):
             cfg['power_state'] = {'mode': 'reboot', 'delay': 'now',
                                   'message': "'rebooting with kexec'"}
 
+        writeline_install_log("Installation finished.")
+        maas_reporter.report_success()
+    except Exception as e:
+        exp_msg = "Installation failed with exception: %s" % e
+        writeline_install_log(exp_msg)
+        LOG.error(exp_msg)
+        maas_reporter.report_failure(exp_msg)
     finally:
         for d in ('sys', 'dev', 'proc'):
             util.do_umount(os.path.join(workingd.target, d))
@@ -280,9 +336,6 @@ def cmd_install(args):
 
     apply_power_state(cfg.get('power_state'))
 
-    LOG.info("Finished installation")
-    print("Installation finished")
-
 
 CMD_ARGUMENTS = (
     ((('-c', '--config'),
diff --git a/curtin/reporter/__init__.py b/curtin/reporter/__init__.py
new file mode 100644
index 00000000..841c0b4a
--- /dev/null
+++ b/curtin/reporter/__init__.py
@@ -0,0 +1,107 @@
+#   Copyright (C) 2014 Canonical Ltd.
+#
+#   Author: Newell Jensen <[email protected]>
+#
+#   Curtin is free software: you can redistribute it and/or modify it under
+#   the terms of the GNU Affero General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Reporter Abstract Base Class."""
+
+# TODO - make python3 compliant
+# str = None
+
+from abc import (
+    ABCMeta,
+    abstractmethod,
+    )
+from curtin.log import LOG
+from curtin.util import (
+    try_import_module,
+    )
+
+INSTALL_LOG = "/var/log/curtin_install.log"
+
+
+class BaseReporter:
+    """Skeleton for a report."""
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def report_progress(self, progress):
+        """Report installation progress."""
+
+    @abstractmethod
+    def report_success(self):
+        """Report installation success."""
+
+    @abstractmethod
+    def report_failure(self, failure):
+        """Report installation failure."""
+
+
+class EmptyReporter(BaseReporter):
+
+    def report_progress(self, progress):
+        """Empty."""
+
+    def report_success(self):
+        """Empty."""
+
+    def report_failure(self, failure):
+        """Empty."""
+
+
+class LoadReporterException(Exception):
+    """Raise exception if desired reporter not loaded."""
+    pass
+
+
+def load_reporter(config):
+    """Loads and returns reporter instance stored in config file."""
+
+    reporter = config.get('reporter')
+    if reporter is None:
+        LOG.info("'reporter' not found in config file.")
+        return EmptyReporter()
+    name, options = reporter.popitem()
+    module = try_import_module('curtin.reporter.%s' % name)
+    if module is None:
+        LOG.error(
+            "Module for %s reporter could not load." % name)
+        return EmptyReporter()
+    try:
+        return module.load_factory(options)
+    except LoadReporterException:
+        LOG.error(
+            "Failed loading %s reporter with %s" % (name, options))
+        return EmptyReporter()
+
+
+def clear_install_log():
+    """Clear the installation log, so no previous installation is present."""
+    try:
+        open(INSTALL_LOG, 'w').close()
+    except IOError:
+        pass
+
+
+def writeline_install_log(output):
+    """Write output into the install log."""
+    if not output.endswith('\n'):
+        output += '\n'
+    try:
+        with open(INSTALL_LOG, 'a') as fp:
+            fp.write(output)
+    except IOError:
+        pass
diff --git a/curtin/reporter/maas.py b/curtin/reporter/maas.py
new file mode 100644
index 00000000..14662df6
--- /dev/null
+++ b/curtin/reporter/maas.py
@@ -0,0 +1,232 @@
+#   Copyright (C) 2014 Canonical Ltd.
+#
+#   Author: Newell Jensen <[email protected]>
+#
+#   Curtin is free software: you can redistribute it and/or modify it under
+#   the terms of the GNU Affero General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
+
+"""MAAS Reporter."""
+
+from curtin.reporter import (
+    BaseReporter,
+    INSTALL_LOG,
+    LoadReporterException,
+    )
+from email.utils import parsedate
+import mimetypes
+import oauth.oauth as oauth
+import os.path
+import random
+import socket
+import string
+import sys
+import time
+import uuid
+import urllib2
+
+
+class MAASReporter(BaseReporter):
+
+    def __init__(self, config):
+        """Load config dictionary and initialize object."""
+        self.url = config['url']
+        self.consumer_key = config['consumer_key']
+        self.consumer_secret = ''
+        self.token_key = config['token_key']
+        self.token_secret = config['token_secret']
+
+    def report_progress(self, progress, files):
+        """Report installation progress."""
+        status = "WORKING"
+        message = "Installation in progress %s" % progress
+        self.report(files, status, message)
+
+    def report_success(self):
+        """Report installation success."""
+        status = "OK"
+        message = "Installation succeeded."
+        self.report([INSTALL_LOG], status, message)
+
+    def report_failure(self, message):
+        """Report installation failure."""
+        status = "FAILED"
+        self.report([INSTALL_LOG], status, message)
+
+    def oauth_headers(self, url, consumer_key, token_key, token_secret,
+                      consumer_secret, clockskew=0):
+        """Build OAuth headers using given credentials."""
+        consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
+        token = oauth.OAuthToken(token_key, token_secret)
+
+        timestamp = int(time.time()) + clockskew
+
+        params = {
+            'oauth_version': "1.0",
+            'oauth_nonce': uuid.uuid4().get_hex(),
+            'oauth_timestamp': timestamp,
+            'oauth_token': token.key,
+            'oauth_consumer_key': consumer.key,
+        }
+        req = oauth.OAuthRequest(http_url=url, parameters=params)
+        req.sign_request(
+            oauth.OAuthSignatureMethod_PLAINTEXT(), consumer, token)
+        return(req.to_header())
+
+    def authenticate_headers(self, url, headers, creds, clockskew):
+        """Update and sign a dict of request headers."""
+        if creds.get('consumer_key', None) is not None:
+            headers.update(self.oauth_headers(
+                url,
+                consumer_key=creds['consumer_key'],
+                token_key=creds['token_key'],
+                token_secret=creds['token_secret'],
+                consumer_secret=creds['consumer_secret'],
+                clockskew=clockskew))
+
+    def encode_multipart_data(self, data, files):
+        """Create a MIME multipart payload from L{data} and L{files}.
+
+        @param data: A mapping of names (ASCII strings) to data (byte string).
+        @param files: A mapping of names (ASCII strings) to file objects ready
+            to be read.
+        @return: A 2-tuple of C{(body, headers)}, where C{body} is a a byte
+            string and C{headers} is a dict of headers to add to the enclosing
+            request in which this payload will travel.
+        """
+        boundary = self._random_string(30)
+
+        lines = []
+        for name in data:
+            lines.extend(self._encode_field(name, data[name], boundary))
+        for name in files:
+            lines.extend(self._encode_file(name, files[name], boundary))
+        lines.extend(('--%s--' % boundary, ''))
+        body = '\r\n'.join(lines)
+
+        headers = {
+            'content-type': 'multipart/form-data; boundary=' + boundary,
+            'content-length': "%d" % len(body),
+        }
+        return body, headers
+
+    def geturl(self, url, creds, headers=None, data=None):
+        """Create MAAS url for sending the report."""
+        if headers is None:
+            headers = {}
+        else:
+            headers = dict(headers)
+
+        clockskew = 0
+
+        exc = Exception("Unexpected Error")
+        for naptime in (1, 1, 2, 4, 8, 16, 32):
+            self.authenticate_headers(url, headers, creds, clockskew)
+            try:
+                req = urllib2.Request(url=url, data=data, headers=headers)
+                return urllib2.urlopen(req).read()
+            except urllib2.HTTPError as exc:
+                if 'date' not in exc.headers:
+                    sys.stderr.write("date field not in %d headers" % exc.code)
+                    pass
+                elif exc.code in (401, 403):
+                    date = exc.headers['date']
+                    try:
+                        ret_time = time.mktime(parsedate(date))
+                        clockskew = int(ret_time - time.time())
+                        sys.stderr.write("updated clock skew to %d" %
+                                         clockskew)
+                    except:
+                        sys.stderr.write("failed to convert date '%s'" % date)
+            except Exception as exc:
+                pass
+
+            sys.stderr.write(
+                "request to %s failed. sleeping %d.: %s" % (url, naptime, exc))
+            time.sleep(naptime)
+
+        raise exc
+
+    def report(self, files, status, message=None):
+        """Send the report."""
+
+        creds = {
+            'consumer_key': self.consumer_key,
+            'token_key': self.token_key,
+            'token_secret': self.token_secret,
+            'consumer_secret': self.consumer_secret,
+            }
+
+        params = {}
+        params['status'] = status
+        if message is not None:
+            params['error'] = message
+
+        install_files = {}
+        for fpath in files:
+            install_files[os.path.basename(fpath)] = open(fpath, "r")
+
+        data, headers = self.encode_multipart_data(params, install_files)
+
+        exc = None
+        msg = ""
+
+        try:
+            payload = self.geturl(self.url, creds=creds, headers=headers,
+                                  data=data)
+            if payload != "OK":
+                raise TypeError("Unexpected result from call: %s" % payload)
+            else:
+                msg = "Success"
+        except urllib2.HTTPError as exc:
+            msg = "http error [%s]" % exc.code
+        except urllib2.URLError as exc:
+            msg = "url error [%s]" % exc.reason
+        except socket.timeout as exc:
+            msg = "socket timeout [%s]" % exc
+        except TypeError as exc:
+            msg = exc.message
+        except Exception as exc:
+            msg = "unexpected error [%s]" % exc
+
+        sys.stderr.write("%s\n" % msg)
+
+    def _encode_field(self, field_name, data, boundary):
+        return (
+            '--' + boundary,
+            'Content-Disposition: form-data; name="%s"' % field_name,
+            '', str(data),
+            )
+
+    def _encode_file(self, name, fileObj, boundary):
+        return (
+            '--' + boundary,
+            'Content-Disposition: form-data; name="%s"; filename="%s"'
+            % (name, name),
+            'Content-Type: %s' % self._get_content_type(name),
+            '',
+            fileObj.read(),
+            )
+
+    def _random_string(self, length):
+        return ''.join(random.choice(string.letters)
+                       for ii in range(length + 1))
+
+    def _get_content_type(self, filename):
+        return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+
+
+def load_factory(options):
+    try:
+        return MAASReporter(options)
+    except Exception:
+        raise LoadReporterException
diff --git a/curtin/util.py b/curtin/util.py
index 6709d771..78ed7850 100644
--- a/curtin/util.py
+++ b/curtin/util.py
@@ -26,39 +26,38 @@ import time
 from .log import LOG
 from . import config
 
-_INSTALLED_HELPERS_PATH = "/usr/lib/curtin/helpers"
-_INSTALLED_MAIN = "/usr/bin/curtin"
+_INSTALLED_HELPERS_PATH = '/usr/lib/curtin/helpers'
+_INSTALLED_MAIN = '/usr/bin/curtin'
 
 
 def subp(args, data=None, rcs=None, env=None, capture=False, shell=False,
          logstring=False):
     if rcs is None:
         rcs = [0]
-    try:
 
+    try:
         if not logstring:
             LOG.debug(("Running command %s with allowed return codes %s"
                        " (shell=%s, capture=%s)"), args, rcs, shell, capture)
         else:
             LOG.debug(("Running hidden command to protect sensitive "
                        "input/output logstring: %s"), logstring)
-
-        if not capture:
-            stdout = None
-            stderr = None
-        else:
+        stdin = None
+        stdout = None
+        stderr = None
+        if capture:
             stdout = subprocess.PIPE
             stderr = subprocess.PIPE
-        stdin = subprocess.PIPE
+        if data is not None:
+            stdin = subprocess.PIPE
         sp = subprocess.Popen(args, stdout=stdout,
                               stderr=stderr, stdin=stdin,
                               env=env, shell=shell)
         (out, err) = sp.communicate(data)
         if isinstance(out, bytes):
-            out = out.decode()
+            out = out.decode('utf-8')
         if isinstance(err, bytes):
-            err = err.decode()
-
+            err = err.decode('utf-8')
     except OSError as e:
         raise ProcessExecutionError(cmd=args, reason=e)
     rc = sp.returncode  # pylint: disable=E1101
@@ -66,7 +65,7 @@ def subp(args, data=None, rcs=None, env=None, capture=False, shell=False,
         raise ProcessExecutionError(stdout=out, stderr=err,
                                     exit_code=rc,
                                     cmd=args)
-    # Just ensure blank instead of none?? (iff capturing)
+    # Just ensure blank instead of none?? (if capturing)
     if not out and capture:
         out = ''
     if not err and capture:
@@ -583,4 +582,19 @@ def human2bytes(size):
 
     return int(num * mpliers[mplier])
 
+
+def import_module(import_str):
+    """Import a module."""
+    __import__(import_str)
+    return sys.modules[import_str]
+
+
+def try_import_module(import_str, default=None):
+    """Try to import a module."""
+    try:
+        return import_module(import_str)
+    except ImportError:
+        return default
+
+
 # vi: ts=4 expandtab syntax=python
diff --git a/debian/control b/debian/control
index b523eb4b..cbe7cb65 100644
--- a/debian/control
+++ b/debian/control
@@ -7,13 +7,15 @@ Build-Depends: debhelper (>= 7),
                pep8,
                pyflakes,
                python-all,
+               python-mock,
                python-nose,
                python-setuptools,
                python-yaml,
                python3,
                python3-nose,
                python3-setuptools,
-               python3-yaml
+               python3-yaml,
+               python3-mock
 Homepage: http://launchpad.net/curtin
 X-Python3-Version: >= 3.2
 
diff --git a/doc/devel/README.txt b/doc/devel/README.txt
index df1bc88c..0ee7f0d1 100644
--- a/doc/devel/README.txt
+++ b/doc/devel/README.txt
@@ -9,15 +9,15 @@ sudo apt-get -qy install kvm libvirt-bin cloud-utils bzr
 ## get cloud image to boot (-disk1.img) and one to install (-root.tar.gz)
 mkdir -p ~/download
 DLDIR=$( cd ~/download && pwd )
-rel="precise"
-rel="raring"
+rel="trusty"
+arch=amd64
 burl="http://cloud-images.ubuntu.com/$rel/current/"
-for f in $rel-server-cloudimg-amd64-root.tar.gz $rel-server-cloudimg-amd64-disk1.img; do
+for f in $rel-server-cloudimg-${arch}-root.tar.gz $rel-server-cloudimg-${arch}-disk1.img; do
   wget "$burl/$f" -O $DLDIR/$f; done
-( cd $DLDIR && qemu-img convert -O qcow $rel-server-cloudimg-amd64-disk1.img $rel-server-cloudimg-amd64-disk1.qcow2)
+( cd $DLDIR && qemu-img convert -O qcow $rel-server-cloudimg-${arch}-disk1.img $rel-server-cloudimg-${arch}-disk1.qcow2)
 
-BOOTIMG="$DLDIR/$rel-server-cloudimg-amd64-disk1.qcow2"
-ROOTTGZ="$DLDIR/$rel-server-cloudimg-amd64-root.tar.gz"
+BOOTIMG="$DLDIR/$rel-server-cloudimg-${arch}-disk1.qcow2"
+ROOTTGZ="$DLDIR/$rel-server-cloudimg-${arch}-root.tar.gz"
 
 ## get curtin
 mkdir -p ~/src
diff --git a/helpers/common b/helpers/common
index 5374a646..e78ec42a 100644
--- a/helpers/common
+++ b/helpers/common
@@ -2,9 +2,14 @@
 TEMP_D=""
 CR="
 "
+VERBOSITY=${VERBOSITY:-${CURTIN_VERBOSITY:-0}}
 
-error() { echo "$@" 1>&1; }
-debug() { [ ${VERBOSITY:-0} -lt "$1" ] || error "$@"; }
+error() { echo "$@" 1>&2; }
+debug() {
+    [ ${VERBOSITY:-0} -ge "$1" ] || return
+    shift
+    error "$@"
+}
 
 partition_main_usage() {
     cat <<EOF
@@ -42,19 +47,45 @@ cleanup() {
 
 wipedev() {
     # wipe the front and end (gpt is at end also)
-    local target="$1" size="" out=""
+    local target="$1" size="" out="" bs="" count="" seek="" mb=$((1024*1024))
+    local info=""
     getsize "$target" ||
         { error "failed to get size of $target"; return 1; }
     size="$_RET"
-    
-    dd if=/dev/zero conv=notrunc of="$target" \
-        bs=$((1024*1024)) count=1 >/dev/null 2>&1 ||
-        { error "failed to zero beginning of $target"; return 1; }
 
-    out=$(dd if=/dev/zero conv=notrunc of="$target" bs=1024 \
-        seek=$(((size/1024)-1024)) count=1024 2>&1)
-    [ $? -eq 0 ] ||
-        { error "failed to wipe end of $target [$size]: $out"; return 1; }
+    # select a block size that evenly divides size. bigger is generally faster.
+    for bs in $mb 4096 1024 512 1; do
+        [ "$((size % bs))" = "0" ] && break
+    done
+    if [ "$bs" = "1" ]; then
+        error "WARN: odd sized '$target' ($size). not divisible by 512."
+    fi
+    [ "$size" -ge "$mb" ] && count=$((mb / bs)) || count=$((size / bs))
+
+    info="size=$size count=$count bs=$bs"
+    debug 1 "wiping start of '$target' with ${info}."
+    # wipe the first MB (up to 'size')
+    out=$(dd if=/dev/zero conv=notrunc "of=$target" \
+            "bs=$bs" "count=$count" 2>&1) || {
+        error "wiping start of '$target' failed."
+        error "  size=$size count=$count bs=$bs: $out"
+        return 1
+    }
+
+    if [ "$size" -gt "$mb" ]; then
+        # do the last 1MB
+        count=$((mb / bs))
+        seek=$(((size / bs) - $count))
+        info="size=$size count=$count bs=$bs seek=$seek"
+        debug 1 "wiping end of '$target' with ${info}."
+        out=$(dd if=/dev/zero conv=notrunc "of=$target" "seek=$seek" \
+            "bs=$bs" "count=$count" 2>&1)
+        if [ $? -ne 0 ]; then
+            error "wiping end of '$target' failed."
+            error "  size=$size count=$count seek=$seek bs=$bs: $out";
+            return 1;
+        fi
+    fi
 
     if [ -b "$target" ]; then
         blockdev --rereadpt "$target"
@@ -62,6 +93,23 @@ wipedev() {
     fi
 }
 
+part2bd() {
+    # part2bd given a partition, return the block device it is on
+    # and the number the partition is.  ie, 'sda2' -> '/dev/sda 2'
+    local dev="$1" fp="" sp="" bd="" ptnum=""
+    dev="/dev/${dev#/dev/}"
+    fp=$(readlink -f "$dev") || return 1
+    sp="/sys/class/block/${fp##*/}"
+    [ -f "$sp/partition" ] || { _RET="$fp 0"; return 0; }
+    read ptnum < "$sp/partition"
+    sp=$(readlink -f "$sp") || return 1
+    # sp now has some /sys/devices/pci..../0:2:0:0/block/sda/sda1
+    bd=${sp##*/block/}
+    bd="${bd%/*}"
+    _RET="/dev/$bd $ptnum"
+    return 0
+}
+
 pt_gpt() {
     local target="$1" end=${2:-""} boot="$3" size="" s512=""
     local start="2048" rootsize="" bootsize="1048576" maxend=""
@@ -218,7 +266,7 @@ pt_prep() {
     [ -b "$target" ] && isblk=true
 
     sgdisk --zap-all "$target" ||
-        fail "failed to clear $target";
+        { error "failed to clear $target"; return 1; }
 
     cmd=(
         sgdisk 
@@ -324,6 +372,7 @@ human2bytes() {
 }
 
 getsize() {
+    # return size of target in bytes
     local target="$1"
     if [ -b "$target" ]; then
         _RET=$(blockdev --getsize64 "$target")
@@ -371,13 +420,11 @@ install_grub() {
     fi
 
     # find the mp device
-    mp_dev=$(awk -v MP=${mp} '$2==MP {print$1}' /proc/mounts)
-    r=$?
-    if [ $r -ne 0 -a $r -ne 1 ]; then
+    mp_dev=$(awk -v MP=${mp} '$2==MP {print$1}' /proc/mounts) || {
         error "unable to determine device for mount $mp";
         return 1;
-    fi
-    [ -b $mp_dev ] || { error "$mp_dev is not a block device!"; return 1; }
+    }
+    [ -b "$mp_dev" ] || { error "$mp_dev is not a block device!"; return 1; }
 
     # get dpkg arch
     local dpkg_arch=""
@@ -441,7 +488,7 @@ install_grub() {
 
     # copy anything after '--' on cmdline to install'd cmdline
     read cmdline < /proc/cmdline
-    local reconf="" newargs=""
+    local newargs=""
 
     tmp="${cmdline##* -- }"
     if [ "$tmp" != "$cmdline" ]; then
@@ -462,25 +509,28 @@ install_grub() {
             [ "${p#console}" = "$p" ] || c="$c $p"; done; echo "${c# }")
     fi
 
-    if [ "${REPLACE_GRUB_LINUX_DEFAULT:-1}" != "0" ]; then
-        local n="GRUB_CMDLINE_LINUX_DEFAULT"
-        local sede="s|$n=.*|$n=\"$newargs\"|"
-        sed -i "$sede" "$mp/etc/default/grub" ||
-            { error "failed to update /etc/default/grub"; return 1; }
-        grep "$n" "$mp/etc/default/grub"
-        reconf="dpkg-reconfigure $grub_name"
-        debug 1 "updating cmdline to '${newargs}'"
-
-        # LP: #1179940 . this fix was applied to raring, which
-        # made changes above not stick.  This might not be the best
-        # way to handle this, but we'll do it for now.
-        local cicfg="etc/default/grub.d/50-cloudimg-settings.cfg"
-        if [ -f "$mp/$cicfg" ]; then
-            debug 1 "moved $cicfg out of the way"
-            mv "$mp/$cicfg" "$mp/$cicfg.disabled"
-        fi
+    local grub_d="etc/default/grub.d"
+    local mygrub_cfg="$grub_d/50-curtin-settings.cfg"
+    [ -d "$mp/$grub_d" ] || mkdir -p "$mp/$grub_d" ||
+        { error "Failed to create $grub_d"; return 1; }
+
+    # LP: #1179940 . The 50-cloudig-settings.cfg file is written by the cloud
+    # images build and defines/override some settings. Disable it.
+    local cicfg="$grub_d/50-cloudimg-settings.cfg"
+    if [ -f "$mp/$cicfg" ]; then
+       debug 1 "moved $cicfg out of the way"
+       mv "$mp/$cicfg" "$mp/$cicfg.disabled"
     fi
 
+    : > "$mp/$mygrub_cfg" ||
+        { error "Failed to write '$mygrub_cfg'"; return 1; }
+    {
+        [ "${REPLACE_GRUB_LINUX_DEFAULT:-1}" = "0" ] ||
+            echo "GRUB_CMDLINE_LINUX_DEFAULT=\"$newargs\""
+        echo "# disable grub os prober that might find other OS installs."
+        echo "GRUB_DISABLE_OS_PROBER=true"
+    } >> "$mp/$mygrub_cfg"
+
     local short="" bd="" grubdev grubdevs_new=""
     grubdevs_new=()
     for grubdev in "${grubdevs[@]}"; do
@@ -489,7 +539,7 @@ install_grub() {
             for bd in "/sys/block/$short/slaves/"/*; do
                 [ -d "$bd" ] || continue
                 bd=${bd##*/}
-                bd="/dev/${bd%[0-9]}" # hack: part2bd
+                bd="/dev/${bd%[0-9]}" # FIXME: part2bd
                 grubdevs_new[${#grubdevs_new[@]}]="$bd"
             done
         else
@@ -501,21 +551,19 @@ install_grub() {
     if [ "$uefi" -ge 1 ]; then
         debug 1 "installing ${grub_name} to: /boot/efi"
         chroot "$mp" env DEBIAN_FRONTEND=noninteractive sh -ec '
-            prober="/etc/grub.d/30_os-prober"
-            [ -x $prober ] && chmod -x "$prober"
-            dpkg-reconfigure "$1"
-            [ -f $prober ] && chmod +x "$prober"
-            grub-install --target=$2 --efi-directory=/boot/efi --bootloader-id=ubuntu --recheck || exit; done' \
-            -- "${grub_name}" "${grub_target}" </dev/null ||
+            pkg="$1"; shift;
+            dpkg-reconfigure "$pkg"
+            update-grub
+            grub-install --target=$2 --efi-directory=/boot/efi \
+                --bootloader-id=ubuntu --recheck' -- \
+            "${grub_name}" "${grub_target}" </dev/null ||
             { error "failed to install grub!"; return 1; }
     else
         debug 1 "installing ${grub_name} to: ${grubdevs[*]}"
         chroot "$mp" env DEBIAN_FRONTEND=noninteractive sh -ec '
             pkg=$1; shift;
-            prober="/etc/grub.d/30_os-prober"
-            [ -x $prober ] && chmod -x "$prober"
             dpkg-reconfigure "$pkg"
-            [ -f $prober ] && chmod +x "$prober"
+            update-grub
             for d in "$@"; do grub-install "$d" || exit; done' \
             -- "${grub_name}" "${grubdevs[@]}" </dev/null ||
             { error "failed to install grub!"; return 1; }
diff --git a/setup.py b/setup.py
index d6a1f84f..85110cb3 100644
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,7 @@ setup(
     author_email='[email protected]',
     license="AGPL",
     url='http://launchpad.net/curtin/',
-    packages=['curtin', 'curtin.commands', 'curtin.block', 'curtin.net'],
+    packages=['curtin', 'curtin.commands', 'curtin.block', 'curtin.net', 'curtin.reporter'],
     scripts=glob('bin/*'),
     data_files=[
         ('/usr/share/doc/curtin',
diff --git a/tests/unittests/test_reporter.py b/tests/unittests/test_reporter.py
new file mode 100644
index 00000000..5f036434
--- /dev/null
+++ b/tests/unittests/test_reporter.py
@@ -0,0 +1,80 @@
+#   Copyright (C) 2014 Canonical Ltd.
+#
+#   Author: Newell Jensen <[email protected]>
+#
+#   Curtin is free software: you can redistribute it and/or modify it under
+#   the terms of the GNU Affero General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+from unittest import TestCase
+from mock import patch
+
+from curtin.reporter import (
+    EmptyReporter,
+    load_reporter,
+    )
+# #XXX: see `XXX` below for details
+# from curtin.reporter.maas import load_factory as maas_load_factory
+# from curtin.reporter.maas import MAASReporter
+
+
+class TestReporter(TestCase):
+
+    @patch('curtin.reporter.LOG')
+    def test_load_reporter_logs_empty_cfg(self, mock_LOG):
+        cfg = {}
+        reporter = load_reporter(cfg)
+        self.assertIsInstance(reporter, EmptyReporter)
+        self.assertTrue(mock_LOG.info.called)
+
+    @patch('curtin.reporter.LOG')
+    def test_load_reporter_logs_cfg_with_no_module(
+            self, mock_LOG):
+        cfg = {'reporter': {'empty': {}}}
+        reporter = load_reporter(cfg)
+        self.assertIsInstance(reporter, EmptyReporter)
+        self.assertTrue(mock_LOG.error.called)
+
+    @patch('curtin.reporter.LOG')
+    def test_load_reporter_logs_cfg_wrong_options(self, mock_LOG):
+        # we are passing wrong config options for maas reporter
+        # to test load_reporter in event reporter options are wrong
+        cfg = {'reporter': {'maas': {'wrong': 'wrong'}}}
+        reporter = load_reporter(cfg)
+        self.assertIsInstance(reporter, EmptyReporter)
+        self.assertTrue(mock_LOG.error.called)
+
+# # XXX newell 2014-09-10 bug=1367493: For Python3 compliance all
+# # oauth usage in MAASReporter will need to be changed to oauthlib
+# # Until this bug is fixed, the below tests will break `make test`
+# # and should be commented out.
+# class TestMAASReporter(TestCase):
+#
+#     def test_load_factory_raises_exception_wrong_options(self):
+#         options = {'wrong': 'wrong'}
+#         self.assertRaises(
+#             LoadReporterException, maas_load_factory, options)
+#
+#     def test_load_factory_returns_maas_reporter_good_options(self):
+#         options = {
+#             'url': 'url', 'consumer_key': 'consumer_key',
+#             'token_key': 'token_key', 'token_secret': 'token_secret'}
+#         reporter = maas_load_factory(options)
+#         self.assertIsInstance(reporter, MAASReporter)
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 9f8ffee8..b84407f2 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -98,4 +98,15 @@ class TestWhich(TestCase):
                            target="/target")
         self.assertEqual(found, "/usr/bin2/fuzz")
 
+
+class TestSubp(TestCase):
+
+    def test_subp_handles_utf8(self):
+        # The given bytes contain utf-8 accented characters as seen in e.g.
+        # the "deja dup" package in Ubuntu.
+        input_bytes = b'd\xc3\xa9j\xc8\xa7'
+        cmd = ['echo', '-n', input_bytes]
+        (out, _err) = util.subp(cmd, capture=True)
+        self.assertEqual(out, input_bytes.decode('utf-8'))
+
 # vi: ts=4 expandtab syntax=python
diff --git a/tools/launch b/tools/launch
index ea4002f1..fc9414eb 100755
--- a/tools/launch
+++ b/tools/launch
@@ -189,8 +189,8 @@ main() {
 			size="${disk##*:}"
 		fi
 		if [ ! -f "$src" ]; then
-			qemu-img create -f raw "${src}" 5G ||
-				{ error "failed create $src"; return 1; }
+			qemu-img create -f raw "${src}" "$size" ||
+				{ error "failed create $src of size $size"; return 1; }
 		fi
 		disk_args=( "${disk_args[@]}"
 			"-drive" "file=${src},if=virtio,cache=unsafe" )
@@ -211,9 +211,9 @@ main() {
 		fpath=$(readlink -f "$src") ||
 			{ error "'$src': failed to get path"; return 1; }
 		if [ -n "$pub" ]; then
-			pub="${fpath##*/}"
+			pub="${src##*/}"
 		fi
-		ln -sf "$src" "${TEMP_D}/${pub}"
+		ln -sf "$fpath" "${TEMP_D}/${pub}"
 	done
 
 	# now replace PUBURL anywhere in cmdargs
author	Scott Moser <[email protected]>	2017-12-19 13:34:22 -0500
committer	Scott Moser <[email protected]>	2017-12-19 13:34:22 -0500
commit	e5c1d36b06042da3c5ca03e130373dbdeb11025d (patch)
tree	106344dec33e99bc589d2fe71cf764ff1ae28b16
parent	c09def7c57d43ee0f5d3662c695c5d7bf752110f (diff)
parent	fce179949da27c3e82c26e223ed3132138426ed6 (diff)