diff options
| author | Corey Bryant <corey.bryant@canonical.com> | 2023-06-08 16:53:16 -0400 |
|---|---|---|
| committer | Corey Bryant <corey.bryant@canonical.com> | 2023-06-08 16:53:16 -0400 |
| commit | b46e1e668afeb37d8d4aece047fdf469bb08c4a4 (patch) | |
| tree | 3afd72a62d97414759ee62ec08ab10ea18c42d74 | |
| parent | 135442e291e8c2cab38e33127935d87af94cbcf6 (diff) | |
New upstream version 3.29.03.29.0upstream-flamingoupstream
64 files changed, 1950 insertions, 200 deletions
diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index d73765c..3e4581b 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -61,6 +61,7 @@ - centos-minimal/8-stream-aarch64-build-succeeds - centos-minimal/9-stream-aarch64-build-succeeds - openeuler-minimal/22.03-LTS-build-succeeds + - rocky-container/rocky-9-aarch64-build-succeeds # # Build and boot functional tests @@ -119,7 +120,7 @@ # mirror: "http://{{ zuul_site_mirror_fqdn }}/centos" - job: - name: dib-nodepool-functional-openstack-fedora-36-containerfile-src + name: dib-nodepool-functional-openstack-fedora-37-containerfile-src description: | Test building and booting an Fedora image from a containerfile with Nodepool and OpenStack (using released @@ -129,11 +130,7 @@ nodepool_debug: true nodepool_diskimage: base_element : fedora-container - release: '36' - # NOTE(ianw) 2022-07-11 : something in jammy has broken running - # podman inside docker. Investigations continue, but force this - # back for now. - nodeset: ubuntu-focal + release: '37' - job: name: dib-nodepool-functional-openstack-rockylinux-8-containerfile-src @@ -147,10 +144,6 @@ nodepool_diskimage: base_element: rocky-container release: '8' - # NOTE(ianw) 2022-07-11 : something in jammy has broken running - # podman inside docker. Investigations continue, but force this - # back for now. - nodeset: ubuntu-focal - job: name: dib-nodepool-functional-openstack-rockylinux-9-containerfile-src @@ -164,10 +157,6 @@ nodepool_diskimage: base_element: rocky-container release: '9' - # NOTE(ianw) 2022-07-11 : something in jammy has broken running - # podman inside docker. Investigations continue, but force this - # back for now. - nodeset: ubuntu-focal - job: name: dib-nodepool-functional-openstack-ubuntu-xenial-src diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index 274cca1..aaeace9 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -15,8 +15,16 @@ nodes: - name: ubuntu-bionic label: ubuntu-bionic - - tox-py38 - - tox-py39 + - tox-py38: + nodeset: + nodes: + - name: ubuntu-focal + label: ubuntu-focal + - tox-py39: + nodeset: + nodes: + - name: ubuntu-focal + label: ubuntu-focal - tox-py310: nodeset: nodes: @@ -28,7 +36,7 @@ - dib-nodepool-functional-openstack-centos-7-src - dib-nodepool-functional-openstack-centos-8-stream-src - dib-nodepool-functional-openstack-centos-9-stream-src - - dib-nodepool-functional-openstack-fedora-36-containerfile-src + - dib-nodepool-functional-openstack-fedora-37-containerfile-src - dib-nodepool-functional-openstack-rockylinux-8-containerfile-src: voting: false - dib-nodepool-functional-openstack-rockylinux-9-containerfile-src: @@ -61,8 +69,16 @@ nodes: - name: ubuntu-bionic label: ubuntu-bionic - - tox-py38 - - tox-py39 + - tox-py38: + nodeset: + nodes: + - name: ubuntu-focal + label: ubuntu-focal + - tox-py39: + nodeset: + nodes: + - name: ubuntu-focal + label: ubuntu-focal - tox-py310: nodeset: nodes: @@ -73,7 +89,7 @@ - dib-nodepool-functional-openstack-centos-7-src - dib-nodepool-functional-openstack-centos-8-stream-src - dib-nodepool-functional-openstack-centos-9-stream-src - - dib-nodepool-functional-openstack-fedora-36-containerfile-src + - dib-nodepool-functional-openstack-fedora-37-containerfile-src - dib-nodepool-functional-openstack-ubuntu-xenial-src - dib-nodepool-functional-openstack-ubuntu-bionic-src - dib-nodepool-functional-openstack-ubuntu-focal-src @@ -87,6 +87,7 @@ Dustin J. Mitchell <dustin@v.igoro.us> Eduardo Santos <eduardo.experimental@gmail.com> Emilien Macchi <emilien@redhat.com> Endre Karlson <endre.karlson@hp.com> +Erik Berg <openstack@slipsprogrammor.no> Erno Kuvaja <jokke@usr.fi> Erwan Velu <erwan@enovance.com> Ethan Gafford <egafford@redhat.com> @@ -95,6 +96,7 @@ Fatih Degirmenci <fdegir@gmail.com> Felipe Alencastro <felipe@alencastro.org> Francois RIGAULT <rigault.francois@gmail.com> Frederic Lepied <frederic.lepied@enovance.com> +Frikin Evgenii <frikin.evgenii@huawei.com> Gabriele Cerami <gcerami@redhat.com> Gary Duan <duanlg@live.cn> Gaëtan Trellu <gaetan.trellu@incloudus.com> @@ -147,6 +149,7 @@ Jiri Stransky <jistr@redhat.com> Joe Gordon <joe.gordon0@gmail.com> Joey D <jobewan@gmail.com> John L. Villalovos <john.l.villalovos@intel.com> +John L. Villalovos <openstack.org@sodarock.com> John Trowbridge <trown@redhat.com> Jon-Paul Sullivan <jonpaul.sullivan@hp.com> Jonas Sticha <jonas.sticha@bmw-carit.de> @@ -295,6 +298,7 @@ Tristan Cacqueray <tdecacqu@redhat.com> Tristan Cacqueray <tristan.cacqueray@enovance.com> Uwe Grawert <uwe.grawert@csc.fi> Van Hung Pham <hungpv@vn.fujitsu.com> +Vanou Ishii <ishii.vanou@fujitsu.com> Victor Lowther <victor.lowther@gmail.com> Vipul Sabhaya <vipuls@gmail.com> Vu Cong Tuan <tuanvc@vn.fujitsu.com> @@ -331,6 +335,7 @@ iberezovskiy <iberezovskiy@mirantis.com> inspurericzhang <zhanglf01@inspur.com> jacky06 <zhang.min@99cloud.net> janonymous <janonymous.codevulture@gmail.com> +jgupta <jayadityagupta11@gmail.com> jodewey <jodewey@cisco.com> justin-hopper <justin.hopper@hp.com> kaiokassiano <kaiokmo@lsd.ufcg.edu.br> @@ -1,6 +1,48 @@ CHANGES ======= +3.29.0 +------ + +* Rename openeuler mirror script +* Check and grow the GPT structure to the end of the volume + +3.28.0 +------ + +* Fix jsonschema version to match lower-constraint +* Fix openeuler mirror problem +* Fix ubuntu-minimal to run autoremove +* Add a FIPS element + +3.27.0 +------ + +* Correct boot path to cover FIPS usage cases +* Update Fedora to 37 +* Removal focal pins for testing +* Document diskimage-builder command +* Switch run\_functests.sh from disk-image-create to diskimage-builder +* A new diskimage-builder command for yaml image builds +* chore: support building Fedora on arm64 AKA aarch64 +* Add swap support +* Fix double-keyed json +* Repeat to umount filesystem when exception occurs + +3.26.0 +------ + +* cache-url: Give up on curl install for Redhat platforms +* Clean up tox.ini for tox v4 +* Reduce thin pool by one more extent +* Grow thin pool metadata by 1GiB +* Add variable for check installing python3 in yum element +* tox jobs: pin to correct nodesets; use host networking for containerfile +* Fix issue in extract image +* Added example configuration +* Added cloud-init growpart element +* Add Rocky 9 ARM64 functional test + 3.25.0 ------ @@ -22,6 +64,7 @@ CHANGES * rocky : create machine-id in 9 * Allow setting ROOT\_LABEL from environment * Do dmsetup remove device in rollback +* Fix wrong yum.conf name of CentOS 9 Stream 3.23.0 ------ @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: diskimage-builder -Version: 3.25.0 +Version: 3.29.0 Summary: Golden Disk Image builder. Home-page: https://docs.openstack.org/diskimage-builder/latest/ Author: OpenStack diff --git a/diskimage_builder.egg-info/PKG-INFO b/diskimage_builder.egg-info/PKG-INFO index f4a0d1a..5b83273 100644 --- a/diskimage_builder.egg-info/PKG-INFO +++ b/diskimage_builder.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: diskimage-builder -Version: 3.25.0 +Version: 3.29.0 Summary: Golden Disk Image builder. Home-page: https://docs.openstack.org/diskimage-builder/latest/ Author: OpenStack diff --git a/diskimage_builder.egg-info/SOURCES.txt b/diskimage_builder.egg-info/SOURCES.txt index f7d1929..6aeb028 100644 --- a/diskimage_builder.egg-info/SOURCES.txt +++ b/diskimage_builder.egg-info/SOURCES.txt @@ -16,6 +16,7 @@ tox.ini bin/dib-lint diskimage_builder/__init__.py diskimage_builder/disk_image_create.py +diskimage_builder/diskimage_builder.py diskimage_builder/element_dependencies.py diskimage_builder/logging_config.py diskimage_builder/paths.py @@ -199,6 +200,12 @@ diskimage_builder/elements/cloud-init-datasources/install.d/05-set-cloud-init-so diskimage_builder/elements/cloud-init-disable-resizefs/README.rst diskimage_builder/elements/cloud-init-disable-resizefs/post-install.d/50-cloud-init-disable-resizefs diskimage_builder/elements/cloud-init-disable-resizefs/post-install.d/51-cloud-init-disable-growpart +diskimage_builder/elements/cloud-init-growpart/README.rst +diskimage_builder/elements/cloud-init-growpart/element-deps +diskimage_builder/elements/cloud-init-growpart/package-installs.yaml +diskimage_builder/elements/cloud-init-growpart/pkg-map +diskimage_builder/elements/cloud-init-growpart/post-install.d/55-growpart +diskimage_builder/elements/cloud-init-growpart/releasenotes/notes/added_growpart_for_lvm-0ce76ba71710c720.yaml diskimage_builder/elements/cloud-init-nocloud/README.rst diskimage_builder/elements/cloud-init-nocloud/element-deps diskimage_builder/elements/cloud-init-nocloud/environment.d/10-cloud-init-nocloud.bash @@ -393,6 +400,7 @@ diskimage_builder/elements/fedora-container/pkg-map diskimage_builder/elements/fedora-container/containerfiles/34 diskimage_builder/elements/fedora-container/containerfiles/35 diskimage_builder/elements/fedora-container/containerfiles/36 +diskimage_builder/elements/fedora-container/containerfiles/37 diskimage_builder/elements/fedora-container/environment.d/10-01-fedora-distro-name.bash diskimage_builder/elements/fedora-container/environment.d/11-yum-dnf.bash diskimage_builder/elements/fedora-container/test-elements/build-succeeds/README.rst @@ -411,6 +419,9 @@ diskimage_builder/elements/fedora/pre-install.d/00-02-set-fedora-mirror diskimage_builder/elements/fedora/pre-install.d/02-set-machine-id diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps +diskimage_builder/elements/fips/README.rst +diskimage_builder/elements/fips/package-installs.yaml +diskimage_builder/elements/fips/pre-install.d/01-setup-fips diskimage_builder/elements/gentoo/README.rst diskimage_builder/elements/gentoo/element-deps diskimage_builder/elements/gentoo/element-provides @@ -556,7 +567,7 @@ diskimage_builder/elements/openeuler-minimal/element-provides diskimage_builder/elements/openeuler-minimal/package-installs.yaml diskimage_builder/elements/openeuler-minimal/environment.d/10-openeuler-distro-name.bash diskimage_builder/elements/openeuler-minimal/environment.d/11-yum-dnf.bash -diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror.bash +diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror diskimage_builder/elements/openeuler-minimal/root.d/00-00-generate-repo diskimage_builder/elements/openeuler-minimal/test-elements/22.03-LTS-build-succeeds/README.rst diskimage_builder/elements/openeuler-minimal/test-elements/22.03-LTS-build-succeeds/element-deps @@ -716,6 +727,9 @@ diskimage_builder/elements/rocky-container/environment.d/10-01-rocky-distro-name diskimage_builder/elements/rocky-container/environment.d/11-yum-dnf.bash diskimage_builder/elements/rocky-container/test-elements/build-succeeds/README.rst diskimage_builder/elements/rocky-container/test-elements/build-succeeds/element-deps +diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/README.rst +diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/element-deps +diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/environment.d/09-set-distro.bash diskimage_builder/elements/rpm-distro/README.rst diskimage_builder/elements/rpm-distro/cleanup.d/99-selinux-fixfiles-restore diskimage_builder/elements/rpm-distro/post-install.d/06-network-config-nonzeroconf @@ -796,6 +810,7 @@ diskimage_builder/elements/ubuntu-common/element-deps diskimage_builder/elements/ubuntu-common/environment.d/10-ubuntu-distro-name.bash diskimage_builder/elements/ubuntu-common/environment.d/11-ubuntu-init-system.bash diskimage_builder/elements/ubuntu-common/install.d/80-disable-rfc3041 +diskimage_builder/elements/ubuntu-common/post-install.d/99-autoremove diskimage_builder/elements/ubuntu-minimal/README.rst diskimage_builder/elements/ubuntu-minimal/element-deps diskimage_builder/elements/ubuntu-minimal/element-provides @@ -850,7 +865,6 @@ diskimage_builder/elements/ubuntu-systemd-container/test-elements/xenial-build-s diskimage_builder/elements/ubuntu-systemd-container/test-elements/xenial-build-succeeds/test-output-formats diskimage_builder/elements/ubuntu-systemd-container/test-elements/xenial-build-succeeds/environment.d/10-set-distro.bash diskimage_builder/elements/ubuntu/environment.d/99-cloud-init-datasources.bash -diskimage_builder/elements/ubuntu/post-install.d/99-autoremove diskimage_builder/elements/ubuntu/pre-install.d/00-remove-apt-xapian-index diskimage_builder/elements/ubuntu/pre-install.d/01-set-ubuntu-mirror diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball @@ -934,6 +948,7 @@ diskimage_builder/lib/ramdisk-functions diskimage_builder/lib/ramdisk-image-create diskimage_builder/tests/__init__.py diskimage_builder/tests/base.py +diskimage_builder/tests/test_diskimage_builder.py diskimage_builder/tests/test_elementdeps.py diskimage_builder/tests/test_loggingconfig.py diskimage_builder/tests/test_no_dup_filenames.py @@ -970,10 +985,12 @@ releasenotes/notes/1.17.0-ef744f36d277dba4.yaml releasenotes/notes/1.18.0-4433d3076627f10d.yaml releasenotes/notes/1.18.1-ceeb514708dcb731.yaml releasenotes/notes/Add-DIB-UBUNTU-KERNEL-5f75a809f3ce9bab.yaml +releasenotes/notes/Fix-ubuntu-minimal-not-running-autoremove-7a4e31de2a525f46.yaml releasenotes/notes/add-6b94d90caea2895e.yaml releasenotes/notes/add-aarch64-support-for-rhel-c62a0e02b1ad9033.yaml releasenotes/notes/add-centos-8-support-4fd28eb04e11ba43.yaml releasenotes/notes/add-efi-packages-70a19464d31ab8d7.yaml +releasenotes/notes/add-fips-element-a5a3e0e3c653f923.yaml releasenotes/notes/add-grub2-efi-x64-modules-for-centos-9-74401ca5ec8201c7.yaml releasenotes/notes/add-modprobe-element-8e3b0287ebb11920.yaml releasenotes/notes/add-openeuler-minimal-element-068dc7d9d94e5098.yaml @@ -993,6 +1010,7 @@ releasenotes/notes/centos-minimal-8-aca6314862741177.yaml releasenotes/notes/centos-minimal-8-stream-97021a2c92463dde.yaml releasenotes/notes/centos-retired-f17ae9f6f03e57e3.yaml releasenotes/notes/cloud-init-resizefs-growpart-e820b8c7f3adb78b.yaml +releasenotes/notes/containerfile-host-networking-0691f3be5c6b11b6.yaml releasenotes/notes/containerfile-root-extraction-430148dd669cb3da.yaml releasenotes/notes/create-945440b17d500bf5.yaml releasenotes/notes/dash-p-after-install-58a87549c1c906c3.yaml @@ -1004,6 +1022,7 @@ releasenotes/notes/dhcp-all-interfaces-add-opt-to-let-NetworkManager-handle-it-f releasenotes/notes/dhcp-all-interfaces-debian-issue-2ec7f1a5186f0b16.yaml releasenotes/notes/dhcp-all-interfaces-networkmanager-b34f2fc8fb1a678f.yaml releasenotes/notes/dhcp-vlan-interfaces-b34ab224b9431e3a.yaml +releasenotes/notes/dib-command-a190f41cbd572b4b.yaml releasenotes/notes/dib-distribution-mirror-8c241c0d3d4a539a.yaml releasenotes/notes/dib-init-system-5647bad17a01c602.yaml releasenotes/notes/dib-init-system_fix_for_debian_jessie-c6f7261ee84dad27.yaml @@ -1024,6 +1043,7 @@ releasenotes/notes/ensure-venv-d7c16a2a04fd3b8f.yaml releasenotes/notes/fedora26-690b9fd9ac3c3d4f.yaml releasenotes/notes/fedora32-9dfa4d9d4dcce1ff.yaml releasenotes/notes/fix-and-update-opensuse-15.3-c7fee120b51e3621.yaml +releasenotes/notes/fix-grub-boot-kernel-commandline-1f94e49884c517c0.yaml releasenotes/notes/fix_rdisc6_loop-32a308a97de99f0f.yaml releasenotes/notes/gpt-esp-partition-size-200568cb6fcc8788.yaml releasenotes/notes/grub-timeout-1cdd14a2b1467d89.yaml @@ -1071,6 +1091,7 @@ releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml releasenotes/notes/specify-bootstrap-packages-via-env-22f5c6982c09f16a.yaml releasenotes/notes/squashfs-output-91c1f0dc37474d3c.yaml releasenotes/notes/start-using-reno-446b3d52a467a273.yaml +releasenotes/notes/swap_support-9ac267f56a0c0424.yaml releasenotes/notes/sysfs-ro-b127a6df2d78e57c.yaml releasenotes/notes/sysprep-f3fd036bc1d2c405.yaml releasenotes/notes/sysprep-fixes-8890b968a8fa7ac1.yaml diff --git a/diskimage_builder.egg-info/entry_points.txt b/diskimage_builder.egg-info/entry_points.txt index 8d27eb0..ceae7bf 100644 --- a/diskimage_builder.egg-info/entry_points.txt +++ b/diskimage_builder.egg-info/entry_points.txt @@ -1,5 +1,6 @@ [console_scripts] disk-image-create = diskimage_builder.disk_image_create:main +diskimage-builder = diskimage_builder.diskimage_builder:main ramdisk-image-create = diskimage_builder.disk_image_create:main [diskimage_builder.block_device.plugin] diff --git a/diskimage_builder.egg-info/pbr.json b/diskimage_builder.egg-info/pbr.json index 347f5ec..3867b31 100644 --- a/diskimage_builder.egg-info/pbr.json +++ b/diskimage_builder.egg-info/pbr.json @@ -1 +1 @@ -{"git_version": "2a25db9e", "is_release": true}
\ No newline at end of file +{"git_version": "48cbe94e", "is_release": true}
\ No newline at end of file diff --git a/diskimage_builder.egg-info/requires.txt b/diskimage_builder.egg-info/requires.txt index 3242ed0..671d7da 100644 --- a/diskimage_builder.egg-info/requires.txt +++ b/diskimage_builder.egg-info/requires.txt @@ -1,5 +1,6 @@ PyYAML>=3.12 flake8<6.0.0,>=3.6.0 +jsonschema>=3.0.2 networkx>=2.3.0 pbr!=2.1.0,>=2.0.0 stevedore>=1.20.0 diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py index 8b85931..99f23be 100644 --- a/diskimage_builder/block_device/blockdevice.py +++ b/diskimage_builder/block_device/blockdevice.py @@ -316,6 +316,20 @@ class BlockDevice(object): print("%s" % root_fs['type']) return 0 + if symbol == "boot-label": + try: + boot_mount = self._config_get_mount("/boot") + boot_fs = self._config_get_mkfs(boot_mount['base']) + # If not explicitly defined, we appear to fallback + # to name for a label, which we can only get from the + # resulting filesystem config. + boot_label = boot_fs.get('label', boot_fs.get('name', '')) + except AssertionError: + boot_label = '' + logger.debug("boot-label [%s]", boot_label) + print("%s" % boot_label) + return 0 + if symbol == 'mount-points': mount_points = self._config_get_all_mount_points() # we return the mountpoints joined by a pipe, because it is not diff --git a/diskimage_builder/block_device/level2/mkfs.py b/diskimage_builder/block_device/level2/mkfs.py index 1f62340..b7e957b 100644 --- a/diskimage_builder/block_device/level2/mkfs.py +++ b/diskimage_builder/block_device/level2/mkfs.py @@ -33,7 +33,8 @@ file_system_max_label_length = { "ext3": 16, "ext4": 16, "xfs": 12, - "vfat": 11 + "vfat": 11, + "swap": 16 } @@ -102,18 +103,20 @@ class FilesystemNode(NodeBase): return (edge_from, edge_to) def create(self): - cmd = ["mkfs"] - - cmd.extend(['-t', self.type]) - if self.opts: - cmd.extend(self.opts) + if self.type in ('swap'): + cmd = ["mkswap"] + else: + cmd = ["mkfs"] + cmd.extend(['-t', self.type]) + if self.opts: + cmd.extend(self.opts) if self.type in ('vfat', 'fat'): cmd.extend(["-n", self.label]) else: cmd.extend(["-L", self.label]) - if self.type in ('ext2', 'ext3', 'ext4'): + if self.type in ('ext2', 'ext3', 'ext4', 'swap'): cmd.extend(['-U', self.uuid]) elif self.type == 'xfs': cmd.extend(['-m', "uuid=%s" % self.uuid]) diff --git a/diskimage_builder/block_device/level3/mount.py b/diskimage_builder/block_device/level3/mount.py index 963f12b..b21f8b6 100644 --- a/diskimage_builder/block_device/level3/mount.py +++ b/diskimage_builder/block_device/level3/mount.py @@ -15,6 +15,7 @@ import functools import logging import os +import time from diskimage_builder.block_device.exception \ import BlockDeviceSetupException @@ -72,17 +73,21 @@ class MountPointNode(NodeBase): return (edge_from, edge_to) def create(self): - logger.debug("mount called [%s]", self.mount_point) - rel_mp = self.mount_point if self.mount_point[0] != '/' \ - else self.mount_point[1:] - mount_point = os.path.join(self.mount_base, rel_mp) - if not os.path.exists(mount_point): - # Need to sudo this because of permissions in the new - # file system tree. - exec_sudo(['mkdir', '-p', mount_point]) - logger.info("Mounting [%s] to [%s]", self.name, mount_point) - exec_sudo(["mount", self.state['filesys'][self.base]['device'], - mount_point]) + if (self.state['filesys'][self.base]['fstype'] == 'swap'): + # No need to mount/activate swap during image build. + mount_point = self.mount_point = 'none' + else: + logger.debug("mount called [%s]", self.mount_point) + rel_mp = self.mount_point if self.mount_point[0] != '/' \ + else self.mount_point[1:] + mount_point = os.path.join(self.mount_base, rel_mp) + if not os.path.exists(mount_point): + # Need to sudo this because of permissions in the new + # file system tree. + exec_sudo(['mkdir', '-p', mount_point]) + logger.info("Mounting [%s] to [%s]", self.name, mount_point) + exec_sudo(["mount", self.state['filesys'][self.base]['device'], + mount_point]) if 'mount' not in self.state: self.state['mount'] = {} @@ -94,6 +99,9 @@ class MountPointNode(NodeBase): self.state['mount_order'].append(self.mount_point) def umount(self): + if (self.state['filesys'][self.base]['fstype'] == 'swap'): + # Swap not mounted/activated during image build. + return logger.info("Called for [%s]", self.name) # Before calling umount, call 'fstrim' on suitable mounted # file systems. This discards unused blocks from the mounted @@ -109,7 +117,21 @@ class MountPointNode(NodeBase): if self.state['filesys'][self.base]['fstype'] != 'vfat': exec_sudo(["fstrim", "--verbose", self.state['mount'][self.mount_point]['path']]) - exec_sudo(["umount", self.state['mount'][self.mount_point]['path']]) + # Even 'fstrim' call sometimes don't solve the issue with 'busy' + # filesystem, so we need to catch the exception and repeat unount. + mount_point = self.state['mount'][self.mount_point]['path'] + catch = None + for try_cnt in range(10, 1, -1): + try: + exec_sudo(["umount", mount_point]) + return + except BlockDeviceSetupException as e: + catch = e + logger.error("umount failed (%s)", e.output.strip()) + time.sleep(3) + + logger.debug("Gave up trying to umount [%s]", mount_point) + raise catch def delete(self): self.umount() diff --git a/diskimage_builder/block_device/level4/fstab.py b/diskimage_builder/block_device/level4/fstab.py index 38f614d..cda8264 100644 --- a/diskimage_builder/block_device/level4/fstab.py +++ b/diskimage_builder/block_device/level4/fstab.py @@ -40,12 +40,15 @@ class FstabNode(NodeBase): if 'fstab' not in self.state: self.state['fstab'] = {} + swap = 'none' in self.state['mount'] and \ + self.state['mount']['none']['name'] == self.base + self.state['fstab'][self.base] = { 'name': self.name, 'base': self.base, - 'options': self.options, - 'dump-freq': self.dump_freq, - 'fsck-passno': self.fsck_passno + 'options': 'sw' if swap else self.options, + 'dump-freq': 0 if swap else self.dump_freq, + 'fsck-passno': 0 if swap else self.fsck_passno } diff --git a/diskimage_builder/block_device/tests/config/lvm_tree_partition_ordering.yaml b/diskimage_builder/block_device/tests/config/lvm_tree_partition_ordering.yaml index 06526b8..e28da14 100644 --- a/diskimage_builder/block_device/tests/config/lvm_tree_partition_ordering.yaml +++ b/diskimage_builder/block_device/tests/config/lvm_tree_partition_ordering.yaml @@ -7,10 +7,10 @@ label: mbr partitions: - name: boot - flags: [ boot,primary ] + flags: [boot, primary] size: 1G - name: root - flags: [ primary ] + flags: [primary] size: 3G - lvm: @@ -27,12 +27,16 @@ lvs: - name: lv_root base: vg - size: 2000M + size: 1900M - name: lv_var base: vg size: 500M + - name: lv_swap + base: vg + size: 100M + - mkfs: base: boot type: ext3 @@ -50,8 +54,8 @@ mount: mount_point: / fstab: - options: "noacl,errors=remount-ro" - fsck-passno: 1 + options: "noacl,errors=remount-ro" + fsck-passno: 1 - mkfs: name: mkfs_var @@ -60,5 +64,15 @@ mount: mount_point: /var fstab: - options: "noacl" - fsck-passno: 2 + options: "noacl" + fsck-passno: 2 + +- mkfs: + name: mkfs_swap + base: lv_swap + type: "swap" + mount: + mount_point: none + fstab: + options: "sw" + fsck-passno: 0 diff --git a/diskimage_builder/block_device/tests/config/multiple_partitions_graph.yaml b/diskimage_builder/block_device/tests/config/multiple_partitions_graph.yaml index fe262a2..1822a38 100644 --- a/diskimage_builder/block_device/tests/config/multiple_partitions_graph.yaml +++ b/diskimage_builder/block_device/tests/config/multiple_partitions_graph.yaml @@ -2,20 +2,23 @@ name: image0 - partitioning: - base: image0 - name: mbr - label: mbr - partitions: - - name: root - base: image0 - flags: [ boot, primary ] - size: 55% - - name: var - base: image0 - size: 40% - - name: var_log - base: image0 - size: 5% + base: image0 + name: mbr + label: mbr + partitions: + - name: root + base: image0 + flags: [boot, primary] + size: 54% + - name: var + base: image0 + size: 40% + - name: var_log + base: image0 + size: 5% + - name: swap + base: image0 + size: 1% - mkfs: base: root @@ -67,3 +70,20 @@ name: fstab_mount_mkfs_var_log fsck-passno: 1 options: defaults + +- mkfs: + base: swap + name: mkfs_swap + type: swap + uuid: swap-uuid-1234 + +- mount: + base: mkfs_swap + name: mount_mkfs_swap + mount_point: none + +- fstab: + base: mount_mkfs_swap + name: fstab_mount_mkfs_swap + fsck-passno: 0 + options: sw diff --git a/diskimage_builder/block_device/tests/config/multiple_partitions_tree.yaml b/diskimage_builder/block_device/tests/config/multiple_partitions_tree.yaml index d60d5a8..39fccd6 100644 --- a/diskimage_builder/block_device/tests/config/multiple_partitions_tree.yaml +++ b/diskimage_builder/block_device/tests/config/multiple_partitions_tree.yaml @@ -2,39 +2,48 @@ name: image0 - partitioning: - base: image0 - name: mbr - label: mbr - partitions: - - name: root - flags: [ boot, primary ] - size: 55% - mkfs: - type: xfs - uuid: root-uuid-1234 - mount: - mount_point: / - fstab: - options: "defaults" - fsck-passno: 1 - - name: var - size: 40% - mkfs: - type: xfs - uuid: var-uuid-1234 - mount: - mount_point: /var - fstab: - options: "defaults" - fsck-passno: 1 - - name: var_log - size: 5% - mkfs: - type: vfat - label: varlog - mount: - mount_point: /var/log - fstab: - options: "defaults" - fsck-passno: 1 - + base: image0 + name: mbr + label: mbr + partitions: + - name: root + flags: [boot, primary] + size: 54% + mkfs: + type: xfs + uuid: root-uuid-1234 + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 + - name: var + size: 40% + mkfs: + type: xfs + uuid: var-uuid-1234 + mount: + mount_point: /var + fstab: + options: "defaults" + fsck-passno: 1 + - name: var_log + size: 5% + mkfs: + type: vfat + label: varlog + mount: + mount_point: /var/log + fstab: + options: "defaults" + fsck-passno: 1 + - name: swap + size: 1% + mkfs: + type: swap + uuid: swap-uuid-1234 + mount: + mount_point: none + fstab: + fsck-passno: 0 + options: "sw" diff --git a/diskimage_builder/block_device/tests/test_mount_order.py b/diskimage_builder/block_device/tests/test_mount_order.py index 097ae7d..06d34e4 100644 --- a/diskimage_builder/block_device/tests/test_mount_order.py +++ b/diskimage_builder/block_device/tests/test_mount_order.py @@ -80,6 +80,7 @@ class TestMountOrder(tc.TestGraphGeneration): state['blockdev']['root'] = {'device': '/dev/loopXp1/root'} state['blockdev']['var'] = {'device': '/dev/loopXp2/var'} state['blockdev']['var_log'] = {'device': '/dev/loopXp3/var_log'} + state['blockdev']['swap'] = {'device': '/dev/loopXp4/swap'} for node in call_order: if isinstance(node, (FilesystemNode, MountPointNode)): @@ -89,7 +90,8 @@ class TestMountOrder(tc.TestGraphGeneration): node.umount() # ensure that partitions were mounted in order root->var->var/log - self.assertListEqual(state['mount_order'], ['/', '/var', '/var/log']) + self.assertListEqual(state['mount_order'], + ['/', '/var', '/var/log', 'none']) # fs creation sequence (note we don't care about order of this # as they're all independent) @@ -101,7 +103,9 @@ class TestMountOrder(tc.TestGraphGeneration): '-m', 'uuid=var-uuid-1234', '-q', '/dev/loopXp2/var']), mock.call(['mkfs', '-t', 'vfat', '-n', 'VARLOG', - '/dev/loopXp3/var_log']) + '/dev/loopXp3/var_log']), + mock.call(['mkswap', '-L', 'mkfs_swap', '-U', 'swap-uuid-1234', + '/dev/loopXp4/swap']) ] self.assertEqual(mock_exec_sudo_mkfs.call_count, len(cmd_sequence)) mock_exec_sudo_mkfs.assert_has_calls(cmd_sequence, any_order=True) @@ -154,6 +158,10 @@ class TestMountOrder(tc.TestGraphGeneration): 'device': '/dev/loopXp3', 'fstype': 'vfat', }, + 'mkfs_swap': { + 'device': '/dev/loopXp4', + 'fstype': 'swap', + }, } for node in call_order: @@ -164,7 +172,9 @@ class TestMountOrder(tc.TestGraphGeneration): node.umount() # ensure that partitions are mounted in order / -> /boot -> /var - self.assertListEqual(state['mount_order'], ['/', '/boot', '/var']) + # swap not mounted in runtime, but should be in 'mount_order' list + self.assertListEqual(state['mount_order'], + ['/', '/boot', '/var', 'none']) cmd_sequence = [ # mount sequence diff --git a/diskimage_builder/diskimage_builder.py b/diskimage_builder/diskimage_builder.py new file mode 100644 index 0000000..3872b0c --- /dev/null +++ b/diskimage_builder/diskimage_builder.py @@ -0,0 +1,574 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import collections +import io +import jsonschema +import os +import os.path +import shlex +import subprocess +import sys +import textwrap +import yaml + +import diskimage_builder.paths + + +class SchemaProperty(object): + """Base class for a basic schema and a help string""" + + key = None + description = None + schema_type = "string" + + def __init__(self, key, description, schema_type=None): + self.key = key + self.description = description + if schema_type: + self.schema_type = schema_type + + def to_schema(self): + return {self.key: {"type": self.schema_type}} + + def type_help(self): + return "Value is a string" + + def to_help(self): + return "%s\n%s\n%s\n(%s)" % ( + self.key, + "-" * len(self.key), + "\n".join(textwrap.wrap(self.description)), + self.type_help(), + ) + + +class Env(SchemaProperty): + """String dict schema for environment variables""" + + def __init__(self, key, description): + super(Env, self).__init__(key, description, schema_type="object") + + def to_schema(self): + schema = super(Env, self).to_schema() + schema[self.key]["additionalProperties"] = {"type": "string"} + return schema + + def type_help(self): + return "Value is a map of strings" + + +class Arg(SchemaProperty): + """Command argument with associated value""" + + arg = None + + def __init__(self, key, description, schema_type=None, arg=None): + super(Arg, self).__init__(key, description, schema_type=schema_type) + self.arg = arg + + def arg_name(self): + if self.arg is None: + return "--%s" % self.key + return self.arg + + def to_argument(self, value=None): + arg = self.arg_name() + if value is not None and value != "": + return [arg, value] + return [] + + +class Flag(Arg): + """Boolean value which does not contribute to arguments""" + + def __init__(self, key, description): + super(Flag, self).__init__(key, description, schema_type="boolean") + + def to_argument(self, value=None): + return [] + + def type_help(self): + return "Value is a boolean" + + +class ArgFlag(Arg): + """Boolean value for a flag argument being set or not""" + + def __init__(self, key, description, arg=None): + super(ArgFlag, self).__init__( + key, description, arg=arg, schema_type="boolean" + ) + + def to_argument(self, value=None): + if value: + return [self.arg_name()] + return [] + + def type_help(self): + return "Value is a boolean" + + +class ArgEnum(Arg): + """String argument constrained to a list of allowed values""" + + enum = None + + def __init__( + self, key, description, schema_type="string", arg=None, enum=None + ): + super(ArgEnum, self).__init__( + key, description, schema_type=schema_type, arg=arg + ) + self.enum = enum and enum or [] + + def to_schema(self): + schema = super(ArgEnum, self).to_schema() + schema[self.key]["enum"] = self.enum + return schema + + def type_help(self): + return "Allowed values: %s" % ", ".join(self.enum) + + +class ArgFlagRepeating(ArgEnum): + """Flag argument which repeats the specified number of times""" + + def __init__(self, key, description, arg=None, max_repeat=0): + enum = list(range(max_repeat + 1)) + super(ArgFlagRepeating, self).__init__( + key, description, schema_type="integer", arg=arg, enum=enum + ) + + def to_argument(self, value): + return [self.arg] * value + + def type_help(self): + return "Allowed values: %s" % ", ".join([str(i) for i in self.enum]) + + +class ArgInt(Arg): + """Integer argument which a minumum constraint""" + + minimum = 1 + + def __init__(self, key, description, arg=None, minimum=1): + super(ArgInt, self).__init__( + key, description, arg=arg, schema_type="integer" + ) + self.minimum = minimum + + def to_schema(self): + schema = super(ArgInt, self).to_schema() + schema[self.key]["minimum"] = self.minimum + return schema + + def to_argument(self, value): + return super(ArgInt, self).to_argument(str(value)) + + def type_help(self): + return "Value is an integer" + + +class ArgList(Arg): + """List of strings converted to comma delimited argument""" + + def __init__(self, key, description, arg=None): + super(ArgList, self).__init__( + key, description, arg=arg, schema_type="array" + ) + + def to_schema(self): + schema = super(ArgList, self).to_schema() + schema[self.key]["items"] = {"type": "string"} + return schema + + def to_argument(self, value): + if not value: + return [] + return super(ArgList, self).to_argument(",".join(value)) + + def type_help(self): + return "Value is a list of strings" + + +class ArgListPositional(ArgList): + """List of strings converted to positional arguments""" + + def __init__(self, key, description): + super(ArgListPositional, self).__init__(key, description) + + def to_argument(self, value): + # it is already a list, just return it + return value + + def type_help(self): + return "Value is a list of strings" + + +class ArgEnumList(ArgList): + """List of strings constrained to a list of allowed values""" + + enum = None + + def __init__(self, key, description, arg=None, enum=None): + super(ArgEnumList, self).__init__(key, description, arg=arg) + self.enum = enum and enum or [] + + def to_schema(self): + schema = super(ArgEnumList, self).to_schema() + schema[self.key]["items"]["enum"] = self.enum + return schema + + def type_help(self): + return ( + "Value is a list of strings with allowed values: %s)" + % ", ".join(self.enum) + ) + + +class ArgDictToString(Arg): + """Dict with string values converted to key=value,key2=value2 argument""" + + def __init__(self, key, description, arg=None): + super(ArgDictToString, self).__init__( + key, description, arg=arg, schema_type="object" + ) + + def to_schema(self): + schema = super(ArgDictToString, self).to_schema() + schema[self.key]["additionalProperties"] = {"type": "string"} + return schema + + def to_argument(self, value): + as_list = [] + for k, v in value.items(): + as_list.append("%s=%s" % (k, v)) + return super(ArgDictToString, self).to_argument(",".join(as_list)) + + def type_help(self): + return "Value is a map of strings" + + +PROPERTIES = [ + Arg("imagename", "Set the imagename of the output image file.", arg="-o"), + ArgEnum( + "arch", + "Set the architecture of the image.", + arg="-a", + enum=[ + "aarch64", + "amd64", + "arm64", + "armhf", + "powerpc", + "ppc64", + "ppc64el", + "ppc64le", + "s390x", + "x86_64", + ], + ), + ArgEnumList( + "types", + "Set the image types of the output image files.", + arg="-t", + enum=[ + "qcow2", + "tar", + "tgz", + "squashfs", + "vhd", + "docker", + "aci", + "raw", + ], + ), + Env( + "environment", + "Environment variables to set during the image build.", + ), + Flag( + "ramdisk", + "Whether to build a ramdisk image.", + ), + ArgFlagRepeating( + "debug-trace", + "Tracing level to log, integer 0 is off.", + arg="-x", + max_repeat=2, + ), + ArgFlag( + "uncompressed", + "Do not compress the image - larger but faster.", + arg="-u", + ), + ArgFlag("clear", "Clear environment before starting work.", arg="-c"), + Arg( + "logfile", + "Save run output to given logfile.", + ), + ArgFlag( + "checksum", + "Generate MD5 and SHA256 checksum files for the created image.", + ), + ArgInt( + "image-size", + "Image size in GB for the created image.", + ), + ArgInt( + "image-extra-size", + "Extra image size in GB for the created image.", + ), + Arg( + "image-cache", + "Location for cached images, defaults to ~/.cache/image-create.", + ), + ArgInt( + "max-online-resize", + "Max number of filesystem blocks to support when resizing. " + "Useful if you want a really large root partition when the " + "image is deployed. Using a very large value may run into a " + "known bug in resize2fs. Setting the value to 274877906944 " + "will get you a 1PB root file system. Making this " + "value unnecessarily large will consume extra disk " + "space on the root partition with extra file system inodes.", + ), + ArgInt( + "min-tmpfs", + "Minimum size in GB needed in tmpfs to build the image.", + ), + ArgInt( + "mkfs-journal-size", + "Filesystem journal size in MB to pass to mkfs.", + ), + Arg( + "mkfs-options", + "Option flags to be passed directly to mkfs.", + ), + ArgFlag("no-tmpfs", "Do not use tmpfs to speed image build."), + ArgFlag("offline", "Do not update cached resources."), + ArgDictToString( + "qemu-img-options", + "Option flags to be passed directly to qemu-img.", + ), + Arg( + "root-label", + 'Label for the root filesystem, defaults to "cloudimg-rootfs".', + ), + Arg( + "ramdisk-element", + "Specify the main element to be used for building ramdisks. " + 'Defaults to "ramdisk". Should be set to "dracut-ramdisk" ' + "for platforms such as RHEL and CentOS that do not package busybox.", + ), + ArgEnum( + "install-type", + "Specify the default installation type.", + enum=["source", "package"], + ), + Arg( + "docker-target", + "Specify the repo and tag to use if the output type is docker, " + "defaults to the value of output imagename.", + ), + ArgList( + "packages", + "Extra packages to install in the image. Runs once, after " + '"install.d" phase. Does not apply when ramdisk is true.', + arg="-p", + ), + ArgFlag( + "skip-base", + 'Skip the default inclusion of the "base" element. ' + "Does not apply when ramdisk is true.", + arg="-n", + ), + ArgListPositional( + "elements", + "list of elements to build the image with", + ), +] + + +SCHEMA_PROPERTIES = {} +for arg in PROPERTIES: + SCHEMA_PROPERTIES.update(arg.to_schema()) + +DIB_SCHEMA = { + "type": "array", + "items": { + "type": "object", + "properties": SCHEMA_PROPERTIES, + "additionalProperties": False, + }, + "additionalProperties": False, +} + + +class Command(object): + script = None + args = None + environ = None + + def __init__(self, script, properties, entry): + self.script = script + self.args = [] + self.environ = {} + for prop in properties: + if prop.key in entry: + value = entry[prop.key] + if isinstance(prop, Env): + self.environ.update(value) + elif isinstance(prop, Arg): + self.args.extend(prop.to_argument(value)) + + def merged_env(self): + environ = os.environ.copy() + # pre-seed some paths for the shell script + environ["_LIB"] = diskimage_builder.paths.get_path("lib") + environ.update(self.environ) + return environ + + def command(self): + return ["bash", self.script] + self.args + + def __repr__(self): + elements = [] + for k, v in self.environ.items(): + elements.append("%s=%s" % (k, shlex.quote(v))) + elements.extend([shlex.quote(a) for a in self.command()]) + return " ".join(elements) + "\n" + + +def help_properties(): + str = io.StringIO() + for prop in PROPERTIES: + str.write(prop.to_help()) + str.write("\n\n") + return str.getvalue() + + +def get_args(): + description = ( + """\ +The file format is YAML which expects a list of image definition maps. + +Supported entries for an image definition are: + +%s +""" + % help_properties() + ) + parser = argparse.ArgumentParser( + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "files", + metavar="<filename>", + nargs="+", + help="Paths to image build definition YAML files", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show the disk-image-create, ramdisk-image-create commands and " + "exit", + ) + parser.add_argument( + "--stop-on-failure", + action="store_true", + help="Stop building images when an image build fails", + ) + args = parser.parse_args(sys.argv[1:]) + return args + + +def merge_entry(merged_entry, entry): + for k, v in entry.items(): + if isinstance(v, list): + # append to existing list + list_value = merged_entry.setdefault(k, []) + list_value.extend(v) + elif isinstance(v, dict): + # update environment dict + dict_value = merged_entry.setdefault(k, {}) + dict_value.update(v) + else: + # update value + merged_entry[k] = v + + +def build_commands(definition): + jsonschema.validate(definition, schema=DIB_SCHEMA) + dib_script = "%s/disk-image-create" % diskimage_builder.paths.get_path( + "lib" + ) + rib_script = "%s/ramdisk-image-create" % diskimage_builder.paths.get_path( + "lib" + ) + + # Start with the default image name, 'image' + previous_imagename = "image" + merged_entries = collections.OrderedDict() + for entry in definition: + imagename = entry.get("imagename", previous_imagename) + previous_imagename = imagename + if imagename not in merged_entries: + merged_entries[imagename] = entry + else: + merge_entry(merged_entries[imagename], entry) + + commands = [] + for entry in merged_entries.values(): + if entry.get("ramdisk", False): + commands.append(Command(rib_script, PROPERTIES, entry)) + else: + commands.append(Command(dib_script, PROPERTIES, entry)) + return commands + + +def main(): + args = get_args() + + # export the path to the current python + if not os.environ.get("DIB_PYTHON_EXEC"): + os.environ["DIB_PYTHON_EXEC"] = sys.executable + + definitions = [] + for file in args.files: + with open(file) as f: + definitions.extend(yaml.safe_load(f)) + commands = build_commands(definitions) + final_returncode = 0 + failed_command = None + for command in commands: + sys.stderr.write(str(command)) + sys.stderr.write("\n") + sys.stderr.flush() + if not args.dry_run: + p = subprocess.Popen(command.command(), env=command.merged_env()) + p.communicate() + if p.returncode != 0: + final_returncode = p.returncode + failed_command = command + if args.stop_on_failure: + break + + if final_returncode != 0: + raise subprocess.CalledProcessError( + final_returncode, failed_command.command() + ) diff --git a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader index 3e24332..64f0681 100755 --- a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader +++ b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader @@ -88,6 +88,52 @@ echo "GRUB_TIMEOUT=${DIB_GRUB_TIMEOUT:-5}" >>/etc/default/grub echo 'GRUB_TERMINAL="serial console"' >>/etc/default/grub echo 'GRUB_GFXPAYLOAD_LINUX=auto' >>/etc/default/grub +# NOTE(TheJulia): We need to remove any boot entry from the /etc/default/grub +# file that may already exist, such as what was added by fips being setup on +# either in the source image or by by an element, as we repack the image. +# with new filesystems. +# Matches any element which looks like " boot=" and the associated value +# in order for us to have a clean starting point to put a value in place, +# if applicable. +# Removes entry trailing with a space, or any entry where boot is set as +# the last argument on the line. +sed -i 's/\ boot=[0-9A-Za-z/=\-]\+//' /etc/default/grub +# NOTE(TheJulia): When using FIPS, dracut wants to evaluate +# the hmac files for the kernel checksum. However, if /boot is +# located on a separate filesystem from the root filesystem, +# than this fails. As a result, we need to identify IF /boot +# is a separate filesystem, and convey this fact as a boot +# argument so dracut does not halt the system on boot. + +if [[ -n "${DIB_BOOT_LABEL}" ]]; then + BOOT_FS="boot=LABEL=${DIB_BOOT_LABEL}" +else + BOOT_FS="" +fi + +# NOTE(TheJulia): While on the subject of FIPS, if there is not an +# explicit /boot partition, then the fips setup command will return +# a successful result, but then also tell you to update your grub +# configuration. This happens specifically with Rocky linux. +# as such, we check/reconcile the flag into place for the kernel +# as the utility will return a result code of 1 if the state is +# inconsistent, i.e. policy in place, but not kernel command line +# argument. + +BOOT_FIPS="" + +if [[ -x /bin/fips-mode-setup ]]; then + set +e + fips-mode-setup --is-enabled + is_fips_enabled=$? + set -e + if [ $is_fips_enabled -eq 1 ]; then + BOOT_FIPS="fips=1" + fi +fi + + + if [[ -n "${DIB_BOOTLOADER_SERIAL_CONSOLE}" ]]; then SERIAL_CONSOLE="${DIB_BOOTLOADER_SERIAL_CONSOLE}" elif [[ "powerpc ppc64 ppc64le" =~ "$ARCH" ]]; then @@ -100,7 +146,7 @@ else fi GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=${SERIAL_CONSOLE} no_timer_check" -echo "GRUB_CMDLINE_LINUX_DEFAULT=\"${GRUB_CMDLINE_LINUX_DEFAULT} ${DIB_BOOTLOADER_DEFAULT_CMDLINE}\"" >>/etc/default/grub +echo "GRUB_CMDLINE_LINUX_DEFAULT=\"${GRUB_CMDLINE_LINUX_DEFAULT} ${DIB_BOOTLOADER_DEFAULT_CMDLINE} ${BOOT_FS} ${BOOT_FIPS}\"" >>/etc/default/grub echo 'GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"' >>/etc/default/grub # os-prober leaks /dev/sda into config file in dual-boot host diff --git a/diskimage_builder/elements/cache-url/README.rst b/diskimage_builder/elements/cache-url/README.rst index 5d30c9a..ec00c7e 100644 --- a/diskimage_builder/elements/cache-url/README.rst +++ b/diskimage_builder/elements/cache-url/README.rst @@ -1,4 +1,10 @@ ========= cache-url ========= + A helper script to download images into a local cache. + +**NOTE** : on RedHat platforms, ensure the curl binary is available. +Due to conflicting differences in platform usage of ```curl-minimal`` +and ```curl``, the usual package dependency methods do not work for +this package. diff --git a/diskimage_builder/elements/cache-url/pkg-map b/diskimage_builder/elements/cache-url/pkg-map index fa4fd95..bde555b 100644 --- a/diskimage_builder/elements/cache-url/pkg-map +++ b/diskimage_builder/elements/cache-url/pkg-map @@ -1,4 +1,9 @@ { + "family": { + "redhat": { + "curl": "" + } + }, "distro": { "gentoo": { "curl": "net-misc/curl" diff --git a/diskimage_builder/elements/centos/pre-install.d/00-02-set-centos-mirror b/diskimage_builder/elements/centos/pre-install.d/00-02-set-centos-mirror index cd02c37..ce97ecd 100755 --- a/diskimage_builder/elements/centos/pre-install.d/00-02-set-centos-mirror +++ b/diskimage_builder/elements/centos/pre-install.d/00-02-set-centos-mirror @@ -14,7 +14,11 @@ DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-} # The others aren't enabled and do not exist on all mirrors if [[ ${DIB_RELEASE} == '7' ]]; then sed -e "s,^#baseurl=http[s]*://mirror.centos.org/\(centos\|altarch\)/,baseurl=$DIB_DISTRIBUTION_MIRROR/,;/^mirrorlist=/d" -i /etc/yum.repos.d/CentOS-Base.repo -# CentOS Stream releases (e.g. 8-stream, 9-stream) +# CentOS Stream releases (9-stream) +elif [[ ${DIB_RELEASE} =~ '9-stream' ]]; then + sed -e "s,^#baseurl=http[s]*://mirror.centos.org/\$contentdir/,baseurl=$DIB_DISTRIBUTION_MIRROR/,;/^mirrorlist=/d" -i /etc/yum.repos.d/centos.repo + sed -e "s,^#baseurl=http[s]*://mirror.centos.org/\$contentdir/,baseurl=$DIB_DISTRIBUTION_MIRROR/,;/^mirrorlist=/d" -i /etc/yum.repos.d/centos-addons.repo +# CentOS Stream releases (8-stream) elif [[ ${DIB_RELEASE} =~ '-stream' ]]; then sed -e "s,^#baseurl=http[s]*://mirror.centos.org/\$contentdir/,baseurl=$DIB_DISTRIBUTION_MIRROR/,;/^mirrorlist=/d" -i /etc/yum.repos.d/CentOS-Stream-BaseOS.repo sed -e "s,^#baseurl=http[s]*://mirror.centos.org/\$contentdir/,baseurl=$DIB_DISTRIBUTION_MIRROR/,;/^mirrorlist=/d" -i /etc/yum.repos.d/CentOS-Stream-AppStream.repo diff --git a/diskimage_builder/elements/cloud-init-growpart/README.rst b/diskimage_builder/elements/cloud-init-growpart/README.rst new file mode 100644 index 0000000..de07b61 --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/README.rst @@ -0,0 +1,22 @@ +=================== +cloud-init-growpart +=================== + +This element enables growpart for OS images. It allows to grow +Specific partitions during the deployment process. +To enable this element simply include it in the elements list. + +**Disclaimer:** This element might not work with some device names supplied, for example when server is deployed and the image is written to a fibre channel device, or a SAS/SATA SSD controller. + +* ``DIB_CLOUD_INIT_GROWPART_DEVICES``: List of partition names that needs to be populated in order to be grown by cloud-init. **Populating this variable is mandatory.** + Cloud-init growpart module documentation - https://cloudinit.readthedocs.io/en/latest/topics/modules.html?highlight=growpart#growpart:: + + DIB_CLOUD_INIT_GROWPART_DEVICES: + - /dev/sda1 + - /dev/vda3 + + +Dependencies: + +* ``/usr/bin/growpart``: **is needed on the system in order to grow the partition**, + however it is part of different packages depending on linux family. That is already taken care of by package-installs. diff --git a/diskimage_builder/elements/cloud-init-growpart/element-deps b/diskimage_builder/elements/cloud-init-growpart/element-deps new file mode 100644 index 0000000..73015c2 --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/element-deps @@ -0,0 +1,2 @@ +package-installs +pkg-map diff --git a/diskimage_builder/elements/cloud-init-growpart/package-installs.yaml b/diskimage_builder/elements/cloud-init-growpart/package-installs.yaml new file mode 100644 index 0000000..46a7087 --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/package-installs.yaml @@ -0,0 +1 @@ +growpart_package: diff --git a/diskimage_builder/elements/cloud-init-growpart/pkg-map b/diskimage_builder/elements/cloud-init-growpart/pkg-map new file mode 100644 index 0000000..4bfbc20 --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/pkg-map @@ -0,0 +1,10 @@ +{ + "family": { + "redhat": { + "growpart_package": "cloud-utils-growpart" + }, + "debian": { + "growpart_package": "cloud-guest-utils" + } + } +} diff --git a/diskimage_builder/elements/cloud-init-growpart/post-install.d/55-growpart b/diskimage_builder/elements/cloud-init-growpart/post-install.d/55-growpart new file mode 100755 index 0000000..0f433b6 --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/post-install.d/55-growpart @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [[ -n ${DIB_CLOUD_INIT_GROWPART_DEVICES} ]]; then + if [ -d /etc/cloud/cloud.cfg.d ]; then + cat > /etc/cloud/cloud.cfg.d/55-growpart.cfg <<EOF +#cloud-config +growpart: + mode: auto + devices: $DIB_CLOUD_INIT_GROWPART_DEVICES + ignore_growroot_disabled: false +EOF + else + echo "The /etc/cloud/cloud.cfg.d directory must exist." + exit 1 + fi +else + echo "Set the device list in DIB_CLOUD_INIT_GROWPART_DEVICES." + exit 1 +fi diff --git a/diskimage_builder/elements/cloud-init-growpart/releasenotes/notes/added_growpart_for_lvm-0ce76ba71710c720.yaml b/diskimage_builder/elements/cloud-init-growpart/releasenotes/notes/added_growpart_for_lvm-0ce76ba71710c720.yaml new file mode 100644 index 0000000..fed8feb --- /dev/null +++ b/diskimage_builder/elements/cloud-init-growpart/releasenotes/notes/added_growpart_for_lvm-0ce76ba71710c720.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added growpart element. It allows for growing specific partitions + during the deployment, which will result in less post deploy actions + needed for the server to be ready for use. +
\ No newline at end of file diff --git a/diskimage_builder/elements/containerfile/root.d/08-containerfile b/diskimage_builder/elements/containerfile/root.d/08-containerfile index 7a852ad..d8ac828 100755 --- a/diskimage_builder/elements/containerfile/root.d/08-containerfile +++ b/diskimage_builder/elements/containerfile/root.d/08-containerfile @@ -28,8 +28,13 @@ if [[ "${DIB_CONTAINERFILE_PODMAN_ROOT:-0}" != '0' ]]; then DIB_CONTAINERFILE_RUNTIME_ROOT=1 fi +# NOTE(ianw) 2022-12-15 : this used to be left blank, but we've found +# with current podman this is the only reliable way to get networking +# in the container we're building (needed for yum update, package +# install, etc.). It's less secure, but we're already running in a +# priviledged container ... if [[ -z "${DIB_CONTAINERFILE_NETWORK_DRIVER:-}" ]]; then - DIB_CONTAINERFILE_RUNTIME_NETWORK="" + DIB_CONTAINERFILE_RUNTIME_NETWORK="--network host" else DIB_CONTAINERFILE_RUNTIME_NETWORK="--network ${DIB_CONTAINERFILE_NETWORK_DRIVER:-}" fi diff --git a/diskimage_builder/elements/fedora-container/containerfiles/37 b/diskimage_builder/elements/fedora-container/containerfiles/37 new file mode 100644 index 0000000..550d2b9 --- /dev/null +++ b/diskimage_builder/elements/fedora-container/containerfiles/37 @@ -0,0 +1,3 @@ +FROM docker.io/library/fedora:37 + +RUN dnf install -y findutils util-linux diff --git a/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash b/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash index 78f0337..bb138d5 100644 --- a/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash +++ b/diskimage_builder/elements/fedora/environment.d/10-fedora-distro-name.bash @@ -1,4 +1,4 @@ export DISTRO_NAME=fedora -export DIB_RELEASE=${DIB_RELEASE:-36} +export DIB_RELEASE=${DIB_RELEASE:-37} export EFI_BOOT_DIR="EFI/fedora" diff --git a/diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image b/diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image index f3eb7bb..4cf69b6 100755 --- a/diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image +++ b/diskimage_builder/elements/fedora/root.d/10-fedora-cloud-image @@ -12,6 +12,9 @@ set -o pipefail if [ 'amd64' = "$ARCH" ] ; then ARCH="x86_64" fi +if [[ "${ARCH}" == "arm64" ]]; then + ARCH="aarch64" +fi DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} diff --git a/diskimage_builder/elements/fips/README.rst b/diskimage_builder/elements/fips/README.rst new file mode 100644 index 0000000..14a0a13 --- /dev/null +++ b/diskimage_builder/elements/fips/README.rst @@ -0,0 +1,24 @@ +==== +fips +==== + +This image element attempts to setup the image so it will boot and operate +in what is often referred to as "FIPS mode", where cryptography policies +and algorithms are enforced to only those which are FIPS approved and +certified. In this context, FIPS is an abbreviation for +Federal Information Processing Standard, specifically publication number +140. You can learn more about FIPS policies at +https://csrc.nist.gov/publications/fips + +This element is a best-effort element and additional software or elements +may be processed after the fact which may impact the work of this element. +It is **generally** regarded as critical to enable FIPS as early as possible, +as cryptography policy can be applied, but may not be fully enforced without +the kernel also operating in FIPS mode. + +If you intend to utilize this element to generate production FIPS images, +it is highly recommended you do so on a host which has already had FIPS +enabled for itself. + +Additionally, not all distributions are explicitly supported. Unsupported +distributions will error providing appropriate guidance, if available. diff --git a/diskimage_builder/elements/fips/package-installs.yaml b/diskimage_builder/elements/fips/package-installs.yaml new file mode 100644 index 0000000..b22545c --- /dev/null +++ b/diskimage_builder/elements/fips/package-installs.yaml @@ -0,0 +1,14 @@ +crypto-policies: + when: + - DISTRO_NAME != ubuntu + - DISTRO_NAME != gentoo +# NOTE(TheJulia): crypto-policies does not exist in: +# - ubuntu +# - gentoo +crypto-policies-scripts: + when: + - DISTRO_NAME != debian + - DISTRO_NAME != ubuntu + - DISTRO_NAME != gentoo +# NOTE(TheJulia): Crypto policies includes the +# fips-mode-setup script in the debian package. diff --git a/diskimage_builder/elements/fips/pre-install.d/01-setup-fips b/diskimage_builder/elements/fips/pre-install.d/01-setup-fips new file mode 100755 index 0000000..5d10318 --- /dev/null +++ b/diskimage_builder/elements/fips/pre-install.d/01-setup-fips @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + + +if [[ "${DISTRO_NAME}" == "ubuntu" ]]; then + echo "ERROR: Setup of FIPS mode with DIB is not supported with Ubuntu." + echo "Please see refer to Ubuntu documentation on how to configure " + echo "FIPS, as it requires an Ubuntu Advantage subscription." + echo "https://ubuntu.com/security/certifications/docs/fips-enablement" + exit 1 +elif [[ "${DISTRO_NAME}" == "gentoo" ]]; then + echo "ERROR: Setup of FIPS mode with DIB is not supported with Gentoo." + echo "Gentoo's documentation appears to largely omit references to" + echo "FIPS setup, and the supportability is unknown." + exit 1 +elif [[ "${DISTRO_NAME}" == "opensuse" ]]; then + echo "ERROR: Setup of FIPS mode with DIB is not supported with OpenSUSE." + echo "Please refer to SUSE documentation on how to perform these actions." + echo "https://www.suse.com/support/kb/doc/?id=000019432" + exit 1 +else + # This command exists in Centos, Fedora, Rocky, and Debian + # and is referenced in documentation and posts about how to setup FIPS. + echo "Attempting to setup FIPS mode utilizing the fips-mode-setup command." + fips-mode-setup --enable + echo "FIPS mode setup completed, please remember this only applies to a" + echo "running operating system nor implies the certification state of the" + echo "resulting running operating system." +fi diff --git a/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols b/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols index 8c0249e..8894bed 100755 --- a/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols +++ b/diskimage_builder/elements/growvols/static/usr/local/sbin/growvols @@ -36,12 +36,22 @@ UNITS = ['%'] UNITS.extend(UNIT_BYTES.keys()) AMOUNT_UNIT_RE = re.compile('^([0-9]+)(%s)$' % '|'.join(UNITS)) +UNIT_BYTES_FORMAT = { + 'B': 1, + 'KiB': 1024, + 'MiB': 1048576, + 'GiB': 1073741824 +} + # Only create growth partition if there is at least 1GiB available MIN_DISK_SPACE_BYTES = UNIT_BYTES['GiB'] # Default LVM physical extent size is 4MiB PHYSICAL_EXTENT_BYTES = 4 * UNIT_BYTES['MiB'] +# Grow the thin pool metadata size to 1GiB +POOL_METADATA_SIZE = UNIT_BYTES['GiB'] + class Command(object): """ An object to represent a command to run with associated comment """ @@ -172,13 +182,13 @@ def printable_cmd(cmd): def convert_bytes(num): - """Format a bytes amount with units MB, GB etc""" - step_unit = 1000.0 - - for x in ['B', 'KB', 'MB', 'GB', 'TB']: - if num < step_unit: - return "%d%s" % (num, x) - num /= step_unit + """Format a bytes amount with units GiB, MiB etc""" + for x in ['GiB', 'MiB', 'KiB', 'B']: + unit = UNIT_BYTES_FORMAT[x] + unit_num = num // unit + if unit_num > 0: + break + return "%d%s" % (unit_num, x) def execute(cmd): @@ -259,6 +269,17 @@ def find_disk(opts, devices): return device +def grow_gpt(disk_name): + device = '/dev/%s' % disk_name + output = execute(['sgdisk', '-v', device]) + + search_str = "it doesn't reside\nat the end of the disk" + if search_str in output: + LOG.info('Fixing GPT structure, moving to end of device %s', device) + execute(['sgdisk', '-e', device]) + execute(['partprobe']) + + def find_space(disk_name): LOG.info('Finding spare space to grow into') dev_path = '/dev/%s' % disk_name @@ -480,6 +501,8 @@ def main(argv): disk = find_disk(opts, devices) disk_name = disk['KNAME'] + grow_gpt(disk_name) + sector_bytes = find_sector_size(disk_name) sector_start, sector_end, size_sectors = find_space(disk_name) @@ -499,9 +522,16 @@ def main(argv): group = find_group(opts) partnum = find_next_partnum(devices, disk_name) devname = find_next_device_name(devices, disk_name, partnum) + thin_pool = find_thin_pool(devices, group) + if thin_pool: + # total size available, reduced by POOL_METADATA_SIZE + # rounded down to whole extent and reduced by 1 extent + # for metadata overhead + size_bytes -= POOL_METADATA_SIZE + size_bytes -= size_bytes % PHYSICAL_EXTENT_BYTES + size_bytes -= PHYSICAL_EXTENT_BYTES dev_path = '/dev/%s' % devname grow_vols = find_grow_vols(opts, devices, group, size_bytes) - thin_pool = find_thin_pool(devices, group) commands = [] @@ -528,14 +558,20 @@ def main(argv): ], 'Add physical volume %s to group %s' % (devname, group))) if thin_pool: - # total size available, rounded down to whole extents - pool_size = size_bytes - size_bytes % PHYSICAL_EXTENT_BYTES commands.append(Command([ 'lvextend', - '-L+%sB' % pool_size, + '--poolmetadatasize', + '+%sB' % POOL_METADATA_SIZE, + thin_pool, + dev_path + ], 'Add %s to thin pool metadata %s' % ( + convert_bytes(POOL_METADATA_SIZE), thin_pool))) + commands.append(Command([ + 'lvextend', + '-L+%sB' % size_bytes, thin_pool, dev_path - ], 'Add %s to thin pool %s' % (convert_bytes(pool_size), + ], 'Add %s to thin pool %s' % (convert_bytes(size_bytes), thin_pool))) for volume_path, size_bytes in grow_vols.items(): diff --git a/diskimage_builder/elements/growvols/tests/test_growvols.py b/diskimage_builder/elements/growvols/tests/test_growvols.py index 7c2deb9..e45c652 100644 --- a/diskimage_builder/elements/growvols/tests/test_growvols.py +++ b/diskimage_builder/elements/growvols/tests/test_growvols.py @@ -110,6 +110,13 @@ DEVICES = [{ SECTOR_START = 79267840 SECTOR_END = 488265727 SGDISK_LARGEST = "%s\n%s\n" % (SECTOR_START, SECTOR_END) +SGDISK_V = """ +Problem: The secondary header's self-pointer indicates that it doesn't reside +at the end of the disk. If you've added a disk to a RAID array, use the 'e' +option on the experts' menu to adjust the secondary header's and partition +table's locations. + +Identified 1 problems!""" # output of vgs --noheadings --options vg_name VGS = " vg\n" @@ -147,10 +154,10 @@ class TestGrowvols(base.BaseTestCase): def test_convert_bytes(self): self.assertEqual('100B', growvols.convert_bytes(100)) - self.assertEqual('1KB', growvols.convert_bytes(1000)) - self.assertEqual('2MB', growvols.convert_bytes(2000000)) - self.assertEqual('3GB', growvols.convert_bytes(3000000000)) - self.assertEqual('4TB', growvols.convert_bytes(4000000000000)) + self.assertEqual('1000B', growvols.convert_bytes(1000)) + self.assertEqual('1MiB', growvols.convert_bytes(2000000)) + self.assertEqual('2GiB', growvols.convert_bytes(3000000000)) + self.assertEqual('3725GiB', growvols.convert_bytes(4000000000000)) @mock.patch('subprocess.Popen') def test_execute(self, mock_popen): @@ -465,6 +472,9 @@ class TestGrowvols(base.BaseTestCase): # noop, only discover block device info mock_execute.side_effect = [ LSBLK, + SGDISK_V, + '', + '', SGDISK_LARGEST, VGS, LVS, @@ -473,6 +483,9 @@ class TestGrowvols(base.BaseTestCase): mock_execute.assert_has_calls([ mock.call(['lsblk', '-Po', 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '-v', '/dev/sda']), + mock.call(['sgdisk', '-e', '/dev/sda']), + mock.call(['partprobe']), mock.call(['sgdisk', '--first-aligned-in-largest', '--end-of-largest', '/dev/sda']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']), @@ -484,6 +497,7 @@ class TestGrowvols(base.BaseTestCase): mock_execute.reset_mock() mock_execute.side_effect = [ LSBLK, + '', SGDISK_LARGEST, VGS, LVS, @@ -493,6 +507,7 @@ class TestGrowvols(base.BaseTestCase): mock_execute.assert_has_calls([ mock.call(['lsblk', '-Po', 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '-v', '/dev/sda']), mock.call(['sgdisk', '--first-aligned-in-largest', '--end-of-largest', '/dev/sda']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']), @@ -512,6 +527,7 @@ class TestGrowvols(base.BaseTestCase): mock_execute.reset_mock() mock_execute.side_effect = [ LSBLK, + '', SGDISK_LARGEST, VGS, LVS, @@ -522,6 +538,7 @@ class TestGrowvols(base.BaseTestCase): mock_execute.assert_has_calls([ mock.call(['lsblk', '-Po', 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '-v', '/dev/sda']), mock.call(['sgdisk', '--first-aligned-in-largest', '--end-of-largest', '/dev/sda']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']), @@ -549,6 +566,7 @@ class TestGrowvols(base.BaseTestCase): sgdisk_largest = "%s\n%s\n" % (sector_start, sector_end) mock_execute.side_effect = [ LSBLK, + '', sgdisk_largest, VGS, LVS, @@ -561,6 +579,7 @@ class TestGrowvols(base.BaseTestCase): # no space to grow, success mock_execute.side_effect = [ LSBLK, + '', sgdisk_largest, VGS, LVS, @@ -579,16 +598,18 @@ class TestGrowvols(base.BaseTestCase): mock_execute.reset_mock() mock_execute.side_effect = [ LSBLK, + '', SGDISK_LARGEST, VGS, LVS_THIN, - '', '', '', '', '', '', '', '', '', '', '' + '', '', '', '', '', '', '', '', '', '', '', '' ] growvols.main(['growvols', '--yes', '--group', 'vg', '/home=20%', 'fs_var=40%']) mock_execute.assert_has_calls([ mock.call(['lsblk', '-Po', 'kname,pkname,name,label,type,fstype,mountpoint']), + mock.call(['sgdisk', '-v', '/dev/sda']), mock.call(['sgdisk', '--first-aligned-in-largest', '--end-of-largest', '/dev/sda']), mock.call(['vgs', '--noheadings', '--options', 'vg_name']), @@ -599,13 +620,15 @@ class TestGrowvols(base.BaseTestCase): mock.call(['partprobe']), mock.call(['pvcreate', '/dev/sda5']), mock.call(['vgextend', 'vg', '/dev/sda5']), - mock.call(['lvextend', '-L+209404821504B', + mock.call(['lvextend', '--poolmetadatasize', '+1073741824B', '/dev/mapper/vg-lv_thinpool', '/dev/sda5']), - mock.call(['lvextend', '--size', '+41880125440B', + mock.call(['lvextend', '-L+208326885376B', + '/dev/mapper/vg-lv_thinpool', '/dev/sda5']), + mock.call(['lvextend', '--size', '+41662021632B', '/dev/mapper/vg-lv_home']), - mock.call(['lvextend', '--size', '+83760250880B', + mock.call(['lvextend', '--size', '+83328237568B', '/dev/mapper/vg-lv_var']), - mock.call(['lvextend', '--size', '+83764445184B', + mock.call(['lvextend', '--size', '+83336626176B', '/dev/mapper/vg-lv_root']), mock.call(['xfs_growfs', '/dev/mapper/vg-lv_home']), mock.call(['xfs_growfs', '/dev/mapper/vg-lv_var']), diff --git a/diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror.bash b/diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror index 312f034..339c896 100644..100755 --- a/diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror.bash +++ b/diskimage_builder/elements/openeuler-minimal/pre-install.d/00-setup-mirror @@ -1,3 +1,11 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + if [ -n "${DIB_DISTRIBUTION_MIRROR:-}" ]; then # Only set the mirror for OS, everything, EPOL and update repositories, # The others (debuginfo and source) aren't mirrored since they do not exist on all mirrors diff --git a/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/README.rst b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/README.rst new file mode 100644 index 0000000..10774f7 --- /dev/null +++ b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/README.rst @@ -0,0 +1 @@ +Verify a Rocky 9 aarch64 image diff --git a/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/element-deps b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/element-deps new file mode 100644 index 0000000..99857b0 --- /dev/null +++ b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/element-deps @@ -0,0 +1,3 @@ +block-device-efi +openstack-ci-mirrors +vm diff --git a/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/environment.d/09-set-distro.bash b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/environment.d/09-set-distro.bash new file mode 100644 index 0000000..65c6e4e --- /dev/null +++ b/diskimage_builder/elements/rocky-container/test-elements/rocky-9-aarch64-build-succeeds/environment.d/09-set-distro.bash @@ -0,0 +1 @@ +export DIB_RELEASE='9' diff --git a/diskimage_builder/elements/sysprep/bin/extract-image b/diskimage_builder/elements/sysprep/bin/extract-image index c460dcc..cc4173e 100755 --- a/diskimage_builder/elements/sysprep/bin/extract-image +++ b/diskimage_builder/elements/sysprep/bin/extract-image @@ -97,9 +97,9 @@ function extract_image() { fi for LOOPDEV in ${LOOPDEVS}; do - fstype=$(lsblk --all --nodeps --noheadings --output FSTYPE $LOOPDEV) - label=$(lsblk --all --nodeps --noheadings --output LABEL $LOOPDEV) - part_type=$(lsblk --all --nodeps --noheadings --output PARTTYPE $LOOPDEV) + fstype=$(sudo blkid -o value -s TYPE -p "${LOOPDEV}" 2>/dev/null) + label=$(sudo blkid -o value -s LABEL -p "${LOOPDEV}" 2>/dev/null) + part_type=$(sudo blkid -o value -s PART_ENTRY_TYPE -p "${LOOPDEV}" 2>/dev/null) if [ -z "${fstype}" ]; then # Ignore block device with no filesystem type diff --git a/diskimage_builder/elements/ubuntu/post-install.d/99-autoremove b/diskimage_builder/elements/ubuntu-common/post-install.d/99-autoremove index e04006d..e04006d 100755 --- a/diskimage_builder/elements/ubuntu/post-install.d/99-autoremove +++ b/diskimage_builder/elements/ubuntu-common/post-install.d/99-autoremove diff --git a/diskimage_builder/elements/yum-minimal/pkg-map b/diskimage_builder/elements/yum-minimal/pkg-map index 45d8b27..edb0416 100644 --- a/diskimage_builder/elements/yum-minimal/pkg-map +++ b/diskimage_builder/elements/yum-minimal/pkg-map @@ -1,4 +1,15 @@ { + "release": { + "centos": { + "7": { + "NetworkManager": "", + "dhcp-client": "" + }, + "9-stream": { + "lsb_release": "ed hostname patch postfix tar time" + } + } + }, "distro": { "openeuler": { "linux-firmware": "", @@ -7,25 +18,9 @@ "redhat-rpm-config": "openEuler-rpm-config" } }, - "release": { - "centos": { - "7": { - "NetworkManager": "", - "dhcp-client": "" - } - } - }, - "release": { - "centos": { - "9-stream": { - "lsb_release": "ed hostname patch postfix tar time" - } - } - }, -"family": { + "family": { "redhat": { "lsb_release": "redhat-lsb-core" } } } - diff --git a/diskimage_builder/elements/yum/README.rst b/diskimage_builder/elements/yum/README.rst index c4442ee..6ee06d4 100644 --- a/diskimage_builder/elements/yum/README.rst +++ b/diskimage_builder/elements/yum/README.rst @@ -22,20 +22,22 @@ The yum repository can also be configured by defining `DIB_YUM_REPO_PACKAGE` as a yum available package or a URL to an rpm file. This package can install repo files with any associated keys and certificates. -Environment Variables for Module Selection during Image Creation ----------------------------------------------------------------- -The following environment variable is used to select module streams to be -enabled during an image build on Yum/DNF based distributions. Any existing -stream for the given module is first disabled prior to enabling -the specified stream. - -#### DIB\_DNF\_MODULE\_STREAMS -This is a space-separated list of module streams to enable prior to any -RPMs being installed. - -Image Build Module Selection Example ------------------------------------- -When using Train release on RHEL/CentOS/Fedora, one must select the appropriate -virt and container-tools module streams: - -DIB_DNF_MODULE_STREAMS='virt:8.2 container-tools:3.0' +Environment Variables +--------------------- + +DIB_DNF_MODULE_STREAMS + :Required: No + :Default: None + :Description: The following environment variable is used to select module streams + to be enabled during an image build on Yum/DNF based distributions.Any existing + stream for the given module is first disabled prior to enabling the specified + stream. + :Example: ``DIB_DNF_MODULE_STREAMS='virt:8.2 container-tools:3.0'`` + +DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE + :Required: No + :Default: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + :Description: The environment variable is used to override default value in pre-install + stage to install epel repository from custom source + :Example: ``DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE=http://repos.example.com/epel/epel-latest-7.noarch.rpm`` + diff --git a/diskimage_builder/elements/yum/pre-install.d/01-00-centos-python3 b/diskimage_builder/elements/yum/pre-install.d/01-00-centos-python3 index 58de4ce..d023353 100755 --- a/diskimage_builder/elements/yum/pre-install.d/01-00-centos-python3 +++ b/diskimage_builder/elements/yum/pre-install.d/01-00-centos-python3 @@ -17,7 +17,7 @@ if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} == 7 ]]; then # early stage. yum install -y python3 # NOTE(dpawlik) The epel-release package is not available in RHEL. - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum install -y ${DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE:-https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm} yum install -y python36-PyYAML yum remove -y epel-release elif [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} > 7 ]]; then diff --git a/diskimage_builder/lib/disk-image-create b/diskimage_builder/lib/disk-image-create index d5a0396..a79686a 100644 --- a/diskimage_builder/lib/disk-image-create +++ b/diskimage_builder/lib/disk-image-create @@ -343,6 +343,11 @@ export DIB_ROOT_LABEL DIB_ROOT_FSTYPE=$(${DIB_BLOCK_DEVICE} getval root-fstype) export DIB_ROOT_FSTYPE +# Need to get the boot device label because, if defined, we may +# need to update boot configuration in some cases +DIB_BOOT_LABEL=$(${DIB_BLOCK_DEVICE} getval boot-label) +export DIB_BOOT_LABEL + # retrieve mount points so we can reuse in elements DIB_MOUNTPOINTS=$(${DIB_BLOCK_DEVICE} getval mount-points) export DIB_MOUNTPOINTS diff --git a/diskimage_builder/lib/ramdisk-image-create b/diskimage_builder/lib/ramdisk-image-create index d5a0396..a79686a 100644 --- a/diskimage_builder/lib/ramdisk-image-create +++ b/diskimage_builder/lib/ramdisk-image-create @@ -343,6 +343,11 @@ export DIB_ROOT_LABEL DIB_ROOT_FSTYPE=$(${DIB_BLOCK_DEVICE} getval root-fstype) export DIB_ROOT_FSTYPE +# Need to get the boot device label because, if defined, we may +# need to update boot configuration in some cases +DIB_BOOT_LABEL=$(${DIB_BLOCK_DEVICE} getval boot-label) +export DIB_BOOT_LABEL + # retrieve mount points so we can reuse in elements DIB_MOUNTPOINTS=$(${DIB_BLOCK_DEVICE} getval mount-points) export DIB_MOUNTPOINTS diff --git a/diskimage_builder/tests/test_diskimage_builder.py b/diskimage_builder/tests/test_diskimage_builder.py new file mode 100644 index 0000000..30b3b7d --- /dev/null +++ b/diskimage_builder/tests/test_diskimage_builder.py @@ -0,0 +1,587 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jsonschema +import mock +import subprocess +import tempfile +import testtools +import os +import yaml + +from diskimage_builder import diskimage_builder as dib + + +class TestDib(testtools.TestCase): + def assert_to_argument(self, expected, prop, value): + self.assertEqual(expected, prop.to_argument(value)) + self.assert_schema_validate(prop, value) + + def assert_schema_validate(self, prop, value, assert_failure=False): + # create a fake dict value and validate the schema against it + value_dict = {prop.key: value} + schema = {"type": "object", "properties": {}} + schema["properties"].update(prop.to_schema()) + if assert_failure: + self.assertRaises( + jsonschema.exceptions.ValidationError, + jsonschema.validate, + value_dict, + schema=schema, + ) + else: + jsonschema.validate(value_dict, schema=schema) + + def test_schema_property(self): + x = dib.SchemaProperty( + "the_key", "the description", schema_type="integer" + ) + self.assertEqual( + """\ +the_key +------- +the description +(Value is a string)""", + x.to_help(), + ) + self.assertEqual({"the_key": {"type": "integer"}}, x.to_schema()) + + def test_env(self): + x = dib.Env( + "environment", + "environment variables to set during the image build", + ) + self.assertEqual( + { + "environment": { + "additionalProperties": {"type": "string"}, + "type": "object", + } + }, + x.to_schema(), + ) + + def test_arg(self): + # no arg + x = dib.Arg("the-key", "") + self.assert_to_argument(["--the-key", "the value"], x, "the value") + + # with arg + x = dib.Arg("the-key", "", arg="--key") + self.assert_to_argument(["--key", "the value"], x, "the value") + + # with empty string value + self.assert_to_argument([], x, "") + + def test_arg_flag(self): + x = dib.ArgFlag("doit", "do it", arg="-d") + self.assertEqual({"doit": {"type": "boolean"}}, x.to_schema()) + + # false value + self.assert_to_argument([], x, False) + + # true value + self.assert_to_argument(["-d"], x, True) + + def test_arg_enum(self): + x = dib.ArgEnum("choice", "", enum=["one", "two", "three"]) + self.assertEqual( + {"choice": {"type": "string", "enum": ["one", "two", "three"]}}, + x.to_schema(), + ) + self.assert_to_argument(["--choice", "one"], x, "one") + self.assert_schema_validate(x, "two") + self.assert_schema_validate(x, "four", assert_failure=True) + + def test_arg_flag_repeating(self): + x = dib.ArgFlagRepeating("log_level", "", arg="-v", max_repeat=3) + self.assertEqual( + {"log_level": {"type": "integer", "enum": [0, 1, 2, 3]}}, + x.to_schema(), + ) + self.assert_to_argument([], x, 0) + self.assert_to_argument(["-v"], x, 1) + self.assert_to_argument(["-v", "-v"], x, 2) + self.assert_to_argument(["-v", "-v", "-v"], x, 3) + + def test_arg_int(self): + x = dib.ArgInt("size", "") + self.assertEqual( + {"size": {"type": "integer", "minimum": 1}}, x.to_schema() + ) + self.assert_to_argument(["--size", "10"], x, 10) + self.assert_schema_validate(x, -1, assert_failure=True) + + def test_arg_list(self): + x = dib.ArgList("packages", "", arg="-p") + self.assertEqual( + {"packages": {"type": "array", "items": {"type": "string"}}}, + x.to_schema(), + ) + self.assert_to_argument( + ["-p", "wget,vim-enhanced"], x, ["wget", "vim-enhanced"] + ) + self.assert_to_argument([], x, []) + + def test_arg_list_position(self): + x = dib.ArgListPositional("elements", "") + self.assert_to_argument( + ["centos", "vm", "bootloader"], x, ["centos", "vm", "bootloader"] + ) + + def test_arg_enum_list(self): + x = dib.ArgEnumList( + "types", "", arg="-t", enum=["qcow2", "tar", "raw"] + ) + self.assertEqual( + { + "types": { + "items": { + "enum": ["qcow2", "tar", "raw"], + "type": "string", + }, + "type": "array", + } + }, + x.to_schema(), + ) + self.assert_to_argument(["-t", "qcow2,raw"], x, ["qcow2", "raw"]) + + def test_arg_dict_to_string(self): + x = dib.ArgDictToString("options", "") + self.assertEqual( + { + "options": { + "additionalProperties": {"type": "string"}, + "type": "object", + } + }, + x.to_schema(), + ) + self.assert_to_argument( + ["--options", "foo=bar,baz=ingo"], + x, + {"foo": "bar", "baz": "ingo"}, + ) + + def test_command_merged_env(self): + c = dib.Command( + "echo", + properties=[dib.Env("environment", "")], + entry={ + "environment": { + "ELEMENTS_PATH": "/path/to/elements", + "DIB_CLOUD_IMAGES": "/path/to/image.qcow2", + } + }, + ) + env = c.merged_env() + self.assertEqual("/path/to/elements", env["ELEMENTS_PATH"]) + self.assertEqual("/path/to/image.qcow2", env["DIB_CLOUD_IMAGES"]) + self.assertIn(needle="_LIB", haystack=env) + # this will be merged with the whole environment, so there will be + # more than 3 values + self.assertGreater(len(env), 3) + + def test_command(self): + c = dib.Command( + "echo", + properties=[ + dib.Env("environment", ""), + dib.Arg("the-key", "", arg="--key"), + dib.ArgFlag("doit", "do it", arg="-d"), + dib.ArgFlagRepeating("verbose", "", arg="-v", max_repeat=3), + dib.ArgDictToString("options", ""), + dib.ArgListPositional("elements", ""), + ], + entry={ + "environment": { + "ELEMENTS_PATH": "/path/to/elements", + "DIB_CLOUD_IMAGES": "~/image.qcow2", + }, + "the-key": "the-value", + "doit": True, + "verbose": 3, + "options": {"foo": "bar"}, + "elements": ["centos", "vm"], + }, + ) + self.assertEqual( + [ + "bash", + "echo", + "--key", + "the-value", + "-d", + "-v", + "-v", + "-v", + "--options", + "foo=bar", + "centos", + "vm", + ], + c.command(), + ) + self.assertEqual( + "ELEMENTS_PATH=/path/to/elements " + "DIB_CLOUD_IMAGES='~/image.qcow2' " + "bash echo --key the-value -d -v -v -v --options foo=bar " + "centos vm\n", + str(c), + ) + + def test_merge_entry(self): + # override normal attributes + merged_entry = { + "imagename": "image1", + "elements": ["one", "two", "three"], + "debug-trace": 1, + } + dib.merge_entry( + merged_entry, + { + "imagename": "image1", + "debug-trace": 2, + "logfile": "image1.log", + }, + ) + self.assertEqual( + { + "imagename": "image1", + "elements": ["one", "two", "three"], + "debug-trace": 2, + "logfile": "image1.log", + }, + merged_entry, + ) + + # append list attributes, update dict attributes + merged_entry = { + "imagename": "image1", + "elements": ["one", "two", "three"], + "environment": { + "DIB_ONE": "1", + "DIB_TWO": "2", + }, + } + dib.merge_entry( + merged_entry, + { + "imagename": "image1", + "elements": ["four", "five"], + "environment": { + "DIB_TWO": "two", + "DIB_THREE": "three", + }, + }, + ) + self.assertEqual( + { + "imagename": "image1", + "elements": ["one", "two", "three", "four", "five"], + "environment": { + "DIB_ONE": "1", + "DIB_TWO": "two", + "DIB_THREE": "three", + }, + }, + merged_entry, + ) + + @mock.patch("diskimage_builder.paths.get_path") + def test_build_commands_simple(self, mock_get_path): + mock_get_path.return_value = "/lib" + commands = dib.build_commands( + [ + { + "imagename": "centos-minimal", + "elements": ["centos", "vm"], + }, + { + "imagename": "ironic-python-agent", + "ramdisk": True, + "elements": [ + "ironic-python-agent-ramdisk", + "extra-hardware", + ], + }, + ] + ) + + self.assertEqual( + [ + "bash", + "/lib/disk-image-create", + "-o", + "centos-minimal", + "centos", + "vm", + ], + commands[0].command(), + ) + self.assertEqual( + [ + "bash", + "/lib/ramdisk-image-create", + "-o", + "ironic-python-agent", + "ironic-python-agent-ramdisk", + "extra-hardware", + ], + commands[1].command(), + ) + + @mock.patch("diskimage_builder.paths.get_path") + def test_build_commands_merged(self, mock_get_path): + mock_get_path.return_value = "/lib" + commands = dib.build_commands( + [ + { # base definition + "imagename": "centos-minimal", + "elements": ["centos", "vm"], + "environment": {"foo": "bar", "zip": "zap"}, + }, + { # merge with previous when no imagename + "elements": ["devuser"], + "logfile": "centos-minimal.log", + }, + { # merge when same imagename + "imagename": "centos-minimal", + "environment": {"foo": "baz", "one": "two"}, + }, + ] + ) + + self.assertEqual(1, len(commands)) + self.assertEqual( + [ + "bash", + "/lib/disk-image-create", + "-o", + "centos-minimal", + "--logfile", + "centos-minimal.log", + "centos", + "vm", + "devuser", + ], + commands[0].command(), + ) + + @mock.patch("diskimage_builder.paths.get_path") + def test_build_commands_all_arguments(self, mock_get_path): + mock_get_path.return_value = "/lib" + commands = dib.build_commands( + [ + { + "arch": "amd64", + "imagename": "everyoption", + "types": ["qcow2"], + "debug-trace": 2, + "uncompressed": True, + "clear": True, + "logfile": "./logfile.log", + "checksum": True, + "image-size": 40, + "image-extra-size": 1, + "image-cache": "~/.cache/dib", + "max-online-resize": 1000, + "min-tmpfs": 7, + "mkfs-journal-size": 1, + "mkfs-options": "-D", + "no-tmpfs": True, + "offline": True, + "qemu-img-options": {"size": "10"}, + "ramdisk-element": "dracut-ramdisk", + "install-type": "package", + "root-label": "root", + "docker-target": "everyoption:latest", + "packages": ["wget", "tmux"], + "skip-base": True, + "elements": ["centos", "vm"], + } + ] + ) + + self.assertEqual( + [ + "bash", + "/lib/disk-image-create", + "-o", + "everyoption", + "-a", + "amd64", + "-t", + "qcow2", + "-x", + "-x", + "-u", + "-c", + "--logfile", + "./logfile.log", + "--checksum", + "--image-size", + "40", + "--image-extra-size", + "1", + "--image-cache", + "~/.cache/dib", + "--max-online-resize", + "1000", + "--min-tmpfs", + "7", + "--mkfs-journal-size", + "1", + "--mkfs-options", + "-D", + "--no-tmpfs", + "--offline", + "--qemu-img-options", + "size=10", + "--root-label", + "root", + "--ramdisk-element", + "dracut-ramdisk", + "--install-type", + "package", + "--docker-target", + "everyoption:latest", + "-p", + "wget,tmux", + "-n", + "centos", + "vm", + ], + commands[0].command(), + ) + + def write_image_definition(self): + image_def = [ + { + "imagename": "centos-minimal", + "elements": ["centos", "vm"], + }, + { + "imagename": "ironic-python-agent", + "ramdisk": True, + "elements": [ + "ironic-python-agent-ramdisk", + "extra-hardware", + ], + }, + ] + with tempfile.NamedTemporaryFile(delete=False) as deffile: + self.addCleanup(os.remove, deffile.name) + self.filelist = [deffile.name] + with open(deffile.name, "w") as f: + f.write(yaml.dump(image_def)) + return deffile.name + + @mock.patch("diskimage_builder.paths.get_path") + @mock.patch("diskimage_builder.diskimage_builder.get_args") + @mock.patch("subprocess.Popen") + def test_main_dry_run(self, mock_popen, mock_get_args, mock_get_path): + mock_get_path.return_value = "/lib" + mock_get_args.return_value = mock.Mock( + dry_run=True, files=[self.write_image_definition()] + ) + dib.main() + mock_popen.assert_not_called() + + @mock.patch("diskimage_builder.paths.get_path") + @mock.patch("diskimage_builder.diskimage_builder.get_args") + @mock.patch("subprocess.Popen") + def test_main(self, mock_popen, mock_get_args, mock_get_path): + mock_get_path.return_value = "/lib" + mock_get_args.return_value = mock.Mock( + dry_run=False, + files=[self.write_image_definition()], + stop_on_failure=False, + ) + + process = mock.Mock() + process.returncode = 0 + mock_popen.return_value = process + + dib.main() + self.assertEqual(2, mock_popen.call_count) + mock_popen.assert_has_calls( + [ + mock.call( + [ + "bash", + "/lib/disk-image-create", + "-o", + "centos-minimal", + "centos", + "vm", + ], + env=mock.ANY, + ), + mock.call( + [ + "bash", + "/lib/ramdisk-image-create", + "-o", + "ironic-python-agent", + "ironic-python-agent-ramdisk", + "extra-hardware", + ], + env=mock.ANY, + ), + ], + any_order=True, + ) + + @mock.patch("diskimage_builder.paths.get_path") + @mock.patch("diskimage_builder.diskimage_builder.get_args") + @mock.patch("subprocess.Popen") + def test_main_stop_on_failure( + self, mock_popen, mock_get_args, mock_get_path + ): + mock_get_path.return_value = "/lib" + mock_get_args.return_value = mock.Mock( + dry_run=False, + files=[self.write_image_definition()], + stop_on_failure=True, + ) + + process = mock.Mock() + process.returncode = -1 + mock_popen.return_value = process + + e = self.assertRaises(subprocess.CalledProcessError, dib.main) + self.assertEqual(1, mock_popen.call_count) + self.assertEqual(-1, e.returncode) + + @mock.patch("diskimage_builder.paths.get_path") + @mock.patch("diskimage_builder.diskimage_builder.get_args") + @mock.patch("subprocess.Popen") + def test_main_continue_on_failure( + self, mock_popen, mock_get_args, mock_get_path + ): + mock_get_path.return_value = "/lib" + mock_get_args.return_value = mock.Mock( + dry_run=False, + files=[self.write_image_definition()], + stop_on_failure=False, + ) + + process = mock.Mock() + process.returncode.side_effect = -1 + mock_popen.return_value = process + + self.assertRaises(subprocess.CalledProcessError, dib.main) + self.assertEqual(2, mock_popen.call_count) diff --git a/doc/source/developer/components.rst b/doc/source/developer/components.rst index df9b675..4f213b8 100644 --- a/doc/source/developer/components.rst +++ b/doc/source/developer/components.rst @@ -24,6 +24,60 @@ Components Ironic no longer supports images created like this. +`diskimage-builder [--dry-run] [--stop-on-failure] [--help] filename.yaml [filename2.yaml...]` + + A YAML defined wrapper over `disk-image-create` and `ramdisk-image-create`. + + To generate kernel+ramdisk pair for use with nova-baremetal, specify the YAML: + + .. code-block:: yaml + + - imagename: deploy.ramdisk + ramdisk: true + elements: + - deploy-baremetal + + Duplicate `imagename` entries are merged into a single entry, allowing customizations over + a base image definition. If an `imagename` is missing, the `imagename` from the previous + entry is implied: + + .. code-block:: yaml + + # base image definition + - imagename: output.qcow + elements: + - vm + - block-device-gpt + - ubuntu-minimal + debug-trace: 1 + environment: + DIB_IMAGE_SIZE: '10' + + # debug logging customization + - imagename: output.qcow + debug-trace: 2 + + # adding element customization + - elements: + - devuser + environment: + DIB_DEV_USER_USERNAME: 'myuser' + DIB_DEV_USER_PWDLESS_SUDO: 'Yes' + DIB_DEV_USER_AUTHORIZED_KEYS: '/home/myuser/.ssh/id_rsa.pub' + + # resulting image entry which will be built + - imagename: output.qcow + elements: + - vm + - block-device-gpt + - ubuntu-minimal + - devuser + debug-trace: 2 + environment: + DIB_DEV_USER_USERNAME: 'myuser' + DIB_DEV_USER_PWDLESS_SUDO: 'Yes' + DIB_DEV_USER_AUTHORIZED_KEYS: '/home/myuser/.ssh/id_rsa.pub' + DIB_IMAGE_SIZE: '10' `element-info` diff --git a/doc/source/user_guide/building_an_image.rst b/doc/source/user_guide/building_an_image.rst index 97c9d2f..fbdb107 100644 --- a/doc/source/user_guide/building_an_image.rst +++ b/doc/source/user_guide/building_an_image.rst @@ -19,6 +19,24 @@ The following command will start our image build (distro must be either This will create a qcow2 file 'image.qcow2' which can then be booted. +Images can also be defined with YAML and built with the `diskimage-builder`. + +With an `image.yaml` file containing: + +.. code-block:: yaml + + - elements: + - <distro> + - vm + +An image is built with: + +:: + + diskimage-builder image.yaml + +Run `diskimage-builder --help` full description of the YAML attributes supported. + Elements -------- @@ -79,6 +97,13 @@ There are currently two defaults: disk-image-create -o output.qcow vm block-device-gpt ubuntu-minimal + Or with `diskimage-builder` YAML + + .. code-block:: yaml + + - imagename: output.qcow + elements: [vm, block-device-gpt, ubuntu-minimal] + * When not using the ``vm`` element a plain filesystem image, without any partitioning, is created. diff --git a/doc/source/user_guide/installation.rst b/doc/source/user_guide/installation.rst index 2ffe2c7..80a1671 100644 --- a/doc/source/user_guide/installation.rst +++ b/doc/source/user_guide/installation.rst @@ -7,8 +7,8 @@ to keep it separate. For details, see the installation section in the :ref:`dev_install`. Once installed, you will be able to :doc:`build images -<building_an_image>` using ``disk-image-create`` and the elements -included in the main ``diskimage-builder`` repository. +<building_an_image>` using ``disk-image-create`` or ``diskimage-builder`` +and the elements included in the main ``diskimage-builder`` repository. Requirements diff --git a/lower-constraints.txt b/lower-constraints.txt index 78efd47..cd45a5a 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -13,6 +13,7 @@ imagesize==0.7.1 iso8601==0.1.11 isort==4.3.4 Jinja2==2.10 +jsonschema==3.0.2 keystoneauth1==3.4.0 lazy-object-proxy==1.3.1 linecache2==1.0.0 diff --git a/releasenotes/notes/Fix-ubuntu-minimal-not-running-autoremove-7a4e31de2a525f46.yaml b/releasenotes/notes/Fix-ubuntu-minimal-not-running-autoremove-7a4e31de2a525f46.yaml new file mode 100644 index 0000000..ce25852 --- /dev/null +++ b/releasenotes/notes/Fix-ubuntu-minimal-not-running-autoremove-7a4e31de2a525f46.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixed the ubuntu-minimal target to also run autoremove at the end of an + image build. diff --git a/releasenotes/notes/add-fips-element-a5a3e0e3c653f923.yaml b/releasenotes/notes/add-fips-element-a5a3e0e3c653f923.yaml new file mode 100644 index 0000000..abbc22c --- /dev/null +++ b/releasenotes/notes/add-fips-element-a5a3e0e3c653f923.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Adds a ``fips`` element which attempts to enable FIPS mode configuration + into the disk image being created. +fixes: + - | + Fixes a possible case with a FIPS enabled disk image where the + cryptographic policies may be applied for enforcement of FIPS mode, + but the underlying kernel command line may be missing. This in particular + was discovered when testing on Rocky Linux. We now test, and add the FIPS + kernel command line flag if it is otherwise enabled. diff --git a/releasenotes/notes/containerfile-host-networking-0691f3be5c6b11b6.yaml b/releasenotes/notes/containerfile-host-networking-0691f3be5c6b11b6.yaml new file mode 100644 index 0000000..876e0bd --- /dev/null +++ b/releasenotes/notes/containerfile-host-networking-0691f3be5c6b11b6.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + The default networking in the ``containerfile`` element is + switched to ``host`` as we have found that more reliable in + generating the initial chroot images. diff --git a/releasenotes/notes/dib-command-a190f41cbd572b4b.yaml b/releasenotes/notes/dib-command-a190f41cbd572b4b.yaml new file mode 100644 index 0000000..088526c --- /dev/null +++ b/releasenotes/notes/dib-command-a190f41cbd572b4b.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + A new command ``diskimage-builder`` is now provided (also aliased to + ``dib``). It provides a YAML file based interface to ``disk-image-create`` + and ``ramdisk-image-create``. All arguments can be specified as yaml + dictionary entries, along with environment variable overrides which are + applied over the provided environment.
\ No newline at end of file diff --git a/releasenotes/notes/fix-grub-boot-kernel-commandline-1f94e49884c517c0.yaml b/releasenotes/notes/fix-grub-boot-kernel-commandline-1f94e49884c517c0.yaml new file mode 100644 index 0000000..51682fe --- /dev/null +++ b/releasenotes/notes/fix-grub-boot-kernel-commandline-1f94e49884c517c0.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixes the GRUB2 bootloader kernel command line where a prior ``boot`` + parameter could cause issues with bootloader configuration when + an image is rebuilt and the ``bootloader`` element is utilized. + We now remove any prior ``boot`` parameter in */etc/default/grub* + when executing the ``bootloader`` element, and then assert a + valid entry ``boot`` parameter if a */boot* partition is defined + as part of the image build. diff --git a/releasenotes/notes/swap_support-9ac267f56a0c0424.yaml b/releasenotes/notes/swap_support-9ac267f56a0c0424.yaml new file mode 100644 index 0000000..56df0c2 --- /dev/null +++ b/releasenotes/notes/swap_support-9ac267f56a0c0424.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds swap as a valid "filesystem" for block devices diff --git a/requirements.txt b/requirements.txt index fa0d5f8..067d95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ PyYAML>=3.12 # MIT stevedore>=1.20.0 # Apache-2.0 # NOTE(ianw) in here because dib-lint uses flake8 flake8<6.0.0,>=3.6.0 # MIT +jsonschema>=3.0.2 # MIT @@ -32,6 +32,7 @@ data_files = console_scripts = disk-image-create = diskimage_builder.disk_image_create:main ramdisk-image-create = diskimage_builder.disk_image_create:main + diskimage-builder = diskimage_builder.diskimage_builder:main diskimage_builder.block_device.plugin = local_loop = diskimage_builder.block_device.level0.localloop:LocalLoop partitioning = diskimage_builder.block_device.level1.partitioning:Partitioning diff --git a/tests/run_functests.sh b/tests/run_functests.sh index bcd1795..cfb388e 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -6,7 +6,7 @@ set -o pipefail BASE_DIR=$(cd $(dirname "$0")/.. && pwd) # then execute tests for elements -export DIB_CMD="disk-image-create" +export DIB_CMD="diskimage-builder" export DIB_ELEMENTS=$(python -c ' import diskimage_builder.paths diskimage_builder.paths.show_path("elements")') @@ -111,13 +111,11 @@ function wait_minus_n { # in results which tests have failed. function logfile_status { local status="$1" - local arg="$2" - local filename - if [[ -z "${arg// }" ]]; then + local filename="$2" + if [[ -z "${filename}" ]]; then return fi - filename="$(echo $arg | cut -f2 -d' ')" echo "Moving ${filename} to ${filename/.log/.$status.log}" mv "$filename" ${filename/.log/.$status.log} } @@ -132,25 +130,30 @@ function run_disk_element_test() { local output_format="$4" local logfile="$5" - local use_tmp_flag="" local dest_dir=$(mktemp -d) if [[ ${KEEP_OUTPUT} -ne 1 ]]; then trap "rm -rf $dest_dir" EXIT fi - if [ "${dont_use_tmp}" = "yes" ]; then - use_tmp_flag="--no-tmpfs" - fi + cat >> $dest_dir/image.yaml << EOF +- imagename: $dest_dir/image + debug-trace: 1 + types: [$output_format] + no-tmpfs: $dont_use_tmp + logfile: $logfile + skip-base: yes + elements: + - $element + - $test_element + environment: + DIB_SHOW_IMAGE_USAGE: "1" +EOF if break="after-error" break_outside_target=1 \ break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \ - DIB_SHOW_IMAGE_USAGE=1 \ ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -x -t ${output_format} \ - ${use_tmp_flag} \ - ${logfile} \ - -o $dest_dir/image -n $element $test_element 2>&1 \ + $DIB_CMD $dest_dir/image.yaml 2>&1 \ | log_with_prefix "${element}/${test_element}"; then if [[ "qcow2" =~ "$output_format" ]]; then @@ -212,16 +215,18 @@ function run_ramdisk_element_test() { local logfile="$5" local dest_dir=$(mktemp -d) - local use_tmp_flag="" - if [ "${dont_use_tmp}" = "yes" ]; then - use_tmp_flag="--no-tmpfs" - fi + cat >> $dest_dir/image.yaml << EOF +- imagename: $dest_dir/image + debug-trace: 1 + no-tmpfs: $dont_use_tmp + logfile: $logfile + elements: + - $element + - $test_element +EOF if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -x -o ${dest_dir}/image \ - ${logfile} \ - ${use_tmp_flag} \ - ${element} ${test_element} 2>&1 \ + $DIB_CMD $dest_dir/image.yaml 2>&1 \ | log_with_prefix "${element}/${test_element}"; then # TODO(dtantsur): test also kernel presence once we sort out its naming # problem (vmlinuz vs kernel) @@ -416,9 +421,9 @@ for test in "${TESTS_TO_RUN[@]}"; do element_output=$(cat ${element_output_override}) fi - log_argument=' ' + log_argument='' if [[ -n "${LOG_DIRECTORY}" ]]; then - log_argument="--logfile ${LOG_DIRECTORY}/${element}_${test_element}.log" + log_argument="${LOG_DIRECTORY}/${element}_${test_element}.log" fi echo "Running $test ($element_type)" @@ -1,12 +1,10 @@ [tox] envlist = pep8,linters,py3 minversion = 3.18.0 -skipsdist = True ignore_basepython_conflict = True [testenv] usedevelop = True -basepython = python3 deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -32,7 +30,6 @@ envdir = {toxworkdir}/venv commands = {toxinidir}/tests/run_functests.sh {posargs} [testenv:cover] -basepython = python3 allowlist_externals = find setenv = PYTHON=coverage run --source diskimage_builder --parallel-mode @@ -50,7 +47,6 @@ deps= -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt -r{toxinidir}/doc/requirements.txt -basepython = python3 allowlist_externals = rm commands = |