more generifying and moving kconfig

01-get-stage3.sh

@@ -3,52 +3,101 @@
 # Note that I use this script to update all my current stages, and rootfs,
 # but this repo is more specifically for Gentoo, so have some Gentoo.
 
-##
-## Vars
-##
-MIRROR="http://mirror.reenigne.net"
-OUTDIR="/var/tmp/catalyst/builds"
-
-mkdir -p ${OUTDIR}
-
-##
-## Gentoo
-##
-
-STAGE3_NAME="stage3-amd64-latest.tar.bz2"
-STAGE3_FILE=$(curl -s "${MIRROR}/gentoo/releases/amd64/autobuilds/latest-stage3-amd64.txt" | awk '/stage3/ { print $1 }')
-LIVE_SHA512=$(curl -s "${MIRROR}/gentoo/releases/amd64/autobuilds/${STAGE3_FILE}.DIGESTS" | awk '/SHA512 HASH/{getline;print}' | grep -iv 'contents' | awk {'print $1'})
-OUR_SHA512=$(sha512sum "${OUTDIR}/${STAGE3_NAME}" | awk {'print $1'})
-
-# download latest stage3 if not the newest
-if [ "${LIVE_SHA512}" != "${OUR_SHA512}" ]
-then
-	echo "Downloading new image - ${STAGE3_NAME}"
-	curl -s "${MIRROR}/gentoo/releases/amd64/autobuilds/${STAGE3_FILE}" > "${OUTDIR}/${STAGE3_NAME}"
+set -e -u -x -o pipefail
+
+# Vars
+export MIRROR=${MIRROR:-"http://gentoo.osuosl.org"}
+export OUTDIR=${OUTDIR:-"/var/tmp/catalyst/builds"}
+export PORTAGE_DIR=${PORTAGE_DIR:-"/var/tmp/catalyst/snapshots"}
+# profiles supported are as follows
+# default/linux/amd64/13.0
+# default/linux/amd64/13.0/no-multilib
+# hardened/linux/amd64
+# hardened/linux/amd64/no-multilib
+# hardened/linux/amd64/selinux (eventually)
+# hardened/linux/amd64/no-multilib/selinux (eventually)
+export PROFILE=${PROFILE:-"default/linux/amd64/13.0"}
+
+mkdir -p "${OUTDIR}"
+
+if [[ "${PROFILE}" == "default/linux/amd64/13.0" ]]; then
+  STAGE3_NAME="stage3-amd64-current.tar.bz2"
+  STAGE3_REAL_PATH=$(curl -s "${MIRROR}/releases/amd64/autobuilds/latest-stage3-amd64.txt" | awk '/stage3/ { print $1 }')
+  STAGE3_REAL_NAME=$(echo -n "${STAGE3_REAL_PATH}" | awk -F/ '{ print $2}')
+  STAGE3_URL="${MIRROR}/releases/amd64/autobuilds/current-stage3-amd64/${STAGE3_REAL_NAME}"
+elif [[ "${PROFILE}" == "default/linux/amd64/13.0/no-multilib" ]]; then
+  STAGE3_NAME="stage3-amd64-nomultilib-current.tar.bz2"
+  STAGE3_REAL_PATH=$(curl -s "${MIRROR}/releases/amd64/autobuilds/latest-stage3-amd64-nomultilib.txt" | awk '/stage3/ { print $1 }')
+  STAGE3_REAL_NAME=$(echo -n "${STAGE3_REAL_PATH}" | awk -F/ '{ print $2}')
+  STAGE3_URL="${MIRROR}/releases/amd64/autobuilds/current-stage3-amd64-nomultilib/${STAGE3_REAL_NAME}"
+elif [[ "${PROFILE}" == "hardened/linux/amd64" ]]; then
+  STAGE3_NAME="stage3-amd64-hardened-current.tar.bz2"
+  STAGE3_REAL_PATH=$(curl -s "${MIRROR}/releases/amd64/autobuilds/latest-stage3-amd64-hardened.txt" | awk '/hardened/ { print $1 }')
+  STAGE3_REAL_NAME=$(echo -n "${STAGE3_REAL_PATH}" | awk -F/ '{ print $3}')
+  STAGE3_URL="${MIRROR}/releases/amd64/autobuilds/current-stage3-amd64-hardened/${STAGE3_REAL_NAME}"
+elif [[ "${PROFILE}" == "hardened/linux/amd64/no-multilib" ]]; then
+  STAGE3_NAME="stage3-amd64-hardened-nomultilib-current.tar.bz2"
+  STAGE3_REAL_PATH=$(curl -s "${MIRROR}/releases/amd64/autobuilds/latest-stage3-amd64-hardened+nomultilib.txt" | awk '/hardened/ { print $1 }')
+  STAGE3_REAL_NAME=$(echo -n "${STAGE3_REAL_PATH}" | awk -F/ '{ print $3}')
+  STAGE3_URL="${MIRROR}/releases/amd64/autobuilds/current-stage3-amd64-hardened+nomultilib/${STAGE3_REAL_NAME}"
 else
-	echo "${STAGE3_NAME} is up to date, skipping"
+  echo 'invalid profile, exiting'
+  exit 1
 fi
 
-# make sure latest stage3 is actually good
-OUR_SHA512=$(sha512sum "${OUTDIR}/${STAGE3_NAME}" | awk {'print $1'})
-if [ "${OUR_SHA512}" != "${OUR_SHA512}" ]; then
-  echo 'downloaded file did not match the sha512 sum'
+curl -s "${STAGE3_URL}.DIGESTS.asc" -o "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc"
+gkeys verify -F "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc"
+STATUS=$?
+if [[ ${STATUS} != 0 ]]; then
+  echo 'stage3 did not verify, removing badness'
+  rm "${OUTDIR}/${STAGE3_REAL_NAME}"
+  rm "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc"
   exit 1
 fi
 
+SHA512=$(grep -A1 SHA512 "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc" | grep stage3 | grep -v CONTENTS | awk '{ print $1 }')
+SHA512_REAL=$(sha512sum "${OUTDIR}/${STAGE3_NAME}" | awk '{ print $1 }')
+if [[ "${SHA512}" != "${SHA512_REAL}" ]]; then
+  echo "Downloading new image - ${STAGE3_REAL_NAME}"
+  curl -s "${STAGE3_URL}" -o "${OUTDIR}/${STAGE3_REAL_NAME}"
+  SHA512=$(grep -A1 SHA512 "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc" | grep stage3 | grep -v CONTENTS | awk '{ print $1 }')
+  SHA512_REAL=$(sha512sum "${OUTDIR}/${STAGE3_REAL_NAME}" | awk '{ print $1 }')
+  if [[ "${SHA512}" != "${SHA512_REAL}" ]]; then
+    echo 'shasum did not match, removing badness'
+    rm "${OUTDIR}/${STAGE3_REAL_NAME}"
+    rm "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc"
+    exit 1
+  fi
+  # otherwise we cleanup and move on
+  if [[ -f "${OUTDIR}/${STAGE3_NAME}" ]]; then
+    rm "${OUTDIR}/${STAGE3_NAME}"
+  fi
+  rm "${OUTDIR}/${STAGE3_REAL_NAME}.DIGESTS.asc"
+  mv "${OUTDIR}/${STAGE3_REAL_NAME}" "${OUTDIR}/${STAGE3_NAME}"
+fi
+
+
 # get the latest portage
-PORTAGE_DIR="/var/tmp/catalyst/snapshots"
-PORTAGE_LIVE_MD5=$(curl -s "${MIRROR}/gentoo/snapshots/portage-latest.tar.bz2.md5sum" | awk '/portage-latest/ {print $1}')
-OUR_MD5=$(md5sum "${PORTAGE_DIR}/portage-latest.tar.bz2" | awk {'print $1'})
+PORTAGE_LIVE_MD5=$(curl -s "${MIRROR}/snapshots/portage-latest.tar.bz2.md5sum" | awk '/portage-latest/ {print $1}')
+OUR_MD5=$(md5sum "${PORTAGE_DIR}/portage-current.tar.bz2" | awk {'print $1'})
 if [[ "${PORTAGE_LIVE_MD5}" != "${OUR_MD5}" ]]; then
   echo 'downloading new portage tarball'
-  curl -s "${MIRROR}/gentoo/snapshots/portage-latest.tar.bz2" > "${PORTAGE_DIR}/portage-latest.tar.bz2"
+  if [[ ! -d "${PORTAGE_DIR}" ]]; then
+    mkdir -p "${PORTAGE_DIR}"
+  fi
+  curl -s "${MIRROR}/snapshots/portage-latest.tar.bz2" -o "${PORTAGE_DIR}/portage-current.tar.bz2"
+  curl -s "${MIRROR}/snapshots/portage-latest.tar.bz2.gpgsig" -o "${PORTAGE_DIR}/portage-current.tar.bz2.gpgsig"
+  gkeys verify -F "${PORTAGE_DIR}/portage-current.tar.bz2"
+  STATUS=$?
+  if [[ ${STATUS} != 0 ]]; then
+    echo 'tarball did not verify, removing badness'
+    rm "${PORTAGE_DIR}/portage-current.tar.bz2"
+    rm "${PORTAGE_DIR}/portage-current.tar.bz2.gpgsig"
+    exit 1
+  elif [[ ${STATUS} == 0 ]]; then
+    echo 'tarball verified'
+    rm "${PORTAGE_DIR}/portage-current.tar.bz2.gpgsig"
+  fi
 else
   echo 'portage tarball is up to date'
 fi
-
-OUR_MD5=$(md5sum "${PORTAGE_DIR}/portage-latest.tar.bz2" | awk {'print $1'})
-if [[ "${PORTAGE_LIVE_MD5}" != "${OUR_MD5}" ]]; then
-  echo 'downloaded file did not match the md5sum'
-  exit 1
-fi

02-catalyst-that-shit.sh

@@ -8,44 +8,111 @@
 # your own scenario.  I have a VM that poops out images for me, and these
 # are the fields I use.
 
-##
-## Vars
-##
-DATE=$(date +%Y%m%d)
-SPECFILE=~/tmp/catalyst/stage4.spec
-OUTDIR=~/tmp/catalyst/gentoo
-OUTFILE="${OUTDIR}/stage4-${DATE}.tar.bz2"
+set -e -u -x -o pipefail
 
+# Vars
+export DATE=${DATE:-"$(date +%Y%m%d)"}
+export OUTDIR=${OUTDIR:-"/root/tmp/catalyst/gentoo"}
+export GIT_BASE_DIR=${GIT_BASE_DIR:-$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )}
+# profiles supported are as follows
+# default/linux/amd64/13.0
+# default/linux/amd64/13.0/no-multilib
+# hardened/linux/amd64
+# hardened/linux/amd64/no-multilib
+# hardened/linux/amd64/selinux (eventually)
+# hardened/linux/amd64/no-multilib/selinux (eventually)
+export PROFILE=${PROFILE:-"default/linux/amd64/13.0"}
+
+
+if [[ "${PROFILE}" == "default/linux/amd64/13.0" ]]; then
+  PROFILE_SHORTNAME="amd64-default"
+  SOURCE_SUBPATH="stage3-amd64-current"
+  KERNEL_SOURCES="gentoo-sources"
+elif [[ "${PROFILE}" == "default/linux/amd64/13.0/no-multilib" ]]; then
+  PROFILE_SHORTNAME="amd64-default-nomultilib"
+  SOURCE_SUBPATH="stage3-amd64-nomultilib-current"
+  KERNEL_SOURCES="gentoo-sources"
+elif [[ "${PROFILE}" == "hardened/linux/musl/amd64" ]]; then
+  PROFILE_SHORTNAME="amd64-hardened-musl"
+  SOURCE_SUBPATH="musl/hardened/amd64/stage3-amd64-musl-hardened"
+  KERNEL_SOURCES="hardened-sources"
+elif [[ "${PROFILE}" == "hardened/linux/amd64" ]]; then
+  PROFILE_SHORTNAME="amd64-hardened"
+  SOURCE_SUBPATH="stage3-amd64-hardened-current"
+  KERNEL_SOURCES="hardened-sources"
+elif [[ "${PROFILE}" == "hardened/linux/amd64/no-multilib" ]]; then
+  PROFILE_SHORTNAME="amd64-hardened-nomultilib"
+  SOURCE_SUBPATH="stage3-amd64-hardened-nomultilib-current"
+  KERNEL_SOURCES="hardened-sources"
+else
+  echo 'invalid profile, exiting'
+  exit 1
+fi
+export OUTFILE=${OUTFILE:-"${OUTDIR}/stage4-${PROFILE_SHORTNAME}-${DATE}.tar.bz2"}
+export SPECFILE=${SPECFILE:-"/root/tmp/catalyst/stage4-${PROFILE_SHORTNAME}.spec"}
 mkdir -p "${OUTDIR}"
 
 # Build the spec file, first
 cat > "${SPECFILE}" << EOF
 subarch: amd64
 target: stage4
-rel_type: default
-profile: default/linux/amd64/13.0
-source_subpath: stage3-amd64-latest
+rel_type: ${PROFILE_SHORTNAME}
+profile: ${PROFILE}
+source_subpath: ${SOURCE_SUBPATH}
 cflags: -O2 -pipe -march=core2
 
-pkgcache_path: /tmp/packages
-kerncache_path: /tmp/kernel
+pkgcache_path: /tmp/packages-${PROFILE_SHORTNAME}
+kerncache_path: /tmp/kernel-${PROFILE_SHORTNAME}
+portage_confdir: ${GIT_BASE_DIR}/portage_overlay
+portage_overlay: /opt/overlays/musl
 
 # Probably best made as parameters
-snapshot: latest
+snapshot: current
 version_stamp: ${DATE}
 
 # Stage 4 stuff
-stage4/use: bash-completion bzip2 idm urandom ipv6 mmx sse sse2 abi_x86_32 abi_x86_64
-stage4/packages: eix dev-vcs/git tmux vim sys-devel/bc cloud-init syslog-ng logrotate vixie-cron dhcpcd net-misc/curl sudo gentoolkit iproute2 grub:0
-stage4/fsscript: /root/gentoo-catalyst/prep.sh
-stage4/root_overlay: /root/gentoo-catalyst/root-overlay
-stage4/rcadd: syslog-ng|default sshd|default vixie-cron|default cloud-config|default cloud-init-local|default cloud-init|default cloud-final|default netmount|default
+stage4/use: bash-completion bzip2 idm ipv6 mmx sse sse2 urandom -nls -fortran
+stage4/packages: app-admin/logrotate app-admin/sudo app-admin/syslog-ng app-editors/vim app-portage/eix app-portage/gentoolkit net-misc/dhcpcd sys-apps/dmidecode sys-apps/gptfdisk sys-apps/iproute2 sys-apps/lsb-release sys-boot/grub:2 sys-devel/bc sys-power/acpid sys-process/cronie
+stage4/fsscript: files/prep.sh
+stage4/root_overlay: root-overlay
+stage4/rcadd: syslog-ng|default sshd|default cronie|default netmount|default acpid|default dhcpcd|default net.lo|default
 
 boot/kernel: gentoo
-boot/kernel/gentoo/sources: gentoo-sources
-boot/kernel/gentoo/config: /root/gentoo-catalyst/kernel.config
-boot/kernel/gentoo/extraversion: reenigne
-boot/kernel/gentoo/gk_kernargs: --all-ramdisk-modules
+boot/kernel/gentoo/sources: ${KERNEL_SOURCES}
+boot/kernel/gentoo/config: files/kernel-${PROFILE_SHORTNAME}.config
+boot/kernel/gentoo/extraversion: openstack
+boot/kernel/gentoo/gk_kernargs: --all-ramdisk-modules --makeopts=-j6
+
+# all of the cleanup...
+stage4/unmerge:
+  sys-kernel/genkernel
+  sys-kernel/gentoo-sources
+  sys-kernel/hardened-sources
+
+stage4/empty:
+  /root/.ccache
+  /tmp
+  /usr/portage/distfiles
+  /usr/src
+  /var/cache/edb/dep
+  /var/cache/genkernel
+  /var/empty
+  /var/run
+  /var/state
+  /var/tmp
+
+stage4/rm:
+  /etc/*-
+  /etc/*.old
+  /etc/ssh/ssh_host_*
+  /root/.*history
+  /root/.lesshst
+  /root/.ssh/known_hosts
+  /root/.viminfo
+  # Remove any generated stuff by genkernel
+  /usr/share/genkernel
+  # This is 3MB of crap for each copy
+  /usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz
 EOF
 
 # Run catalyst
@@ -55,4 +122,4 @@ catalyst -f "${SPECFILE}"
 rm "${SPECFILE}"
 
 # Move the outputted image
-mv "/var/tmp/catalyst/builds/default/stage4-amd64-${DATE}.tar.bz2" "${OUTFILE}"
+mv "/var/tmp/catalyst/builds/${PROFILE_SHORTNAME}/stage4-amd64-${DATE}.tar.bz2" "${OUTFILE}"

03-prep-that-image.sh

@@ -1,64 +1,80 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Okay, so here's some real meat.  We take a drive (as 02 said, I use a VM),
 # and we spray that stage4 all over it.  Then we rub some grub (0.97) all over
 # it to make it feel better, and then we box it up and ship it out.
 
-##
-## Vars
-##
-TARGET_DISK=vdd
-TEMP_DIR=/image-prep/gentoo
-TARGET_IMAGE=/var/www/reenigne-gentoo-`date +%Y-%m-%d`
-MOUNT_DIR=/mnt
-ORIG_DIR=`pwd`
-TARBALL=/image-prep/gentoo/stage4.tar.bz2
+set -e -u -x -o pipefail
+
+# Vars
+export TEMP_DIR=${TEMP_DIR:-'/root/tmp/catalyst/gentoo'}
+export MOUNT_DIR=${MOUNT_DIR:-'/mnt'}
+export DATE=${DATE:-"$(date +%Y%m%d)"}
+export PORTAGE_DIR=${PORTAGE_DIR:-"/var/tmp/catalyst/snapshots"}
+# profiles supported are as follows
+# default/linux/amd64/13.0
+# default/linux/amd64/13.0/no-multilib
+# hardened/linux/amd64
+# hardened/linux/amd64/no-multilib
+# hardened/linux/amd64/selinux (eventually)
+# hardened/linux/amd64/no-multilib/selinux (eventually)
+export PROFILE=${PROFILE:-"default/linux/amd64/13.0"}
+if [[ "${PROFILE}" == "default/linux/amd64/13.0" ]]; then
+  PROFILE_SHORTNAME="amd64-default"
+elif [[ "${PROFILE}" == "default/linux/amd64/13.0/no-multilib" ]]; then
+  PROFILE_SHORTNAME="amd64-default-nomultilib"
+elif [[ "${PROFILE}" == "hardened/linux/amd64" ]]; then
+  PROFILE_SHORTNAME="amd64-hardened"
+elif [[ "${PROFILE}" == "hardened/linux/amd64/no-multilib" ]]; then
+  PROFILE_SHORTNAME="amd64-hardened-nomultilib"
+else
+  echo 'invalid profile, exiting'
+  exit 1
+fi
+export TARBALL=${TARBALL:-"/root/tmp/catalyst/gentoo/stage4-${PROFILE_SHORTNAME}-${DATE}.tar.bz2"}
+export TEMP_IMAGE=${TEMP_IMAGE:-"gentoo-${PROFILE_SHORTNAME}.img"}
+export TARGET_IMAGE=${TARGET_IMAGE:-"/root/openstack-${PROFILE_SHORTNAME}-${DATE}.qcow2"}
+
+# create a raw partition and do stuff with it
+fallocate -l 1G "${TEMP_DIR}/${TEMP_IMAGE}"
+BLOCK_DEV=$(losetup -f --show "${TEMP_DIR}/${TEMP_IMAGE}")
 
 # Okay, we have the disk, let's prep it
 echo 'Building disk'
-parted -s /dev/$TARGET_DISK mklabel msdos
-parted -s --align=none /dev/$TARGET_DISK mkpart primary 2048s 100%
-parted -s /dev/$TARGET_DISK set 1 boot on
-mkfs.ext4 -F /dev/${TARGET_DISK}1
+parted -s "${BLOCK_DEV}" mklabel gpt
+parted -s --align=none "${BLOCK_DEV}" mkpart bios_boot 0 2M
+parted -s --align=none "${BLOCK_DEV}" mkpart primary 2M 100%
+parted -s "${BLOCK_DEV}" set 1 boot on
+parted -s "${BLOCK_DEV}" set 1 bios_grub on
+mkfs.ext4 -F "${BLOCK_DEV}p2"
+e2label "${BLOCK_DEV}p2" cloudimg-rootfs
 
 # Mount it
 echo 'Mounting disk'
-mount /dev/${TARGET_DISK}1 $MOUNT_DIR
-
-# Let's localize commands now
-cd $MOUNT_DIR
+mkdir -p "${MOUNT_DIR}/${PROFILE_SHORTNAME}"
+mount "${BLOCK_DEV}p2" "${MOUNT_DIR}/${PROFILE_SHORTNAME}"
 
 # Expand the stage
 echo 'Expanding tarball'
-tar xjpf $TARBALL -C ./
+tar --xattrs -xjpf "${TARBALL}" -C "${MOUNT_DIR}/${PROFILE_SHORTNAME}"
 
-# Throw in a resolv.conf (because we download portage next)
-cp /etc/resolv.conf etc/resolv.conf
+#echo 'Adding in /usr/portage'
+#tar --xattrs -xjpf "${PORTAGE_DIR}/portage-latest.tar.bz2" -C "${MOUNT_DIR}/${PROFILE_SHORTNAME}/usr"
 
-# Catalyst doesn't give us portage, so that's cool
-echo 'Downloading portage'
-curl -s http://mirror.reenigne.net/gentoo/snapshots/portage-latest.tar.bz2 > portage-latest.tar.bz2
-echo 'Expanding portage'
-tar xjf portage-latest.tar.bz2 -C usr/
-rm portage-latest.tar.bz2
+# Install grub
+echo 'Installing grub'
+grub2-install "${BLOCK_DEV}" --root-directory "${MOUNT_DIR}/${PROFILE_SHORTNAME}/"
 
 # Clean up
 echo 'Syncing; unmounting'
 sync
-sleep 5; # To unmount, just in case.  5 seconds is nothing next to the dd below
-cd $ORIG_DIR
-umount $MOUNT_DIR
-
-# Install grub
-echo 'Installing grub'
-printf "device (hd0) /dev/${TARGET_DISK}\nroot (hd0,0)\nsetup (hd0)\nquit\n" | grub --batch
+umount "${MOUNT_DIR}/${PROFILE_SHORTNAME}"
 
-# Now it's unmounted, but we need to make an image!
-echo 'dding image'
-dd if=/dev/$TARGET_DISK of=${TEMP_DIR}/temp.raw
+# get rid of block mapping
+losetup -d "${BLOCK_DEV}"
 
-echo 'Converting dd image to qcow2'
-qemu-img convert -c -f raw -O qcow2 ${TEMP_DIR}/temp.raw ${TARGET_IMAGE}.qcow2
+echo 'Converting raw image to qcow2'
+qemu-img convert -c -f raw -O qcow2 "${TEMP_DIR}/${TEMP_IMAGE}" "${TARGET_IMAGE}"
 
 echo 'Cleaning up'
-rm ${TEMP_DIR}/temp.raw
+rm "${TEMP_DIR}/${TEMP_IMAGE}"