Merge multiple PRs in one

Jenkinsfile

@@ -86,15 +86,14 @@ try {
                         throw e
                     } finally {
                         set_gh_status_pending(is_pr, 'Store images')
-                        params = ["${script_env}", "docker/publish-project.sh"].join("\n")
                         stage('Store images') {
+                            params = ["${script_env}", "docker/publish-project.sh"].join("\n")
+                            sh "${params}"
+                            params = ["${script_env}", "docker/publish-sstate.sh"].join("\n")
                             sh "${params}"
                         }
                     }
                 } // docker_image
-                archiveArtifacts allowEmptyArchive: true,
-                                 artifacts: 'build*/TestResults_*/TEST-*.xml'
-                step_xunit('build*/TestResults_*/TEST-*.xml')
                 tester_script = readFile "docker/tester-exec.sh"
                 tester_summary = readFile "docker/tester-create-summary.sh"
                 qemu_script = readFile "docker/run-qemu.exp"
@@ -111,6 +110,18 @@ try {
                 docker.image(image_name).inside(run_args) {
                     params = ["${script_env}", "docker/post-build.sh"].join("\n")
                     sh "${params}"
+                    params = ["${script_env}", "docker/publish-sstate.sh"].join("\n")
+                    sh "${params}"
+                }
+                // note wildcard: handle pre-build reports in build.pre/ as well
+                lock(resource: "global_data") {
+                    summary += sh(returnStdout: true,
+                                  script: "docker/tester-create-summary.sh 'oe-selftest: post-build' '' build/TestResults_*/TEST- 0")
+                    archiveArtifacts allowEmptyArchive: true,
+                                     artifacts: 'build*/TestResults_*/TEST-*.xml'
+                }
+                lock(resource: "step-xunit") {
+                    step_xunit('build*/TestResults_*/TEST-*.xml')
                 }
             }
         }

doc/howtos/OSTree.rst

@@ -0,0 +1,150 @@
+Handling System Updates Using OSTree
+####################################
+
+IoT Reference OS Kit ('refkit') has support for updating devices running
+one of the refkit profile images using an HTTP/HTTPs server and OSTree.
+OSTree is a library and suite of command line tools, largely inspired by
+git, that provides version control and a distribution mechanism for bootable
+OS filesystem trees, or other binaries. For a comprehensive introduction
+to OSTree, an overview of its architecture and feature set, please refer
+to its `documentation <http://ostree.readthedocs.io>_`.
+
+If enabled, OSTree support in refkit
+
+ * provides A/B-versioning (within a single block device partition)
+ * creates a per-image OSTree repository for each image built
+ * exposes builds in a common OSTree repository as a series of commits
+ * can sign each commit (which is then verified during updates)
+ * can provide a service for automatic image updates
+
+
+Enabling OSTree Support
+#######################
+
+To enable end-to-end OSTree support with automatic image updates, you will
+need to
+
+ * enable OSTree support for your builds
+ * ideally generate and use a pair of signing keys for your builds/updates
+ * expose the OSTree repository of your builds over HTTP/HTTPS
+ * point your clients to the exposed OSTree repository
+
+To enable OSTree support, turn on the 'ostree' image feature. You can
+do this by incuding the following configuration snippet in your local.conf
+or other suitable global configuration file::
+
+    REFKIT_IMAGE_EXTRA_FEATURES += "ostree"
+
+For development images (*REFKIT_IMAGE_MODE* set to *'development'*), there
+is no need to generate any keys. The default configuration uses pregenerated
+keys already present in the git repository for such images. For production
+images you can generate a pair of signing keys with the following command::
+
+    meta-refkit-core/files/gnupg/generate-keys.sh $(pwd)/gpg release@example.org
+
+This will generate a pair of private and public GPG keys with the key ID
+release@example.org, put them into the GPG keyring at $(pwd)/gpg and also
+export them to the files $(pwd)/release@example.org.{pub,sec}.
+
+To use these newly generated keys for signing and verifying the OSTree
+repository commits, add the following to your local.conf or other suitable
+global configuration file (replace $(pwd) with the path to your
+top-level intel-iot-refkit directory)::
+
+    OSTREE_GPGDIR = "$(pwd)/gpg"
+    OSTREE_GPGID  = "release@example.org"
+
+Assuming you want to use your build machine, build.example.org, in this
+example also as your update server, you can readily point the image for
+updates to your update/build server by adding the following to your
+local.conf or other suitable global configuration file::
+
+    OSTREE_REMOTE = "http://build.example.org/ostree/"
+
+You can also use HTTPS instead of HTTP if you want to and your server is
+properly configured for serving HTTPS requests.
+
+Next you need to expose the OSTree repository your builds are exported to
+over HTTP/HTTPS for clients to consume. By default this repository is
+located in build/tmp-glibc/deploy/ostree-repo, but you can change this
+location by adding the following to your local.conf or other suitable
+global configuration file::
+
+    OSTREE_EXPORT = "<path-to-the-repository>"
+
+Now assuming, you did not change the location, and you use Apache for
+service HTTP/HTTPS requests, you can expose this repository with Apache
+by adding the following to your Apache configuration::
+
+    Alias "/ostree/" "<path-to-intel-iot-refkit>/build/tmp-glibc/deploy/ostree-repo/"
+
+    <Directory <path-to-inte-iot-refkit>/build/tmp-glibc/deploy/ostree-repo>
+        Options Indexes FollowSymLinks
+        Require all granted
+    </Directory>
+
+Finally you should restart (or start) your Apache server to activate the
+configuration changes. This might be a good time to also make sure that
+any firewall rules you might have will allow your clients access to the
+HTTP port of the server.
+
+Another alternative is to use the built-in trivial HTTP server in ostree
+available as the *ostree trivial-httpd* command (if it is enabled at
+compile time). With that you could serve out the repository with the
+following commands::
+
+    cd build/tmp-glibc/deploy
+    ln -sf ostree-repo ostree
+    ostree trivial-httpd --port 80
+
+A third alternative is to use a simple Python HTTP server, for instance
+the `one <http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/meta/lib/oeqa/utils/httpserver.py>_` from OpenEmbedded Core.
+
+Now with the above configuration in place, and an HTTP server running,
+subsequent builds should get automatically exported and pulled in as
+updates by the clients running one of your refkit images.
+
+
+Disabling Automatic Updates
+###########################
+
+If you prefer not to pull in updates automatically to the clients, disable
+the refkit-update systemd service. You can do this by running the following
+command on the client device::
+
+    systemctl stop refkit-update.service
+    systemctl disable refkit-update.service
+
+
+Pulling In Updates Manually
+###########################
+
+If you want to manually pull any potentially available updates, you can do
+so by running the following command on a client device::
+
+    refkit-ostree-update --one-shot
+
+This will check the server for available updates, pull in any such one,
+and request a reboot to activate the changes if the update was successfully
+installed.
+
+
+Preventing/Delaying Automatic Reboot
+####################################
+
+Note that by default after an update has been installed the system will be
+rebooted to activate the latest changes. Any entity that needs to prevent
+or delay the reboot to a more convenient time in the future should use
+systemd-inhibit or the corresponding systemd(-logind) interfaces for doing
+so.
+
+For instance, if you have an interactive shell (or a login session via ssh)
+while the updater is running, or you are running it yourself manually, and
+you don't want the system to get rebooted under you in case an update does
+get pulled in, you should do a::
+
+    systemd-inhibit --what=shutdown $SHELL
+
+Once you're done with whatever you were doing and want to allow any pending
+updates to proceed to reboot, you can simply exit the innermost shell.
+

doc/system-update.rst

@@ -91,9 +91,10 @@ OSTree tool
   working in the running OS, the initramfs also overrides :file:`/proc/cmdline`
   with a version that has the expected ``ostree`` boot parameter.
 
-``refkit-ostree_git.bb``
-  Some on-target helper script, partly used by the initramfs and partly
-  used as wrapper around the actual :command:`ostree`.
+``refkit-ostree.bb``
+  On-target helper scripts and binaries. Partly (/usr/bin/refkit-ostree) used
+  by the initramfs ostree module (/etc/init.d/91-ostree), the rest is and is
+  used by the refkit ostree updater (/usr/bin/refkit-ostree-update).
 
 ``ostree-image.bbclass``
   This helper class gets inherited automatically by ``refkit-image.bbclass``
@@ -117,18 +118,17 @@ OSTree Usage
 See the comments in ``ostree-image.bbclass`` for instructions on how
 to configure the image creation. In particular, image signing and
 publishing the permanent OSTree repository require some planning and
-customization.
+customization. Also check the :file:`howtos/OSTree.rst` for more instructions
+and tips on how to use OSTree for system updates.
 
-In development images, the default is to use a generated GPG key from
-:file:`tmp-glibc/deploy/gnupg/` and a "permanent" OSTree repository in
-:file:`tmp-glibc/deploy/ostree-repo/`. In other words, removing
-:file:`tmp-glibc` really starts from scratch.
+In development images, the default is to use a pregenerated GPG key from
+the pregenerated GPG keyring in :file:`meta-refkit-core/files/gnupg` and a
+"permanent" OSTree repository in :file:`tmp-glibc/deploy/ostree-repo/`.
+In other words, removing :file:`tmp-glibc` really starts from scratch.
 
 Extra work is necessary when images from previous builds are still
 meant to be updateable:
 
-#. The GPG key must be stored elsewhere (see ``OSTREE_GPGDIR`` and
-   ``OSTREE_GPGID``).
 #. The public OSTree repo must be stored elsewhere (see ``OSTREE_REPO``) *or*
 #. after a successful build, the new commit in :file:`tmp-glibc/deploy/ostree-repo/`
    must be moved to a different, more permanent OSTree repo with the
@@ -144,13 +144,14 @@ into the release process for a product.
 
 .. deltas: https://ostree.readthedocs.io/en/latest/manual/repository-management/#derived-data-static-deltas-and-the-summary-file
 
-Once a device has booted into an OSTree-enabled image, the
-:command:`ostree` command can be used as usual. Updates are configured
-in :file:`/ostree/repo/config` to pull new OS releases from the
-``OSTREE_REMOTE`` URL that was set at build time.
-
-Beware that system updates should be done with :command:`refkit-ostree
-update`, because that will also update the UEFI combo app.
+Once a device has booted into an OSTree-enabled image, if the rekit-update
+service has been enabled and running (which it is by default), any updates
+should get automatically pulled in and activated by a reboot of the device.
+If manual updates are preferred, the refkit-update service should be stopped
+and/or disabled. Manual updates can be triggered by running
+:command:`refkit-ostree-update --one-shot`. Updates are configured in
+:file:`/ostree/repo/config` to pull new OS releases from the ``OSTREE_REMOTE``
+URL that was set at build time.
 
 OSTree Filesystem
 -----------------

docker/publish-project.sh

@@ -114,17 +114,6 @@ if [ -f "${LOG}" ]; then
     rsync -avz ${LOG}* ${_RSYNC_DEST}/
 fi
 
-if [ -d sstate-cache ]; then
-  if [ ! -z ${BUILD_CACHE_DIR+x} ]; then
-    if [ -d ${BUILD_CACHE_DIR}/sstate ]; then
-      # populate shared sstate from local sstate:
-      _src=sstate-cache
-      _dst=${RSYNC_PUBLISH_DIR}/bb-cache/sstate
-      find ${_src} -mindepth 1 -maxdepth 1 -type d -exec rsync -a --ignore-existing {} ${_dst}/ \;
-    fi
-  fi
-fi
-
 ## for debugging signatures: publish stamps
 if [ -d ${_BRESULT}/stamps ]; then
     create_remote_dirs ${_RSYNC_DEST} .stamps/${TARGET_MACHINE}/

docker/publish-sstate.sh

@@ -0,0 +1,26 @@
+#!/bin/sh -xeu
+#
+# publish-project.sh: Publish local sstate into global sstate
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+cd $WORKSPACE/build
+
+if [ -d sstate-cache ]; then
+  if [ ! -z ${BUILD_CACHE_DIR+x} ]; then
+    if [ -d ${BUILD_CACHE_DIR}/sstate ]; then
+      # populate shared sstate from local sstate, show names for tracability
+      _src=sstate-cache
+      _dst=${RSYNC_PUBLISH_DIR}/bb-cache/sstate
+      find ${_src} -mindepth 1 -maxdepth 1 -type d -exec rsync -a --info=name --ignore-existing {} ${_dst}/ \;
+    fi
+  fi
+fi

docker/tester-create-summary.sh

@@ -1,3 +1,4 @@
+#!/bin/bash -ue
 #
 # tester-create-summary.sh: tester creates summary information
 # Copyright (c) 2016, Intel Corporation.

docker/tester-exec.sh

@@ -111,6 +111,12 @@ testimg() {
 
     daft ${DEVICE} ${FILENAME} --record
     TEST_EXIT_CODE=$?
+    if [ "$TEST_EXIT_CODE" = 1 ]; then
+      echo "WARNING: daft=1 would lead to UNSTABLE: wipe results, retry daft"
+      rm -f *.log *.log.raw *.xml
+      daft ${DEVICE} ${FILENAME} --record
+      TEST_EXIT_CODE=$?
+    fi
   fi
 
   # delete symlinks, these point outside of local set and are useless

meta-flatpak/classes/flatpak-config.bbclass

@@ -60,9 +60,11 @@ FLATPAK_BRANCH ?= "${DISTRO_VERSION}"
 FLATPAK_LATEST ?= "${DISTRO}/${FLATPAK_PN}/latest"
 FLATPAK_BUILD  ?= "${DISTRO}/${FLATPAK_PN}/build/${BUILD_ID}"
 
-# This is the GPG key id of our repository signing key. If you set this to
-# empty, signing is disabled altogether.
-FLATPAK_GPGID ?= "refkit-signing@key"
+# This is the GPG homedir and the key ID for signing repository commits. If
+# you set the key ID to empty, signing is disabled altogether.
+FLATPAK_GPGDIR ?= "${FLATPAKBASE}/../meta-refkit-core/files/gnupg"
+FLATPAK_GPGID  ?= "${@d.getVar('DISTRO').replace(' ', '_') + \
+                        '-development-signing@key'}"
 
 # We can pre-populate the image during build with a set of pre-declared
 # flatpak repositories and associated dedicated flatpak-session users.

meta-flatpak/classes/flatpak-repository.bbclass

@@ -4,9 +4,6 @@ REQUIRED_DISTRO_FEATURES_append = " usrmerge systemd pam"
 
 inherit flatpak-config
 
-REFKIT_SIGNING_KEYS += "${FLATPAK_GPGID}"
-inherit refkit-signing-keys
-
 # These are lists of files we check to determine the flatpak
 # runtime type of an image if it is not directly visible from
 # the image name. This did not used to be necessary before we
@@ -64,8 +61,7 @@ fakeroot do_flatpak_populate_repository () {
    echo "${IMAGE_BASENAME} is a flatpak $RUNTIME_TYPE image"
 
    if [ -n "${FLATPAK_GPGID}" ]; then
-       GPG_SIGN="--gpg-home ${REFKIT_SIGNING_GPGDIR} \
-                 --gpg-id ${FLATPAK_GPGID}"
+       GPG_SIGN="--gpg-home ${FLATPAK_GPGDIR} --gpg-id ${FLATPAK_GPGID}"
    else
        GPG_SIGN=""
    fi
@@ -153,8 +149,7 @@ fakeroot do_flatpak_export_repository () {
    echo "${IMAGE_BASENAME} is a flatpak $RUNTIME_TYPE image"
 
    if [ -n "${FLATPAK_GPGID}" ]; then
-       GPG_SIGN="--gpg-home ${REFKIT_SIGNING_GPGDIR} \
-                 --gpg-id ${FLATPAK_GPGID}"
+       GPG_SIGN="--gpg-home ${FLATPAK_GPGDIR} --gpg-id ${FLATPAK_GPGID}"
    else
        GPG_SIGN=""
    fi