Issue #259 improved

.github/workflows/build.yaml

@@ -41,24 +41,32 @@ jobs:
       id: kernel-submodule
       run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT
 
-    - name: ccache
-      uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92
+    - name: restore ccache
+      uses: actions/cache/restore@v4
+      id: restore-cache
       with:
+        path: |
+          .ccache
         key: kernel-${{ steps.kernel-submodule.outputs.ref }}
         restore-keys: kernel-
 
-    - name: Build kernel
-      run: ./build_kernel.sh
-
     - name: Update VERSION
       if: github.event_name == 'pull_request'
       run: |
         echo -n $(cat VERSION)-$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-7) > VERSION
 
-    - name: Build system
+    - name: Build
       env:
         GIT_HASH: ${{ github.event.pull_request.head.sha }}
-      run: ./build_system.sh
+      run: ./build.sh
+
+    - name: save ccache
+      uses: actions/cache/save@v4
+      if: always() && steps.restore-cache.outputs.cache-hit != 'true'
+      with:
+        path: |
+          .ccache
+        key: ${{ steps.restore-cache.outputs.cache-primary-key }}
 
     - name: Statistics
       id: stats

Dockerfile.agnos

@@ -82,8 +82,8 @@ COPY ./userspace/base_setup.sh /tmp/agnos
 RUN /tmp/agnos/base_setup.sh
 
 # Install openpilot dependencies
-COPY ./userspace/openpilot_dependencies.sh /tmp/agnos/
-RUN /tmp/agnos/openpilot_dependencies.sh
+# Moved to base_setup.sh
+
 COPY ./userspace/openpilot_python_dependencies.sh /tmp/agnos/
 RUN /tmp/agnos/openpilot_python_dependencies.sh
 
@@ -96,11 +96,6 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" install -yq \
     /tmp/agnos/libssl1.1_1.1.1f-1ubuntu2.22_arm64.deb \
     /tmp/agnos/libffi6_3.2.1-8_arm64.deb
 
-# ################### #
-# ###### AGNOS ###### #
-# ################### #
-FROM agnos-base
-
 # Hardware setup
 RUN mkdir -p /tmp/agnos/debs
 COPY ./userspace/debs /tmp/agnos/debs
@@ -113,8 +108,8 @@ COPY --from=agnos-compiler-modemmanager /tmp/modemmanager.deb /tmp/
 COPY --from=agnos-compiler-lpac /tmp/lpac.deb /tmp/
 
 RUN cd /tmp && \
-    apt-get update && \
-    apt-get install -yq --no-install-recommends \
+    apt-fast update && \
+    apt-fast install -yq --no-install-recommends \
     python3 \
     python3-dev \
     gir1.2-qmi-1.0 \
@@ -188,8 +183,9 @@ RUN touch -r /lib/systemd/systemd /etc/fstab
 # populate /usr
 COPY ./userspace/usr/comma/ /usr/$USERNAME/
 COPY ./userspace/usr/share/fonts/* /usr/share/fonts/
-COPY ./userspace/libs/* /usr/lib/aarch64-linux-gnu/
-COPY ./userspace/libs32/* /usr/lib/arm-linux-gnueabihf/
+# avoid using * in the file list to keep symlinks
+COPY ./userspace/libs/ /usr/lib/aarch64-linux-gnu/
+COPY ./userspace/libs32/ /usr/lib/arm-linux-gnueabihf/
 
 # this is big, only enable when we need it
 # kernel headers for the AGNOS kernel (built on device)
@@ -202,7 +198,8 @@ COPY ./userspace/files/gl-renderer.so /usr/lib/arm-linux-gnueabihf/weston
 
 # Setup systemd services
 COPY ./userspace/services.sh /tmp/agnos
-RUN /tmp/agnos/services.sh
+# Ignore errors in case services are not present
+RUN set +e; /tmp/agnos/services.sh && exit 0; set -e;
 
 # MOTD
 RUN rm -r /etc/update-motd.d/*
@@ -242,12 +239,18 @@ COPY ./userspace/files/CAMERA_ICP.elf /usr/lib/firmware
 RUN sed -i 's/hosts:          files dns myhostname/hosts:          files myhostname dns/g' /etc/nsswitch.conf
 
 # TODO: move this to base_setup.sh or build gcc from source
-# Remove unused architectures & bins from arm-none-eabi
-RUN cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ && \
-    find thumb/ -maxdepth 1 -type d ! \( -name 'thumb' -o -name 'v7e-m+fp' -o -name 'v7e-m+dp' \) -exec rm -rf {} + && \
-    rm cc1plus g++-mapper-server && \
-    find /usr/bin -maxdepth 1 -type f -name 'arm-none-eabi-*' ! \( -name 'arm-none-eabi-gcc' -o -name 'arm-none-eabi-objcopy' -o -name 'arm-none-eabi-objdump' \) -delete
+# TODO: may remove more packages not needed for runtime to reduce final disk image size
+RUN apt-get remove -yq \
+    build-essential \
+    gcc gcc-13 gcc-13-aarch64-linux-gnu gcc-aarch64-linux-gnu \
+    cpp-13-aarch64-linux-gnu \
+    && apt-get -yq autoremove \
+    && apt-get -yq clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Remove dev packages to save space
+# Ignore any errors on dpkg
+RUN dpkg --remove $(dpkg -l '*-dev' | awk '/^ii/ {print $2}'); exit 0
 
 # keep this last
 RUN ldconfig

build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+./build_kernel.sh &
+./build_system.sh &
+wait

build_kernel.sh

@@ -72,7 +72,9 @@ build_kernel() {
   echo "-- First make --"
   make $DEFCONFIG O=out
   echo "-- Second make: $(nproc --all) cores --"
-  make -j$(nproc --all) O=out  # Image.gz-dtb
+  # Build on CPUs * 1.5
+  NPROCS=$(printf "%.0f" $(echo "$(nproc) * 1.5" | bc -l))
+  make -j$NPROCS O=out  # Image.gz-dtb
 
   # Turn on if you want perf
   # LDFLAGS=-static make -j$(nproc --all) -C tools/perf

build_system.sh

@@ -103,8 +103,8 @@ docker container rm -f $CONTAINER_ID $MOUNT_CONTAINER_ID" EXIT
 
 # Extract image
 echo "Extracting docker image"
-docker container export -o $BUILD_DIR/filesystem.tar $CONTAINER_ID
-exec_as_root tar -xf $BUILD_DIR/filesystem.tar -C $ROOTFS_DIR > /dev/null
+# Extract to disk directly to avoid disk space issues
+docker container export $CONTAINER_ID | docker exec -i $MOUNT_CONTAINER_ID tar -xf - -C $ROOTFS_DIR > /dev/null
 
 # Avoid detecting as container
 echo "Removing .dockerenv file"

userspace/base_setup.sh

@@ -50,6 +50,22 @@ echo "comma - nice -10" >> /etc/security/limits.conf
 locale-gen en_US.UTF-8
 update-locale LANG=en_US.UTF-8
 
+# Create dirs
+mkdir /data && chown $USERNAME:$USERNAME /data
+mkdir /persist && chown $USERNAME:$USERNAME /persist
+mkdir /config && chown root:root /config
+
+# Disable pstore service that moves files out of /sys/fs/pstore
+systemctl disable systemd-pstore.service
+
+# Nopasswd sudo
+echo "comma ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# setup /bin/sh symlink
+ln -sf /bin/bash /bin/sh
+
+# Install necessary libs
+# TODO: find only needed libs and tools to build runtime image
 apt-fast upgrade -yq
 apt-fast install --no-install-recommends -yq \
     alsa-utils \
@@ -58,75 +74,60 @@ apt-fast install --no-install-recommends -yq \
     build-essential \
     bzip2 \
     curl \
-    cpuset \
     dfu-util \
-    evtest \
-    git \
-    git-core \
-    git-lfs \
-    gdb \
-    htop \
-    i2c-tools \
-    ifmetric \
     ifupdown \
     iptables-persistent \
     jq \
     landscape-common \
-    libi2c-dev \
     libqmi-utils \
     libtool \
-    libncursesw5-dev \
     libnss-myhostname \
-    libgdbm-dev \
-    libc6-dev \
-    libsqlite3-dev \
-    libssl-dev \
-    libffi-dev \
-    llvm \
-    nano \
-    net-tools \
     nload \
     network-manager \
-    nvme-cli \
     openssl \
     ppp \
-    smartmontools \
-    speedtest-cli \
     ssh \
     sshfs \
-    sudo \
     systemd-resolved \
     systemd-timesyncd \
     traceroute \
-    tk-dev \
     ubuntu-minimal \
     ubuntu-server \
     ubuntu-standard \
     udev \
     udhcpc \
     wget \
-    wireless-tools \
-    zlib1g-dev
-
-rm -rf /var/lib/apt/lists/*
-
-# Create dirs
-mkdir /data && chown $USERNAME:$USERNAME /data
-mkdir /persist && chown $USERNAME:$USERNAME /persist
-mkdir /config && chown root:root /config
-
-# Disable pstore service that moves files out of /sys/fs/pstore
-systemctl disable systemd-pstore.service
-
-# Nopasswd sudo
-echo "comma ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-# setup /bin/sh symlink
-ln -sf /bin/bash /bin/sh
-
-# Install necessary libs
-apt-fast update -yq
-apt-fast install --no-install-recommends -yq \
+    autoconf \
+    automake \
+    casync \
+    clang \
+    clinfo \
+    cmake \
+    cppcheck \
+    curl \
+    darkstat \
+    dkms \
+    gpiod \
+    libglib2.0-0t64 \
+    libportaudio2 \
+    libtool \
+    libxcb-icccm4 \
+    libxcb-image0 \
+    libxcb-keysyms1 \
+    libxcb-render-util0 \
+    libxcb-xinerama0 \
+    libxcb-xinput0 \
+    libxcb-xkb1 \
+    libxkbcommon-x11-0 \
+    locales \
+    nethogs \
+    ocl-icd-libopencl1 \
+    opencl-headers \
+    pkg-config \
+    texinfo \
+    vnstat \
+    xvfb \
+    zstd \
     libacl1:armhf \
     libasan6-armhf-cross \
     libatomic1-armhf-cross \
@@ -135,7 +136,6 @@ apt-fast install --no-install-recommends -yq \
     libblkid1:armhf \
     libc6:armhf \
     libc6-armhf-cross \
-    libc6-dev:armhf \
     libc6-dev-armhf-cross \
     libcairo2:armhf \
     libcap2:armhf \
@@ -151,25 +151,19 @@ apt-fast install --no-install-recommends -yq \
     libgomp1-armhf-cross \
     libgudev-1.0-0:armhf \
     libinput-bin:armhf \
-    libinput-dev:armhf \
     libinput10:armhf \
-    libjpeg-dev:armhf \
     libjpeg-turbo8:armhf \
-    libjpeg-turbo8-dev:armhf \
     libjpeg8:armhf \
-    libjpeg8-dev:armhf \
     libkmod2:armhf \
     libmtdev1t64:armhf \
     libpam0g:armhf \
-    libpam0g-dev:armhf \
     libpcre3:armhf \
     libpixman-1-0:armhf \
     libpng16-16t64:armhf \
     libselinux1:armhf \
     libstdc++6:armhf \
     libstdc++6-armhf-cross \
     libubsan1-armhf-cross \
-    libudev-dev:armhf \
     libudev1:armhf \
     libuuid1:armhf \
     libwacom9:armhf \
@@ -182,15 +176,11 @@ apt-fast install --no-install-recommends -yq \
     libxext6:armhf \
     libxkbcommon0:armhf \
     libxrender1:armhf \
-    linux-libc-dev:armhf \
     linux-libc-dev-armhf-cross \
     zlib1g:armhf \
     libegl1 \
-    libegl-dev \
     libgles1 \
     libgles2 \
-    libgles-dev \
-    libx264-dev \
     openssh-server \
     dnsmasq-base \
     isc-dhcp-client \
@@ -199,5 +189,7 @@ apt-fast install --no-install-recommends -yq \
     kmod \
     wpasupplicant \
     hostapd \
-    libgtk2.0-dev \
     libxml2:armhf \
+    portaudio19-dev \
+    libdbus-1-dev \
+