Building native engine distribution with Bazel

.github/workflows/bazel-build.yml

@@ -9,7 +9,8 @@ jobs:
     name: Run bazel GUI build
     # if: github.ref != 'refs/heads/develop'
     runs-on:
-      - ubuntu-latest
+      - self-hosted
+      - Linux
     steps:
       - uses: bazel-contrib/setup-bazel@0.15.0
         with:
@@ -53,19 +54,20 @@ jobs:
   build-engine:
     name: Run bazel Engine build
     runs-on:
-      - ubuntu-latest
+      - self-hosted
+      - Linux
     steps:
       - uses: bazel-contrib/setup-bazel@0.15.0
         with:
           bazelisk-version: 1.x
           bazelisk-cache: true
           bazelrc: build --remote_cache=grpcs://${{ vars.ENSO_BAZEL_CACHE_URI }} --remote_cache_header="authorization=Basic ${{ secrets.ENSO_BAZEL_CACHE_TOKEN }}"
       - uses: actions/checkout@v4
-      - run: bazel build //:sbt_build_engine_distribution
+      - run: bazel build --verbose_failures //:sbt_build_native_engine_distribution
       - name: Get Engine build output location
         id: get_bazel_output
         run: |
-          OUTPUT_SYMLINK=$(bazel cquery --output=files //:sbt_build_engine_distribution)
+          OUTPUT_SYMLINK=$(bazel cquery --output=files //:sbt_build_native_engine_distribution)
           BAZEL_OUTPUT=$(readlink -f "$OUTPUT_SYMLINK")
           echo "location=$BAZEL_OUTPUT" >> $GITHUB_OUTPUT
       - uses: actions/upload-artifact@v4

BUILD.bazel

@@ -2,7 +2,7 @@ load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files")
 load("@aspect_rules_ts//ts:defs.bzl", "ts_config")
 load("@buildifier_prebuilt//:rules.bzl", "buildifier")
 load("@npm//:defs.bzl", "npm_link_all_packages")
-load("//:bazel_scripts/run_engine_distribution.bzl", "run_enso")
+load("//:bazel_scripts/run_engine_distribution.bzl", "ensure_native_enso", "run_enso")
 load("//:bazel_scripts/stdlibs.bzl", "stdlib_source_patterns")
 load("//toolchains/sbt:run_sbt.bzl", "run_sbt")
 
@@ -61,14 +61,23 @@ buildifier(
 ### Engine distribution ###
 ###########################
 
-STDLIB_SOURCES = glob(stdlib_source_patterns())
+STDLIB_SOURCES = glob(
+    stdlib_source_patterns(),
+    allow_empty = True,
+)
 
-# TODO: some dependencies might be unnecessary here, but checking everything is time-consuming.
-SRCS = STDLIB_SOURCES + [
+SBT_PROJECT_FILES = [
     ".scalafmt.conf",
     "build.sbt",
     "project/plugins.sbt",
     "project/build.properties",
+] + glob([
+    "project/*.scala",
+    "project/src/**",
+])
+
+# TODO: some dependencies might be unnecessary here, but checking everything is time-consuming.
+ENGINE_DIST_SOURCES = STDLIB_SOURCES + SBT_PROJECT_FILES + [
     "distribution/manifest.template.yaml",
     "distribution/launcher-manifest.yaml",
     "distribution/enso.bundle.template",
@@ -81,7 +90,6 @@ SRCS = STDLIB_SOURCES + [
     "//app/ydoc-server-polyglot:dist",
 ] + glob([
     "engine/common/src/**",
-    "engine/interpreter-dsl-test/src/**",
     "engine/language-server/src/**",
     "engine/launcher/src/**",
     "engine/polyglot-api/src/**",
@@ -90,25 +98,22 @@ SRCS = STDLIB_SOURCES + [
     "engine/runner/src/**",
     "engine/runner-common/src/**",
     "engine/runtime/src/**",
-    "engine/runtime-benchmarks/src/**",
     "engine/runtime-compiler/src/**",
     "engine/runtime-compiler-dump/src/**",
     "engine/runtime-compiler-dump-igv/src/**",
     "engine/runtime-instrument-common/src/**",
     "engine/runtime-instrument-id-execution/src/**",
     "engine/runtime-instrument-repl-debugger/src/**",
     "engine/runtime-instrument-runtime-server/src/**",
-    "engine/runtime-integration-tests/src/**",
     "engine/runtime-language-arrow/src/**",
     "engine/runtime-language-epb/src/**",
     "engine/runtime-parser/src/**",
     "engine/runtime-parser-dsl/src/**",
     "engine/runtime-parser-processor/src/**",
-    "engine/runtime-parser-processor-tests/src/**",
     "engine/runtime-suggestions/src/**",
     "engine/runtime-test-instruments/src/**",
+    "distribution/engine/THIRD-PARTY/**",
     "lib/java/akka-wrapper/src/**",
-    "lib/java/benchmarks-common/src/**",
     "lib/java/fansi-wrapper/src/**",
     "lib/java/interpreter-dsl/src/**",
     "lib/java/jna-wrapper/src/**",
@@ -124,12 +129,10 @@ SRCS = STDLIB_SOURCES + [
     "lib/java/poi-wrapper/src/**",
     "lib/java/runtime-utils/src/**",
     "lib/java/scala-libs-wrapper/src/**",
-    "lib/java/test-utils/src/**",
     "lib/java/ydoc-polyfill/src/**",
     "lib/java/ydoc-server/src/**",
     "lib/java/ydoc-server-registration/src/**",
     "lib/java/zio-wrapper/src/**",
-    "lib/scala/bench-processor/src/**",
     "lib/scala/cli/src/**",
     "lib/scala/common-polyglot-core-utils/src/**",
     "lib/scala/connected-lock-manager/src/**",
@@ -142,7 +145,6 @@ SRCS = STDLIB_SOURCES + [
     "lib/scala/filewatcher/src/**",
     "lib/scala/json-rpc-server/src/**",
     "lib/scala/library-manager/src/**",
-    "lib/scala/locking-test-helper/src/**",
     "lib/scala/logging-config/src/**",
     "lib/scala/logging-service/src/**",
     "lib/scala/logging-service-logback/src/**",
@@ -161,14 +163,13 @@ SRCS = STDLIB_SOURCES + [
     "lib/scala/text-buffer/src/**",
     "lib/scala/version-output/src/**",
     "lib/scala/yaml/src/**",
-    "project/*.scala",
-    "project/src/**",
     "std-bits/aws/src/**",
     "std-bits/base/src/**",
     "std-bits/benchmarks/src/**",
     "std-bits/database/src/**",
     "std-bits/google/src/**",
     "std-bits/image/src/**",
+    "std-bits/duckdb/src/**",
     "std-bits/microsoft/src/**",
     "std-bits/snowflake/src/**",
     "std-bits/table/src/**",
@@ -220,16 +221,26 @@ EXTRA_SYSTEM_PROPS = [
     "-Denso.BazelSupport.ydocServer.polyglotMainJs=$(location //app/ydoc-server-polyglot:dist)",
 ]
 
+SMALL_JDK_SYSTEM_PROPS = [
+    "-Denso.BazelSupport.enabled=true",
+    "-Denso.BazelSupport.home=$(BINDIR)/home",
+]
+
 ENV = {
     "JAVA_HOME": "$(JAVABASE)",
     "PATH": "$(FLATC_PATH):$(JAVABASE)/bin:$$PATH",
     "LC_ALL": "C.UTF-8",
     "SBT_SERVER_FORCESTART": "1",
 }
 
+NATIVE_IMAGE_ENV = dict(
+    ENV,
+    ENSO_LAUNCHER = "native",
+)
+
 run_sbt(
     name = "sbt_build_engine_distribution",
-    srcs = SRCS,
+    srcs = ENGINE_DIST_SOURCES,
     args = [
         "buildEngineDistribution",
     ],
@@ -239,24 +250,43 @@ run_sbt(
     visibility = ["//visibility:public"],
 )
 
+run_sbt(
+    name = "sbt_build_native_engine_distribution",
+    srcs = ENGINE_DIST_SOURCES,
+    args = [
+        "buildEngineDistribution",
+    ],
+    env = NATIVE_IMAGE_ENV,
+    out_dir = "built-distribution-native",
+    system_props = SBT_SYSTEM_PROPS + EXTRA_SYSTEM_PROPS,
+    visibility = ["//visibility:public"],
+)
+
 run_sbt(
     name = "sbt_build_small_jdk",
-    srcs = SRCS,
+    srcs = SBT_PROJECT_FILES,
     args = [
         "buildSmallJdkForRelease",
     ],
     env = ENV,
     out_dir = "built-small-jdk",
-    system_props = SBT_SYSTEM_PROPS + EXTRA_SYSTEM_PROPS,
+    system_props = SBT_SYSTEM_PROPS + SMALL_JDK_SYSTEM_PROPS,
     visibility = ["//visibility:public"],
 )
 
 # This is just a sanity check that runs equivalent of `enso --run test/Base_Tests/**/Maybe_Spec.enso`.
 run_enso(
     name = "run_enso_test",
     src = "//test/Base_Tests:maybe_spec",
-    distribution = ":sbt_build_engine_distribution",
+    distribution = ":sbt_build_native_engine_distribution",
     run_args = [
         "--no-ir-caches",
     ],
 )
+
+# Sanity check that ensure that the distribution was built with native image.
+ensure_native_enso(
+    name = "ensure_native_enso",
+    distribution = ":sbt_build_native_engine_distribution",
+    visibility = ["//visibility:public"],
+)

MODULE.bazel

@@ -19,6 +19,8 @@ bazel_dep(name = "aspect_rules_ts", version = "3.7.0")
 bazel_dep(name = "rules_nodejs", version = "6.3.2")
 bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
 bazel_dep(name = "bazel_skylib", version = "1.8.1")
+bazel_dep(name = "rules_cc", version = "0.2.12")
+bazel_dep(name = "zlib", version = "1.3.1.bcr.7")
 bazel_dep(name = "aspect_rules_esbuild", version = "0.22.1")
 
 ## Node.js

bazel_scripts/run_engine_distribution.bzl

@@ -5,7 +5,7 @@ Simple rule that runs Enso distribution via a shell script.
 def _run_enso_impl(ctx):
     distribution = ctx.attr.distribution[DefaultInfo].files
     binary = ctx.actions.declare_file(ctx.label.name + ".sh")
-    dist_dir = distribution.to_list()[0].path
+    dist_dir = distribution.to_list()[0].basename
     src_file = ctx.file.src.path
     java_toolchain_type = ctx.toolchains["@bazel_tools//tools/jdk:runtime_toolchain_type"]
     java_home = java_toolchain_type.java_runtime.java_home_runfiles_path
@@ -15,7 +15,7 @@ def _run_enso_impl(ctx):
         content = """#!/bin/bash
         export JAVA_HOME="{java_home}"
         export PATH="$JAVA_HOME/bin:$PATH"
-        binary_path=built-distribution/enso-engine-*/enso-*/bin/enso
+        binary_path={dist}/enso-engine-*/enso-*/bin/enso
         if [ ! -f $binary_path ]; then
             echo "Error: Could not find enso binary in {dist}"
             exit 1
@@ -42,6 +42,51 @@ def _run_enso_impl(ctx):
         runfiles = all_runfiles,
     )]
 
+def _ensure_native_enso_impl(ctx):
+    """ Runs enso executable with `--version` argument to ensure it was built with native image """
+    distribution = ctx.attr.distribution[DefaultInfo].files
+    binary = ctx.actions.declare_file(ctx.label.name + ".sh")
+    dist_dir = distribution.to_list()[0].basename
+    ctx.actions.write(
+        output = binary,
+        content = """#!/bin/bash
+        binary_path={dist}/enso-engine-*/enso-*/bin/enso
+        if [ ! -f $binary_path ]; then
+            echo "Error: Could not find enso binary in {dist}"
+            exit 1
+        fi
+        $PWD/$binary_path --version
+        file --mime-encoding $PWD/$binary_path | grep -q "binary"
+        if [ $? -ne 0 ]; then
+            echo "Error: enso binary is not a native image build"
+            exit 1
+        fi
+        """.format(
+            dist = dist_dir,
+        ),
+        is_executable = True,
+    )
+
+    all_runfiles = ctx.runfiles(
+        files = distribution.to_list(),
+    )
+
+    return [DefaultInfo(
+        executable = binary,
+        runfiles = all_runfiles,
+    )]
+
+ensure_native_enso = rule(
+    implementation = _ensure_native_enso_impl,
+    attrs = {
+        "distribution": attr.label(
+            mandatory = True,
+            allow_files = True,
+        ),
+    },
+    executable = True,
+)
+
 run_enso = rule(
     implementation = _run_enso_impl,
     toolchains = [

bazel_scripts/stdlibs.bzl

@@ -34,5 +34,8 @@ def stdlib_source_patterns():
         glob_patterns += [
             "distribution/lib/Standard/{}/0.0.0-dev/src/**".format(lib_name),
             "distribution/lib/Standard/{}/0.0.0-dev/package.yaml".format(lib_name),
+            "distribution/lib/Standard/{}/0.0.0-dev/data/**".format(lib_name),
+            "distribution/lib/Standard/{}/0.0.0-dev/docs/**".format(lib_name),
+            "distribution/lib/Standard/{}/0.0.0-dev/THIRD-PARTY/**".format(lib_name),
         ]
     return glob_patterns