Merge remote-tracking branch 'origin/sycl' into private/udit/property_parsing

Author: uditagarwal97

.github/workflows/sycl-coverity.yml

@@ -1,6 +1,18 @@
 name: Coverity
 on:
   workflow_dispatch:
+    inputs:
+      description:
+        description: 'Custom build description'
+        required: true
+        type: string
+        default: 'Out-of-order Coverity scan'
+      ref:
+        description: 'Git ref to checkout and scan'
+        required: true
+        type: string
+        default: 'sycl'
+
   schedule:
     - cron: '0 0 * * 0'
 
@@ -27,7 +39,7 @@ jobs:
     - uses: ./devops/actions/cached_checkout
       with:
         path: src
-        ref: ${{ github.sha }}
+        ref: ${{ inputs.ref || github.sha }}
         cache_path: "/__w/repo_cache/"
 
     - name: Get coverity tool
@@ -56,10 +68,15 @@ jobs:
 
     - name: Submit build
       run: |
+        default_description="${{ inputs.description }}"
+        if [[ -z $default_description ]]; then
+          default_description="Regular build"
+        fi
+
         # Initialize a build. Fetch a cloud upload url.
         curl -X POST \
-        -d version="sycl: ${{ github.sha }}" \
-        -d description="Regular build" \
+        -d version="${{ inputs.ref || github.sha }}" \
+        -d description="$default_description" \
         -d email=${{ secrets.COVERITY_EMAIL }} \
         -d token=${{ secrets.COVERITY_TOKEN }} \
         -d file_name="intel_llvm.tgz" \

.github/workflows/sycl-linux-precommit.yml

@@ -23,7 +23,8 @@ on:
     - '.github/workflows/sycl-nightly.yml'
     - '.github/workflows/sycl-rel-nightly.yml'
     - '.github/workflows/sycl-rel-nightly-launch.yml'
-    - '.github/workflows/trivy.yml'
+    - '.github/workflows/sycl-trivy.yml'
+    - '.github/workflows/sycl-coverity.yml'
     - 'devops/containers/**'
     - 'devops/actions/build_container/**'
     - 'unified-runtime/examples/**'
@@ -241,6 +242,7 @@ jobs:
       benchmark_upload_results: false
       benchmark_preset: 'Minimal'
       benchmark_dry_run: true
+      benchmark_exit_on_failure: true
       repo_ref: ${{ github.sha }}
       toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}

.github/workflows/sycl-linux-run-tests.yml

@@ -140,6 +140,12 @@ on:
         type: string
         default: 'false'
         required: False
+      benchmark_exit_on_failure:
+        description: |
+          Whether or not to fail the workflow upon a failure.
+        type: string
+        default: 'false'
+        required: False
 
   workflow_dispatch:
     inputs:
@@ -211,6 +217,12 @@ on:
           - "build-only"
           - "run-only"
 
+      benchmark_exit_on_failure:
+        type: choice
+        options:
+          - "true"
+          - "false"
+
 permissions:
   contents: read
   packages: read
@@ -358,6 +370,7 @@ jobs:
         save_name: ${{ inputs.benchmark_save_name }}
         preset: ${{ inputs.benchmark_preset }}
         dry_run: ${{ inputs.benchmark_dry_run }}
+        exit_on_failure: ${{ inputs.benchmark_exit_on_failure }}
         build_ref: ${{ inputs.repo_ref }}
       env:
         RUNNER_TAG: ${{ inputs.runner }}

.github/workflows/trivy.yml renamed to .github/workflows/sycl-trivy.yml

@@ -41,6 +41,12 @@ on:
         description: |
           Upload results to https://intel.github.io/llvm/benchmarks/.
         required: true
+      exit_on_failure:
+        type: string # true/false: workflow_dispatch does not support booleans
+        description: |
+          Fail benchmark script on any error. Limit number of iterations to just test correctness.
+        required: false
+        default: 'false'
       runner:
         type: string
         required: true
@@ -90,6 +96,13 @@ on:
           - false
           - true
         default: true
+      exit_on_failure:
+        description: Fail benchmark script on any error. Limit number of iterations to just test correctness.
+        type: choice
+        options:
+          - false
+          - true
+        default: false
       runner:
         description: Self-hosted runner to use for the benchmarks
         type: choice
@@ -193,6 +206,7 @@ jobs:
       benchmark_upload_results: ${{ inputs.upload_results }}
       benchmark_save_name: ${{ needs.sanitize_inputs.outputs.benchmark_save_name }}
       benchmark_preset: ${{ inputs.preset }}
+      benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
       repo_ref: ${{ needs.sanitize_inputs.outputs.build_ref }}
       toolchain_artifact: ${{ needs.build_sycl.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_sycl.outputs.toolchain_artifact_filename }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -22,7 +22,8 @@ on:
     - '.github/workflows/sycl-nightly.yml'
     - '.github/workflows/sycl-rel-nightly.yml'
     - '.github/workflows/sycl-rel-nightly-launch.yml'
-    - '.github/workflows/trivy.yml'
+    - '.github/workflows/sycl-trivy.yml'
+    - '.github/workflows/sycl-coverity.yml'
     - 'devops/containers/**'
     - 'devops/actions/build_container/**'
     - 'devops/compat_ci_exclude.sycl-rel-6_2'

.github/workflows/sycl-windows-precommit.yml

@@ -32,6 +32,9 @@ inputs:
   dry_run:
     type: string
     required: False
+  exit_on_failure:
+    type: string
+    required: False
 
 runs:
   using: "composite"
@@ -192,8 +195,10 @@ runs:
       sycl-ls
       echo "-----"
 
-      taskset -c "$CORES" ./devops/scripts/benchmarks/main.py \
-        "$(realpath ./llvm_test_workdir)" \
+      WORKDIR="$(realpath ./llvm_test_workdir)"
+      if [ -n "$WORKDIR" ] && [ -d "$WORKDIR" ] && [[ "$WORKDIR" == *llvm_test_workdir* ]]; then rm -rf "$WORKDIR" ; fi
+
+      taskset -c "$CORES" ./devops/scripts/benchmarks/main.py "$WORKDIR" \
         --sycl "$(realpath ./toolchain)" \
         --ur "$(realpath ./ur/install)" \
         --adapter "$FORCELOAD_ADAPTER" \
@@ -204,7 +209,8 @@ runs:
         --preset "$PRESET" \
         --timestamp-override "$SAVE_TIMESTAMP" \
         --detect-version sycl,compute_runtime \
-        --flamegraph inclusive
+        ${{ inputs.exit_on_failure == 'true' && '--exit-on-failure --iterations 1' || '' }}
+      # TODO: add back: "--flamegraph inclusive" once works properly
 
       echo "-----"
       python3 ./devops/scripts/benchmarks/compare.py to_hist \

devops/actions/run-tests/benchmark/action.yml

@@ -36,6 +36,10 @@ using the built compiler located in `~/llvm/build/` and
 installed Unified Runtime in directory `~/ur_install`,
 and then **run** the benchmarks for `adapter_name` adapter.
 
+The scripts will try to reuse the files stored in `~/benchmarks_workdir/`. 
+If any dependant projects binaries are already built, they will not be rebuilt
+again if their tags match tags specified by benchmarks source code.
+
 >NOTE: By default `level_zero` adapter is used.
 
 >NOTE: Pay attention to the `--ur` parameter. It points directly to the directory where UR is installed.  
@@ -48,10 +52,6 @@ $ cmake --build ~/ur_build -j $(nproc)
 $ cmake --install ~/ur_build
 ```
 
-### Rebuild
-The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time.  
-To avoid that, use `--no-rebuild` option.
-
 ## Results
 
 By default, the benchmark results are not stored.  

devops/scripts/benchmarks/README.md

@@ -66,9 +66,13 @@ def setup(self) -> None:
                 self.git_tag(),
                 Path(options.workdir),
                 "onednn",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name} skipped")
+            return
+
         extra_cmake_args = [
             f"-DCMAKE_PREFIX_PATH={options.sycl}",
             "-DCMAKE_CXX_COMPILER=clang++",
@@ -80,7 +84,6 @@ def setup(self) -> None:
         ]
         self.project.configure(
             extra_cmake_args,
-            install_prefix=False,
             add_sycl=True,
         )
         self.project.build(

devops/scripts/benchmarks/benches/benchdnn.py

@@ -14,6 +14,7 @@
 from git_project import GitProject
 from options import options
 from utils.result import BenchmarkMetadata, Result
+from utils.logger import log
 
 from .base import Benchmark, Suite, TracingType
 from .compute_metadata import ComputeMetadataGenerator
@@ -74,9 +75,13 @@ def setup(self) -> None:
                 self.git_hash(),
                 Path(options.workdir),
                 "compute-benchmarks",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name} skipped")
+            return
+
         extra_args = [
             f"-DBUILD_SYCL=ON",
             f"-DSYCL_COMPILER_ROOT={options.sycl}",
@@ -96,7 +101,7 @@ def setup(self) -> None:
                 f"-Dunified-runtime_DIR={options.ur}/lib/cmake/unified-runtime",
             ]
 
-        self.project.configure(extra_args, install_prefix=False, add_sycl=True)
+        self.project.configure(extra_args, add_sycl=True)
         self.project.build(add_sycl=True)
 
     def additional_metadata(self) -> dict[str, BenchmarkMetadata]:
@@ -266,12 +271,13 @@ def benchmarks(self) -> list[Benchmark]:
 
         # Add UR-specific benchmarks
         benches += [
-            MemcpyExecute(self, RUNTIMES.UR, 400, 1, 102400, 10, 1, 1, 1, 1, 0),
-            MemcpyExecute(self, RUNTIMES.UR, 400, 1, 102400, 10, 0, 1, 1, 1, 0),
-            MemcpyExecute(self, RUNTIMES.UR, 100, 4, 102400, 10, 1, 1, 0, 1, 0),
-            MemcpyExecute(self, RUNTIMES.UR, 100, 4, 102400, 10, 1, 1, 0, 0, 0),
-            MemcpyExecute(self, RUNTIMES.UR, 4096, 4, 1024, 10, 0, 1, 0, 1, 0),
-            MemcpyExecute(self, RUNTIMES.UR, 4096, 4, 1024, 10, 0, 1, 0, 1, 1),
+            # TODO: multithread_benchmark_ur fails with segfault
+            # MemcpyExecute(self, RUNTIMES.UR, 400, 1, 102400, 10, 1, 1, 1, 1, 0),
+            # MemcpyExecute(self, RUNTIMES.UR, 400, 1, 102400, 10, 0, 1, 1, 1, 0),
+            # MemcpyExecute(self, RUNTIMES.UR, 100, 4, 102400, 10, 1, 1, 0, 1, 0),
+            # MemcpyExecute(self, RUNTIMES.UR, 100, 4, 102400, 10, 1, 1, 0, 0, 0),
+            # MemcpyExecute(self, RUNTIMES.UR, 4096, 4, 1024, 10, 0, 1, 0, 1, 0),
+            # MemcpyExecute(self, RUNTIMES.UR, 4096, 4, 1024, 10, 0, 1, 0, 1, 1),
             UsmMemoryAllocation(self, RUNTIMES.UR, "Device", 256, "Both"),
             UsmMemoryAllocation(self, RUNTIMES.UR, "Device", 256 * 1024, "Both"),
             UsmBatchMemoryAllocation(self, RUNTIMES.UR, "Device", 128, 256, "Both"),
@@ -342,11 +348,12 @@ def cpu_count_str(self, separator: str = "") -> str:
 
     def get_iters(self, run_trace: TracingType):
         """Returns the number of iterations to run for the given tracing type."""
-        return (
-            self.iterations_trace
-            if run_trace != TracingType.NONE
-            else self.iterations_regular
-        )
+        if options.exit_on_failure:
+            # we are just testing that the benchmark runs successfully
+            return 3
+        if run_trace == TracingType.NONE:
+            return self.iterations_regular
+        return self.iterations_trace
 
     def supported_runtimes(self) -> list[RUNTIMES]:
         """Base runtimes supported by this benchmark, can be overridden."""
@@ -764,6 +771,7 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
             "--contents=Zeros",
             "--multiplier=1",
             "--vectorSize=1",
+            "--lws=256",
         ]