Skip to content

fix(opensearch) #10293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jhheider wants to merge 2 commits into from
Closed

fix(opensearch) #10293

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
29f8701
fix(opensearch)
jhheider Aug 22, 2025
1fdae9f
try using repo for k-NN
jhheider Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 76 additions & 61 deletions projects/opensearch.org/package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,68 +38,80 @@ build:
working-directory: ${{prefix}}/config

# checkout k-NN plugin
- run:
- |
if [ -d .git ]; then
git fetch
else
git clone https://github.com/opensearch-project/k-NN .
fi
- git checkout {{version}}.0
- git reset --hard
- git submodule foreach --recursive git reset --hard
- git submodule update --init --recursive
- git config --global user.email "hello@pkgx.dev"
- git config --global user.name "pkgx"
working-directory: k-NN

# workarounds for m1 build. see: https://github.com/opensearch-project/k-NN/blob/main/DEVELOPER_GUIDE.md#extra-setup-for-mac-m1-machines
- run:
- sed -i -e 's/-march=native/-mcpu=apple-m1/g' nmslib/similarity_search/CMakeLists.txt
- sed -i -e 's/-mcpu=apple-a14/-mcpu=apple-m1/g' nmslib/python_bindings/setup.py
- sed -i -e 's/__aarch64__/__undefine_aarch64__/g' faiss/faiss/utils/distances_simd.cpp
if: darwin/aarch64
working-directory: k-NN/jni/external
# - run:
# - |
# if [ -d .git ]; then
# git fetch
# else
# git clone https://github.com/opensearch-project/k-NN .
# fi
# - git checkout {{version}}.0
# - git reset --hard
# - git submodule foreach --recursive git reset --hard
# - git submodule update --init --recursive
# - git config --global user.email "hello@pkgx.dev"
# - git config --global user.name "pkgx"
# working-directory: k-NN

- run:
- sed -i -f $PROP jni/CMakeLists.txt jni/cmake/*.cmake
- export CC=clang
- export CXX=clang++
prop: |
s|/usr/local/opt/libomp/|{{deps.openmp.llvm.org.prefix}}/|g
s|/opt/homebrew/opt/libomp/|{{deps.openmp.llvm.org.prefix}}/|g
if: darwin
working-directory: k-NN

# this is recommended in https://github.com/opensearch-project/k-NN/blob/45e9e542aef60ef7073ee726e6ac14dec27bfa04/scripts/build.sh#L91-L94
- run: |
if test "{{hw.arch}}" = "x86-64"; then
sed -i -e 's/-march=native/-march=x86-64/g' CMakeLists.txt
fi
working-directory: k-NN/jni/external/nmslib/similarity_search
if: ^2
- run:
- cmake .
- make
working-directory: k-NN/jni
- run: |
for LIB in *.jnilib; do
install_name_tool -add_rpath @loader_path $LIB
done
working-directory: k-NN/jni/release
if: darwin
- run:
- ./gradlew build --refresh-dependencies -x integTest -x test -DskipTests=true -Dopensearch.version={{version}} -Dbuild.snapshot=false -Dbuild.version_qualifier=
- ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version={{version}} -Dbuild.snapshot=false -Dbuild.version_qualifier=
- ./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=false -Dbuild.version_qualifier= -Dopensearch.version={{version}}
working-directory: k-NN
- run:
- mkdir -p ./build/distributions/lib
- cp -v ./jni/release/libopensearchknn* ./build/distributions/lib
- cd ./build/distributions
- zip -r opensearch-knn-{{version}}.0.zip lib/
- '{{prefix}}/bin/opensearch-plugin install --batch file:`pwd`/opensearch-knn-{{version}}.0.zip'
working-directory: k-NN
# # workarounds for m1 build. see: https://github.com/opensearch-project/k-NN/blob/main/DEVELOPER_GUIDE.md#extra-setup-for-mac-m1-machines
# - run:
# - sed -i -e 's/-march=native/-mcpu=apple-m1/g' nmslib/similarity_search/CMakeLists.txt
# - sed -i -e 's/-mcpu=apple-a14/-mcpu=apple-m1/g' nmslib/python_bindings/setup.py
# - sed -i -e 's/__aarch64__/__undefine_aarch64__/g' faiss/faiss/utils/distances_simd.cpp
# if: darwin/aarch64
# working-directory: k-NN/jni/external

# - run:
# - sed -i -f $PROP jni/CMakeLists.txt jni/cmake/*.cmake
# - export CC=clang
# - export CXX=clang++
# prop: |
# s|/usr/local/opt/libomp/|{{deps.openmp.llvm.org.prefix}}/|g
# s|/opt/homebrew/opt/libomp/|{{deps.openmp.llvm.org.prefix}}/|g
# if: darwin
# working-directory: k-NN

# # this is recommended in https://github.com/opensearch-project/k-NN/blob/45e9e542aef60ef7073ee726e6ac14dec27bfa04/scripts/build.sh#L91-L94
# - run: |
# if test "{{hw.arch}}" = "x86-64"; then
# sed -i -e 's/-march=native/-march=x86-64/g' CMakeLists.txt
# fi
# working-directory: k-NN/jni/external/nmslib/similarity_search
# if: ^2

# # this is blowing up the build on aarch64, so we disable it
# # /Users/builder/actions-runner/_work/pantry/pantry/builds/opensearch.org-2.19.2/k-NN/jni/external/nmslib/similarity_search/src/distcomp_scalar.cc:85:9: error: pragma message requires parenthesized string
# # 85 | #pragma message WARN("ScalarProductSIMD<float>: SSE2 is not available, defaulting to pure C++ implementation!")
# - run: sed -i -f $PROP distcomp_scalar.cc
# working-directory: k-NN/jni/external/nmslib/similarity_search/src
# if: ^2

# prop: |
# s|#pragma message WARN|// #pragma message WARN|g

# - run:
# - cmake . $CMAKE_JNI_ARGS
# - make
# working-directory: k-NN/jni
# - run: |
# for LIB in *.jnilib; do
# install_name_tool -add_rpath @loader_path $LIB
# done
# working-directory: k-NN/jni/release
# if: darwin
# - run:
# - ./gradlew build --refresh-dependencies -x integTest -x test -DskipTests=true -Dopensearch.version={{version}} -Dbuild.snapshot=false -Dbuild.version_qualifier=
# - ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version={{version}} -Dbuild.snapshot=false -Dbuild.version_qualifier=
# - ./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=false -Dbuild.version_qualifier= -Dopensearch.version={{version}}
# working-directory: k-NN
# - run:
# - mkdir -p ./build/distributions/lib
# - cp -v ./jni/release/libopensearchknn* ./build/distributions/lib
# - cd ./build/distributions
# - zip -r opensearch-knn-{{version}}.0.zip lib/
# - '{{prefix}}/bin/opensearch-plugin install --batch file:`pwd`/opensearch-knn-{{version}}.0.zip'
# working-directory: k-NN
- ${{prefix}}/bin/opensearch-plugin install --batch org.opensearch.plugin:opensearch-knn:{{version}}.0
- run: echo 'export JAVA_LIBRARY_PATH="$OPENSEARCH_HOME/plugins/opensearch-knn/lib:$JAVA_LIBRARY_PATH"' >> opensearch-env
working-directory: ${{prefix}}/bin
if: darwin
Expand All @@ -110,6 +122,9 @@ build:
linux/x86-64:
# /opt/gnu.org/binutils/v2.43.1/bin/ld: release/libopensearchknn_faiss_avx2.so: undefined reference to `_gfortran_concat_string'
LDFLAGS: $LDFLAGS -Wl,-lgfortran
# linux:
# CMAKE_JNI_ARGS:
# - -DBLAS_LIBRARIES={{deps.openblas.net.prefix}}/lib/libopenblas.a

provides:
- bin/opensearch
Expand Down
Loading