Skip to content

Update nn-hal and integrate remote inference support #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
akodanka wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Update nn-hal and integrate remote inference support #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
0722c03
Resolved segfault while running CTS for nnhal
rairatne Aug 29, 2022
e19bead
Disable operations to resolve segfault in CTS
rairatne Aug 29, 2022
c61c84d
Added manifest changes to enable nnhal 1.3
rairatne Sep 12, 2022
21ef5eb
Compile NN-Hal for Android 12
Nagamani71 Jan 24, 2022
cbdc8f1
Enable operations required for object detection
rairatne Sep 27, 2022
699fea8
Re-enable Operations that were disabled due to CTS errors
akodanka Nov 22, 2022
d8627f3
Added remote infrencing logic using grpc
rairatne Jun 14, 2022
194df1b
Use unix sockets for grpc communication with AI Dispatcher
akodanka Jan 19, 2023
d3962fb
Fix build warnings at NN-HAL
akodanka Jan 23, 2023
dc79b2e
Remove hardcoding for the nn-hal path
akodanka Jan 27, 2023
c5ae931
Fix issues raised by Coverity
akodanka Jan 27, 2023
20868f0
Cleanup log messages to prevent leakage of data
akodanka Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 70 additions & 8 deletions Android.bp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ cc_library_shared {
srcs: [
"Driver.cpp",
"BasePreparedModel.cpp",
"DetectionClient.cpp",
"utils.cpp",
"IENetwork.cpp",
"ModelManager.cpp",
Expand All @@ -24,10 +25,20 @@ cc_library_shared {
],

include_dirs: [
"frameworks/ml/nn/common/include",
"frameworks/ml/nn/runtime/include",
"packages/modules/NeuralNetworks/common/include",
"packages/modules/NeuralNetworks/runtime/include",
"frameworks/native/libs/nativewindow/include",
"external/mesa3d/include/android_stub"
"external/mesa3d/include/android_stub",
"external/grpc-grpc",
"external/grpc-grpc/include",
"external/grpc-grpc/third_party/cares",
"external/grpc-grpc/third_party/cares/config_android",
"external/grpc-grpc/src/core/ext/filters/client_channel",
"external/grpc-grpc/third_party/nanopb",
"external/protobuf",
"external/protobuf/src",
"external/protobuf/config",
"external/protobuf/android"
],

header_libs: [
Expand Down Expand Up @@ -90,10 +101,13 @@ cc_library_shared {
"libhidlmemory",
"liblog",
"libnativewindow",
"libtextclassifier_hash",
"libutils",
"libinference_engine",
"libngraph",
"libMKLDNNPlugin"
"libMKLDNNPlugin",
"libgrpc++",
"libprotobuf-cpp-full"
],

static_libs: [
Expand All @@ -102,10 +116,51 @@ cc_library_shared {
"libngraph_creator",
],

generated_headers: [
"ObjectDetectionProtoStub_h",
],
generated_sources: [
"ObjectDetectionProtoStub_cc",
],

defaults: [
"neuralnetworks_defaults"
]
],

vintf_fragments: ["android.hardware.neuralnetworks@1.3.xml"]

}

genrule {
name: "ObjectDetectionProtoStub_h",
tools: [
"aprotoc",
"protoc-gen-grpc-cpp-plugin",
],
cmd: "$(location aprotoc) -I$$(dirname $(in)) -Iexternal/protobuf/src --plugin=protoc-gen-grpc=$(location protoc-gen-grpc-cpp-plugin) $(in) --grpc_out=$(genDir) --cpp_out=$(genDir)",
srcs: [
"proto/nnhal_object_detection.proto",
],
out: [
"nnhal_object_detection.pb.h",
"nnhal_object_detection.grpc.pb.h",
],
}

genrule {
name: "ObjectDetectionProtoStub_cc",
tools: [
"aprotoc",
"protoc-gen-grpc-cpp-plugin",
],
cmd: "$(location aprotoc) -I$$(dirname $(in)) -Iexternal/protobuf/src --plugin=protoc-gen-grpc=$(location protoc-gen-grpc-cpp-plugin) $(in) --grpc_out=$(genDir) --cpp_out=$(genDir)",
srcs: [
"proto/nnhal_object_detection.proto",
],
out: [
"nnhal_object_detection.pb.cc",
"nnhal_object_detection.grpc.pb.cc",
],
}

//##############################################################
Expand All @@ -120,8 +175,8 @@ cc_binary {
srcs: ["service.cpp"],

include_dirs: [
"frameworks/ml/nn/common/include",
"frameworks/ml/nn/runtime/include",
"packages/modules/NeuralNetworks/common/include",
"packages/modules/NeuralNetworks/runtime/include",
"frameworks/native/libs/nativewindow/include",
"external/mesa3d/include/android_stub"
],
Expand Down Expand Up @@ -155,4 +210,11 @@ cc_binary {
],

compile_multilib: "64",
}
}

//##############################################################
cc_library_headers {
name: "libnnhal_headers",
vendor: true,
export_include_dirs: ["."],
}
1 change: 1 addition & 0 deletions BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ shared_library("intel_nnhal") {
"ModelManager.cpp",
"cpu/CpuPreparedModel.cpp",
"BasePreparedModel.cpp",
"DetectionClient.cpp",
]

include_dirs = [
Expand Down
181 changes: 123 additions & 58 deletions BasePreparedModel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ namespace nnhal {
using namespace android::nn;

static const Timing kNoTiming = {.timeOnDevice = UINT64_MAX, .timeInDriver = UINT64_MAX};
bool mRemoteCheck = false;
std::shared_ptr<DetectionClient> mDetectionClient;

void BasePreparedModel::deinitialize() {
ALOGV("Entering %s", __func__);
Expand All @@ -56,6 +58,45 @@ bool BasePreparedModel::initialize() {
return true;
}

bool BasePreparedModel::checkRemoteConnection() {
char grpc_prop[PROPERTY_VALUE_MAX] = "";
bool is_success = false;
if(getGrpcIpPort(grpc_prop)) {
ALOGV("Attempting GRPC via TCP : %s", grpc_prop);
mDetectionClient = std::make_shared<DetectionClient>(
grpc::CreateChannel(grpc_prop, grpc::InsecureChannelCredentials()));
if(mDetectionClient) {
auto reply = mDetectionClient->prepare(is_success);
ALOGI("GRPC(TCP) prepare response is %d : %s", is_success, reply.c_str());
}
}
if (!is_success && getGrpcSocketPath(grpc_prop)) {
ALOGV("Attempting GRPC via unix : %s", grpc_prop);
mDetectionClient = std::make_shared<DetectionClient>(
grpc::CreateChannel(std::string("unix:") + grpc_prop, grpc::InsecureChannelCredentials()));
if(mDetectionClient) {
auto reply = mDetectionClient->prepare(is_success);
ALOGI("GRPC(unix) prepare response is %d : %s", is_success, reply.c_str());
}
}
mRemoteCheck = is_success;
return is_success;
}

bool BasePreparedModel::loadRemoteModel() {
ALOGI("Entering %s", __func__);
bool is_success = false;
if(mDetectionClient) {
auto reply = mDetectionClient->sendIRs(is_success);
ALOGI("sendIRs response GRPC %d %s", is_success, reply.c_str());
}
else {
ALOGE("%s mDetectionClient is null",__func__);
}
mRemoteCheck = is_success;
return is_success;
}

static Return<void> notify(const sp<V1_0::IExecutionCallback>& callback, const ErrorStatus& status,
const hidl_vec<OutputShape>&, Timing) {
return callback->notify(status);
Expand Down Expand Up @@ -201,7 +242,7 @@ void asyncExecute(const Request& request, MeasureTiming measure, BasePreparedMod
ALOGD("Ignorning output at index(%d), since it is invalid", outIndex);
continue;
}
ALOGD("Output index: %d layername : %s", outIndex, outputNodeName.c_str());
ALOGV("Output index: %d layername : %s", outIndex, outputNodeName.c_str());
auto srcBlob = plugin->getBlob(outputNodeName);
auto operandType = modelInfo->getOperandType(outIndex);
uint32_t actualLength = srcBlob->byteSize();
Expand Down Expand Up @@ -337,49 +378,64 @@ static std::tuple<ErrorStatus, hidl_vec<V1_2::OutputShape>, Timing> executeSynch
ALOGD("Ignorning input at index(%d), since it is invalid", inIndex);
continue;
}
ALOGD("Input index: %d layername : %s", inIndex, inputNodeName.c_str());
auto destBlob = plugin->getBlob(inputNodeName);
if (modelInfo->getOperandType(inIndex) == OperandType::TENSOR_FLOAT16) {
float* dest = destBlob->buffer().as<float*>();
_Float16* src = (_Float16*)srcPtr;

for (unsigned int i = 0; i < len / 2; i++) {
dest[i] = src[i];
}
ALOGV("Input index: %d layername : %s", inIndex, inputNodeName.c_str());
//check if remote infer is available
//TODO: Need to add FLOAT16 support for remote inferencing
if(mRemoteCheck && mDetectionClient) {
mDetectionClient->add_input_data(inputNodeName, (uint8_t*)srcPtr, ngraphNw->getOutputShape(inIndex));
} else {
uint8_t* dest = destBlob->buffer().as<uint8_t*>();
std::memcpy(dest, (uint8_t*)srcPtr, len);
auto destBlob = plugin->getBlob(inputNodeName);
if (modelInfo->getOperandType(inIndex) == OperandType::TENSOR_FLOAT16) {
float* dest = destBlob->buffer().as<float*>();
_Float16* src = (_Float16*)srcPtr;

for (unsigned int i = 0; i < len / 2; i++) {
dest[i] = src[i];
}
} else {
uint8_t* dest = destBlob->buffer().as<uint8_t*>();
std::memcpy(dest, (uint8_t*)srcPtr, len);
}
}

}

ALOGD("%s Run", __func__);
ALOGV("%s Run", __func__);

if (measure == MeasureTiming::YES) deviceStart = now();
try {
plugin->infer();
} catch (const std::exception& ex) {
ALOGE("%s Exception !!! %s", __func__, ex.what());
return {ErrorStatus::GENERAL_FAILURE, {}, kNoTiming};
if(mRemoteCheck) {
ALOGI("%s GRPC Remote Infer", __func__);
auto reply = mDetectionClient->remote_infer();
ALOGI("***********GRPC server response************* %s", reply.c_str());
}
if (!mRemoteCheck || !mDetectionClient->get_status()){
try {
ALOGV("%s Client Infer", __func__);
plugin->infer();
} catch (const std::exception& ex) {
ALOGE("%s Exception !!! %s", __func__, ex.what());
return {ErrorStatus::GENERAL_FAILURE, {}, kNoTiming};
}
}
if (measure == MeasureTiming::YES) deviceEnd = now();

for (size_t i = 0; i < request.outputs.size(); i++) {
auto outIndex = modelInfo->getModelOutputIndex(i);
ALOGI("OutputIndex: %d", outIndex);
ALOGV("OutputIndex: %d", outIndex);
const std::string& outputNodeName = ngraphNw->getNodeName(outIndex);
if (outputNodeName == "") {
ALOGD("Ignorning output at index(%d), since it is invalid", outIndex);
continue;
}
ALOGD("Output index: %d layername : %s", outIndex, outputNodeName.c_str());
ALOGV("Output index: %d layername : %s", outIndex, outputNodeName.c_str());
auto srcBlob = plugin->getBlob(outputNodeName);
auto operandType = modelInfo->getOperandType(outIndex);
uint32_t actualLength = srcBlob->byteSize();
uint32_t expectedLength = 0;
void* destPtr = modelInfo->getBlobFromMemoryPoolOut(request, i, expectedLength);
auto outputBlobDims = srcBlob->getTensorDesc().getDims();

ALOGD("output precision: %d", static_cast<int>(srcBlob->getTensorDesc().getPrecision()));
ALOGV("output precision: %d", static_cast<int>(srcBlob->getTensorDesc().getPrecision()));

switch (operandType) {
case OperandType::TENSOR_BOOL8:
Expand Down Expand Up @@ -420,51 +476,60 @@ static std::tuple<ErrorStatus, hidl_vec<V1_2::OutputShape>, Timing> executeSynch
"OUTPUT_INSUFFICIENT_SIZE error");
return {ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, modelInfo->getOutputShapes(), kNoTiming};
}

switch (operandType) {
case OperandType::TENSOR_INT32:
case OperandType::TENSOR_FLOAT32: {
std::memcpy((uint8_t*)destPtr, srcBlob->buffer().as<uint8_t*>(),
srcBlob->byteSize());
break;
}
case OperandType::TENSOR_BOOL8: {
floatToUint8(srcBlob->buffer().as<float*>(), (uint8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT8_ASYMM: {
floatToUint8(srcBlob->buffer().as<float*>(), (uint8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT8_SYMM:
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
case OperandType::TENSOR_QUANT8_ASYMM_SIGNED: {
floatToint8(srcBlob->buffer().as<float*>(), (int8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_FLOAT16: {
floatToFloat16(srcBlob->buffer().as<float*>(), (_Float16*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT16_SYMM: {
floatToInt16(srcBlob->buffer().as<float*>(), (int16_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT16_ASYMM: {
floatToUInt16(srcBlob->buffer().as<float*>(), (uint16_t*)destPtr, srcBlob->size());
break;
//copy output from remote infer
//TODO: Add support for other OperandType
if (mRemoteCheck && mDetectionClient && mDetectionClient->get_status()) {
mDetectionClient->get_output_data(outputNodeName, (uint8_t*)destPtr, ngraphNw->getOutputShape(outIndex));
} else {
switch (operandType) {
case OperandType::TENSOR_INT32:
case OperandType::TENSOR_FLOAT32: {
std::memcpy((uint8_t*)destPtr, srcBlob->buffer().as<uint8_t*>(),
srcBlob->byteSize());
break;
}
case OperandType::TENSOR_BOOL8: {
floatToUint8(srcBlob->buffer().as<float*>(), (uint8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT8_ASYMM: {
floatToUint8(srcBlob->buffer().as<float*>(), (uint8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT8_SYMM:
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
case OperandType::TENSOR_QUANT8_ASYMM_SIGNED: {
floatToint8(srcBlob->buffer().as<float*>(), (int8_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_FLOAT16: {
floatToFloat16(srcBlob->buffer().as<float*>(), (_Float16*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT16_SYMM: {
floatToInt16(srcBlob->buffer().as<float*>(), (int16_t*)destPtr, srcBlob->size());
break;
}
case OperandType::TENSOR_QUANT16_ASYMM: {
floatToUInt16(srcBlob->buffer().as<float*>(), (uint16_t*)destPtr, srcBlob->size());
break;
}
default:
std::memcpy((uint8_t*)destPtr, srcBlob->buffer().as<uint8_t*>(),
srcBlob->byteSize());
break;
}
default:
std::memcpy((uint8_t*)destPtr, srcBlob->buffer().as<uint8_t*>(),
srcBlob->byteSize());
break;

}
}

if (!modelInfo->updateRequestPoolInfos()) {
ALOGE("Failed to update the request pool infos");
return {ErrorStatus::GENERAL_FAILURE, {}, kNoTiming};
}
if (mRemoteCheck && mDetectionClient && mDetectionClient->get_status()) {
mDetectionClient->clear_data();
}

if (measure == MeasureTiming::YES) {
driverEnd = now();
Expand Down
Loading