diff --git a/.gitignore b/.gitignore index b2674c9..24f4df0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ asn1c_combined/libasncodec.a # Ignore files added by CLion .idea/ cmake-build-*/ + +# Ignore Python bytecode files (generated during `do_kafka_test.sh` execution) +*.pyc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 121dc5f..ba134fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,18 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.18 as runtime-deps +USER root +WORKDIR /asn1_codec +VOLUME ["/asn1_codec_share"] + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + + # === BUILDER IMAGE === -FROM alpine:3.18 as builder +FROM runtime-deps as builder USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] @@ -9,9 +22,6 @@ RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ make \ - bash \ - librdkafka \ - librdkafka-dev \ asio-dev # Install pugixml @@ -50,18 +60,13 @@ RUN echo "export CC=gcc" >> ~/.bashrc # Build acm. RUN mkdir -p /build && cd /build && cmake /asn1_codec && make + # === RUNTIME IMAGE === -FROM alpine:3.18 +FROM runtime-deps USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # copy the built files from the builder COPY --from=builder /asn1_codec /asn1_codec COPY --from=builder /build /build diff --git a/Dockerfile.amazonlinux b/Dockerfile.amazonlinux index 8eda7df..de342aa 100644 --- a/Dockerfile.amazonlinux +++ b/Dockerfile.amazonlinux @@ -1,9 +1,11 @@ -# === BUILDER IMAGE for ACM === -FROM amazonlinux:2023 as builder +# === RUNTIME DEPENDENCIES IMAGE === +FROM amazonlinux:2023 as runtime-deps USER root WORKDIR /asn1_codec -## add build dependencies -RUN yum install -y cmake g++ make bash automake libtool autoconf +VOLUME ["/asn1_codec_share"] + +# # add runtime dependencies +RUN yum install -y bash # Install librdkafka from Confluent repo RUN rpm --import https://packages.confluent.io/rpm/7.6/archive.key @@ -11,6 +13,15 @@ COPY ./confluent.repo /etc/yum.repos.d RUN yum clean all RUN yum install -y librdkafka-devel + +# === BUILDER IMAGE for ACM === +FROM runtime-deps as builder +USER root +WORKDIR /asn1_codec + +## add build dependencies +RUN yum install -y cmake g++ make bash automake libtool autoconf + # Install asio from Fedora repo (build dependency only) COPY ./fedora.repo /etc/yum.repos.d RUN dnf --enablerepo=fedora install asio-devel -y @@ -46,22 +57,12 @@ ADD ./data /asn1_codec/data RUN mkdir -p /build && cd /build && cmake /asn1_codec && make - # === RUNTIME IMAGE === -FROM amazonlinux:2023 +FROM runtime-deps USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] -# # add runtime dependencies -RUN yum install -y bash - -# Install librdkafka from Confluent repo -RUN rpm --import https://packages.confluent.io/rpm/7.6/archive.key -COPY ./confluent.repo /etc/yum.repos.d -RUN yum clean all -RUN yum install -y librdkafka-devel - # # copy the built files from the builder COPY --from=builder /asn1_codec /asn1_codec COPY --from=builder /build /build diff --git a/Dockerfile.debug b/Dockerfile.debug index eed2ecd..dd39c4c 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -1,5 +1,18 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.18 as runtime-deps +USER root +WORKDIR /asn1_codec +VOLUME ["/asn1_codec_share"] + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + + # === BUILDER IMAGE === -FROM alpine:3.18 as builder +FROM runtime-deps as builder USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] @@ -9,9 +22,6 @@ RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ make \ - bash \ - librdkafka \ - librdkafka-dev \ asio-dev # Install pugixml @@ -23,8 +33,6 @@ RUN export LD_LIBRARY_PATH=/usr/local/lib ADD ./asn1c_combined /asn1_codec/asn1c_combined RUN cd /asn1_codec/asn1c_combined && bash doIt.sh - - # Remove any lingering .asn files RUN rm -rf /asn1c_codec/asn1c_combined/j2735-asn-files RUN rm -rf /asn1c_codec/asn1c_combined/semi-asn-files @@ -53,20 +61,12 @@ RUN echo "export CC=gcc" >> ~/.bashrc RUN mkdir -p /build && cd /build && cmake /asn1_codec && make - - # === RUNTIME IMAGE === -FROM alpine:3.18 +FROM runtime-deps USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # Use jemalloc RUN apk add --upgrade --no-cache jemalloc ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2 diff --git a/Dockerfile.dev b/Dockerfile.dev index 3850a0d..27e3c5a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,18 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.18 as runtime-deps +USER root +WORKDIR /asn1_codec +VOLUME ["/asn1_codec_share"] + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + + # === BUILDER IMAGE === -FROM alpine:3.18 as builder +FROM runtime-deps as builder USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] @@ -9,9 +22,6 @@ RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ make \ - bash \ - librdkafka \ - librdkafka-dev \ asio-dev # Install pugixml @@ -49,18 +59,13 @@ RUN echo "export CC=gcc" >> ~/.bashrc # Build acm. RUN mkdir -p /build && cd /build && cmake /asn1_codec && make + # === RUNTIME IMAGE === -FROM alpine:3.18 +FROM runtime-deps USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # Use jemalloc RUN apk add --upgrade --no-cache jemalloc ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2 diff --git a/Dockerfile.standalone b/Dockerfile.standalone index dcb47f5..7409255 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,5 +1,18 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.18 as runtime-deps +USER root +WORKDIR /asn1_codec +VOLUME ["/asn1_codec_share"] + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + librdkafka \ + librdkafka-dev + + # === BUILDER IMAGE === -FROM alpine:3.18 as builder +FROM runtime-deps as builder USER root WORKDIR /asn1_codec @@ -8,9 +21,6 @@ RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ make \ - bash \ - librdkafka \ - librdkafka-dev \ asio-dev # Install pugixml @@ -43,17 +53,12 @@ ADD ./data /asn1_codec/data # Build acm. RUN mkdir -p /build && cd /build && cmake /asn1_codec && make + # === RUNTIME IMAGE === -FROM alpine:3.18 +FROM runtime-deps USER root WORKDIR /asn1_codec -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - librdkafka \ - librdkafka-dev - # Use jemalloc RUN apk add --upgrade --no-cache jemalloc ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2 diff --git a/Dockerfile.testing b/Dockerfile.testing index 34913b6..6ec0d94 100644 --- a/Dockerfile.testing +++ b/Dockerfile.testing @@ -1,5 +1,19 @@ +# === RUNTIME DEPENDENCIES IMAGE === +FROM alpine:3.18 as runtime-deps +USER root +WORKDIR /asn1_codec +VOLUME ["/asn1_codec_share"] + +# add runtime dependencies +RUN apk add --upgrade --no-cache \ + bash \ + python3 \ + librdkafka \ + librdkafka-dev + + # === BUILDER IMAGE === -FROM alpine:3.18 as builder +FROM runtime-deps as builder USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] @@ -9,9 +23,6 @@ RUN apk add --upgrade --no-cache --virtual .build-deps \ cmake \ g++ \ make \ - bash \ - librdkafka \ - librdkafka-dev \ asio-dev # Install pugixml @@ -52,18 +63,11 @@ RUN echo "export CC=gcc" >> ~/.bashrc RUN mkdir -p /build && cd /build && cmake /asn1_codec && make # === RUNTIME IMAGE === -FROM alpine:3.18 +FROM runtime-deps USER root WORKDIR /asn1_codec VOLUME ["/asn1_codec_share"] -# add runtime dependencies -RUN apk add --upgrade --no-cache \ - bash \ - python3 \ - librdkafka \ - librdkafka-dev - # Use jemalloc RUN apk add --upgrade --no-cache jemalloc ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2 diff --git a/do_kafka_test.sh b/do_kafka_test.sh index 6ac1850..5a2172f 100755 --- a/do_kafka_test.sh +++ b/do_kafka_test.sh @@ -43,39 +43,66 @@ setup() { waitForKafkaToCreateTopics() { maxAttempts=100 attempts=0 - KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka) + + # Wait for kafka container to appear and broker to respond + while true; do + attempts=$((attempts+1)) + KAFKA_CONTAINER_NAME=$(docker ps --format '{{.Names}}' | grep kafka | head -n 1 || true) + + if [ -z "$KAFKA_CONTAINER_NAME" ]; then + echo "Kafka container not running yet (attempt $attempts/$maxAttempts)..." + else + # try a non-interactive topics list to confirm broker is responsive + if docker exec "$KAFKA_CONTAINER_NAME" /opt/kafka/bin/kafka-topics.sh --list --zookeeper $DOCKER_HOST_IP >/dev/null 2>&1; then + echo "Kafka broker responded on container '$KAFKA_CONTAINER_NAME'" + break + else + echo "Kafka container found but broker not yet responding (attempt $attempts/$maxAttempts)..." + fi + fi + + if [ $attempts -ge $maxAttempts ]; then + echo "Kafka did not become ready after $maxAttempts attempts. Exiting." + ./stop_kafka.sh + exit 1 + fi + sleep 1 + done + + # Now wait for required topics to be created + attempts=0 while true; do attempts=$((attempts+1)) - if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) = "0" ]; then + if [ $(docker ps | grep "$KAFKA_CONTAINER_NAME" | wc -l) = "0" ]; then echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting." ./stop_kafka.sh exit 1 fi - ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1) + ltopics=$(docker exec "$KAFKA_CONTAINER_NAME" /opt/kafka/bin/kafka-topics.sh --list --zookeeper $DOCKER_HOST_IP 2>/dev/null || true) allTopicsCreated=true - if [ $(echo $ltopics | grep "topic.Asn1DecoderInput" | wc -l) = "0" ]; then + if [ $(echo "$ltopics" | grep "topic.Asn1DecoderInput" | wc -l) = "0" ]; then allTopicsCreated=false - elif [ $(echo $ltopics | grep "topic.Asn1DecoderOutput" | wc -l) = "0" ]; then + elif [ $(echo "$ltopics" | grep "topic.Asn1DecoderOutput" | wc -l) = "0" ]; then allTopicsCreated=false - elif [ $(echo $ltopics | grep "topic.Asn1EncoderInput" | wc -l) = "0" ]; then + elif [ $(echo "$ltopics" | grep "topic.Asn1EncoderInput" | wc -l) = "0" ]; then allTopicsCreated=false - elif [ $(echo $ltopics | grep "topic.Asn1EncoderOutput" | wc -l) = "0" ]; then + elif [ $(echo "$ltopics" | grep "topic.Asn1EncoderOutput" | wc -l) = "0" ]; then allTopicsCreated=false fi - + if [ $allTopicsCreated = true ]; then echo "Kafka has created all required topics" break fi - sleep 1 - if [ $attempts -ge $maxAttempts ]; then echo "Kafka has not created all required topics after $maxAttempts attempts. Exiting." ./stop_kafka.sh exit 1 fi + + sleep 1 done }