byk dev to rtc dev

.env

@@ -1,7 +1,7 @@
 AWS_ACCESS_KEY_ID=your_aws_access_key_id
 AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key
 BEDROCK_AWS_REGION=eu-west-1
-AZURE_OPENAI_API_KEY=your_azure_openai_api_key
-AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint
-AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4o
+AZURE_OPENAI_API_KEY=your_openai_api_key
+AZURE_OPENAI_ENDPOINT=your_openai_endpoint
+AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4o-mini
 PROVIDER_NAME=azure-openai

DSL/CronManager/DSL/mock_signed_url_generation.yml

@@ -0,0 +1,5 @@
+mock_signed_url_generate:
+  trigger: off
+  type: exec
+  command: "../app/scripts/presigned_url_generate.sh"
+  allowedEnvs: ['centopsAgencies']

DSL/CronManager/script/callback_format.sh

@@ -14,16 +14,16 @@ log() {
 }
 PROGRESS_UPDATE_URL="http://ruuter-public:8086/global-classifier/datasets/progress/update"
 # Debug: Check Python environment
-log "🔍 Python version: $(python3 --version)"
-log "🔍 Python path: $(which python3)"
+log "Python version: $(python3 --version)"
+log "Python path: $(which python3)"
 
 # Install required packages
-log "🔍 Installing required Python packages..."
+log "Installing required Python packages..."
 python3 -m pip install --quiet --no-cache-dir requests pydantic pandas || {
-    log "❌ Failed to install packages"
+    log "Failed to install packages"
     exit 1
 }
-log "✅ Required packages installed"
+log "Required packages installed"
 
 log "Dataset generation callback processing started"
 log "File path: $filePath"
@@ -35,7 +35,7 @@ log "Extracted dataset ID: $dataset_id"
 # Direct Python script path for processing generation callback (inside container)
 CALLBACK_SCRIPT="/app/src/s3_dataset_processor/dataset_generation_callback_processor.py"
 
-log "🔍 Calling direct Python script to process generation callback..."
+log "Calling direct Python script to process generation callback..."
 
 # Create temporary file for response
 temp_response="/tmp/callback_response.json"
@@ -65,77 +65,77 @@ python3 "$CALLBACK_SCRIPT" \
   > /tmp/callback_stdout.log 2> /tmp/callback_stderr.log
 exit_code=$?
 
-log "🪵 Python STDOUT:"
+log "Python STDOUT:"
 cat /tmp/callback_stdout.log
 
-log "🪵 Python STDERR:"
+log "Python STDERR:"
 cat /tmp/callback_stderr.log
 
-log "🔍 Python script exit code: $exit_code"
+log "Python script exit code: $exit_code"
 
 if [ -f "$temp_response" ]; then
-    log "📄 Contents of output JSON:"
+    log "Contents of output JSON:"
     cat "$temp_response"
 else
-    log "⚠️ No output JSON file was generated."
+    log "No output JSON file was generated."
 fi
 
 # Check if script execution was successful
 if [ "$exit_code" -eq 0 ] && [ -f "$temp_response" ]; then
-    log "✅ Python script execution successful"
+    log "Python script execution successful"
 
     response_body=$(cat "$temp_response")
-    log "🔍 Response: $response_body"
+    log "Response: $response_body"
 
     # Parse the response to get status information
     if command -v jq >/dev/null 2>&1; then
         status=$(echo "$response_body" | jq -r '.status // "unknown"')
         message=$(echo "$response_body" | jq -r '.message // "unknown"')
 
-        log "📊 Callback Processing Status:"
+        log "Callback Processing Status:"
         log "  - Status: $status"
         log "  - Message: $message"
         log "  - Dataset ID: $dataset_id"
 
     else
         # Fallback parsing without jq
-        log "⚠️ jq not available, using grep/sed for parsing"
+        log "jq not available, using grep/sed for parsing"
 
         status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | sed 's/.*"status":"\([^"]*\)".*/\1/' || echo "unknown")
         message=$(echo "$response_body" | grep -o '"message":"[^"]*"' | sed 's/.*"message":"\([^"]*\)".*/\1/' || echo "unknown")
 
-        log "📊 Callback Processing Status:"
+        log "Callback Processing Status:"
         log "  - Status: $status"
         log "  - Message: $message"
         log "  - Dataset ID: $dataset_id"
     fi
 
     # Check if callback processing was completed
     if [ "$status" = "completed" ]; then
-        log "✅ Dataset generation callback processed successfully"
-        log "🔄 Callback payload has been sent to status update endpoint"
+        log "Dataset generation callback processed successfully"
+        log "Callback payload has been sent to status update endpoint"
         log "   - agencies: [{agencyId: X, syncStatus: Synced_with_CKB/Sync_with_CKB_Failed}, ...]"
         log "   - datasetId: $dataset_id"
         log "   - generationStatus: Generation_Success/Generation_Failed"
 
     else
-        log "⚠️ Unexpected status received: $status"
-        log "⚠️ Message: $message"
+        log "Unexpected status received: $status"
+        log "Message: $message"
     fi
 
     # Cleanup temp file
     rm -f "$temp_response"
 
 else
-    log "❌ Python script execution failed with exit code: $exit_code"
+    log "Python script execution failed with exit code: $exit_code"
     if [ -f "$temp_response" ]; then
         log "Error response: $(cat $temp_response)"
         rm -f "$temp_response"
     fi
     exit 1
 fi
 
-log "✅ Dataset generation callback processing completed successfully"
-log "📋 Summary: Dataset ID: $dataset_id, Request Status: $status"
+log "Dataset generation callback processing completed successfully"
+log "Summary: Dataset ID: $dataset_id, Request Status: $status"
 
 exit 0