Skip to content

Commit 3f2f0c6

Browse files
tjazerzentjazerzen
authored
Feat: Update project setup files (#36)
Update project setup files. There are many independent changes, but they were motivated by one of the following: - Efficiency - Convenience - Cache clearing - To not get into corrupted state when conformance tests fail Issue: https://app.devrev.ai/devrev/works/ISS-210921
1 parent abe4cd5 commit 3f2f0c6

File tree

3 files changed

+112
-69
lines changed

3 files changed

+112
-69
lines changed

‎jest.setup.js‎

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ const resetMockServer = async () => {
8888
port: 8003,
8989
path: '/reset-mock-server',
9090
method: 'POST',
91+
agent: false,
9192
headers: {
9293
'Content-Type': 'application/json',
9394
'Content-Length': Buffer.byteLength(postData)
@@ -120,19 +121,62 @@ const resetMockServer = async () => {
120121
}
121122
};
122123

123-
const originalBeforeEach = global.beforeEach;
124+
// Function to end rate limiting
125+
const endRateLimiting = async () => {
126+
try {
127+
const http = require('http');
128+
129+
// Simple HTTP request that properly closes the connection
130+
const postData = '';
131+
const options = {
132+
hostname: 'localhost',
133+
port: 8004,
134+
path: '/end_rate_limiting',
135+
method: 'POST',
136+
agent: false,
137+
headers: {
138+
'Content-Type': 'application/json',
139+
'Content-Length': Buffer.byteLength(postData)
140+
}
141+
};
124142

125-
global.beforeEach = (fn, timeout) => {
126-
originalBeforeEach(async () => {
127-
try {
128-
// Reset the mock server state before each test
129-
await resetMockServer();
143+
const response = await new Promise((resolve, reject) => {
144+
const req = http.request(options, (res) => {
145+
let data = '';
146+
res.on('data', (chunk) => data += chunk);
147+
res.on('end', () => resolve({ status: res.statusCode, data }));
148+
});
130149

131-
// Execute the original beforeEach function
132-
await fn();
133-
} catch (err) {
134-
expect(`beforeEach failed: ${err.message}`).toBe('beforeEach should not fail');
135-
throw err;
150+
req.on('error', reject);
151+
req.on('timeout', () => {
152+
req.destroy();
153+
reject(new Error('Request timeout'));
154+
});
155+
156+
req.setTimeout(5000);
157+
req.write(postData);
158+
req.end();
159+
});
160+
161+
if (response.status !== 200) {
162+
console.warn(`Failed to end rate limiting: ${response.status}`);
136163
}
137-
}, timeout);
164+
} catch (error) {
165+
console.warn(`Could not connect to rate limiting server: ${error.message}`);
166+
}
138167
};
168+
169+
// This hook runs BEFORE each test in every test file.
170+
beforeEach(async () => {
171+
try {
172+
await resetMockServer();
173+
await endRateLimiting();
174+
} catch (error) {
175+
originalConsole.error(
176+
'[jest.setup.js] beforeEach: Failed to reset mock server. ' +
177+
`Error: ${error instanceof Error ? error.message : String(error)}`
178+
);
179+
// Don't throw - allow tests to run even if reset fails
180+
// The internal resetMockServer() already handles errors gracefully
181+
}
182+
});

‎mock_devrev_server.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,4 +378,5 @@ async def download_jsonl_gz_file(file_name: str):
378378

379379
if __name__ == "__main__":
380380
import uvicorn
381+
print(f"Starting DevRev server on http://localhost:8003")
381382
uvicorn.run(app, host="localhost", port=8003)

‎run_devrev_snapin_conformance_tests.sh‎

Lines changed: 55 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ NPM_INSTALL_OUTPUT_FILTER="up to date in|added [0-9]* packages, removed [0-9]* p
66
ANSI_ESCAPE_PATTERN="s/\x1b\[[0-9;]*[mK]//g"
77

88
# Maximum number of characters to display from log files
9-
SNAP_IN_LOG_MAX_CHARS=80000
10-
DEVREV_SERVER_LOG_MAX_CHARS=40000
11-
PROXY_SERVER_LOG_MAX_CHARS=20000
9+
SNAP_IN_LOG_MAX_CHARS=100000
10+
DEVREV_SERVER_LOG_MAX_CHARS=600000
11+
PROXY_SERVER_LOG_MAX_CHARS=30000
1212

1313
# Function to print a log file, truncating it if it's too large
1414
print_log_file() {
@@ -52,19 +52,13 @@ source "$EXEC_DIR/.env"
5252
set +a # stop automatically exporting
5353

5454
# Function to check and kill any Node process running on port 8000 (React development server)
55-
check_and_kill_node_server() {
55+
check_and_kill_processed_on_port() {
5656
local port=${1:-8000} # Default to port 8000 if no port is provided
5757
# Find process listening on specified port
5858
local pid=$(lsof -i :$port -t 2>/dev/null)
5959
if [ ! -z "$pid" ]; then
60-
if ps -p $pid | grep -q "node"; then
6160
printf "Found server running on port $port. Killing it...\n"
62-
kill $pid 2>/dev/null
63-
sleep 1 # Give the process time to terminate
64-
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
65-
printf "Node server terminated.\n"
66-
fi
67-
fi
61+
kill -9 "$pid" 2>/dev/null
6862
fi
6963
}
7064

@@ -84,10 +78,20 @@ get_children() {
8478

8579
# Function to start the mock DevRev server
8680
start_mock_devrev_server() {
87-
python3 "$SCRIPT_DIR/mock_devrev_server.py" > "$MOCK_DEVREV_SERVER_LOG" 2>&1 &
81+
python3 -u "$SCRIPT_DIR/mock_devrev_server.py" > "$MOCK_DEVREV_SERVER_LOG" 2>&1 &
8882
MOCK_SERVER_PID=$!
89-
sleep 2 # Give the server time to start
90-
printf "\n"
83+
84+
# Wait until the mock DevRev server prints its startup line to the log
85+
while true; do
86+
printf "Waiting for the DevRev server to start...\n"
87+
# If process already died, surface error and logs
88+
if grep -iq -E "Starting DevRev server on|Uvicorn running on" "$MOCK_DEVREV_SERVER_LOG" 2>/dev/null; then
89+
break
90+
fi
91+
sleep 0.1
92+
done
93+
94+
printf "DevRev server is up and running!\n\n"
9195
}
9296

9397
start_proxy_server() {
@@ -96,11 +100,19 @@ start_proxy_server() {
96100
printf "Error: rate_limiting_proxy.py file not found in $EXEC_DIR/rate_limiting_proxy.py. This file should exist (and should be adopted for 3rd party service's rate limiting response format).\n"
97101
exit 69
98102
fi
99-
python3 "$EXEC_DIR/rate_limiting_proxy.py" > "$PROXY_SERVER_LOG" 2>&1 &
103+
# The -u flag is critical here to disable python's output buffering, ensuring logs are written immediately.
104+
python3 -u "$EXEC_DIR/rate_limiting_proxy.py" > "$PROXY_SERVER_LOG" 2>&1 &
100105
PROXY_SERVER_PID=$!
101-
sleep 2 # Give the server time to start
102106

103-
# Check if the proxy server started successfully.
107+
# Wait until the proxy server prints its startup line to the log
108+
while true; do
109+
printf "Waiting for the proxy server to start...\n"
110+
if grep -iq -E "Starting proxy server on" "$PROXY_SERVER_LOG" 2>/dev/null; then
111+
break
112+
fi
113+
sleep 0.1
114+
done
115+
104116
if ! kill -0 "$PROXY_SERVER_PID" > /dev/null 2>&1; then
105117
wait "$PROXY_SERVER_PID"
106118
EXIT_CODE=$?
@@ -110,15 +122,16 @@ start_proxy_server() {
110122
exit 69
111123
fi
112124
fi
113-
printf "\n"
125+
126+
printf "Proxy server is up and running!\n\n"
114127
}
115128

116129
# Cleanup function to ensure all processes are terminated
117130
cleanup() {
118131
# Kill any running npm processes started by this script
119132
if [ ! -z "${NPM_PID+x}" ]; then
120-
pkill -P $NPM_PID 2>/dev/null
121-
kill $NPM_PID 2>/dev/null
133+
pkill -9 -P $NPM_PID > /dev/null 2>&1
134+
kill -9 $NPM_PID > /dev/null 2>&1
122135
fi
123136

124137
# Kill React app and its children if they exist
@@ -127,12 +140,12 @@ cleanup() {
127140
get_children $SNAP_IN_PID
128141

129142
# Kill the main process
130-
kill $SNAP_IN_PID 2>/dev/null
143+
kill -9 $SNAP_IN_PID > /dev/null 2>&1
131144

132145
# Kill all the subprocesses
133146
for pid in "${processes_to_kill[@]}"
134147
do
135-
kill $pid 2>/dev/null
148+
kill -9 $pid > /dev/null 2>&1
136149
done
137150

138151
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
@@ -142,15 +155,15 @@ cleanup() {
142155

143156
# Kill mock DevRev server if it exists
144157
if [ ! -z "${MOCK_SERVER_PID+x}" ]; then
145-
kill $MOCK_SERVER_PID 2>/dev/null
158+
kill -9 $MOCK_SERVER_PID > /dev/null 2>&1
146159
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
147160
printf "Mock DevRev server terminated!\n"
148161
fi
149162
fi
150163

151164
# Kill proxy server if it exists
152165
if [ ! -z "${PROXY_SERVER_PID+x}" ]; then
153-
kill $PROXY_SERVER_PID 2>/dev/null
166+
kill -9 $PROXY_SERVER_PID > /dev/null 2>&1
154167
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
155168
printf "Proxy server terminated!\n"
156169
fi
@@ -166,32 +179,15 @@ cleanup() {
166179
trap cleanup EXIT SIGINT SIGTERM
167180

168181
# Check for and kill any existing servers from previous runs
169-
check_and_kill_node_server 8000
170-
check_and_kill_node_server 8002
171-
172-
# Ensure nothing is running on port 8003, then start the mock DevRev server
173-
existing_pids=$(lsof -i :8003 -t 2>/dev/null)
174-
if [ ! -z "$existing_pids" ]; then
175-
printf "Killing existing process(es) on port 8003: %s\n" "$existing_pids"
176-
for pid in $existing_pids; do
177-
kill $pid 2>/dev/null
178-
done
179-
sleep 1
180-
fi
182+
check_and_kill_processed_on_port 8000
183+
check_and_kill_processed_on_port 8002
184+
check_and_kill_processed_on_port 8003
185+
check_and_kill_processed_on_port 8004
186+
181187
start_mock_devrev_server
182188

183189
# Set HTTPS_PROXY environment variable to point to proxy server
184190
export HTTPS_PROXY="http://localhost:8004"
185-
186-
# Ensure nothing is running on port 8004, then start the proxy server
187-
existing_pids_8004=$(lsof -i :8004 -t 2>/dev/null)
188-
if [ ! -z "$existing_pids_8004" ]; then
189-
printf "Killing existing process(es) on port 8004: %s\n" "$existing_pids_8004"
190-
for pid in $existing_pids_8004; do
191-
kill $pid 2>/dev/null
192-
done
193-
sleep 1
194-
fi
195191
start_proxy_server
196192

197193
# Check if chef-cli binary exists at CHEF_CLI_PATH
@@ -207,9 +203,9 @@ if [ -z "$EXTRACTED_FILES_FOLDER_PATH" ]; then
207203
fi
208204

209205
# Check if EXTRACTED_FILES_FOLDER_PATH does not end with "node_$1/build"
210-
if [[ "$EXTRACTED_FILES_FOLDER_PATH" != *"node_$1/extracted_files" ]]; then
206+
if [[ "$EXTRACTED_FILES_FOLDER_PATH" != *"node_build/extracted_files" ]]; then
211207
echo "Error: EXTRACTED_FILES_FOLDER_PATH should end with 'node_$1/extracted_files'."
212-
echo "Note: The value of EXTRACTED_FILES_FOLDER_PATH should be <path_to_directory_where_you_rendered_the_snap-in>/node_$1/extracted_files."
208+
echo "Note: The value of EXTRACTED_FILES_FOLDER_PATH should be <path_to_directory_where_you_rendered_the_snap-in>/node_build/extracted_files."
213209
exit 69 # EXIT_SERVICE_UNAVAILABLE
214210
fi
215211

@@ -247,11 +243,11 @@ fi
247243

248244
# Check if the node subfolder exists
249245
if [ -d "$NODE_SUBFOLDER" ]; then
250-
# Find and delete all files and folders except "node_modules", "build", and "package-lock.json"
251-
find "$NODE_SUBFOLDER" -mindepth 1 ! -path "$NODE_SUBFOLDER/node_modules*" ! -path "$NODE_SUBFOLDER/build*" ! -name "package-lock.json" -exec rm -rf {} +
246+
# Find and delete all files and folders except "node_modules"
247+
find "$NODE_SUBFOLDER" -mindepth 1 ! -path "$NODE_SUBFOLDER/node_modules*" -exec rm -rf {} +
252248

253249
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
254-
printf "Cleanup completed, keeping 'node_modules' and 'package-lock.json'.\n"
250+
printf "Cleanup completed, keeping 'node_modules'.\n"
255251
fi
256252
else
257253
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
@@ -333,11 +329,11 @@ fi
333329

334330
# Check if the conformance tests node subfolder exists
335331
if [ -d "$NODE_CONFORMANCE_TESTS_SUBFOLDER" ]; then
336-
# Find and delete all files and folders except "node_modules", "build", and "package-lock.json"
337-
find "$NODE_CONFORMANCE_TESTS_SUBFOLDER" -mindepth 1 ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/node_modules*" ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/build*" ! -name "package-lock.json" -exec rm -rf {} +
332+
# Find and delete all files and folders except "node_modules"
333+
find "$NODE_CONFORMANCE_TESTS_SUBFOLDER" -mindepth 1 ! -path "$NODE_CONFORMANCE_TESTS_SUBFOLDER/node_modules*" -exec rm -rf {} +
338334

339335
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
340-
printf "Cleanup completed, keeping 'node_modules' and 'package-lock.json'.\n"
336+
printf "Cleanup completed, keeping 'node_modules'.\n"
341337
fi
342338
else
343339
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
@@ -367,19 +363,21 @@ fi
367363

368364
printf "\n#### Running conformance tests...\n"
369365

370-
npm test -- --runInBand --setupFilesAfterEnv="$SCRIPT_DIR/jest.setup.js" --detectOpenHandles 2>&1 | sed -E "$ANSI_ESCAPE_PATTERN"
366+
npm test -- --runInBand --setupFilesAfterEnv="$SCRIPT_DIR/jest.setup.js" --detectOpenHandles --forceExit 2>&1 | sed -E "$ANSI_ESCAPE_PATTERN"
371367
conformance_tests_result=$?
372368

373369
printf "\n#### Output of the DevRev server log file:\n\n"
374370
print_log_file "$MOCK_DEVREV_SERVER_LOG" "$DEVREV_SERVER_LOG_MAX_CHARS"
375371
printf "\n#### Output of The Snap-In log file:\n"
376372
print_log_file "$NODE_SUBFOLDER/app.log" "$SNAP_IN_LOG_MAX_CHARS"
377-
printf "\n"
373+
# uncomment if you need the logs of the proxy server
374+
# printf "\n#### Output of the proxy server log file:\n\n"
375+
# print_log_file "$PROXY_SERVER_LOG" "$PROXY_SERVER_LOG_MAX_CHARS"
378376
printf "\n"
379377

380378
if [ $conformance_tests_result -ne 0 ]; then
381379
if [ "${VERBOSE:-}" -eq 1 ] 2>/dev/null; then
382380
printf "Error: Conformance tests have failed.\n"
383381
fi
384382
exit 2
385-
fi
383+
fi

0 commit comments

Comments
 (0)