Skip to content

Commit c768c83

Browse files
aldotobingaldotobing
committed
improve security detection
1 parent f7dd88e commit c768c83

File tree

4 files changed

+308
-103
lines changed

4 files changed

+308
-103
lines changed

‎components/LiveDashboard.tsx‎

Lines changed: 55 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -5,75 +5,9 @@ import { aggregateLogData } from '../lib/log-aggregator';
55
import { Dashboard } from './Dashboard';
66
import { Eye, EyeOff, XCircle } from 'lucide-react';
77

8-
// A more robust parser for common Nginx log formats.
9-
const parseLogLine = (line: string) => {
10-
try {
11-
// This regex handles cases where the request field might be empty ("") or malformed.
12-
// It also handles extra fields at the end (like the "-" at the end of some Nginx logs)
13-
const match = line.match(/^(?<ip>[\d.]+) - (?<user>\S+) \[(?<timestamp>.+?)\] "(?<method>\S+) (?<path>[^"]*) HTTP\/\d+\.\d+" (?<code>\d+) (?<size>\d+) "(?<referrer>[^"]*)" "(?<agent>[^"]*)" "(?<xForwardedFor>[^"]*)"$/);
14-
15-
if (!match || !match.groups) {
16-
console.warn("Could not parse log line:", line);
17-
return null;
18-
}
19-
20-
const { ip, user, timestamp, method, path, code, size, referrer, agent, xForwardedFor } = match.groups;
21-
22-
// Define valid HTTP methods
23-
const validHttpMethods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT'];
24-
25-
// Initialize method and url
26-
let finalMethod = 'N/A';
27-
let url = 'N/A';
28-
29-
// Only parse method and URL if request is not empty
30-
if (method && path) {
31-
// Check if the method is a valid HTTP method
32-
if (validHttpMethods.includes(method)) {
33-
finalMethod = method;
34-
url = path || 'N/A';
35-
} else {
36-
// Check for known attack patterns or clearly malformed requests
37-
const isAttackPattern = method.includes('%') ||
38-
method.includes(';') ||
39-
method.includes('wget') ||
40-
method.includes('curl') ||
41-
method.length > 100;
42-
43-
// Check for clearly non-HTTP method patterns
44-
const isNonHttpPattern = method.includes('_DUPLEX_') ||
45-
method.startsWith('SSTP_') ||
46-
method.includes('Mozi') ||
47-
method.includes('->');
48-
49-
if (isAttackPattern || isNonHttpPattern) {
50-
finalMethod = 'MALFORMED';
51-
} else if (method.length <= 25) {
52-
// Short, non-attack patterns are classified as OTHER
53-
finalMethod = 'OTHER';
54-
} else {
55-
// Long patterns are classified as MALFORMED
56-
finalMethod = 'MALFORMED';
57-
}
58-
59-
url = `${method} ${path}`; // Keep the full request for analysis
60-
}
61-
}
62-
63-
return {
64-
ipAddress: ip,
65-
timestamp,
66-
method: finalMethod,
67-
path: url,
68-
status: code,
69-
bodyBytesSent: size,
70-
referer: referrer,
71-
userAgent: agent,
72-
};
73-
} catch (error) {
74-
console.error('Error parsing log line:', error);
75-
return null;
76-
}
8+
// Create a worker for single line parsing
9+
const createSingleLineParserWorker = () => {
10+
return new Worker(new URL('../workers/singleLineParser.js', import.meta.url));
7711
};
7812

7913
export interface LogData {
@@ -128,6 +62,7 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
12862
const pongTimeoutRef = useRef<NodeJS.Timeout | null>(null);
12963
const lastMessageTime = useRef<number>(Date.now());
13064
const reconnectDelay = useRef(BASE_RECONNECT_DELAY);
65+
const logParserWorkerRef = useRef<Worker | null>(null);
13166

13267
// Update wsUrl and newWsUrl when initialWsUrl changes
13368
useEffect(() => {
@@ -139,17 +74,44 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
13974
return Math.min(BASE_RECONNECT_DELAY * Math.pow(2, attempt), MAX_RECONNECT_DELAY);
14075
};
14176

77+
78+
14279
const handleNewLogLine = useCallback((line: string) => {
143-
const parsedLog = parseLogLine(line);
144-
if (parsedLog) {
145-
setParsedLines(prevLines => {
146-
// Keep all logs, just prepend the new one
147-
const updatedLines = [parsedLog, ...prevLines];
148-
const aggregatedData = aggregateLogData(updatedLines);
149-
setLogData(aggregatedData);
150-
return updatedLines;
151-
});
152-
}
80+
// console.log('Received log line:', line);
81+
82+
// Create a new worker for each line to avoid queueing issues
83+
const worker = createSingleLineParserWorker();
84+
85+
worker.onmessage = (event) => {
86+
if (event.data.error) {
87+
console.error('Error parsing log line:', event.data.error);
88+
} else if (event.data.parsedLine) {
89+
const parsedLine = event.data.parsedLine;
90+
// console.log('Parsed line:', parsedLine);
91+
92+
if (parsedLine.attackType) {
93+
// console.log('ATTACK DETECTED:', parsedLine.attackType, 'in line:', line);
94+
}
95+
96+
setParsedLines(prevLines => {
97+
const updatedLines = [parsedLine, ...prevLines];
98+
setLogData(aggregateLogData(updatedLines));
99+
return updatedLines;
100+
});
101+
}
102+
worker.terminate();
103+
};
104+
105+
worker.onerror = (error) => {
106+
console.error('Worker error:', error);
107+
worker.terminate();
108+
};
109+
110+
// Send the log line to the worker for parsing
111+
worker.postMessage({
112+
line: line,
113+
format: 'nginx' // or make this configurable
114+
});
153115
}, []);
154116

155117
const setupPingPong = useCallback(() => {
@@ -214,7 +176,7 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
214176
wsRef.current = ws;
215177

216178
ws.onopen = () => {
217-
console.log('WebSocket connected');
179+
// console.log('WebSocket connected to:', url);
218180
// Check if component is still mounted before updating state
219181
if (!wsRef.current) return;
220182
setIsConnected(true);
@@ -234,6 +196,7 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
234196

235197
if (typeof event.data === 'string') {
236198
lastMessageTime.current = Date.now();
199+
// console.log('WebSocket message received:', event.data);
237200

238201
try {
239202
const data = JSON.parse(event.data);
@@ -242,17 +205,19 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
242205
clearTimeout(pongTimeoutRef.current);
243206
pongTimeoutRef.current = null;
244207
}
208+
// console.log('Pong received');
245209
return;
246210
}
247211
} catch (e) {
248212
// Not a JSON message, treat as log line
213+
// console.log('Treating message as log line');
249214
handleNewLogLine(event.data);
250215
}
251216
}
252217
};
253218

254219
ws.onclose = (event) => {
255-
console.log('WebSocket closed:', event.code, event.reason);
220+
// console.log('WebSocket closed:', event.code, event.reason);
256221
// Don't update state if component is unmounting
257222
if (!wsRef.current) return;
258223

@@ -271,7 +236,7 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
271236

272237
// Special handling for our forced reconnection
273238
if (event.code === 4001) { // Our custom code for visibility change
274-
console.log('Reconnecting after visibility change...');
239+
// console.log('Reconnecting after visibility change...');
275240
connectWebSocket();
276241
return;
277242
}
@@ -285,7 +250,7 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
285250
reconnectDelay.current = calculateReconnectDelay(nextAttempt);
286251

287252
reconnectTimeoutRef.current = setTimeout(() => {
288-
console.log(`Attempting to reconnect (${nextAttempt + 1}/${MAX_RECONNECT_ATTEMPTS})...`);
253+
// console.log(`Attempting to reconnect (${nextAttempt + 1}/${MAX_RECONNECT_ATTEMPTS})...`);
289254
connectWebSocket();
290255
}, reconnectDelay.current);
291256

@@ -573,7 +538,13 @@ export function LiveDashboard({ wsUrl: initialWsUrl }: { wsUrl: string }) {
573538
</div>
574539
)}
575540

576-
{logData && parsedLines.length > 0 ? (
541+
{isConnected && !logData && parsedLines.length === 0 ? (
542+
<div className="flex items-center justify-center h-64 bg-gray-50 dark:bg-gray-800/50 rounded-xl">
543+
<div className="text-center">
544+
<p className="text-gray-500 dark:text-gray-400">Connected to WebSocket. Waiting for log data...</p>
545+
</div>
546+
</div>
547+
) : logData && parsedLines.length > 0 ? (
577548
<Dashboard stats={logData} parsedLines={parsedLines} />
578549
) : (
579550
<div className="flex items-center justify-center h-64 bg-gray-50 dark:bg-gray-800/50 rounded-xl">

‎lib/log-aggregator.ts‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ interface SuspiciousIpInfo {
2020
}
2121

2222
export const aggregateLogData = (parsedLines: any[], filters: Record<string, any> = {}) => {
23+
// console.log('Aggregating log data, parsedLines:', parsedLines.length);
24+
2325
const stats = {
2426
requestStats: {
2527
totalRequests: 0,
@@ -67,6 +69,8 @@ export const aggregateLogData = (parsedLines: any[], filters: Record<string, any
6769
//console.log('=== Starting to process', parsedLines.length, 'log entries with filters:', filters);
6870

6971
parsedLines.forEach((parsedLine, index) => {
72+
// console.log(`Processing line ${index}:`, parsedLine);
73+
7074
// Apply filters if any
7175
let filterMatch = true;
7276

@@ -115,7 +119,11 @@ export const aggregateLogData = (parsedLines: any[], filters: Record<string, any
115119
}
116120
}
117121

118-
if (!filterMatch) return;
122+
if (!filterMatch) {
123+
// console.log(`Line ${index} filtered out`);
124+
return;
125+
}
126+
119127
const {
120128
ipAddress,
121129
remoteUser,
@@ -129,7 +137,18 @@ export const aggregateLogData = (parsedLines: any[], filters: Record<string, any
129137
attackType,
130138
} = parsedLine;
131139

132-
if (!ipAddress || !method || !status) return;
140+
// console.log(`Line ${index} details:`, {
141+
// ipAddress,
142+
// method,
143+
// path,
144+
// status,
145+
// attackType
146+
// });
147+
148+
if (!ipAddress || !method || !status) {
149+
// console.log(`Line ${index} missing required fields, skipping`);
150+
return;
151+
}
133152

134153
initIpStats(ipAddress);
135154
const bytes = parseInt(bodyBytesSent, 10) || 0;
@@ -184,26 +203,42 @@ export const aggregateLogData = (parsedLines: any[], filters: Record<string, any
184203
const hour = parseInt(hourStr, 10);
185204
if (!isNaN(hour) && hour >= 0 && hour < 24) {
186205
stats.trafficOverTime[hour].count++;
187-
return; // Successfully processed
206+
// Removed the return statement that was preventing attack counting
188207
}
189208
}
190209
}
191-
console.warn('Could not parse timestamp:', timestamp);
210+
// console.warn('Could not parse timestamp:', timestamp);
192211
} catch (error) {
193212
console.error('Error parsing timestamp:', timestamp, error);
194213
}
195214

196215
if (attackType) {
216+
// console.log('ATTACK COUNTING SECTION REACHED for:', attackType);
217+
console.log('Aggregator found attack:', attackType, 'for line:', {
218+
ipAddress,
219+
method,
220+
path,
221+
attackType
222+
});
223+
197224
stats.attackDistribution[attackType]++;
198225
stats.requestStats.totalAttackAttempts++;
199226
stats.ipStats.attackCounts[ipAddress]++;
227+
200228
stats.recentAttacks.push({
201229
timestamp,
202230
ipAddress,
203231
remoteUser,
204232
attackType: attackType,
205233
requestPath: path,
206234
});
235+
} else {
236+
// console.log('No attack type found for line:', {
237+
// ipAddress,
238+
// method,
239+
// path,
240+
// attackType
241+
// });
207242
}
208243
});
209244

@@ -214,7 +249,7 @@ export const aggregateLogData = (parsedLines: any[], filters: Record<string, any
214249
}));
215250

216251
//console.log('Processed traffic data:', processedTrafficData.filter(x => x.count > 0));
217-
252+
218253
return {
219254
requestStats: {
220255
...stats.requestStats,

‎workers/logParser.js‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ self.onmessage = (e) => {
6161
"(?:%27|'|--|;|--\\s|/\\*.*?\\*/|#|%23|%2527)",
6262
// SQL commands and patterns
6363
"\\b(?:UNION\\s+(?:ALL\\s+)?SELECT|SELECT\\s+.*?\\bFROM|INSERT\\s+INTO|DELETE\\s+FROM|UPDATE\\s+\\w+\\s+SET|DROP\\s+(?:TABLE|DATABASE)|CREATE\\s+(?:TABLE|DATABASE)|ALTER\\s+TABLE)\\b",
64-
// Logical operators and comparisons
65-
"\\b(?:OR|AND|X?OR|NOT|LIKE|RLIKE|REGEXP)\\s+['\\d]\\s*[=<>~!]?=",
64+
// Logical operators and comparisons (both in query params and general)
65+
"(?:[?&][^=]*=(?:%27|'|%2527).*?(?:OR|AND|X?OR|NOT).*?=|\\b(?:OR|AND|X?OR|NOT)\\s+['\\d]\\s*[=<>~!]?=)",
6666
// Database functions and procedures
6767
"\\b(?:EXEC(?:UTE)?(?:\\(|\\s)|EXEC\\s+SP_|XP_|sp_|xp_|WAITFOR\\s+DELAY|SLEEP\\s*\\(|BENCHMARK\\s*\\(|PG_SLEEP\\s*\\(|UPDATEXML\\s*\\(|EXTRACTVALUE\\s*\\()",
6868
// String operations
@@ -88,7 +88,7 @@ self.onmessage = (e) => {
8888
"Command Injection": new RegExp([
8989
// More comprehensive command injection patterns
9090
"\\b(?:cat|ls|uname|whoami|pwd|rm|touch|wget|curl|scp|rsync|ftp|nc|ncat|nmap|ping|traceroute|telnet|ssh|bash|sh|zsh|dash|powershell|cmd\\.exe|cmd\\/c|\\|\\||&&|;)\\b",
91-
"\\$\s*\\(.*\\)", // $(command)
91+
"\\$\\s*\\(.*\\)", // $(command)
9292
"`.*`", // `command`
9393
"\\|\\|\\s*\\w+", // || command
9494
"&&\\s*\\w+", // && command
@@ -98,23 +98,22 @@ self.onmessage = (e) => {
9898
"\\b(?:exec|system|passthru|shell_exec|popen|proc_open|pcntl_exec)\\s*\\("
9999
].join('|'), 'i'),
100100

101-
"Directory Traversal": new RegExp(
101+
"Directory Traversal": new RegExp([
102102
// Match directory traversal sequences with at least two levels up
103-
'(?:^|/|\\\\)(?:\\.{1,2}[\./\\]){2,}' +
103+
"(?:^|/|\\\\)(?:\\.{1,2}[\\./\\\\]){2,}",
104104
// Match encoded traversal sequences
105-
'|(?:^|/|\\)(?:%2e%2e|%252e%252e|%c0%ae%c0%ae|%u002e%u002e)[/\\\\]' +
105+
"(?:^|/|\\\\)(?:%2e%2e|%252e%252e|%c0%ae%c0%ae|%u002e%u002e)[/\\\\]",
106106
// Match absolute paths to sensitive files
107-
'|/(?:etc/(?:passwd|shadow|group|hosts)|proc/self/environ|windows/win\.ini|boot\.ini|php\.ini|my\.cnf)(?:/|$|\\0)' +
107+
"/(?:etc/(?:passwd|shadow|group|hosts)|proc/self/environ|windows/win\\.ini|boot\\.ini|php\\.ini|my\\.cnf)(?:/|$|\\x00)",
108108
// Match Windows-style absolute paths
109-
'|^[a-zA-Z]:\\\\[^\\/]+' +
109+
"^[a-zA-Z]:\\\\[^/]+",
110110
// Match common path traversal patterns
111-
'|\\.\\.(?:%2f|%252f|%5c|%255c)' +
111+
"\\.\\.\\.(?:%2f|%252f|%5c|%255c)",
112112
// Match null byte injection
113-
'|\\x00|%00|\\0' +
113+
"\\\\x00|%00|\\\\0",
114114
// Match double encoding
115-
'|%25(?:2e|25|5c|2f|5f|3d|3f|26|3a|3b)',
116-
'i'
117-
),
115+
"%25(?:2e|25|5c|2f|5f|3d|3f|26|3a|3b)"
116+
].join('|'), 'i'),
118117

119118
"Brute Force": new RegExp([
120119
// More specific brute force patterns - focusing on common attack paths
@@ -419,4 +418,4 @@ self.onmessage = (e) => {
419418
};
420419

421420
self.postMessage({ stats: finalStats, parsedLines });
422-
};
421+
};

0 commit comments

Comments
 (0)