From 373f97784aa61b4dda6490fa9a0dae60bcc5ec4b Mon Sep 17 00:00:00 2001 From: serega6531 Date: Sat, 4 Apr 2020 01:20:42 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20=D1=80=D0=B0=D1=81=D0=BF=D0=B0=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=BE=D0=B9=20websocket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 4 + gradle/wrapper/gradle-wrapper.properties | 2 +- .../serega6531/packmate/model/CtfService.java | 2 + .../packmate/service/StreamOptimizer.java | 172 ++++++++++++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 17babb3..586f0c6 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,9 @@ configurations { repositories { mavenCentral() + + // удалить после выхода стабильной версии Java-WebSocket + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } } dependencies { @@ -32,6 +35,7 @@ dependencies { compile 'org.pcap4j:pcap4j-core:1.8.2' compile 'org.pcap4j:pcap4j-packetfactory-static:1.8.2' compile group: 'com.google.guava', name: 'guava', version: '28.2-jre' + compile group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.0-SNAPSHOT' compileOnly 'org.projectlombok:lombok' runtimeOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'org.postgresql:postgresql' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a2bf131..a4b4429 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index 01ddf6e..5eb5719 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -22,4 +22,6 @@ public class CtfService { private boolean mergeAdjacentPackets; + private boolean inflateWebSockets = true; //TODO + } \ No newline at end of file diff --git a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java index 1b0a7c4..6274a97 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java @@ -5,6 +5,13 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.ArrayUtils; +import org.java_websocket.drafts.Draft_6455; +import org.java_websocket.exceptions.InvalidDataException; +import org.java_websocket.exceptions.InvalidHandshakeException; +import org.java_websocket.extensions.permessage_deflate.PerMessageDeflateExtension; +import org.java_websocket.framing.Framedata; +import org.java_websocket.handshake.HandshakeImpl1Client; +import org.java_websocket.handshake.HandshakeImpl1Server; import ru.serega6531.packmate.model.CtfService; import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.utils.Bytes; @@ -13,9 +20,12 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URLDecoder; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; +import java.util.regex.Matcher; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; @@ -28,6 +38,20 @@ public class StreamOptimizer { private static final byte[] GZIP_HEADER = {0x1f, (byte) 0x8b, 0x08}; + private static final java.util.regex.Pattern WEBSOCKET_KEY_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Key: (.+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_EXTENSIONS_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Extensions?: (.+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_VERSION_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Version: (\\d+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_ACCEPT_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Accept: (.+)\\r\\n"); + + private static final String WEBSOCKET_EXTENSION_HEADER = "Sec-WebSocket-Extension: permessage-deflate"; + private static final String WEBSOCKET_EXTENSIONS_HEADER = "Sec-WebSocket-Extensions: permessage-deflate"; + private static final String WEBSOCKET_UPGRADE_HEADER = "Upgrade: websocket\r\n"; + private static final String WEBSOCKET_CONNECTION_HEADER = "Connection: Upgrade\r\n"; + /** * Вызвать для выполнения оптимизаций на переданном списке пакетов. */ @@ -40,6 +64,10 @@ public class StreamOptimizer { urldecodeRequests(); } + if (service.isInflateWebSockets()) { + inflateWebSocket(); + } + if (service.isMergeAdjacentPackets()) { mergeAdjacentPackets(); } @@ -226,4 +254,148 @@ public class StreamOptimizer { return null; } + private void inflateWebSocket() { + if (!new String(packets.get(0).getContent()).contains("HTTP/")) { + return; + } + + final List clientHandshakePackets = packets.stream() + .takeWhile(Packet::isIncoming) + .collect(Collectors.toList()); + + final String clientHandshake = getHandshake(clientHandshakePackets); + if (clientHandshake == null) { + return; + } + + final List serverHandshakePackets = packets.stream() + .skip(clientHandshakePackets.size()) + .takeWhile(p -> !p.isIncoming()) + .collect(Collectors.toList()); + + final String serverHandshake = getHandshake(serverHandshakePackets); + if (serverHandshake == null) { + return; + } + + HandshakeImpl1Server serverHandshakeImpl = fillServerHandshake(serverHandshake); + HandshakeImpl1Client clientHandshakeImpl = fillClientHandshake(clientHandshake); + + if (serverHandshakeImpl == null || clientHandshakeImpl == null) { + return; + } + + final List wsPackets = this.packets.subList( + clientHandshakePackets.size() + serverHandshakePackets.size(), + this.packets.size()); + + final byte[] wsContent = wsPackets.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .orElse(null); + + if (wsContent == null) { + return; + } + + final ByteBuffer frame = ByteBuffer.wrap(wsContent); + Draft_6455 draft = new Draft_6455(new PerMessageDeflateExtension()); + + try { + draft.acceptHandshakeAsServer(clientHandshakeImpl); + draft.acceptHandshakeAsClient(clientHandshakeImpl, serverHandshakeImpl); + } catch (InvalidHandshakeException e) { + log.warn("WebSocket handshake", e); + return; + } + + try { + final List list = draft.translateFrame(frame); + log.info(list.toString()); + } catch (InvalidDataException e) { + log.warn("WebSocket data", e); + } + } + + private String getHandshake(final List packets) { + final String handshake = packets.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .map(String::new) + .orElse(null); + + if (handshake == null || + !handshake.contains(WEBSOCKET_CONNECTION_HEADER) || + !handshake.contains(WEBSOCKET_UPGRADE_HEADER)) { + return null; + } + + if (!handshake.contains(WEBSOCKET_EXTENSION_HEADER) && + !handshake.contains(WEBSOCKET_EXTENSIONS_HEADER)) { + return null; + } + + return handshake; + } + + private HandshakeImpl1Client fillClientHandshake(String clientHandshake) { + Matcher matcher = WEBSOCKET_VERSION_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String version = matcher.group(1); + + matcher = WEBSOCKET_KEY_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String key = matcher.group(1); + + matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String extensions = matcher.group(1); + + HandshakeImpl1Client clientHandshakeImpl = new HandshakeImpl1Client(); + + clientHandshakeImpl.put("Upgrade", "websocket"); + clientHandshakeImpl.put("Connection", "Upgrade"); + clientHandshakeImpl.put("Sec-WebSocket-Version", version); + clientHandshakeImpl.put("Sec-WebSocket-Key", key); + clientHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); + + return clientHandshakeImpl; + } + + private HandshakeImpl1Server fillServerHandshake(String serverHandshake) { + Matcher matcher = WEBSOCKET_VERSION_PATTERN.matcher(serverHandshake); + if (!matcher.find()) { + return null; + } + String version = matcher.group(1); + + matcher = WEBSOCKET_ACCEPT_PATTERN.matcher(serverHandshake); + if (!matcher.find()) { + return null; + } + String accept = matcher.group(1); + + matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(serverHandshake); + if (!matcher.find()) { + return null; + } + String extensions = matcher.group(1); + + HandshakeImpl1Server serverHandshakeImpl = new HandshakeImpl1Server(); + + serverHandshakeImpl.put("Upgrade", "websocket"); + serverHandshakeImpl.put("Connection", "Upgrade"); + serverHandshakeImpl.put("Sec-WebSocket-Version", version); + serverHandshakeImpl.put("Sec-WebSocket-Accept", accept); + serverHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); + + return serverHandshakeImpl; + } + } From e25e66d727cdb8ecf0c2096729c768b19fd4291c Mon Sep 17 00:00:00 2001 From: serega6531 Date: Sat, 4 Apr 2020 22:53:29 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20=D1=80=D0=B0=D1=81=D0=BF=D0=B0=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=BE=D0=B9=20websocket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ru/serega6531/packmate/model/Packet.java | 10 + .../packmate/service/StreamOptimizer.java | 181 ++------------ .../packmate/service/StreamService.java | 4 +- .../packmate/service/WebSocketsParser.java | 221 ++++++++++++++++++ .../packmate/StreamOptimizerTest.java | 10 +- 5 files changed, 252 insertions(+), 174 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index cc60bd2..603202f 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -54,4 +54,14 @@ public class Packet { private byte[] content; + @Transient + @JsonIgnore + public String getContentString() { + return new String(content); + } + + public String toString() { + return "Packet(id=" + id + ", content=" + getContentString() + ")"; + } + } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java index 6274a97..a161afe 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java @@ -5,13 +5,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.ArrayUtils; -import org.java_websocket.drafts.Draft_6455; -import org.java_websocket.exceptions.InvalidDataException; -import org.java_websocket.exceptions.InvalidHandshakeException; -import org.java_websocket.extensions.permessage_deflate.PerMessageDeflateExtension; -import org.java_websocket.framing.Framedata; -import org.java_websocket.handshake.HandshakeImpl1Client; -import org.java_websocket.handshake.HandshakeImpl1Server; import ru.serega6531.packmate.model.CtfService; import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.utils.Bytes; @@ -20,12 +13,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URLDecoder; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.ZipException; @@ -34,43 +24,31 @@ import java.util.zip.ZipException; public class StreamOptimizer { private final CtfService service; - private final List packets; + private List packets; private static final byte[] GZIP_HEADER = {0x1f, (byte) 0x8b, 0x08}; - private static final java.util.regex.Pattern WEBSOCKET_KEY_PATTERN = - java.util.regex.Pattern.compile("Sec-WebSocket-Key: (.+)\\r\\n"); - private static final java.util.regex.Pattern WEBSOCKET_EXTENSIONS_PATTERN = - java.util.regex.Pattern.compile("Sec-WebSocket-Extensions?: (.+)\\r\\n"); - private static final java.util.regex.Pattern WEBSOCKET_VERSION_PATTERN = - java.util.regex.Pattern.compile("Sec-WebSocket-Version: (\\d+)\\r\\n"); - private static final java.util.regex.Pattern WEBSOCKET_ACCEPT_PATTERN = - java.util.regex.Pattern.compile("Sec-WebSocket-Accept: (.+)\\r\\n"); - - private static final String WEBSOCKET_EXTENSION_HEADER = "Sec-WebSocket-Extension: permessage-deflate"; - private static final String WEBSOCKET_EXTENSIONS_HEADER = "Sec-WebSocket-Extensions: permessage-deflate"; - private static final String WEBSOCKET_UPGRADE_HEADER = "Upgrade: websocket\r\n"; - private static final String WEBSOCKET_CONNECTION_HEADER = "Connection: Upgrade\r\n"; - /** * Вызвать для выполнения оптимизаций на переданном списке пакетов. */ - public void optimizeStream() { + public List optimizeStream() { if (service.isUngzipHttp()) { unpackGzip(); } - if (service.isUrldecodeHttpRequests()) { - urldecodeRequests(); - } - if (service.isInflateWebSockets()) { inflateWebSocket(); } + if (service.isUrldecodeHttpRequests()) { + urldecodeRequests(); + } + if (service.isMergeAdjacentPackets()) { mergeAdjacentPackets(); } + + return packets; } /** @@ -136,7 +114,7 @@ public class StreamOptimizer { for (Packet packet : packets) { if (packet.isIncoming()) { - String content = new String(packet.getContent()); + String content = packet.getContentString(); if (content.contains("HTTP/")) { httpStarted = true; } @@ -173,7 +151,7 @@ public class StreamOptimizer { i = gzipStartPacket + 1; // продвигаем указатель на следующий после склеенного блок } } else if (!packet.isIncoming()) { - String content = new String(packet.getContent()); + String content = packet.getContentString(); int contentPos = content.indexOf("\r\n\r\n"); boolean http = content.startsWith("HTTP/"); @@ -255,147 +233,16 @@ public class StreamOptimizer { } private void inflateWebSocket() { - if (!new String(packets.get(0).getContent()).contains("HTTP/")) { + if (!packets.get(0).getContentString().contains("HTTP/")) { return; } - final List clientHandshakePackets = packets.stream() - .takeWhile(Packet::isIncoming) - .collect(Collectors.toList()); - - final String clientHandshake = getHandshake(clientHandshakePackets); - if (clientHandshake == null) { + final WebSocketsParser parser = new WebSocketsParser(packets); + if(!parser.isParsed()) { return; } - final List serverHandshakePackets = packets.stream() - .skip(clientHandshakePackets.size()) - .takeWhile(p -> !p.isIncoming()) - .collect(Collectors.toList()); - - final String serverHandshake = getHandshake(serverHandshakePackets); - if (serverHandshake == null) { - return; - } - - HandshakeImpl1Server serverHandshakeImpl = fillServerHandshake(serverHandshake); - HandshakeImpl1Client clientHandshakeImpl = fillClientHandshake(clientHandshake); - - if (serverHandshakeImpl == null || clientHandshakeImpl == null) { - return; - } - - final List wsPackets = this.packets.subList( - clientHandshakePackets.size() + serverHandshakePackets.size(), - this.packets.size()); - - final byte[] wsContent = wsPackets.stream() - .map(Packet::getContent) - .reduce(ArrayUtils::addAll) - .orElse(null); - - if (wsContent == null) { - return; - } - - final ByteBuffer frame = ByteBuffer.wrap(wsContent); - Draft_6455 draft = new Draft_6455(new PerMessageDeflateExtension()); - - try { - draft.acceptHandshakeAsServer(clientHandshakeImpl); - draft.acceptHandshakeAsClient(clientHandshakeImpl, serverHandshakeImpl); - } catch (InvalidHandshakeException e) { - log.warn("WebSocket handshake", e); - return; - } - - try { - final List list = draft.translateFrame(frame); - log.info(list.toString()); - } catch (InvalidDataException e) { - log.warn("WebSocket data", e); - } - } - - private String getHandshake(final List packets) { - final String handshake = packets.stream() - .map(Packet::getContent) - .reduce(ArrayUtils::addAll) - .map(String::new) - .orElse(null); - - if (handshake == null || - !handshake.contains(WEBSOCKET_CONNECTION_HEADER) || - !handshake.contains(WEBSOCKET_UPGRADE_HEADER)) { - return null; - } - - if (!handshake.contains(WEBSOCKET_EXTENSION_HEADER) && - !handshake.contains(WEBSOCKET_EXTENSIONS_HEADER)) { - return null; - } - - return handshake; - } - - private HandshakeImpl1Client fillClientHandshake(String clientHandshake) { - Matcher matcher = WEBSOCKET_VERSION_PATTERN.matcher(clientHandshake); - if (!matcher.find()) { - return null; - } - String version = matcher.group(1); - - matcher = WEBSOCKET_KEY_PATTERN.matcher(clientHandshake); - if (!matcher.find()) { - return null; - } - String key = matcher.group(1); - - matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(clientHandshake); - if (!matcher.find()) { - return null; - } - String extensions = matcher.group(1); - - HandshakeImpl1Client clientHandshakeImpl = new HandshakeImpl1Client(); - - clientHandshakeImpl.put("Upgrade", "websocket"); - clientHandshakeImpl.put("Connection", "Upgrade"); - clientHandshakeImpl.put("Sec-WebSocket-Version", version); - clientHandshakeImpl.put("Sec-WebSocket-Key", key); - clientHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); - - return clientHandshakeImpl; - } - - private HandshakeImpl1Server fillServerHandshake(String serverHandshake) { - Matcher matcher = WEBSOCKET_VERSION_PATTERN.matcher(serverHandshake); - if (!matcher.find()) { - return null; - } - String version = matcher.group(1); - - matcher = WEBSOCKET_ACCEPT_PATTERN.matcher(serverHandshake); - if (!matcher.find()) { - return null; - } - String accept = matcher.group(1); - - matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(serverHandshake); - if (!matcher.find()) { - return null; - } - String extensions = matcher.group(1); - - HandshakeImpl1Server serverHandshakeImpl = new HandshakeImpl1Server(); - - serverHandshakeImpl.put("Upgrade", "websocket"); - serverHandshakeImpl.put("Connection", "Upgrade"); - serverHandshakeImpl.put("Sec-WebSocket-Version", version); - serverHandshakeImpl.put("Sec-WebSocket-Accept", accept); - serverHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); - - return serverHandshakeImpl; + packets = parser.getParsedPackets(); } } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 19afa74..8d0fc25 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -93,7 +93,7 @@ public class StreamService { countingService.countStream(service.getPort(), packets.size()); - new StreamOptimizer(service, packets).optimizeStream(); + packets = new StreamOptimizer(service, packets).optimizeStream(); processUserAgent(packets, stream); Stream savedStream = save(stream); @@ -110,7 +110,7 @@ public class StreamService { private void processUserAgent(List packets, Stream stream) { String ua = null; for (Packet packet : packets) { - String content = new String(packet.getContent()); + String content = packet.getContentString(); final Matcher matcher = userAgentPattern.matcher(content); if (matcher.find()) { ua = matcher.group(1); diff --git a/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java new file mode 100644 index 0000000..d8bde27 --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java @@ -0,0 +1,221 @@ +package ru.serega6531.packmate.service; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.java_websocket.drafts.Draft_6455; +import org.java_websocket.exceptions.InvalidDataException; +import org.java_websocket.exceptions.InvalidHandshakeException; +import org.java_websocket.extensions.permessage_deflate.PerMessageDeflateExtension; +import org.java_websocket.framing.DataFrame; +import org.java_websocket.framing.Framedata; +import org.java_websocket.handshake.HandshakeImpl1Client; +import org.java_websocket.handshake.HandshakeImpl1Server; +import ru.serega6531.packmate.model.Packet; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.stream.Collectors; + +@Slf4j +public class WebSocketsParser { + + private static final java.util.regex.Pattern WEBSOCKET_KEY_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Key: (.+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_EXTENSIONS_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Extensions?: (.+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_VERSION_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Version: (\\d+)\\r\\n"); + private static final java.util.regex.Pattern WEBSOCKET_ACCEPT_PATTERN = + java.util.regex.Pattern.compile("Sec-WebSocket-Accept: (.+)\\r\\n"); + + private static final String WEBSOCKET_EXTENSION_HEADER = "Sec-WebSocket-Extension: permessage-deflate"; + private static final String WEBSOCKET_EXTENSIONS_HEADER = "Sec-WebSocket-Extensions: permessage-deflate"; + private static final String WEBSOCKET_UPGRADE_HEADER = "upgrade: websocket\r\n"; + private static final String WEBSOCKET_CONNECTION_HEADER = "connection: upgrade\r\n"; + + private final List packets; + private List frames; + + @Getter + private boolean parsed = false; + private int httpEnd = -1; + + public WebSocketsParser(List packets) { + this.packets = packets; + detectWebSockets(); + } + + private void detectWebSockets() { + final List clientHandshakePackets = packets.stream() + .takeWhile(Packet::isIncoming) + .collect(Collectors.toList()); + + final String clientHandshake = getHandshake(clientHandshakePackets); + if (clientHandshake == null) { + return; + } + + for (int i = clientHandshakePackets.size(); i < packets.size(); i++) { + if (packets.get(i).getContentString().endsWith("\r\n\r\n")) { + httpEnd = i + 1; + break; + } + } + + if (httpEnd == -1) { + return; + } + + final List serverHandshakePackets = packets.subList(clientHandshakePackets.size(), httpEnd); + final String serverHandshake = getHandshake(serverHandshakePackets); + if (serverHandshake == null) { + return; + } + + HandshakeImpl1Server serverHandshakeImpl = fillServerHandshake(serverHandshake); + HandshakeImpl1Client clientHandshakeImpl = fillClientHandshake(clientHandshake); + + if (serverHandshakeImpl == null || clientHandshakeImpl == null) { + return; + } + + Draft_6455 draft = new Draft_6455(new PerMessageDeflateExtension()); + + try { + draft.acceptHandshakeAsServer(clientHandshakeImpl); + draft.acceptHandshakeAsClient(clientHandshakeImpl, serverHandshakeImpl); + } catch (InvalidHandshakeException e) { + log.warn("WebSocket handshake", e); + return; + } + + final List wsPackets = this.packets.subList( + httpEnd, + this.packets.size()); + + final byte[] wsContent = wsPackets.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .orElse(null); + + if (wsContent == null) { + return; + } + + final ByteBuffer frame = ByteBuffer.wrap(wsContent); + + try { + frames = draft.translateFrame(frame); + } catch (InvalidDataException e) { + log.warn("WebSocket data", e); + return; + } + + parsed = true; + } + + public List getParsedPackets() { + if (!parsed) { + throw new IllegalStateException("WS is not parsed"); + } + + final List handshakes = packets.subList(0, httpEnd); + List newPackets = new ArrayList<>(handshakes.size() + frames.size()); + newPackets.addAll(handshakes); + + final Packet lastPacket = packets.get(packets.size() - 1); + + for (Framedata frame : frames) { + if(frame instanceof DataFrame) { + newPackets.add(Packet.builder() + .content(frame.getPayloadData().array()) + .incoming(true) //TODO + .timestamp(lastPacket.getTimestamp()) + .ttl(lastPacket.getTtl()) + .ungzipped(lastPacket.isUngzipped()) + .build() + ); + } + } + + return newPackets; + } + + private String getHandshake(final List packets) { + final String handshake = packets.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .map(String::new) + .orElse(null); + + if (handshake == null || + !handshake.toLowerCase().contains(WEBSOCKET_CONNECTION_HEADER) || + !handshake.toLowerCase().contains(WEBSOCKET_UPGRADE_HEADER)) { + return null; + } + + if (!handshake.contains(WEBSOCKET_EXTENSION_HEADER) && + !handshake.contains(WEBSOCKET_EXTENSIONS_HEADER)) { + return null; + } + + return handshake; + } + + private HandshakeImpl1Client fillClientHandshake(String clientHandshake) { + Matcher matcher = WEBSOCKET_VERSION_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String version = matcher.group(1); + + matcher = WEBSOCKET_KEY_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String key = matcher.group(1); + + matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(clientHandshake); + if (!matcher.find()) { + return null; + } + String extensions = matcher.group(1); + + HandshakeImpl1Client clientHandshakeImpl = new HandshakeImpl1Client(); + + clientHandshakeImpl.put("Upgrade", "websocket"); + clientHandshakeImpl.put("Connection", "Upgrade"); + clientHandshakeImpl.put("Sec-WebSocket-Version", version); + clientHandshakeImpl.put("Sec-WebSocket-Key", key); + clientHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); + + return clientHandshakeImpl; + } + + private HandshakeImpl1Server fillServerHandshake(String serverHandshake) { + Matcher matcher = WEBSOCKET_ACCEPT_PATTERN.matcher(serverHandshake); + if (!matcher.find()) { + return null; + } + String accept = matcher.group(1); + + matcher = WEBSOCKET_EXTENSIONS_PATTERN.matcher(serverHandshake); + if (!matcher.find()) { + return null; + } + String extensions = matcher.group(1); + + HandshakeImpl1Server serverHandshakeImpl = new HandshakeImpl1Server(); + + serverHandshakeImpl.put("Upgrade", "websocket"); + serverHandshakeImpl.put("Connection", "Upgrade"); + serverHandshakeImpl.put("Sec-WebSocket-Accept", accept); + serverHandshakeImpl.put("Sec-WebSocket-Extensions", extensions); + + return serverHandshakeImpl; + } + +} diff --git a/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java b/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java index ecdbf5c..67bde4a 100644 --- a/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java +++ b/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java @@ -28,8 +28,8 @@ class StreamOptimizerTest { List list = new ArrayList<>(); list.add(p); - new StreamOptimizer(service, list).optimizeStream(); - final String processed = new String(list.get(0).getContent()); + list = new StreamOptimizer(service, list).optimizeStream(); + final String processed = list.get(0).getContentString(); assertTrue(processed.contains("aaabbb")); } @@ -42,8 +42,8 @@ class StreamOptimizerTest { List list = new ArrayList<>(); list.add(p); - new StreamOptimizer(service, list).optimizeStream(); - final String processed = new String(list.get(0).getContent()); + list = new StreamOptimizer(service, list).optimizeStream(); + final String processed = list.get(0).getContentString(); assertTrue(processed.contains("а б")); } @@ -67,7 +67,7 @@ class StreamOptimizerTest { list.add(p5); list.add(p6); - new StreamOptimizer(service, list).optimizeStream(); + list = new StreamOptimizer(service, list).optimizeStream(); assertEquals(4, list.size()); assertEquals(2, list.get(1).getContent().length); From 2688e1fc566918b1614dfdf732fbbcba57d31cf7 Mon Sep 17 00:00:00 2001 From: serega6531 Date: Sun, 5 Apr 2020 00:36:52 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20=D1=80=D0=B0=D1=81=D0=BF=D0=B0=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=BE=D0=B9=20websocket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/service/WebSocketsParser.java | 98 ++++++++++++------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java index d8bde27..45ab970 100644 --- a/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java +++ b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java @@ -37,11 +37,10 @@ public class WebSocketsParser { private static final String WEBSOCKET_CONNECTION_HEADER = "connection: upgrade\r\n"; private final List packets; - private List frames; @Getter private boolean parsed = false; - private int httpEnd = -1; + private List parsedPackets; public WebSocketsParser(List packets) { this.packets = packets; @@ -58,6 +57,7 @@ public class WebSocketsParser { return; } + int httpEnd = -1; for (int i = clientHandshakePackets.size(); i < packets.size(); i++) { if (packets.get(i).getContentString().endsWith("\r\n\r\n")) { httpEnd = i + 1; @@ -92,56 +92,88 @@ public class WebSocketsParser { return; } - final List wsPackets = this.packets.subList( + final List wsPackets = packets.subList( httpEnd, - this.packets.size()); + packets.size()); - final byte[] wsContent = wsPackets.stream() - .map(Packet::getContent) - .reduce(ArrayUtils::addAll) - .orElse(null); - - if (wsContent == null) { + if(wsPackets.isEmpty()) { return; } - final ByteBuffer frame = ByteBuffer.wrap(wsContent); - - try { - frames = draft.translateFrame(frame); - } catch (InvalidDataException e) { - log.warn("WebSocket data", e); - return; - } + final List handshakes = packets.subList(0, httpEnd); + parse(wsPackets, handshakes, draft); parsed = true; } + private void parse(final List wsPackets, final List handshakes, Draft_6455 draft) { + List> sides = sliceToSides(wsPackets); + parsedPackets = new ArrayList<>(handshakes); + + for (List side : sides) { + final Packet lastPacket = side.get(0); + + final byte[] wsContent = side.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .get(); + + final ByteBuffer buffer = ByteBuffer.wrap(wsContent); + List frames; + + try { + frames = draft.translateFrame(buffer); + } catch (InvalidDataException e) { + log.warn("WebSocket data", e); + return; + } + + for (Framedata frame : frames) { + if(frame instanceof DataFrame) { + parsedPackets.add(Packet.builder() + .content(frame.getPayloadData().array()) + .incoming(lastPacket.isIncoming()) + .timestamp(lastPacket.getTimestamp()) + .ttl(lastPacket.getTtl()) + .ungzipped(lastPacket.isUngzipped()) + .build() + ); + } + } + } + } + public List getParsedPackets() { if (!parsed) { throw new IllegalStateException("WS is not parsed"); } - final List handshakes = packets.subList(0, httpEnd); - List newPackets = new ArrayList<>(handshakes.size() + frames.size()); - newPackets.addAll(handshakes); + return parsedPackets; + } - final Packet lastPacket = packets.get(packets.size() - 1); + private List> sliceToSides(List packets) { + List> result = new ArrayList<>(); + List side = new ArrayList<>(); + boolean incoming = true; - for (Framedata frame : frames) { - if(frame instanceof DataFrame) { - newPackets.add(Packet.builder() - .content(frame.getPayloadData().array()) - .incoming(true) //TODO - .timestamp(lastPacket.getTimestamp()) - .ttl(lastPacket.getTtl()) - .ungzipped(lastPacket.isUngzipped()) - .build() - ); + for (Packet packet : packets) { + if(packet.isIncoming() != incoming) { + incoming = packet.isIncoming(); + + if(!side.isEmpty()) { + result.add(side); + side = new ArrayList<>(); + } } + + side.add(packet); } - return newPackets; + if(!side.isEmpty()) { + result.add(side); + } + + return result; } private String getHandshake(final List packets) { From d33ca5e66714c29f8b67e0d71ed568ba35041a34 Mon Sep 17 00:00:00 2001 From: serega6531 Date: Sun, 5 Apr 2020 01:51:33 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20=D1=80=D0=B0=D1=81=D0=BF=D0=B0=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=BE=D0=B9=20websocket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/serega6531/packmate/model/CtfService.java | 2 +- src/main/java/ru/serega6531/packmate/model/Packet.java | 2 ++ .../java/ru/serega6531/packmate/service/StreamOptimizer.java | 3 +++ .../java/ru/serega6531/packmate/service/WebSocketsParser.java | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index 5eb5719..58bef42 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -22,6 +22,6 @@ public class CtfService { private boolean mergeAdjacentPackets; - private boolean inflateWebSockets = true; //TODO + private boolean inflateWebSockets; } \ No newline at end of file diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 603202f..b0a6f7b 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -52,6 +52,8 @@ public class Packet { private boolean ungzipped; + private boolean webSocketInflated; + private byte[] content; @Transient diff --git a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java index a161afe..7050fc8 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamOptimizer.java @@ -89,6 +89,7 @@ public class StreamOptimizer { final List cut = packets.subList(start, end); final long timestamp = cut.get(0).getTimestamp(); final boolean ungzipped = cut.stream().anyMatch(Packet::isUngzipped); + final boolean webSocketInflated = cut.stream().anyMatch(Packet::isWebSocketInflated); boolean incoming = cut.get(0).isIncoming(); //noinspection OptionalGetWithoutIsPresent final byte[] content = cut.stream() @@ -101,6 +102,7 @@ public class StreamOptimizer { .incoming(incoming) .timestamp(timestamp) .ungzipped(ungzipped) + .webSocketInflated(webSocketInflated) .content(content) .build()); } @@ -221,6 +223,7 @@ public class StreamOptimizer { .incoming(false) .timestamp(cut.get(0).getTimestamp()) .ungzipped(true) + .webSocketInflated(false) .content(newContent) .build(); } catch (ZipException e) { diff --git a/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java index 45ab970..0486001 100644 --- a/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java +++ b/src/main/java/ru/serega6531/packmate/service/WebSocketsParser.java @@ -136,6 +136,7 @@ public class WebSocketsParser { .timestamp(lastPacket.getTimestamp()) .ttl(lastPacket.getTtl()) .ungzipped(lastPacket.isUngzipped()) + .webSocketInflated(true) .build() ); } From bb5bfb5d65e7b23ec7ce1d2e29cb6a2c11177928 Mon Sep 17 00:00:00 2001 From: serega6531 Date: Sun, 5 Apr 2020 01:52:27 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D1=84=D1=80=D0=BE=D0=BD=D1=82=D0=B5=D0=BD=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend b/frontend index 6883d55..4e6a685 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 6883d5566bf61d837d06f5e8eed1834c1ddc4218 +Subproject commit 4e6a685e698ad764a69a0b5ce182b60bf130f3b5