From 938031f1dedfbedabef5b049368ed5b6ba6a900a Mon Sep 17 00:00:00 2001 From: Sergey Date: Mon, 31 Jul 2023 15:42:17 +0000 Subject: [PATCH] Use RawHTTP library to process HTTP streams (packmate/Packmate!23) --- build.gradle.kts | 2 + frontend | 2 +- .../serega6531/packmate/model/CtfService.java | 4 +- .../ru/serega6531/packmate/model/Packet.java | 10 +- .../packmate/model/pojo/PacketDto.java | 1 + .../packmate/model/pojo/ServiceCreateDto.java | 3 +- .../packmate/model/pojo/ServiceDto.java | 3 +- .../packmate/model/pojo/ServiceUpdateDto.java | 3 +- .../packmate/pcap/FilePcapWorker.java | 13 +- .../optimization/HttpChunksProcessor.java | 180 ------------------ .../optimization/HttpGzipProcessor.java | 121 ------------ .../service/optimization/HttpProcessor.java | 65 +++++++ .../optimization/HttpUrldecodeProcessor.java | 6 +- .../service/optimization/PacketsMerger.java | 33 ++-- .../service/optimization/StreamOptimizer.java | 79 ++------ .../service/optimization/TlsDecryptor.java | 15 +- .../optimization/WebSocketsParser.java | 33 +--- .../packmate/utils/PacketUtils.java | 18 +- .../packmate/StreamOptimizerTest.java | 13 +- 19 files changed, 158 insertions(+), 446 deletions(-) delete mode 100644 src/main/java/ru/serega6531/packmate/service/optimization/HttpChunksProcessor.java delete mode 100644 src/main/java/ru/serega6531/packmate/service/optimization/HttpGzipProcessor.java create mode 100644 src/main/java/ru/serega6531/packmate/service/optimization/HttpProcessor.java diff --git a/build.gradle.kts b/build.gradle.kts index 7e81b7e..811e5cc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -46,6 +46,8 @@ dependencies { implementation(group = "org.bouncycastle", name = "bcprov-jdk15on", version = "1.70") implementation(group = "org.bouncycastle", name = "bctls-jdk15on", version = "1.70") implementation(group = "org.modelmapper", name = "modelmapper", version = "3.1.1") + implementation("com.athaydes.rawhttp:rawhttp-core:2.5.2") + compileOnly("org.jetbrains:annotations:24.0.1") compileOnly("org.projectlombok:lombok") runtimeOnly("org.springframework.boot:spring-boot-devtools") diff --git a/frontend b/frontend index 079e8e9..8f23d97 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 079e8e95b69dee67c2fd5c29a164eebd833313b0 +Subproject commit 8f23d97100f11cd6671bb62aca5ce9b6a898b865 diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index c58ae71..99bcd6d 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -25,9 +25,7 @@ public class CtfService { private boolean decryptTls; - private boolean processChunkedEncoding; - - private boolean ungzipHttp; + private boolean http; private boolean urldecodeHttpRequests; diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index d3e8d96..453d605 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -24,7 +24,7 @@ import java.util.Set; } ) @AllArgsConstructor -@Builder +@Builder(toBuilder = true) @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") }) public class Packet { @@ -49,11 +49,13 @@ public class Packet { private boolean incoming; // true если от клиента к серверу, иначе false - private boolean ungzipped; + private boolean httpProcessed = false; - private boolean webSocketParsed; + private boolean webSocketParsed = false; - private boolean tlsDecrypted; + private boolean tlsDecrypted = false; + + private boolean hasHttpBody = false; @Column(nullable = false) private byte[] content; diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/PacketDto.java b/src/main/java/ru/serega6531/packmate/model/pojo/PacketDto.java index b276786..3e6d697 100644 --- a/src/main/java/ru/serega6531/packmate/model/pojo/PacketDto.java +++ b/src/main/java/ru/serega6531/packmate/model/pojo/PacketDto.java @@ -14,6 +14,7 @@ public class PacketDto { private boolean ungzipped; private boolean webSocketParsed; private boolean tlsDecrypted; + private boolean hasHttpBody; private byte[] content; } diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceCreateDto.java b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceCreateDto.java index 98ce728..ff00042 100644 --- a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceCreateDto.java +++ b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceCreateDto.java @@ -8,8 +8,7 @@ public class ServiceCreateDto { private int port; private String name; private boolean decryptTls; - private boolean processChunkedEncoding; - private boolean ungzipHttp; + private boolean http; private boolean urldecodeHttpRequests; private boolean mergeAdjacentPackets; private boolean parseWebSockets; diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceDto.java b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceDto.java index 0955ecd..8d00ec9 100644 --- a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceDto.java +++ b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceDto.java @@ -8,8 +8,7 @@ public class ServiceDto { private int port; private String name; private boolean decryptTls; - private boolean processChunkedEncoding; - private boolean ungzipHttp; + private boolean http; private boolean urldecodeHttpRequests; private boolean mergeAdjacentPackets; private boolean parseWebSockets; diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceUpdateDto.java b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceUpdateDto.java index db46765..d9a89c5 100644 --- a/src/main/java/ru/serega6531/packmate/model/pojo/ServiceUpdateDto.java +++ b/src/main/java/ru/serega6531/packmate/model/pojo/ServiceUpdateDto.java @@ -8,8 +8,7 @@ public class ServiceUpdateDto { private int port; private String name; private boolean decryptTls; - private boolean processChunkedEncoding; - private boolean ungzipHttp; + private boolean http; private boolean urldecodeHttpRequests; private boolean mergeAdjacentPackets; private boolean parseWebSockets; diff --git a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java index a3c6c82..58807dc 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java @@ -22,6 +22,8 @@ import java.net.UnknownHostException; @Slf4j public class FilePcapWorker extends AbstractPcapWorker { + private final File directory = new File("pcaps"); + private final SubscriptionService subscriptionService; private final File file; @@ -33,11 +35,8 @@ public class FilePcapWorker extends AbstractPcapWorker { super(servicesService, streamService, localIp); this.subscriptionService = subscriptionService; - File directory = new File("pcaps"); file = new File(directory, filename); - if (!file.exists()) { - throw new PcapFileNotFoundException(file, directory); - } + validateFileExists(); processorExecutorService = new InlineExecutorService(); } @@ -87,4 +86,10 @@ public class FilePcapWorker extends AbstractPcapWorker { public String getExecutorState() { return "inline"; } + + private void validateFileExists() { + if (!file.exists()) { + throw new PcapFileNotFoundException(file, directory); + } + } } diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/HttpChunksProcessor.java b/src/main/java/ru/serega6531/packmate/service/optimization/HttpChunksProcessor.java deleted file mode 100644 index 99e91f8..0000000 --- a/src/main/java/ru/serega6531/packmate/service/optimization/HttpChunksProcessor.java +++ /dev/null @@ -1,180 +0,0 @@ -package ru.serega6531.packmate.service.optimization; - -import com.google.common.primitives.Bytes; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import ru.serega6531.packmate.model.Packet; -import ru.serega6531.packmate.utils.BytesUtils; -import ru.serega6531.packmate.utils.PacketUtils; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Slf4j -@RequiredArgsConstructor -public class HttpChunksProcessor { - - private static final String CHUNKED_HTTP_HEADER = "transfer-encoding: chunked\r\n"; - private final List packets; - - private int position; - private boolean chunkStarted = false; - private final List chunkPackets = new ArrayList<>(); - - public void processChunkedEncoding() { - int start = -1; - - for (position = 0; position < packets.size(); position++) { - Packet packet = packets.get(position); - if (!packet.isIncoming()) { - String content = packet.getContentString(); - - boolean http = content.startsWith("HTTP/"); - int contentPos = content.indexOf("\r\n\r\n"); - - if (http && contentPos != -1) { // начало body - String headers = content.substring(0, contentPos + 2); // захватываем первые \r\n - boolean chunked = headers.toLowerCase().contains(CHUNKED_HTTP_HEADER); - if (chunked) { - chunkStarted = true; - start = position; - chunkPackets.add(packet); - - checkCompleteChunk(chunkPackets, start); - } else { - chunkStarted = false; - chunkPackets.clear(); - } - } else if (chunkStarted) { - chunkPackets.add(packet); - checkCompleteChunk(chunkPackets, start); - } - } - } - } - - private void checkCompleteChunk(List packets, int start) { - boolean end = Arrays.equals(packets.get(packets.size() - 1).getContent(), "0\r\n\r\n".getBytes()) || - BytesUtils.endsWith(packets.get(packets.size() - 1).getContent(), "\r\n0\r\n\r\n".getBytes()); - - if (end) { - processChunk(packets, start); - } - } - - @SneakyThrows - private void processChunk(List packets, int start) { - //noinspection OptionalGetWithoutIsPresent - final byte[] content = PacketUtils.mergePackets(packets).get(); - - ByteArrayOutputStream output = new ByteArrayOutputStream(content.length); - - final int contentStart = Bytes.indexOf(content, "\r\n\r\n".getBytes()) + 4; - output.write(content, 0, contentStart); - - ByteBuffer buf = ByteBuffer.wrap(Arrays.copyOfRange(content, contentStart, content.length)); - - while (true) { - final int chunkSize = readChunkSize(buf); - - switch (chunkSize) { - case -1 -> { - log.warn("Failed to merge chunks, next chunk size not found"); - resetChunk(); - return; - } - case 0 -> { - buildWholePacket(packets, start, output); - return; - } - default -> { - if (!readChunk(buf, chunkSize, output)) return; - if (!readTrailer(buf)) return; - } - } - } - } - - private boolean readChunk(ByteBuffer buf, int chunkSize, ByteArrayOutputStream output) throws IOException { - if (chunkSize > buf.remaining()) { - log.warn("Failed to merge chunks, chunk size too big: {} + {} > {}", - buf.position(), chunkSize, buf.capacity()); - resetChunk(); - return false; - } - - byte[] chunk = new byte[chunkSize]; - buf.get(chunk); - output.write(chunk); - return true; - } - - private boolean readTrailer(ByteBuffer buf) { - if (buf.remaining() < 2) { - log.warn("Failed to merge chunks, chunk doesn't end with \\r\\n"); - resetChunk(); - return false; - } - - int c1 = buf.get(); - int c2 = buf.get(); - - if (c1 != '\r' || c2 != '\n') { - log.warn("Failed to merge chunks, chunk trailer is not equal to \\r\\n"); - resetChunk(); - return false; - } - - return true; - } - - private void buildWholePacket(List packets, int start, ByteArrayOutputStream output) { - Packet result = Packet.builder() - .incoming(false) - .timestamp(packets.get(0).getTimestamp()) - .ungzipped(false) - .webSocketParsed(false) - .tlsDecrypted(packets.get(0).isTlsDecrypted()) - .content(output.toByteArray()) - .build(); - - this.packets.removeAll(packets); - this.packets.add(start, result); - - resetChunk(); - position = start + 1; - } - - private void resetChunk() { - chunkStarted = false; - chunkPackets.clear(); - } - - private int readChunkSize(ByteBuffer buf) { - StringBuilder sb = new StringBuilder(); - - while (buf.remaining() > 2) { - byte b = buf.get(); - - if ((b >= '0' && b <= '9') || (b >= 'a' && b <= 'f')) { - sb.append((char) b); - } else if (b == '\r') { - if (buf.get() == '\n') { - return Integer.parseInt(sb.toString(), 16); - } else { - return -1; // после \r не идет \n - } - } else { - return -1; - } - } - - return -1; - } - -} diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/HttpGzipProcessor.java b/src/main/java/ru/serega6531/packmate/service/optimization/HttpGzipProcessor.java deleted file mode 100644 index fd02180..0000000 --- a/src/main/java/ru/serega6531/packmate/service/optimization/HttpGzipProcessor.java +++ /dev/null @@ -1,121 +0,0 @@ -package ru.serega6531.packmate.service.optimization; - -import com.google.common.primitives.Bytes; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.ArrayUtils; -import ru.serega6531.packmate.model.Packet; -import ru.serega6531.packmate.utils.PacketUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.zip.GZIPInputStream; -import java.util.zip.ZipException; - -@Slf4j -@RequiredArgsConstructor -public class HttpGzipProcessor { - - private static final String GZIP_HTTP_HEADER = "content-encoding: gzip\r\n"; - private static final byte[] GZIP_HEADER = {0x1f, (byte) 0x8b, 0x08}; - - private final List packets; - - boolean gzipStarted = false; - private int position; - - /** - * Попытаться распаковать GZIP из исходящих http пакетов.
- * GZIP поток начинается на найденном HTTP пакете с заголовком Content-Encoding: gzip - * (при этом заголовок HTTP может быть в другом пакете)
- * Поток заканчивается при обнаружении нового HTTP заголовка, - * при смене стороны передачи или при окончании всего стрима - */ - public void unpackGzip() { - int gzipStartPacket = 0; - - for (position = 0; position < packets.size(); position++) { - Packet packet = packets.get(position); - - if (packet.isIncoming() && gzipStarted) { // поток gzip закончился - extractGzip(gzipStartPacket, position - 1); - } else if (!packet.isIncoming()) { - String content = packet.getContentString(); - - int contentPos = content.indexOf("\r\n\r\n"); - boolean http = content.startsWith("HTTP/"); - - if (http && gzipStarted) { // начался новый http пакет, заканчиваем старый gzip поток - extractGzip(gzipStartPacket, position - 1); - } - - if (contentPos != -1) { // начало body - String headers = content.substring(0, contentPos + 2); // захватываем первые \r\n - boolean gziped = headers.toLowerCase().contains(GZIP_HTTP_HEADER); - if (gziped) { - gzipStarted = true; - gzipStartPacket = position; - } - } - } - } - - if (gzipStarted) { // стрим закончился gzip пакетом - extractGzip(gzipStartPacket, packets.size() - 1); - } - } - - /** - * Попытаться распаковать кусок пакетов с gzip body и вставить результат на их место - */ - private void extractGzip(int gzipStartPacket, int gzipEndPacket) { - List cut = packets.subList(gzipStartPacket, gzipEndPacket + 1); - - Packet decompressed = decompressGzipPackets(cut); - if (decompressed != null) { - packets.removeAll(cut); - packets.add(gzipStartPacket, decompressed); - - gzipStarted = false; - position = gzipStartPacket + 1; // продвигаем указатель на следующий после склеенного блок - } - } - - private Packet decompressGzipPackets(List cut) { - //noinspection OptionalGetWithoutIsPresent - final byte[] content = PacketUtils.mergePackets(cut).get(); - - final int gzipStart = Bytes.indexOf(content, GZIP_HEADER); - byte[] httpHeader = Arrays.copyOfRange(content, 0, gzipStart); - byte[] gzipBytes = Arrays.copyOfRange(content, gzipStart, content.length); - - try { - final GZIPInputStream gzipStream = new GZIPInputStream(new ByteArrayInputStream(gzipBytes)); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copy(gzipStream, out); - byte[] newContent = ArrayUtils.addAll(httpHeader, out.toByteArray()); - - log.debug("GZIP decompressed: {} -> {} bytes", gzipBytes.length, out.size()); - - return Packet.builder() - .incoming(false) - .timestamp(cut.get(0).getTimestamp()) - .ungzipped(true) - .webSocketParsed(false) - .tlsDecrypted(cut.get(0).isTlsDecrypted()) - .content(newContent) - .build(); - } catch (ZipException e) { - log.warn("Failed to decompress gzip, leaving as it is: {}", e.getMessage()); - } catch (IOException e) { - log.error("decompress gzip", e); - } - - return null; - } - -} diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/HttpProcessor.java b/src/main/java/ru/serega6531/packmate/service/optimization/HttpProcessor.java new file mode 100644 index 0000000..a2bf638 --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/service/optimization/HttpProcessor.java @@ -0,0 +1,65 @@ +package ru.serega6531.packmate.service.optimization; + +import lombok.extern.slf4j.Slf4j; +import rawhttp.core.HttpMessage; +import rawhttp.core.RawHttp; +import rawhttp.core.RawHttpOptions; +import rawhttp.core.body.BodyReader; +import rawhttp.core.errors.InvalidHttpHeader; +import rawhttp.core.errors.InvalidHttpRequest; +import rawhttp.core.errors.InvalidHttpResponse; +import rawhttp.core.errors.InvalidMessageFrame; +import rawhttp.core.errors.UnknownEncodingException; +import ru.serega6531.packmate.model.Packet; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +@Slf4j +public class HttpProcessor { + + private static final RawHttp rawHttp = new RawHttp(RawHttpOptions.strict()); + + public void process(List packets) { + packets.stream() + .filter(p -> !p.isWebSocketParsed()) + .forEach(this::processPacket); + } + + private void processPacket(Packet packet) { + try { + ByteArrayInputStream contentStream = new ByteArrayInputStream(packet.getContent()); + HttpMessage message; + + if (packet.isIncoming()) { + message = rawHttp.parseRequest(contentStream).eagerly(); + } else { + message = rawHttp.parseResponse(contentStream).eagerly(); + } + + packet.setContent(getDecodedMessage(message)); + packet.setHasHttpBody(message.getBody().isPresent()); + } catch (IOException | InvalidHttpRequest | InvalidHttpResponse | InvalidHttpHeader | InvalidMessageFrame | + UnknownEncodingException e) { + log.warn("Could not parse http packet", e); + } + } + + private byte[] getDecodedMessage(HttpMessage message) throws IOException { + ByteArrayOutputStream os = new ByteArrayOutputStream(256); + + message.getStartLine().writeTo(os); + message.getHeaders().writeTo(os); + + Optional body = message.getBody(); + if (body.isPresent()) { + body.get().writeDecodedTo(os, 256); + } + + return os.toByteArray(); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/HttpUrldecodeProcessor.java b/src/main/java/ru/serega6531/packmate/service/optimization/HttpUrldecodeProcessor.java index 90810ae..0f99fd4 100644 --- a/src/main/java/ru/serega6531/packmate/service/optimization/HttpUrldecodeProcessor.java +++ b/src/main/java/ru/serega6531/packmate/service/optimization/HttpUrldecodeProcessor.java @@ -1,6 +1,5 @@ package ru.serega6531.packmate.service.optimization; -import lombok.AllArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import ru.serega6531.packmate.model.Packet; @@ -9,17 +8,14 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.List; -@AllArgsConstructor @Slf4j public class HttpUrldecodeProcessor { - private final List packets; - /** * Декодирование urlencode с http пакета до смены стороны или окончания стрима */ @SneakyThrows - public void urldecodeRequests() { + public void urldecodeRequests(List packets) { boolean httpStarted = false; for (Packet packet : packets) { diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/PacketsMerger.java b/src/main/java/ru/serega6531/packmate/service/optimization/PacketsMerger.java index 498492a..a66a17e 100644 --- a/src/main/java/ru/serega6531/packmate/service/optimization/PacketsMerger.java +++ b/src/main/java/ru/serega6531/packmate/service/optimization/PacketsMerger.java @@ -1,30 +1,25 @@ package ru.serega6531.packmate.service.optimization; -import lombok.AllArgsConstructor; import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.utils.PacketUtils; import java.util.List; -@AllArgsConstructor public class PacketsMerger { - private final List packets; - /** - * Сжать соседние пакеты в одном направлении в один. - * Выполняется после других оптимизаций чтобы правильно определять границы пакетов. + * Сжать соседние пакеты в одном направлении в один. Не склеивает WS и не-WS пакеты. */ - public void mergeAdjacentPackets() { + public void mergeAdjacentPackets(List packets) { int start = 0; int packetsInRow = 0; - boolean incoming = true; + Packet previous = null; for (int i = 0; i < packets.size(); i++) { Packet packet = packets.get(i); - if (packet.isIncoming() != incoming) { + if (previous == null || !shouldBeInSameBatch(packet, previous)) { if (packetsInRow > 1) { - compress(start, i); + compress(packets, start, i); i = start + 1; // продвигаем указатель на следующий после склеенного блок } @@ -34,36 +29,40 @@ public class PacketsMerger { packetsInRow++; } - incoming = packet.isIncoming(); + previous = packet; } if (packetsInRow > 1) { - compress(start, packets.size()); + compress(packets, start, packets.size()); } } /** * Сжать кусок со start по end в один пакет */ - private void compress(int start, int end) { + private void compress(List packets, int start, int end) { final List cut = packets.subList(start, end); final long timestamp = cut.get(0).getTimestamp(); - final boolean ungzipped = cut.stream().anyMatch(Packet::isUngzipped); + final boolean httpProcessed = cut.stream().anyMatch(Packet::isHttpProcessed); final boolean webSocketParsed = cut.stream().anyMatch(Packet::isWebSocketParsed); final boolean tlsDecrypted = cut.get(0).isTlsDecrypted(); final boolean incoming = cut.get(0).isIncoming(); - //noinspection OptionalGetWithoutIsPresent - final byte[] content = PacketUtils.mergePackets(cut).get(); + final byte[] content = PacketUtils.mergePackets(cut); packets.removeAll(cut); packets.add(start, Packet.builder() .incoming(incoming) .timestamp(timestamp) - .ungzipped(ungzipped) + .httpProcessed(httpProcessed) .webSocketParsed(webSocketParsed) .tlsDecrypted(tlsDecrypted) .content(content) .build()); } + private boolean shouldBeInSameBatch(Packet p1, Packet p2) { + return p1.isIncoming() == p2.isIncoming() && + p1.isWebSocketParsed() == p2.isWebSocketParsed(); + } + } diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/StreamOptimizer.java b/src/main/java/ru/serega6531/packmate/service/optimization/StreamOptimizer.java index 5d735dd..e86038d 100644 --- a/src/main/java/ru/serega6531/packmate/service/optimization/StreamOptimizer.java +++ b/src/main/java/ru/serega6531/packmate/service/optimization/StreamOptimizer.java @@ -1,7 +1,6 @@ package ru.serega6531.packmate.service.optimization; import lombok.AllArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import ru.serega6531.packmate.model.CtfService; import ru.serega6531.packmate.model.Packet; @@ -16,6 +15,11 @@ public class StreamOptimizer { private final CtfService service; private List packets; + private final PacketsMerger merger = new PacketsMerger(); + private final HttpUrldecodeProcessor urldecodeProcessor = new HttpUrldecodeProcessor(); + private final HttpProcessor httpProcessor = new HttpProcessor(); + + /** * Вызвать для выполнения оптимизаций на переданном списке пакетов. */ @@ -29,51 +33,42 @@ public class StreamOptimizer { } } - if (service.isProcessChunkedEncoding()) { - try { - processChunkedEncoding(); - } catch (Exception e) { - log.warn("Error optimizing stream (chunks)", e); - return packets; - } - } - - if (service.isUngzipHttp()) { - try { - unpackGzip(); - } catch (Exception e) { - log.warn("Error optimizing stream (gzip)", e); - return packets; - } - } - if (service.isParseWebSockets()) { try { parseWebSockets(); } catch (Exception e) { - log.warn("Error optimizing stream (websocketss)", e); + log.warn("Error optimizing stream (websockets)", e); return packets; } } if (service.isUrldecodeHttpRequests()) { try { - urldecodeRequests(); + urldecodeProcessor.urldecodeRequests(packets); } catch (Exception e) { log.warn("Error optimizing stream (urldecode)", e); return packets; } } - if (service.isMergeAdjacentPackets()) { + if (service.isMergeAdjacentPackets() || service.isHttp()) { try { - mergeAdjacentPackets(); + merger.mergeAdjacentPackets(packets); } catch (Exception e) { log.warn("Error optimizing stream (adjacent)", e); return packets; } } + if (service.isHttp()) { + try { + httpProcessor.process(packets); + } catch (Exception e) { + log.warn("Error optimizing stream (http)", e); + return packets; + } + } + return packets; } @@ -86,44 +81,6 @@ public class StreamOptimizer { } } - /** - * Сжать соседние пакеты в одном направлении в один. - * Выполняется после других оптимизаций чтобы правильно определять границы пакетов. - */ - private void mergeAdjacentPackets() { - final PacketsMerger merger = new PacketsMerger(packets); - merger.mergeAdjacentPackets(); - } - - /** - * Декодирование urlencode с http пакета до смены стороны или окончания стрима - */ - @SneakyThrows - private void urldecodeRequests() { - final HttpUrldecodeProcessor processor = new HttpUrldecodeProcessor(packets); - processor.urldecodeRequests(); - } - - /** - * Chunked transfer encoding - */ - private void processChunkedEncoding() { - HttpChunksProcessor processor = new HttpChunksProcessor(packets); - processor.processChunkedEncoding(); - } - - /** - * Попытаться распаковать GZIP из исходящих http пакетов.
- * GZIP поток начинается на найденном HTTP пакете с заголовком Content-Encoding: gzip - * (при этом заголовок HTTP может быть в другом пакете)
- * Поток заканчивается при обнаружении нового HTTP заголовка, - * при смене стороны передачи или при окончании всего стрима - */ - private void unpackGzip() { - final HttpGzipProcessor processor = new HttpGzipProcessor(packets); - processor.unpackGzip(); - } - private void parseWebSockets() { if (!packets.get(0).getContentString().contains("HTTP/")) { return; diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/TlsDecryptor.java b/src/main/java/ru/serega6531/packmate/service/optimization/TlsDecryptor.java index d743ec9..0bbe318 100644 --- a/src/main/java/ru/serega6531/packmate/service/optimization/TlsDecryptor.java +++ b/src/main/java/ru/serega6531/packmate/service/optimization/TlsDecryptor.java @@ -182,15 +182,12 @@ public class TlsDecryptor { decoded = clearDecodedData(decoded); - result.add(Packet.builder() - .content(decoded) - .incoming(packet.isIncoming()) - .timestamp(packet.getTimestamp()) - .ungzipped(false) - .webSocketParsed(false) - .tlsDecrypted(true) - .ttl(packet.getTtl()) - .build()); + result.add( + packet.toBuilder() + .content(decoded) + .tlsDecrypted(true) + .build() + ); } } } diff --git a/src/main/java/ru/serega6531/packmate/service/optimization/WebSocketsParser.java b/src/main/java/ru/serega6531/packmate/service/optimization/WebSocketsParser.java index c4c0759..f7ce4d7 100644 --- a/src/main/java/ru/serega6531/packmate/service/optimization/WebSocketsParser.java +++ b/src/main/java/ru/serega6531/packmate/service/optimization/WebSocketsParser.java @@ -120,14 +120,9 @@ public class WebSocketsParser { } private Packet mimicPacket(Packet packet, byte[] content, boolean ws) { - return Packet.builder() + return packet.toBuilder() .content(content) - .incoming(packet.isIncoming()) - .timestamp(packet.getTimestamp()) - .ttl(packet.getTtl()) - .ungzipped(packet.isUngzipped()) .webSocketParsed(ws) - .tlsDecrypted(packet.isTlsDecrypted()) .build(); } @@ -138,8 +133,7 @@ public class WebSocketsParser { for (List side : sides) { final Packet lastPacket = side.get(0); - //noinspection OptionalGetWithoutIsPresent - final byte[] wsContent = PacketUtils.mergePackets(side).get(); + final byte[] wsContent = PacketUtils.mergePackets(side); final ByteBuffer buffer = ByteBuffer.wrap(wsContent); List frames; @@ -153,15 +147,11 @@ public class WebSocketsParser { for (Framedata frame : frames) { if (frame instanceof DataFrame) { - parsedPackets.add(Packet.builder() - .content(frame.getPayloadData().array()) - .incoming(lastPacket.isIncoming()) - .timestamp(lastPacket.getTimestamp()) - .ttl(lastPacket.getTtl()) - .ungzipped(lastPacket.isUngzipped()) - .webSocketParsed(true) - .tlsDecrypted(lastPacket.isTlsDecrypted()) - .build() + parsedPackets.add( + lastPacket.toBuilder() + .content(frame.getPayloadData().array()) + .webSocketParsed(true) + .build() ); } } @@ -179,13 +169,10 @@ public class WebSocketsParser { } private String getHandshake(final List packets) { - final String handshake = PacketUtils.mergePackets(packets) - .map(String::new) - .orElse(null); + final String handshake = new String(PacketUtils.mergePackets(packets)); - if (handshake == null || - !handshake.toLowerCase().contains(WEBSOCKET_CONNECTION_HEADER) || - !handshake.toLowerCase().contains(WEBSOCKET_UPGRADE_HEADER)) { + if (!handshake.toLowerCase().contains(WEBSOCKET_CONNECTION_HEADER) + || !handshake.toLowerCase().contains(WEBSOCKET_UPGRADE_HEADER)) { return null; } diff --git a/src/main/java/ru/serega6531/packmate/utils/PacketUtils.java b/src/main/java/ru/serega6531/packmate/utils/PacketUtils.java index cd4a057..7c37420 100644 --- a/src/main/java/ru/serega6531/packmate/utils/PacketUtils.java +++ b/src/main/java/ru/serega6531/packmate/utils/PacketUtils.java @@ -1,20 +1,28 @@ package ru.serega6531.packmate.utils; import lombok.experimental.UtilityClass; -import org.apache.commons.lang3.ArrayUtils; import ru.serega6531.packmate.model.Packet; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.List; -import java.util.Optional; @UtilityClass public class PacketUtils { - public Optional mergePackets(List cut) { - return cut.stream() + public byte[] mergePackets(List cut) { + int size = cut.stream() .map(Packet::getContent) - .reduce(ArrayUtils::addAll); + .mapToInt(c -> c.length) + .sum(); + + ByteArrayOutputStream os = new ByteArrayOutputStream(size); + + cut.stream() + .map(Packet::getContent) + .forEach(os::writeBytes); + + return os.toByteArray(); } public List> sliceToSides(List packets) { diff --git a/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java b/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java index c7d01e0..b58de32 100644 --- a/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java +++ b/src/test/java/ru/serega6531/packmate/StreamOptimizerTest.java @@ -3,8 +3,7 @@ package ru.serega6531.packmate; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.Test; import ru.serega6531.packmate.model.Packet; -import ru.serega6531.packmate.service.optimization.HttpChunksProcessor; -import ru.serega6531.packmate.service.optimization.HttpGzipProcessor; +import ru.serega6531.packmate.service.optimization.HttpProcessor; import ru.serega6531.packmate.service.optimization.HttpUrldecodeProcessor; import ru.serega6531.packmate.service.optimization.PacketsMerger; @@ -27,18 +26,18 @@ class StreamOptimizerTest { List list = new ArrayList<>(); list.add(p); - new HttpGzipProcessor(list).unpackGzip(); + new HttpProcessor().process(list); final String processed = list.get(0).getContentString(); assertTrue(processed.contains("aaabbb")); } @Test void testUrldecodeRequests() { - Packet p = createPacket("GET /?q=%D0%B0+%D0%B1 HTTP/1.1\r\n\r\n".getBytes(), true); + Packet p = createPacket("GET /?q=%D0%B0+%D0%B1 HTTP/1.1\r\nHost: localhost:8080\r\n\r\n".getBytes(), true); List list = new ArrayList<>(); list.add(p); - new HttpUrldecodeProcessor(list).urldecodeRequests(); + new HttpUrldecodeProcessor().urldecodeRequests(list); final String processed = list.get(0).getContentString(); assertTrue(processed.contains("а б")); } @@ -60,7 +59,7 @@ class StreamOptimizerTest { list.add(p5); list.add(p6); - new PacketsMerger(list).mergeAdjacentPackets(); + new PacketsMerger().mergeAdjacentPackets(list); assertEquals(4, list.size()); assertEquals(2, list.get(1).getContent().length); @@ -74,7 +73,7 @@ class StreamOptimizerTest { "6\r\nChunk1\r\n6\r\nChunk2\r\n0\r\n\r\n"; List packets = new ArrayList<>(List.of(createPacket(content.getBytes(), false))); - new HttpChunksProcessor(packets).processChunkedEncoding(); + new HttpProcessor().process(packets); assertEquals(1, packets.size()); assertTrue(packets.get(0).getContentString().contains("Chunk1Chunk2"));