From 17e3d3cb48ea47cb3c0fa0b13c829cbfb35a572b Mon Sep 17 00:00:00 2001 From: serega6531 Date: Mon, 3 Feb 2020 02:25:45 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/repository/StreamRepository.java | 7 + .../packmate/service/StreamService.java | 140 +++++++++--------- 2 files changed, 75 insertions(+), 72 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index 53eed39..be306d0 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -2,7 +2,14 @@ package ru.serega6531.packmate.repository; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; import ru.serega6531.packmate.model.Stream; public interface StreamRepository extends JpaRepository, JpaSpecificationExecutor { + + @Query("UPDATE Stream SET favorite = :favorite WHERE id = :id") + @Modifying + void setFavorite(long id, boolean favorite); + } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index af134ee..761369f 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -103,6 +103,21 @@ public class StreamService { } } + optimizeStream(packets, service); + processUserAgent(packets, stream); + + Stream savedStream = save(stream); + + Set foundPatterns = getFoundPatterns(packets, savedStream); + savedStream.setFoundPatterns(foundPatterns); + savedStream.setPackets(packetService.saveAll(packets)); + savedStream = save(savedStream); + + subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, savedStream)); + return true; + } + + private void optimizeStream(List packets, CtfService service) { if (service.isUngzipHttp()) { unpackGzip(packets); } @@ -114,7 +129,9 @@ public class StreamService { if (service.isMergeAdjacentPackets()) { mergeAdjacentPackets(packets); } + } + private void processUserAgent(List packets, Stream stream) { String ua = null; for (Packet packet : packets) { String content = new String(packet.getContent()); @@ -128,12 +145,12 @@ public class StreamService { if (ua != null) { stream.setUserAgentHash(calculateUserAgentHash(ua)); } + } - Stream savedStream = save(stream); - + private Set getFoundPatterns(List packets, Stream savedStream) { Set foundPatterns = new HashSet<>(); - for (ru.serega6531.packmate.model.Packet packet : packets) { + for (Packet packet : packets) { packet.setStream(savedStream); final Set matches = patternService.findMatches(packet.getContent(), packet.isIncoming()); packet.setMatches(matches); @@ -143,12 +160,7 @@ public class StreamService { .collect(Collectors.toList())); } - savedStream.setFoundPatterns(foundPatterns); - savedStream.setPackets(packetService.saveAll(packets)); - savedStream = save(savedStream); - - subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, savedStream)); - return true; + return foundPatterns; } private void mergeAdjacentPackets(List packets) { @@ -161,21 +173,7 @@ public class StreamService { if (packet.isIncoming() != incoming) { if (packetsInRow > 1) { final List cut = packets.subList(start, i); - final long timestamp = cut.get(0).getTimestamp(); - final boolean ungzipped = cut.stream().anyMatch(Packet::isUngzipped); - //noinspection OptionalGetWithoutIsPresent - final byte[] content = cut.stream() - .map(Packet::getContent) - .reduce(ArrayUtils::addAll) - .get(); - - packets.removeAll(cut); - packets.add(start, Packet.builder() - .incoming(incoming) - .timestamp(timestamp) - .ungzipped(ungzipped) - .content(content) - .build()); + compress(packets, cut, incoming); i++; } @@ -190,24 +188,28 @@ public class StreamService { if (packetsInRow > 1) { final List cut = packets.subList(start, packets.size()); - final long timestamp = cut.get(0).getTimestamp(); - final boolean ungzipped = cut.stream().anyMatch(Packet::isUngzipped); - //noinspection OptionalGetWithoutIsPresent - final byte[] content = cut.stream() - .map(Packet::getContent) - .reduce(ArrayUtils::addAll) - .get(); - - packets.removeAll(cut); - packets.add(Packet.builder() - .incoming(incoming) - .timestamp(timestamp) - .ungzipped(ungzipped) - .content(content) - .build()); + compress(packets, cut, incoming); } } + private void compress(List packets, List cut, boolean incoming) { + final long timestamp = cut.get(0).getTimestamp(); + final boolean ungzipped = cut.stream().anyMatch(Packet::isUngzipped); + //noinspection OptionalGetWithoutIsPresent + final byte[] content = cut.stream() + .map(Packet::getContent) + .reduce(ArrayUtils::addAll) + .get(); + + packets.removeAll(cut); + packets.add(Packet.builder() + .incoming(incoming) + .timestamp(timestamp) + .ungzipped(ungzipped) + .content(content) + .build()); + } + @SneakyThrows private void urldecodeRequests(List packets) { boolean httpStarted = false; @@ -229,6 +231,9 @@ public class StreamService { } } + /** + * Попытаться распаковать gzip из исходящих http пакетов + */ private void unpackGzip(List packets) { boolean gzipStarted = false; int gzipStartPacket = 0; @@ -237,15 +242,9 @@ public class StreamService { for (int i = 0; i < packets.size(); i++) { Packet packet = packets.get(i); - if (packet.isIncoming() && gzipStarted) { + if (packet.isIncoming() && gzipStarted) { // поток gzip закончился gzipEndPacket = i - 1; - - List cut = packets.subList(gzipStartPacket, gzipEndPacket + 1); - - Packet decompressed = decompressGzipPackets(cut); - if (decompressed != null) { - packets.removeAll(cut); - packets.add(gzipStartPacket, decompressed); + if(extractGzip(packets, gzipStartPacket, gzipEndPacket)) { gzipStarted = false; i = gzipStartPacket + 1; } @@ -255,14 +254,9 @@ public class StreamService { int contentPos = content.indexOf("\r\n\r\n"); boolean http = content.startsWith("HTTP/"); - if (http && gzipStarted) { + if (http && gzipStarted) { // начался новый http пакет, заканчиваем старый gzip поток gzipEndPacket = i - 1; - List cut = packets.subList(gzipStartPacket, gzipEndPacket + 1); - - Packet decompressed = decompressGzipPackets(cut); - if (decompressed != null) { - packets.removeAll(cut); - packets.add(gzipStartPacket, decompressed); + if(extractGzip(packets, gzipStartPacket, gzipEndPacket)) { gzipStarted = false; i = gzipStartPacket + 1; } @@ -279,18 +273,27 @@ public class StreamService { } } - if (gzipStarted) { - gzipEndPacket = packets.size() - 1; - List cut = packets.subList(gzipStartPacket, gzipEndPacket + 1); - - Packet decompressed = decompressGzipPackets(cut); - if (decompressed != null) { - packets.removeAll(cut); - packets.add(gzipStartPacket, decompressed); - } + if (gzipStarted) { // стрим закончился gzip пакетом + extractGzip(packets, gzipStartPacket, packets.size() - 1); } } + /** + * @return получилось ли распаковать + */ + private boolean extractGzip(List packets, int gzipStartPacket, int gzipEndPacket) { + List cut = packets.subList(gzipStartPacket, gzipEndPacket + 1); + + Packet decompressed = decompressGzipPackets(cut); + if (decompressed != null) { + packets.removeAll(cut); + packets.add(gzipStartPacket, decompressed); + return true; + } + + return false; + } + private Packet decompressGzipPackets(List packets) { //noinspection OptionalGetWithoutIsPresent final byte[] content = packets.stream() @@ -350,15 +353,8 @@ public class StreamService { @SuppressWarnings("UnusedReturnValue") @Transactional - public Stream setFavorite(long id, boolean favorite) { - final Optional streamOptional = repository.findById(id); - if (streamOptional.isPresent()) { - final Stream stream = streamOptional.get(); - stream.setFavorite(favorite); - return repository.save(stream); - } - - return null; + public void setFavorite(long id, boolean favorite) { + repository.setFavorite(id, favorite); } public List findAll(Pagination pagination, Optional service, boolean onlyFavorites) {