From 560e0d444fedbc3034de2a2bfa725be0ba3def69 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sat, 29 Jan 2022 23:07:55 +0300 Subject: [PATCH 01/17] =?UTF-8?q?=D0=92=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6?= =?UTF-8?q?=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20=D1=83=D0=B4=D0=B0=D0=BB=D1=8F?= =?UTF-8?q?=D1=82=D1=8C=20=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=B5=20=D0=B4?= =?UTF-8?q?=D0=B0=D0=BD=D0=BD=D1=8B=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 9 ++-- docker/Dockerfile_db | 2 +- .../packmate/OldStreamsCleanupTask.java | 34 ++++++++++++++ .../serega6531/packmate/model/CtfService.java | 23 ++++++++-- .../packmate/model/FoundPattern.java | 17 ++++++- .../ru/serega6531/packmate/model/Packet.java | 27 ++++++++--- .../ru/serega6531/packmate/model/Pattern.java | 32 +++++++++---- .../ru/serega6531/packmate/model/Stream.java | 33 +++++++++++-- .../packmate/repository/StreamRepository.java | 2 + .../packmate/service/StreamService.java | 46 +++++++++++-------- src/main/resources/application.yml | 11 +++-- 11 files changed, 183 insertions(+), 53 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java diff --git a/docker-compose.yml b/docker-compose.yml index d0c4ab7..7ea93d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,9 @@ services: PCAP_FILE: ${PACKMATE_PCAP_FILE} WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears} WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456} + OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false} + OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5} + OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240} env_file: - .env container_name: packmate-app @@ -25,11 +28,12 @@ services: "--spring.datasource.username=$${DB_USER}", "--spring.datasource.password=$${DB_PASSWORD}", "--capture-mode=$${MODE}", "--pcap-file=$${PCAP_FILE}", "--interface-name=$${INTERFACE}", "--local-ip=$${LOCAL_IP}", "--account-login=$${WEB_LOGIN}", + "--old-streams-cleanup-enabled=$${OLD_STREAMS_CLEANUP_ENABLED}", "--cleanup-interval=$${OLD_STREAMS_CLEANUP_INTERVAL}", + "--old-streams-threshold=$${OLD_STREAMS_CLEANUP_THRESHOLD}", "--account-password=$${WEB_PASSWORD}", "--server.port=65000", "--server.address=0.0.0.0" ] depends_on: - db - restart: unless-stopped db: container_name: packmate-db build: @@ -44,5 +48,4 @@ services: volumes: - "./data/postgres_data:/var/lib/postgresql/data" network_mode: "host" - image: packmate-db:v1 - restart: unless-stopped + image: packmate-db:v1 \ No newline at end of file diff --git a/docker/Dockerfile_db b/docker/Dockerfile_db index 262e3ca..b6a246c 100644 --- a/docker/Dockerfile_db +++ b/docker/Dockerfile_db @@ -1,4 +1,4 @@ -FROM postgres:13.3-alpine +FROM postgres:14.1-alpine ARG POSTGRES_USER ARG POSTGRES_PASSWORD diff --git a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java new file mode 100644 index 0000000..48fd059 --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java @@ -0,0 +1,34 @@ +package ru.serega6531.packmate; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.service.StreamService; + +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +@Component +@Slf4j +@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true") +public class OldStreamsCleanupTask { + + private final StreamService service; + private final int oldStreamsThreshold; + + public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) { + this.service = service; + this.oldStreamsThreshold = oldStreamsThreshold; + } + + @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M") + public void cleanup() { + ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES); + log.info("Cleaning up old streams (before {})", before); + long deleted = service.cleanupOldStreams(before); + log.info("Deleted {} rows", deleted); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index 7b0788e..a20a8f4 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -1,18 +1,23 @@ package ru.serega6531.packmate.model; -import lombok.Data; +import lombok.*; +import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import java.util.Objects; -@Data +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @Table(name = "service") public class CtfService { @Id - private int port; + private Integer port; private String name; @@ -28,4 +33,16 @@ public class CtfService { private boolean parseWebSockets; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + CtfService that = (CtfService) o; + return port != null && Objects.equals(port, that.port); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } \ No newline at end of file diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index 4d54e33..1bb1e02 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -1,9 +1,11 @@ package ru.serega6531.packmate.model; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import javax.persistence.*; +import java.util.Objects; @Entity @GenericGenerator( @@ -20,12 +22,11 @@ import javax.persistence.*; @Builder @Getter @ToString -@EqualsAndHashCode(exclude = "packet") public class FoundPattern { @Id @GeneratedValue(generator = "found_pattern_generator") - private int id; + private Long id; @ManyToOne @JoinColumn(name = "packet_id", nullable = false) @@ -38,6 +39,18 @@ public class FoundPattern { private int endPosition; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + FoundPattern that = (FoundPattern) o; + return id != null && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 8d3c7ee..6bccc94 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -1,12 +1,17 @@ package ru.serega6531.packmate.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import javax.persistence.*; +import java.util.Objects; import java.util.Set; -@Data +@Getter +@Setter +@RequiredArgsConstructor @Entity @GenericGenerator( name = "packet_generator", @@ -17,11 +22,9 @@ import java.util.Set; @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") } ) -@NoArgsConstructor @AllArgsConstructor @Builder @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") }) -@EqualsAndHashCode(exclude = "stream") public class Packet { @Id @@ -34,9 +37,9 @@ public class Packet { @Transient private int ttl; - @ManyToOne - @JoinColumn(name = "stream_id", nullable = false) - private Stream stream; + @Column(name = "stream_id") + @JsonIgnore + private Long streamId; @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER) private Set matches; @@ -62,4 +65,16 @@ public class Packet { return "Packet(id=" + id + ", content=" + getContentString() + ")"; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Packet packet = (Packet) o; + return id != null && Objects.equals(id, packet.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Pattern.java b/src/main/java/ru/serega6531/packmate/model/Pattern.java index f108752..9017934 100644 --- a/src/main/java/ru/serega6531/packmate/model/Pattern.java +++ b/src/main/java/ru/serega6531/packmate/model/Pattern.java @@ -1,17 +1,21 @@ package ru.serega6531.packmate.model; -import lombok.Data; -import lombok.ToString; +import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; import ru.serega6531.packmate.model.enums.PatternSearchType; -import javax.persistence.*; -import java.util.List; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import java.util.Objects; -@Data -@ToString(exclude = "matchedStreams") +@Getter +@Setter +@RequiredArgsConstructor +@ToString @Entity @GenericGenerator( name = "pattern_generator", @@ -26,7 +30,7 @@ public class Pattern { @Id @GeneratedValue(generator = "pattern_generator") - private int id; + private Integer id; private boolean enabled; @@ -46,8 +50,16 @@ public class Pattern { private long searchStartTimestamp; - @ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY) - private List matchedStreams; - + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Pattern pattern = (Pattern) o; + return id != null && Objects.equals(id, pattern.id); + } + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index ad48d1b..ef8585d 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -1,17 +1,20 @@ package ru.serega6531.packmate.model; -import lombok.Data; -import lombok.ToString; +import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; -@Data -@ToString(exclude = "packets") +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @GenericGenerator( name = "stream_generator", @@ -33,8 +36,10 @@ public class Stream { private Protocol protocol; - @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL) + @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true) + @JoinColumn(name = "stream_id") @OrderBy("id") + @ToString.Exclude private List packets; private long startTimestamp; @@ -42,6 +47,12 @@ public class Stream { private long endTimestamp; @ManyToMany + @JoinTable( + name = "stream_found_patterns", + joinColumns = @JoinColumn(name = "stream_id"), + inverseJoinColumns = @JoinColumn(name = "pattern_id") + ) + @ToString.Exclude private Set foundPatterns = new HashSet<>(); private boolean favorite; @@ -52,4 +63,16 @@ public class Stream { @Column(columnDefinition = "char(3)") private String userAgentHash; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Stream stream = (Stream) o; + return id != null && Objects.equals(id, stream.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index be306d0..240ea60 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -12,4 +12,6 @@ public interface StreamRepository extends JpaRepository, JpaSpecif @Modifying void setFavorite(long id, boolean favorite); + long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); + } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 2e6a48f..cebba57 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -20,6 +20,7 @@ import ru.serega6531.packmate.repository.StreamRepository; import ru.serega6531.packmate.service.optimization.RsaKeysHolder; import ru.serega6531.packmate.service.optimization.StreamOptimizer; +import java.time.ZonedDateTime; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -90,6 +91,15 @@ public class StreamService { } } + countingService.countStream(service.getPort(), packets.size()); + + List optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + + if (isStreamIgnored(optimizedPackets, service)) { + log.debug("New stream is ignored"); + return false; + } + Optional firstIncoming = packets.stream() .filter(Packet::isIncoming) .findFirst(); @@ -101,27 +111,15 @@ public class StreamService { stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp()); stream.setService(service.getPort()); - countingService.countStream(service.getPort(), packets.size()); + String userAgentHash = getUserAgentHash(optimizedPackets); + stream.setUserAgentHash(userAgentHash); - packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + Set foundPatterns = matchPatterns(optimizedPackets, service); + stream.setFoundPatterns(foundPatterns); + stream.setPackets(optimizedPackets); - if (isStreamIgnored(packets, service)) { - log.debug("New stream is ignored"); - return false; - } - - processUserAgent(packets, stream); Stream savedStream = save(stream); - for (Packet packet : packets) { - packet.setStream(savedStream); - } - - Set foundPatterns = matchPatterns(packets, service); - savedStream.setFoundPatterns(foundPatterns); - savedStream.setPackets(packets); - savedStream = save(savedStream); - subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream))); return true; } @@ -143,7 +141,7 @@ public class StreamService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId())); } - private void processUserAgent(List packets, Stream stream) { + private String getUserAgentHash(List packets) { String ua = null; for (Packet packet : packets) { String content = packet.getContentString(); @@ -155,7 +153,9 @@ public class StreamService { } if (ua != null) { - stream.setUserAgentHash(calculateUserAgentHash(ua)); + return calculateUserAgentHash(ua); + } else { + return null; } } @@ -242,6 +242,14 @@ public class StreamService { return repository.findById(id); } + /** + * @return Number of deleted rows + */ + @Transactional + public long cleanupOldStreams(ZonedDateTime before) { + return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000); + } + @Transactional public void setFavorite(long id, boolean favorite) { repository.setFavorite(id, favorite); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 4abd2b1..37067a5 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -15,13 +15,16 @@ spring: enable-capture: true -capture-mode: LIVE # LIVE, FILE +capture-mode: LIVE # LIVE, FILE, VIEW interface-name: enp0s31f6 pcap-file: file.pcap local-ip: "192.168.0.125" account-login: BinaryBears account-password: 123456 -udp-stream-timeout: 20 # секунд -tcp-stream-timeout: 40 # секунд -timeout-stream-check-interval: 10 # секунд +udp-stream-timeout: 20 # seconds +tcp-stream-timeout: 40 # seconds +timeout-stream-check-interval: 10 # seconds +old-streams-cleanup-enabled: true +old-streams-threshold: 240 # minutes +cleanup-interval: 5 # minutes ignore-empty-packets: true \ No newline at end of file From 3d06b25d7a356d69a48b2bba76e9f915d25c27b2 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Mon, 31 Jan 2022 02:09:27 +0300 Subject: [PATCH 02/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=BE=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 4 ++-- gradle/wrapper/gradle-wrapper.properties | 2 +- .../ru/serega6531/packmate/OldStreamsCleanupTask.java | 2 +- .../java/ru/serega6531/packmate/model/FoundPattern.java | 2 +- src/main/java/ru/serega6531/packmate/model/Packet.java | 9 ++++----- src/main/java/ru/serega6531/packmate/model/Stream.java | 8 ++------ .../ru/serega6531/packmate/pcap/AbstractPcapWorker.java | 9 +++++---- .../ru/serega6531/packmate/service/StreamService.java | 4 ++++ 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7ea93d0..26db4b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,10 @@ services: DB_USER: ${PACKMATE_DB_USER:-packmate} DB_PASSWORD: ${PACKMATE_DB_PASSWORD:-K604YnL3G1hp2RDkCZNjGpxbyNpNHTRb} DB_NAME: ${PACKMATE_DB_NAME:-packmate} - INTERFACE: ${PACKMATE_INTERFACE} + INTERFACE: ${PACKMATE_INTERFACE:-} LOCAL_IP: ${PACKMATE_LOCAL_IP} MODE: ${PACKMATE_MODE:-LIVE} - PCAP_FILE: ${PACKMATE_PCAP_FILE} + PCAP_FILE: ${PACKMATE_PCAP_FILE:-} WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears} WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456} OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 4d9ca16..ec991f9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java index 48fd059..729fffd 100644 --- a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java +++ b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java @@ -26,7 +26,7 @@ public class OldStreamsCleanupTask { @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M") public void cleanup() { ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES); - log.info("Cleaning up old streams (before {})", before); + log.info("Cleaning up old non-favorite streams (before {})", before); long deleted = service.cleanupOldStreams(before); log.info("Deleted {} rows", deleted); } diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index 1bb1e02..8ec151a 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -28,7 +28,7 @@ public class FoundPattern { @GeneratedValue(generator = "found_pattern_generator") private Long id; - @ManyToOne + @ManyToOne(optional = false) @JoinColumn(name = "packet_id", nullable = false) @Setter private Packet packet; diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 6bccc94..6ed12be 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -1,6 +1,5 @@ package ru.serega6531.packmate.model; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; @@ -37,11 +36,11 @@ public class Packet { @Transient private int ttl; - @Column(name = "stream_id") - @JsonIgnore - private Long streamId; + @ManyToOne(optional = false) + @JoinColumn(name = "stream_id", nullable = false) + private Stream stream; - @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, orphanRemoval = true) private Set matches; private long timestamp; diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index ef8585d..9da546b 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -6,10 +6,7 @@ import org.hibernate.annotations.GenericGenerator; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.util.*; @Getter @Setter @@ -36,8 +33,7 @@ public class Stream { private Protocol protocol; - @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true) - @JoinColumn(name = "stream_id") + @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL, orphanRemoval = true) @OrderBy("id") @ToString.Exclude private List packets; diff --git a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java index 1f8eaa8..7056019 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java @@ -5,6 +5,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.pcap4j.core.BpfProgram; import org.pcap4j.core.PacketListener; import org.pcap4j.core.PcapHandle; @@ -46,8 +47,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { private final ListMultimap unfinishedUdpStreams = ArrayListMultimap.create(); // в следующих мапах в значениях находится srcIp соответствующего пакета - private final SetMultimap> fins = HashMultimap.create(); - private final SetMultimap> acks = HashMultimap.create(); + private final SetMultimap> fins = HashMultimap.create(); + private final SetMultimap> acks = HashMultimap.create(); protected AbstractPcapWorker(ServicesService servicesService, StreamService streamService, @@ -182,8 +183,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { * Udp не имеет фазы закрытия, поэтому закрывается только по таймауту */ private void checkTcpTermination(boolean ack, boolean fin, boolean rst, - ImmutablePair sourceIpAndPort, - ImmutablePair destIpAndPort, + Pair sourceIpAndPort, + Pair destIpAndPort, UnfinishedStream stream) { if (fin) { diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index cebba57..5256dde 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -118,6 +118,10 @@ public class StreamService { stream.setFoundPatterns(foundPatterns); stream.setPackets(optimizedPackets); + for (Packet packet : optimizedPackets) { + packet.setStream(stream); + } + Stream savedStream = save(stream); subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream))); From f5dcdf12aba0916a1ef827d334fdcf91d0715868 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Mon, 31 Jan 2022 21:52:11 +0300 Subject: [PATCH 03/17] =?UTF-8?q?=D0=9E=D0=BF=D1=82=D0=B8=D0=BC=D0=B8?= =?UTF-8?q?=D0=B7=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=BE=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=BB=D1=83=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=B0=D0=BA?= =?UTF-8?q?=D0=B5=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/controller/PacketController.java | 16 +++++----------- .../ru/serega6531/packmate/model/Packet.java | 2 +- .../packmate/repository/StreamRepository.java | 9 +++++++++ .../packmate/service/StreamService.java | 8 ++++++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/controller/PacketController.java b/src/main/java/ru/serega6531/packmate/controller/PacketController.java index 4d9db1b..0192996 100644 --- a/src/main/java/ru/serega6531/packmate/controller/PacketController.java +++ b/src/main/java/ru/serega6531/packmate/controller/PacketController.java @@ -5,13 +5,11 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import ru.serega6531.packmate.model.Stream; +import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.pojo.PacketDto; import ru.serega6531.packmate.service.StreamService; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; @RestController @@ -27,14 +25,10 @@ public class PacketController { @PostMapping("/{streamId}") public List getPacketsForStream(@PathVariable long streamId) { - final Optional stream = streamService.find(streamId); - if (stream.isPresent()) { - return stream.get().getPackets().stream() - .map(streamService::packetToDto) - .collect(Collectors.toList()); - } else { - return Collections.emptyList(); - } + List packets = streamService.getPackets(streamId); + return packets.stream() + .map(streamService::packetToDto) + .collect(Collectors.toList()); } } diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 6ed12be..2f168b6 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -36,7 +36,7 @@ public class Packet { @Transient private int ttl; - @ManyToOne(optional = false) + @ManyToOne(fetch = FetchType.LAZY, optional = false) @JoinColumn(name = "stream_id", nullable = false) private Stream stream; diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index 240ea60..4c7ff2a 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -6,6 +6,8 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import ru.serega6531.packmate.model.Stream; +import java.util.Optional; + public interface StreamRepository extends JpaRepository, JpaSpecificationExecutor { @Query("UPDATE Stream SET favorite = :favorite WHERE id = :id") @@ -14,4 +16,11 @@ public interface StreamRepository extends JpaRepository, JpaSpecif long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); + @Query("SELECT s FROM Stream s " + + "LEFT JOIN FETCH s.packets AS packets " + + "LEFT JOIN FETCH packets.matches " + + "WHERE s.id = :id" + ) + Optional getStreamWithPackets(long id); + } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 5256dde..6a254a9 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -7,10 +7,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; +import org.springframework.http.HttpStatus; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.server.ResponseStatusException; import ru.serega6531.packmate.model.*; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; @@ -242,8 +244,10 @@ public class StreamService { return saved; } - public Optional find(long id) { - return repository.findById(id); + public List getPackets(long streamId) { + return repository.getStreamWithPackets(streamId) + .map(Stream::getPackets) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); } /** From bfdbdba1509bcab39d4ab4c246d8e6370a11109b Mon Sep 17 00:00:00 2001 From: sshkurov Date: Wed, 2 Feb 2022 04:48:43 +0300 Subject: [PATCH 04/17] Batch inserts --- src/main/resources/application.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 37067a5..3272e0b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,6 +9,9 @@ spring: ddl-auto: update properties: hibernate: + jdbc: + batch_size: 20 + order_inserts: true temp: use_jdbc_metadata_defaults: false database-platform: org.hibernate.dialect.PostgreSQLDialect From 9838777b74bd1814e2b2786d23f5490e65780aad Mon Sep 17 00:00:00 2001 From: sshkurov Date: Wed, 2 Feb 2022 06:00:31 +0300 Subject: [PATCH 05/17] =?UTF-8?q?=D0=93=D0=B5=D0=BD=D0=B5=D1=80=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BD=D0=B5=D1=81=D0=BA?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20id=20=D0=BF=D0=B0=D0=BA=D0=B5?= =?UTF-8?q?=D1=82=D0=BE=D0=B2=20=D0=B7=D0=B0=20=D1=80=D0=B0=D0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ru/serega6531/packmate/model/FoundPattern.java | 9 +++++---- .../java/ru/serega6531/packmate/model/Packet.java | 8 +++++--- .../java/ru/serega6531/packmate/model/Pattern.java | 12 ++++++++---- .../java/ru/serega6531/packmate/model/Stream.java | 7 ++++--- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index 8ec151a..d7b365b 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -3,6 +3,7 @@ package ru.serega6531.packmate.model; import lombok.*; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import javax.persistence.*; import java.util.Objects; @@ -12,9 +13,9 @@ import java.util.Objects; name = "found_pattern_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "found_pattern_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "found_pattern_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "1") } ) @NoArgsConstructor @@ -25,7 +26,7 @@ import java.util.Objects; public class FoundPattern { @Id - @GeneratedValue(generator = "found_pattern_generator") + @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "found_pattern_generator") private Long id; @ManyToOne(optional = false) diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 2f168b6..91b6a1d 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -3,6 +3,7 @@ package ru.serega6531.packmate.model; import lombok.*; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import javax.persistence.*; import java.util.Objects; @@ -16,9 +17,10 @@ import java.util.Set; name = "packet_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "packet_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "packet_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "5"), + @Parameter(name = "optimizer", value = "hilo") } ) @AllArgsConstructor diff --git a/src/main/java/ru/serega6531/packmate/model/Pattern.java b/src/main/java/ru/serega6531/packmate/model/Pattern.java index 9017934..46bcba1 100644 --- a/src/main/java/ru/serega6531/packmate/model/Pattern.java +++ b/src/main/java/ru/serega6531/packmate/model/Pattern.java @@ -1,8 +1,12 @@ package ru.serega6531.packmate.model; -import lombok.*; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; import ru.serega6531.packmate.model.enums.PatternSearchType; @@ -21,9 +25,9 @@ import java.util.Objects; name = "pattern_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "pattern_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "pattern_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "1") } ) public class Pattern { diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index 9da546b..ad060dc 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -3,6 +3,7 @@ package ru.serega6531.packmate.model; import lombok.*; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; @@ -17,9 +18,9 @@ import java.util.*; name = "stream_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "stream_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "stream_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "1") } ) public class Stream { From a9b97d076b2972fc4aa267cbbd26eefd1a27dfaf Mon Sep 17 00:00:00 2001 From: sshkurov Date: Wed, 2 Feb 2022 07:54:55 +0300 Subject: [PATCH 06/17] =?UTF-8?q?=D0=93=D0=B5=D0=BD=D0=B5=D1=80=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BD=D0=B5=D1=81=D0=BA?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20id=20=D0=B4=D0=BB=D1=8F=20?= =?UTF-8?q?=D0=B2=D1=81=D0=B5=D1=85=20entity=20=D0=B7=D0=B0=20=D1=80=D0=B0?= =?UTF-8?q?=D0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/serega6531/packmate/model/FoundPattern.java | 3 ++- src/main/java/ru/serega6531/packmate/model/Packet.java | 2 +- src/main/java/ru/serega6531/packmate/model/Stream.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index d7b365b..fa8d477 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -15,7 +15,8 @@ import java.util.Objects; parameters = { @Parameter(name = "sequence_name", value = "found_pattern_seq"), @Parameter(name = "initial_value", value = "1"), - @Parameter(name = "increment_size", value = "1") + @Parameter(name = "increment_size", value = "2000"), + @Parameter(name = "optimizer", value = "hilo") } ) @NoArgsConstructor diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 91b6a1d..4095fcf 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -19,7 +19,7 @@ import java.util.Set; parameters = { @Parameter(name = "sequence_name", value = "packet_seq"), @Parameter(name = "initial_value", value = "1"), - @Parameter(name = "increment_size", value = "5"), + @Parameter(name = "increment_size", value = "20000"), @Parameter(name = "optimizer", value = "hilo") } ) diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index ad060dc..f12bb2f 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -20,7 +20,8 @@ import java.util.*; parameters = { @Parameter(name = "sequence_name", value = "stream_seq"), @Parameter(name = "initial_value", value = "1"), - @Parameter(name = "increment_size", value = "1") + @Parameter(name = "increment_size", value = "1000"), + @Parameter(name = "optimizer", value = "hilo") } ) public class Stream { From 61cabaaa3b6cff50995dc0f83a1cde2322089eef Mon Sep 17 00:00:00 2001 From: sshkurov Date: Thu, 3 Feb 2022 03:27:09 +0300 Subject: [PATCH 07/17] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20gradle=20=D0=B8=20=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8?= =?UTF-8?q?=D0=BE=D1=82=D0=B5=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 24 +++---- docker/Dockerfile_app | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../ApplicationConfiguration.java | 66 +------------------ .../configuration/SecurityConfiguration.java | 63 ++++++++++++++++++ .../configuration/WebSocketConfiguration.java | 27 ++++++++ 6 files changed, 106 insertions(+), 78 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java create mode 100644 src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java diff --git a/build.gradle b/build.gradle index 5cb5758..ed300cd 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'org.springframework.boot' version '2.4.1' + id 'org.springframework.boot' version '2.6.3' id 'java' } @@ -27,19 +27,19 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-security" implementation "org.springframework.boot:spring-boot-starter-websocket" implementation 'org.springframework.session:spring-session-core' - compile 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' - compile group: 'commons-io', name: 'commons-io', version: '2.7' - compile 'org.pcap4j:pcap4j-core:1.8.2' - compile 'org.pcap4j:pcap4j-packetfactory-static:1.8.2' - compile group: 'com.google.guava', name: 'guava', version: '30.1-jre' - compile group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.68' - compile group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.68' - compile group: 'org.modelmapper', name: 'modelmapper', version: '2.3.0' + implementation 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.11.0' + implementation 'org.pcap4j:pcap4j-core:1.8.2' + implementation 'org.pcap4j:pcap4j-packetfactory-static:1.8.2' + implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + implementation group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.70' + implementation group: 'org.modelmapper', name: 'modelmapper', version: '2.4.5' compileOnly 'org.projectlombok:lombok' runtimeOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'org.postgresql:postgresql' annotationProcessor 'org.projectlombok:lombok' - testCompile 'org.junit.jupiter:junit-jupiter:5.6.2' + testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2' } diff --git a/docker/Dockerfile_app b/docker/Dockerfile_app index 1d44a46..f7ebd65 100644 --- a/docker/Dockerfile_app +++ b/docker/Dockerfile_app @@ -12,5 +12,5 @@ RUN ./gradlew --no-daemon build -x test FROM adoptopenjdk/openjdk15:alpine-jre WORKDIR /app RUN apk --no-cache add libpcap -COPY --from=1 /tmp/compile/build/libs/packmate-*.jar app.jar +COPY --from=1 /tmp/compile/build/libs/packmate-*-SNAPSHOT.jar app.jar EXPOSE 65000:65000 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ec991f9..2e6e589 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java index 3dbf80a..f86522c 100644 --- a/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java +++ b/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java @@ -1,25 +1,14 @@ package ru.serega6531.packmate.configuration; -import lombok.extern.slf4j.Slf4j; import org.pcap4j.core.PcapNativeException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent; -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.web.socket.config.annotation.EnableWebSocket; -import org.springframework.web.socket.config.annotation.WebSocketConfigurer; -import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; -import ru.serega6531.packmate.WebSocketHandler; import ru.serega6531.packmate.model.enums.CaptureMode; import ru.serega6531.packmate.pcap.FilePcapWorker; import ru.serega6531.packmate.pcap.LivePcapWorker; @@ -32,25 +21,9 @@ import ru.serega6531.packmate.service.SubscriptionService; import java.net.UnknownHostException; @Configuration -@EnableWebSecurity @EnableScheduling -@EnableWebSocket @EnableAsync -@Slf4j -public class ApplicationConfiguration extends WebSecurityConfigurerAdapter implements WebSocketConfigurer { - - @Value("${account-login}") - private String login; - - @Value("${account-password}") - private String password; - - private final WebSocketHandler webSocketHandler; - - @Autowired - public ApplicationConfiguration(WebSocketHandler webSocketHandler) { - this.webSocketHandler = webSocketHandler; - } +public class ApplicationConfiguration { @Bean(destroyMethod = "stop") @Autowired @@ -64,48 +37,13 @@ public class ApplicationConfiguration extends WebSecurityConfigurerAdapter imple return switch (captureMode) { case LIVE -> new LivePcapWorker(servicesService, streamService, localIpString, interfaceName); case FILE -> new FilePcapWorker(servicesService, streamService, subscriptionService, localIpString, filename); - default -> new NoOpPcapWorker(); + case VIEW -> new NoOpPcapWorker(); }; } - @Autowired - public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception { - auth.inMemoryAuthentication() - .withUser(login) - .password(passwordEncoder().encode(password)) - .authorities("ROLE_USER"); - } - - @Override - protected void configure(HttpSecurity http) throws Exception { - http.csrf() - .disable() - .authorizeRequests() - .antMatchers("/site.webmanifest") - .permitAll() - .anyRequest().authenticated() - .and() - .httpBasic() - .and() - .headers() - .frameOptions() - .sameOrigin(); - } - @Bean public PasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); } - @EventListener - public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) { - log.info("Login failed for user {}, password {}", - e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials()); - } - - @Override - public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(webSocketHandler, "/api/ws") - .withSockJS(); - } } diff --git a/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java new file mode 100644 index 0000000..0d8a2bf --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java @@ -0,0 +1,63 @@ +package ru.serega6531.packmate.configuration; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.EventListener; +import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +@EnableWebSecurity +@Slf4j +public class SecurityConfiguration extends WebSecurityConfigurerAdapter { + + @Value("${account-login}") + private String login; + + @Value("${account-password}") + private String password; + + private final PasswordEncoder passwordEncoder; + + @Autowired + public SecurityConfiguration(PasswordEncoder passwordEncoder) { + this.passwordEncoder = passwordEncoder; + } + + @Autowired + public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception { + auth.inMemoryAuthentication() + .withUser(login) + .password(passwordEncoder.encode(password)) + .authorities("ROLE_USER"); + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + http.csrf() + .disable() + .authorizeRequests() + .antMatchers("/site.webmanifest") + .permitAll() + .anyRequest().authenticated() + .and() + .httpBasic() + .and() + .headers() + .frameOptions() + .sameOrigin(); + } + + @EventListener + public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) { + log.info("Login failed for user {}, password {}", + e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials()); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java new file mode 100644 index 0000000..1b494e1 --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java @@ -0,0 +1,27 @@ +package ru.serega6531.packmate.configuration; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +import ru.serega6531.packmate.WebSocketHandler; + +@EnableWebSocket +@Configuration +public class WebSocketConfiguration implements WebSocketConfigurer { + + private final WebSocketHandler webSocketHandler; + + @Autowired + public WebSocketConfiguration(WebSocketHandler webSocketHandler) { + this.webSocketHandler = webSocketHandler; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(webSocketHandler, "/api/ws") + .withSockJS(); + } + +} From cf90c1e7586f47d16a13b94adb282180185e0f2f Mon Sep 17 00:00:00 2001 From: sshkurov Date: Thu, 3 Feb 2022 03:27:58 +0300 Subject: [PATCH 08/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D1=8B=20=D1=82=D0=B5=D1=81=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/PatternMatcherTest.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java b/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java index f550c0f..f4e450f 100644 --- a/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java +++ b/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java @@ -1,7 +1,7 @@ package ru.serega6531.packmate; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.springframework.util.Assert; import ru.serega6531.packmate.model.FoundPattern; import ru.serega6531.packmate.model.Pattern; import ru.serega6531.packmate.model.enums.PatternSearchType; @@ -27,13 +27,14 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("[a-f]{3}"); pattern.setSearchType(PatternSearchType.REGEX); final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); } @Test @@ -50,13 +51,14 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("bbb"); pattern.setSearchType(PatternSearchType.SUBSTRING); final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); } @Test @@ -73,13 +75,27 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("AAaa"); pattern.setSearchType(PatternSearchType.SUBBYTES); final PatternMatcher matcher = new PatternMatcher(content, List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); + } + + private void assertMatchesAreCorrect(Set correctMatches, Set foundMatches) { + Assertions.assertEquals(correctMatches.size(), foundMatches.size()); + + Assertions.assertTrue(correctMatches.stream().allMatch(correct -> + foundMatches.stream().anyMatch(found -> matchesEqual(correct, found)) + )); + } + + private boolean matchesEqual(FoundPattern one, FoundPattern two) { + return one.getStartPosition() == two.getStartPosition() && + one.getEndPosition() == two.getEndPosition(); } } From d2555845550e51dba63bced64ade2ac17e31b124 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Fri, 4 Feb 2022 08:10:24 +0300 Subject: [PATCH 09/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D1=8B=20=D0=B4=D1=83=D0=B1=D0=BB=D0=B8=D1=80?= =?UTF-8?q?=D1=83=D1=8E=D1=89=D0=B8=D0=B5=D1=81=D1=8F=20=D0=BF=D0=B0=D0=BA?= =?UTF-8?q?=D0=B5=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ru/serega6531/packmate/repository/StreamRepository.java | 2 +- src/main/java/ru/serega6531/packmate/service/StreamService.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index 4c7ff2a..1eecf35 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -17,7 +17,7 @@ public interface StreamRepository extends JpaRepository, JpaSpecif long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); @Query("SELECT s FROM Stream s " + - "LEFT JOIN FETCH s.packets AS packets " + + "JOIN FETCH s.packets AS packets " + "LEFT JOIN FETCH packets.matches " + "WHERE s.id = :id" ) diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 6a254a9..4563f4d 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -247,6 +247,7 @@ public class StreamService { public List getPackets(long streamId) { return repository.getStreamWithPackets(streamId) .map(Stream::getPackets) + .map(packets -> packets.stream().distinct().collect(Collectors.toList())) .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); } From 70449e435ac4fab6caf1fd4231132da567e0686d Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sat, 5 Feb 2022 04:03:41 +0300 Subject: [PATCH 10/17] =?UTF-8?q?=D0=9E=D0=BF=D1=82=D0=B8=D0=BC=D0=B8?= =?UTF-8?q?=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D1=8B=20=D1=81=20=D0=91=D0=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 1 + .../packmate/model/FoundPattern.java | 1 + .../ru/serega6531/packmate/model/Stream.java | 11 ++++++++-- .../packmate/model/pojo/Pagination.java | 5 ++++- .../packmate/repository/StreamRepository.java | 20 +++++++++---------- .../packmate/service/StreamService.java | 20 +++++++++---------- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index ed300cd..01658eb 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' implementation group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.70' implementation group: 'org.modelmapper', name: 'modelmapper', version: '2.4.5' + compileOnly 'org.jetbrains:annotations:22.0.0' compileOnly 'org.projectlombok:lombok' runtimeOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'org.postgresql:postgresql' diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index fa8d477..27b4500 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -24,6 +24,7 @@ import java.util.Objects; @Builder @Getter @ToString +@Table(indexes = { @Index(name = "found_pattern_packet_id_index", columnList = "packet_id DESC") }) public class FoundPattern { @Id diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index f12bb2f..7ec6edf 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -1,13 +1,19 @@ package ru.serega6531.packmate.model; -import lombok.*; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import org.hibernate.annotations.Parameter; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; @Getter @Setter @@ -24,6 +30,7 @@ import java.util.*; @Parameter(name = "optimizer", value = "hilo") } ) +@Table(indexes = {@Index(name = "stream_id_desc_index", columnList = "id DESC")}) public class Stream { @Id diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java index 0b2baec..30fa38d 100644 --- a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java +++ b/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java @@ -1,6 +1,7 @@ package ru.serega6531.packmate.model.pojo; import lombok.Data; +import org.jetbrains.annotations.Nullable; import org.springframework.data.domain.Sort; import ru.serega6531.packmate.model.Pattern; @@ -9,12 +10,14 @@ public class Pagination { private Sort.Direction direction; - private long startingFrom; + @Nullable + private Long startingFrom; private int pageSize; private boolean favorites; // только для стримов, определяет, искать только избранные стримы или все + @Nullable private Pattern pattern; // только для стримов, если не null, ищем стримы с этим паттерном } diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index 1eecf35..c971322 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -1,12 +1,11 @@ package ru.serega6531.packmate.repository; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.*; +import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.Stream; -import java.util.Optional; +import javax.persistence.QueryHint; +import java.util.List; public interface StreamRepository extends JpaRepository, JpaSpecificationExecutor { @@ -16,11 +15,12 @@ public interface StreamRepository extends JpaRepository, JpaSpecif long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); - @Query("SELECT s FROM Stream s " + - "JOIN FETCH s.packets AS packets " + - "LEFT JOIN FETCH packets.matches " + - "WHERE s.id = :id" + @Query("SELECT DISTINCT p FROM Packet p " + + "LEFT JOIN FETCH p.matches " + + "WHERE p.stream.id = :streamId " + + "ORDER BY p.id" ) - Optional getStreamWithPackets(long id); + @QueryHints(@QueryHint(name = org.hibernate.jpa.QueryHints.HINT_PASS_DISTINCT_THROUGH, value = "false")) + List getPackets(long streamId); } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 4563f4d..752c03d 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -7,12 +7,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; -import org.springframework.http.HttpStatus; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.server.ResponseStatusException; import ru.serega6531.packmate.model.*; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; @@ -245,10 +243,7 @@ public class StreamService { } public List getPackets(long streamId) { - return repository.getStreamWithPackets(streamId) - .map(Stream::getPackets) - .map(packets -> packets.stream().distinct().collect(Collectors.toList())) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + return repository.getPackets(streamId); } /** @@ -267,11 +262,14 @@ public class StreamService { public List findAll(Pagination pagination, Optional service, boolean onlyFavorites) { PageRequest page = PageRequest.of(0, pagination.getPageSize(), pagination.getDirection(), "id"); - Specification spec; - if (pagination.getDirection() == Sort.Direction.ASC) { - spec = streamIdGreaterThan(pagination.getStartingFrom()); - } else { - spec = streamIdLessThan(pagination.getStartingFrom()); + Specification spec = Specification.where(null); + + if (pagination.getStartingFrom() != null) { + if (pagination.getDirection() == Sort.Direction.ASC) { + spec = spec.and(streamIdGreaterThan(pagination.getStartingFrom())); + } else { + spec = spec.and(streamIdLessThan(pagination.getStartingFrom())); + } } if (service.isPresent()) { From 6bfa09d960d7abbf7ade50919241816e56ff3840 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 08:48:29 +0300 Subject: [PATCH 11/17] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D1=8B=20not=20null=20=D0=B0=D0=BD=D0=BD=D0=BE?= =?UTF-8?q?=D1=82=D0=B0=D1=86=D0=B8=D0=B8=20=D0=BD=D0=B0=20=D1=81=D1=83?= =?UTF-8?q?=D1=89=D0=BD=D0=BE=D1=81=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ru/serega6531/packmate/model/CtfService.java | 2 ++ .../java/ru/serega6531/packmate/model/Packet.java | 1 + .../java/ru/serega6531/packmate/model/Pattern.java | 13 ++++++++++--- .../java/ru/serega6531/packmate/model/Stream.java | 6 ++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index a20a8f4..737f2fb 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -3,6 +3,7 @@ package ru.serega6531.packmate.model; import lombok.*; import org.hibernate.Hibernate; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; @@ -19,6 +20,7 @@ public class CtfService { @Id private Integer port; + @Column(nullable = false) private String name; private boolean decryptTls; diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 4095fcf..c35af64 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -55,6 +55,7 @@ public class Packet { private boolean tlsDecrypted; + @Column(nullable = false) private byte[] content; @Transient diff --git a/src/main/java/ru/serega6531/packmate/model/Pattern.java b/src/main/java/ru/serega6531/packmate/model/Pattern.java index 46bcba1..6d5acad 100644 --- a/src/main/java/ru/serega6531/packmate/model/Pattern.java +++ b/src/main/java/ru/serega6531/packmate/model/Pattern.java @@ -11,9 +11,7 @@ import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; import ru.serega6531.packmate.model.enums.PatternSearchType; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; +import javax.persistence.*; import java.util.Objects; @Getter @@ -38,16 +36,25 @@ public class Pattern { private boolean enabled; + @Column(nullable = false) private String name; + @Column(nullable = false) private String value; + @Column(nullable = false) private String color; // для вставки в css + @Enumerated + @Column(nullable = false) private PatternSearchType searchType; + @Enumerated + @Column(nullable = false) private PatternDirectionType directionType; + @Enumerated + @Column(nullable = false) private PatternActionType actionType; private Integer serviceId; diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index 7ec6edf..685fc3f 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -37,9 +37,11 @@ public class Stream { @GeneratedValue(generator = "stream_generator") private Long id; - @Column(name = "service_id") + @Column(name = "service_id", nullable = false) private int service; + @Enumerated + @Column(nullable = false) private Protocol protocol; @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL, orphanRemoval = true) @@ -62,7 +64,7 @@ public class Stream { private boolean favorite; - @Column(columnDefinition = "smallint") + @Column(nullable = false, columnDefinition = "smallint") private int ttl; @Column(columnDefinition = "char(3)") From 56ed2f5878f738aaeab61a79edca39e72a0c61a5 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 08:52:09 +0300 Subject: [PATCH 12/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=BE=20=D0=B7=D0=B0=D0=BF=D0=BE=D0=BB=D0=BD?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20pcap=20=D1=84=D0=B8=D0=BB=D1=8C?= =?UTF-8?q?=D1=82=D1=80=D0=B0=20=D0=BF=D1=80=D0=B8=20=D0=B7=D0=B0=D0=BF?= =?UTF-8?q?=D1=83=D1=81=D0=BA=D0=B5,=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D0=B2=D1=8B=D0=B2=D0=BE=D0=B4=20=D1=81?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=BE=D1=8F=D0=BD=D0=B8=D1=8F=20=D0=BE=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B4=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/PackmateApplication.java | 20 ---------- .../packmate/pcap/AbstractPcapWorker.java | 1 - .../packmate/pcap/FilePcapWorker.java | 5 +++ .../packmate/pcap/LivePcapWorker.java | 10 ++++- .../packmate/pcap/NoOpPcapWorker.java | 5 +++ .../serega6531/packmate/pcap/PcapWorker.java | 1 + .../packmate/service/PatternService.java | 4 ++ .../packmate/service/PcapService.java | 4 ++ .../packmate/service/ServicesService.java | 17 +++++--- .../tasks/ExecutorStateLoggerTask.java | 23 +++++++++++ .../{ => tasks}/OldStreamsCleanupTask.java | 2 +- .../packmate/tasks/StartupListener.java | 40 +++++++++++++++++++ .../{ => tasks}/TimeoutStreamsSaver.java | 2 +- 13 files changed, 103 insertions(+), 31 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java rename src/main/java/ru/serega6531/packmate/{ => tasks}/OldStreamsCleanupTask.java (97%) create mode 100644 src/main/java/ru/serega6531/packmate/tasks/StartupListener.java rename src/main/java/ru/serega6531/packmate/{ => tasks}/TimeoutStreamsSaver.java (97%) diff --git a/src/main/java/ru/serega6531/packmate/PackmateApplication.java b/src/main/java/ru/serega6531/packmate/PackmateApplication.java index ddad525..a1ae6f3 100644 --- a/src/main/java/ru/serega6531/packmate/PackmateApplication.java +++ b/src/main/java/ru/serega6531/packmate/PackmateApplication.java @@ -1,33 +1,13 @@ package ru.serega6531.packmate; -import org.pcap4j.core.PcapNativeException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import ru.serega6531.packmate.model.enums.CaptureMode; -import ru.serega6531.packmate.service.PcapService; @SpringBootApplication public class PackmateApplication { - @Value("${enable-capture}") - private boolean enableCapture; - - @Value("${capture-mode}") - private CaptureMode captureMode; - public static void main(String[] args) { SpringApplication.run(PackmateApplication.class, args); } - @EventListener(ApplicationReadyEvent.class) - public void afterStartup(ApplicationReadyEvent event) throws PcapNativeException { - if (enableCapture && captureMode == CaptureMode.LIVE) { - final PcapService pcapService = event.getApplicationContext().getBean(PcapService.class); - pcapService.start(); - } - } - } diff --git a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java index 7056019..b7d619a 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java @@ -257,7 +257,6 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { } @Override - @SneakyThrows public void setFilter(String filter) { this.filter = filter; applyFilter(); diff --git a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java index 730d552..0f9e178 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java @@ -79,4 +79,9 @@ public class FilePcapWorker extends AbstractPcapWorker { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.PCAP_STOPPED, null)); } + + @Override + public String getExecutorState() { + return "inline"; + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java index 0802e91..f225851 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java @@ -10,7 +10,9 @@ import ru.serega6531.packmate.service.ServicesService; import ru.serega6531.packmate.service.StreamService; import java.net.UnknownHostException; -import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; @Slf4j public class LivePcapWorker extends AbstractPcapWorker { @@ -30,7 +32,7 @@ public class LivePcapWorker extends AbstractPcapWorker { BasicThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern("pcap-processor").build(); - processorExecutorService = Executors.newSingleThreadExecutor(factory); + processorExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), factory); } public void start() throws PcapNativeException { @@ -63,4 +65,8 @@ public class LivePcapWorker extends AbstractPcapWorker { log.info("Intercept stopped"); } + @Override + public String getExecutorState() { + return processorExecutorService.toString(); + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java index 0cad155..f26c48b 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java @@ -24,4 +24,9 @@ public class NoOpPcapWorker implements PcapWorker { @Override public void setFilter(String filter) { } + + @Override + public String getExecutorState() { + return "none"; + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java index 91f8fde..eb81c78 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java @@ -20,4 +20,5 @@ public interface PcapWorker { void setFilter(String filter); + String getExecutorState(); } diff --git a/src/main/java/ru/serega6531/packmate/service/PatternService.java b/src/main/java/ru/serega6531/packmate/service/PatternService.java index 6ff6ab6..3af8832 100644 --- a/src/main/java/ru/serega6531/packmate/service/PatternService.java +++ b/src/main/java/ru/serega6531/packmate/service/PatternService.java @@ -15,6 +15,7 @@ import ru.serega6531.packmate.model.pojo.PatternDto; import ru.serega6531.packmate.model.pojo.SubscriptionMessage; import ru.serega6531.packmate.repository.PatternRepository; +import javax.annotation.PostConstruct; import java.time.Instant; import java.util.*; import java.util.concurrent.TimeUnit; @@ -40,7 +41,10 @@ public class PatternService { this.streamService = streamService; this.subscriptionService = subscriptionService; this.modelMapper = modelMapper; + } + @PostConstruct + public void init() { repository.findAll().forEach(p -> patterns.put(p.getId(), p)); log.info("Loaded {} patterns", patterns.size()); } diff --git a/src/main/java/ru/serega6531/packmate/service/PcapService.java b/src/main/java/ru/serega6531/packmate/service/PcapService.java index 528fc77..c8f665f 100644 --- a/src/main/java/ru/serega6531/packmate/service/PcapService.java +++ b/src/main/java/ru/serega6531/packmate/service/PcapService.java @@ -54,4 +54,8 @@ public class PcapService { worker.setFilter(filter); } + public String getExecutorState() { + return worker.getExecutorState(); + } + } diff --git a/src/main/java/ru/serega6531/packmate/service/ServicesService.java b/src/main/java/ru/serega6531/packmate/service/ServicesService.java index 014e950..3d8454b 100644 --- a/src/main/java/ru/serega6531/packmate/service/ServicesService.java +++ b/src/main/java/ru/serega6531/packmate/service/ServicesService.java @@ -12,12 +12,10 @@ import ru.serega6531.packmate.model.pojo.ServiceDto; import ru.serega6531.packmate.model.pojo.SubscriptionMessage; import ru.serega6531.packmate.repository.ServiceRepository; +import javax.annotation.PostConstruct; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; @Service @Slf4j @@ -43,7 +41,10 @@ public class ServicesService { this.pcapService = pcapService; this.modelMapper = modelMapper; this.localIp = InetAddress.getByName(localIpString); + } + @PostConstruct + public void init() { repository.findAll().forEach(s -> services.put(s.getPort(), s)); log.info("Loaded {} services", services.size()); } @@ -78,7 +79,7 @@ public class ServicesService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.DELETE_SERVICE, port)); - pcapService.updateFilter(findAll()); + updateFilter(); } public CtfService save(CtfService service) { @@ -89,11 +90,15 @@ public class ServicesService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.SAVE_SERVICE, toDto(saved))); - pcapService.updateFilter(findAll()); + updateFilter(); return saved; } + public void updateFilter() { + pcapService.updateFilter(findAll()); + } + public ServiceDto toDto(CtfService service) { return modelMapper.map(service, ServiceDto.class); } diff --git a/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java b/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java new file mode 100644 index 0000000..d022c0d --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java @@ -0,0 +1,23 @@ +package ru.serega6531.packmate.tasks; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.service.PcapService; + +@Component +@Slf4j +public class ExecutorStateLoggerTask { + + private final PcapService service; + + public ExecutorStateLoggerTask(PcapService service) { + this.service = service; + } + + @Scheduled(fixedDelayString = "PT1M", initialDelayString = "PT1M") + public void cleanup() { + log.info("Executor state: {}", service.getExecutorState()); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java b/src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java similarity index 97% rename from src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java rename to src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java index 729fffd..7635a3b 100644 --- a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java +++ b/src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java @@ -1,4 +1,4 @@ -package ru.serega6531.packmate; +package ru.serega6531.packmate.tasks; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java b/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java new file mode 100644 index 0000000..16e923c --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java @@ -0,0 +1,40 @@ +package ru.serega6531.packmate.tasks; + +import org.pcap4j.core.PcapNativeException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.model.enums.CaptureMode; +import ru.serega6531.packmate.service.PcapService; +import ru.serega6531.packmate.service.ServicesService; + +@Component +public class StartupListener { + + @Value("${enable-capture}") + private boolean enableCapture; + + @Value("${capture-mode}") + private CaptureMode captureMode; + + private final PcapService pcapService; + private final ServicesService servicesService; + + public StartupListener(PcapService pcapService, ServicesService servicesService) { + this.pcapService = pcapService; + this.servicesService = servicesService; + } + + @EventListener(ApplicationReadyEvent.class) + public void afterStartup() throws PcapNativeException { + if (enableCapture) { + servicesService.updateFilter(); + + if (captureMode == CaptureMode.LIVE) { + pcapService.start(); + } + } + } + +} diff --git a/src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java b/src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java similarity index 97% rename from src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java rename to src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java index d251fb7..466074b 100644 --- a/src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java +++ b/src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java @@ -1,4 +1,4 @@ -package ru.serega6531.packmate; +package ru.serega6531.packmate.tasks; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; From b48e23fdc531d9dee7deb8f9ce5379bc5e2cc016 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 10:50:32 +0300 Subject: [PATCH 13/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BF=D0=B0=D0=B3=D0=B8=D0=BD=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8F=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/controller/PacketController.java | 10 ++++---- .../packmate/controller/StreamController.java | 6 ++--- .../packmate/model/pojo/PacketPagination.java | 14 +++++++++++ .../packmate/model/pojo/Pagination.java | 23 ------------------- .../packmate/model/pojo/StreamPagination.java | 20 ++++++++++++++++ .../packmate/repository/StreamRepository.java | 4 +++- .../packmate/service/StreamService.java | 17 +++++++------- 7 files changed, 52 insertions(+), 42 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java delete mode 100644 src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java create mode 100644 src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java diff --git a/src/main/java/ru/serega6531/packmate/controller/PacketController.java b/src/main/java/ru/serega6531/packmate/controller/PacketController.java index 0192996..e70a38f 100644 --- a/src/main/java/ru/serega6531/packmate/controller/PacketController.java +++ b/src/main/java/ru/serega6531/packmate/controller/PacketController.java @@ -1,12 +1,10 @@ package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.pojo.PacketDto; +import ru.serega6531.packmate.model.pojo.PacketPagination; import ru.serega6531.packmate.service.StreamService; import java.util.List; @@ -24,8 +22,8 @@ public class PacketController { } @PostMapping("/{streamId}") - public List getPacketsForStream(@PathVariable long streamId) { - List packets = streamService.getPackets(streamId); + public List getPacketsForStream(@PathVariable long streamId, @RequestBody PacketPagination pagination) { + List packets = streamService.getPackets(streamId, pagination.getStartingFrom(), pagination.getPageSize()); return packets.stream() .map(streamService::packetToDto) .collect(Collectors.toList()); diff --git a/src/main/java/ru/serega6531/packmate/controller/StreamController.java b/src/main/java/ru/serega6531/packmate/controller/StreamController.java index e8697b8..e0a77fa 100644 --- a/src/main/java/ru/serega6531/packmate/controller/StreamController.java +++ b/src/main/java/ru/serega6531/packmate/controller/StreamController.java @@ -2,7 +2,7 @@ package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import ru.serega6531.packmate.model.pojo.Pagination; +import ru.serega6531.packmate.model.pojo.StreamPagination; import ru.serega6531.packmate.model.pojo.StreamDto; import ru.serega6531.packmate.service.StreamService; @@ -22,14 +22,14 @@ public class StreamController { } @PostMapping("/all") - public List getStreams(@RequestBody Pagination pagination) { + public List getStreams(@RequestBody StreamPagination pagination) { return service.findAll(pagination, Optional.empty(), pagination.isFavorites()).stream() .map(service::streamToDto) .collect(Collectors.toList()); } @PostMapping("/{port}") - public List getStreams(@PathVariable int port, @RequestBody Pagination pagination) { + public List getStreams(@PathVariable int port, @RequestBody StreamPagination pagination) { return service.findAll(pagination, Optional.of(port), pagination.isFavorites()).stream() .map(service::streamToDto) .collect(Collectors.toList()); diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java new file mode 100644 index 0000000..4f97c7e --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java @@ -0,0 +1,14 @@ +package ru.serega6531.packmate.model.pojo; + +import lombok.Data; +import org.jetbrains.annotations.Nullable; + +@Data +public class PacketPagination { + + @Nullable + private Long startingFrom; + + private int pageSize; + +} diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java deleted file mode 100644 index 30fa38d..0000000 --- a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java +++ /dev/null @@ -1,23 +0,0 @@ -package ru.serega6531.packmate.model.pojo; - -import lombok.Data; -import org.jetbrains.annotations.Nullable; -import org.springframework.data.domain.Sort; -import ru.serega6531.packmate.model.Pattern; - -@Data -public class Pagination { - - private Sort.Direction direction; - - @Nullable - private Long startingFrom; - - private int pageSize; - - private boolean favorites; // только для стримов, определяет, искать только избранные стримы или все - - @Nullable - private Pattern pattern; // только для стримов, если не null, ищем стримы с этим паттерном - -} diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java new file mode 100644 index 0000000..d78606f --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java @@ -0,0 +1,20 @@ +package ru.serega6531.packmate.model.pojo; + +import lombok.Data; +import org.jetbrains.annotations.Nullable; +import ru.serega6531.packmate.model.Pattern; + +@Data +public class StreamPagination { + + @Nullable + private Long startingFrom; + + private int pageSize; + + private boolean favorites; // определяет, искать только избранные стримы или все + + @Nullable + private Pattern pattern; // если не null, ищем стримы с этим паттерном + +} diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index c971322..1af9dce 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -1,5 +1,6 @@ package ru.serega6531.packmate.repository; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.*; import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.Stream; @@ -18,9 +19,10 @@ public interface StreamRepository extends JpaRepository, JpaSpecif @Query("SELECT DISTINCT p FROM Packet p " + "LEFT JOIN FETCH p.matches " + "WHERE p.stream.id = :streamId " + + "AND (:startingFrom IS NULL OR p.id > :startingFrom) " + "ORDER BY p.id" ) @QueryHints(@QueryHint(name = org.hibernate.jpa.QueryHints.HINT_PASS_DISTINCT_THROUGH, value = "false")) - List getPackets(long streamId); + List getPackets(long streamId, Long startingFrom, Pageable pageable); } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 752c03d..845ecc5 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -1,10 +1,12 @@ package ru.serega6531.packmate.service; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; import org.modelmapper.ModelMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; import org.springframework.scheduling.annotation.Async; @@ -242,8 +244,9 @@ public class StreamService { return saved; } - public List getPackets(long streamId) { - return repository.getPackets(streamId); + public List getPackets(long streamId, @Nullable Long startingFrom, int pageSize) { +// long safeStartingFrom = startingFrom != null ? startingFrom : 0; + return repository.getPackets(streamId, startingFrom, Pageable.ofSize(pageSize)); } /** @@ -259,17 +262,13 @@ public class StreamService { repository.setFavorite(id, favorite); } - public List findAll(Pagination pagination, Optional service, boolean onlyFavorites) { - PageRequest page = PageRequest.of(0, pagination.getPageSize(), pagination.getDirection(), "id"); + public List findAll(StreamPagination pagination, Optional service, boolean onlyFavorites) { + PageRequest page = PageRequest.of(0, pagination.getPageSize(), Sort.Direction.DESC, "id"); Specification spec = Specification.where(null); if (pagination.getStartingFrom() != null) { - if (pagination.getDirection() == Sort.Direction.ASC) { - spec = spec.and(streamIdGreaterThan(pagination.getStartingFrom())); - } else { - spec = spec.and(streamIdLessThan(pagination.getStartingFrom())); - } + spec = spec.and(streamIdLessThan(pagination.getStartingFrom())); } if (service.isPresent()) { From 3b5662548f5cf89b9a4a0da65e519c6e6fd0bf8f Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 11:07:20 +0300 Subject: [PATCH 14/17] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D1=84=D1=80=D0=BE=D0=BD=D1=82=D0=B5=D0=BD=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend | 2 +- .../packmate/configuration/WebSocketConfiguration.java | 2 +- .../serega6531/packmate/{ => controller}/WebSocketHandler.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/main/java/ru/serega6531/packmate/{ => controller}/WebSocketHandler.java (95%) diff --git a/frontend b/frontend index 9ff0cef..b5d3074 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 9ff0cefd57a40d515d1b729b014a6a887b5bce04 +Subproject commit b5d3074f7694276cca1c98a251935a88c0bdbd96 diff --git a/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java index 1b494e1..1e2f55f 100644 --- a/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java +++ b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java @@ -5,7 +5,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; -import ru.serega6531.packmate.WebSocketHandler; +import ru.serega6531.packmate.controller.WebSocketHandler; @EnableWebSocket @Configuration diff --git a/src/main/java/ru/serega6531/packmate/WebSocketHandler.java b/src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java similarity index 95% rename from src/main/java/ru/serega6531/packmate/WebSocketHandler.java rename to src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java index c0075a0..2c77404 100644 --- a/src/main/java/ru/serega6531/packmate/WebSocketHandler.java +++ b/src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java @@ -1,4 +1,4 @@ -package ru.serega6531.packmate; +package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; From ae19082418223c0035b4a662dfb9c441af9ca5d6 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 11:53:15 +0300 Subject: [PATCH 15/17] =?UTF-8?q?=D0=9F=D0=BE=D1=87=D0=B8=D1=89=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=BA=D0=BE=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/serega6531/packmate/model/Stream.java | 2 +- .../java/ru/serega6531/packmate/service/StreamService.java | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index 685fc3f..4c2eb19 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -53,7 +53,7 @@ public class Stream { private long endTimestamp; - @ManyToMany + @ManyToMany(fetch = FetchType.EAGER) @JoinTable( name = "stream_found_patterns", joinColumns = @JoinColumn(name = "stream_id"), diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 845ecc5..d43101f 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -307,10 +307,6 @@ public class StreamService { return (root, query, cb) -> cb.equal(root.get("favorite"), true); } - private Specification streamIdGreaterThan(long id) { - return (root, query, cb) -> cb.greaterThan(root.get("id"), id); - } - private Specification streamIdLessThan(long id) { return (root, query, cb) -> cb.lessThan(root.get("id"), id); } From 23a308d583eba757bf9bcae134170b2428a32307 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sun, 6 Feb 2022 13:21:50 +0300 Subject: [PATCH 16/17] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D1=8B=20=D1=84=D0=B8=D0=BB=D1=8C=D1=82=D1=80?= =?UTF-8?q?=D1=8B=20=D0=B4=D0=BB=D1=8F=20=D0=BF=D1=83=D1=81=D1=82=D0=BE?= =?UTF-8?q?=D0=B3=D0=BE=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=B0=20=D1=81?= =?UTF-8?q?=D0=B5=D1=80=D0=B2=D0=B8=D1=81=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../packmate/service/PcapService.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/serega6531/packmate/service/PcapService.java b/src/main/java/ru/serega6531/packmate/service/PcapService.java index c8f665f..b633055 100644 --- a/src/main/java/ru/serega6531/packmate/service/PcapService.java +++ b/src/main/java/ru/serega6531/packmate/service/PcapService.java @@ -41,13 +41,19 @@ public class PcapService { } public void updateFilter(Collection services) { - final String ports = services.stream() - .map(CtfService::getPort) - .map(p -> "port " + p) - .collect(Collectors.joining(" or ")); + String filter; - final String format = "(tcp or udp) and (%s)"; - String filter = String.format(format, ports); + if (services.isEmpty()) { + filter = "tcp or udp"; + } else { + final String ports = services.stream() + .map(CtfService::getPort) + .map(p -> "port " + p) + .collect(Collectors.joining(" or ")); + + final String format = "(tcp or udp) and (%s)"; + filter = String.format(format, ports); + } log.debug("New filter: " + filter); From 86f43ea3d14ac7e3f1383e71b0ed7c4fc45c9e59 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Tue, 8 Feb 2022 00:21:12 +0300 Subject: [PATCH 17/17] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 +++++++++++ README_EN.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/README.md b/README.md index 2b3a478..2b5fc40 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,17 @@ PACKMATE_PCAP_FILE=dump.pcap PACKMATE_MODE=VIEW ``` +При захвате живого трафика рекомендуется включать удаление старых стримов, иначе ближе к концу +соревнования анализатор будет медленнее работать. +```dotenv +PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true +# Интервал удаления старых стримов (в минутах). +# Лучше ставить маленькое число, чтобы стримы удалялись маленькими кусками, и это не нагружало систему +PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1 +# Насколько старым стрим должен быть для удаления (в минутах от текущего времени) +PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240 +``` + Чтобы использовать расшифровку TLS, нужно положить соответствующий приватный ключ, который использовался для генерации сертификата, в папку `rsa_keys`. diff --git a/README_EN.md b/README_EN.md index 098ce0c..abec29e 100644 --- a/README_EN.md +++ b/README_EN.md @@ -76,6 +76,17 @@ PACKMATE_MODE=FILE PACKMATE_PCAP_FILE=dump.pcap ``` +When capturing live traffic it's better to turn on old streams removal. Otherwise, after some time Packmate +will start working slower. +```dotenv +PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true +# Old streams removal interval (in minutes). +# It's better to use small numbers so the streams are removed in small chunks and don't overload the server. +PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1 +# How old the stream must be to be removed (in minutes before current time) +PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240 +``` + To decrypt TLS, put the private key used to generate a certificate into the `rsa_keys` folder. ### Launch