From 560e0d444fedbc3034de2a2bfa725be0ba3def69 Mon Sep 17 00:00:00 2001 From: sshkurov Date: Sat, 29 Jan 2022 23:07:55 +0300 Subject: [PATCH] =?UTF-8?q?=D0=92=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D1=8C=20=D1=83=D0=B4=D0=B0=D0=BB=D1=8F=D1=82?= =?UTF-8?q?=D1=8C=20=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=B5=20=D0=B4=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D1=8B=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 9 ++-- docker/Dockerfile_db | 2 +- .../packmate/OldStreamsCleanupTask.java | 34 ++++++++++++++ .../serega6531/packmate/model/CtfService.java | 23 ++++++++-- .../packmate/model/FoundPattern.java | 17 ++++++- .../ru/serega6531/packmate/model/Packet.java | 27 ++++++++--- .../ru/serega6531/packmate/model/Pattern.java | 32 +++++++++---- .../ru/serega6531/packmate/model/Stream.java | 33 +++++++++++-- .../packmate/repository/StreamRepository.java | 2 + .../packmate/service/StreamService.java | 46 +++++++++++-------- src/main/resources/application.yml | 11 +++-- 11 files changed, 183 insertions(+), 53 deletions(-) create mode 100644 src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java diff --git a/docker-compose.yml b/docker-compose.yml index d0c4ab7..7ea93d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,9 @@ services: PCAP_FILE: ${PACKMATE_PCAP_FILE} WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears} WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456} + OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false} + OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5} + OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240} env_file: - .env container_name: packmate-app @@ -25,11 +28,12 @@ services: "--spring.datasource.username=$${DB_USER}", "--spring.datasource.password=$${DB_PASSWORD}", "--capture-mode=$${MODE}", "--pcap-file=$${PCAP_FILE}", "--interface-name=$${INTERFACE}", "--local-ip=$${LOCAL_IP}", "--account-login=$${WEB_LOGIN}", + "--old-streams-cleanup-enabled=$${OLD_STREAMS_CLEANUP_ENABLED}", "--cleanup-interval=$${OLD_STREAMS_CLEANUP_INTERVAL}", + "--old-streams-threshold=$${OLD_STREAMS_CLEANUP_THRESHOLD}", "--account-password=$${WEB_PASSWORD}", "--server.port=65000", "--server.address=0.0.0.0" ] depends_on: - db - restart: unless-stopped db: container_name: packmate-db build: @@ -44,5 +48,4 @@ services: volumes: - "./data/postgres_data:/var/lib/postgresql/data" network_mode: "host" - image: packmate-db:v1 - restart: unless-stopped + image: packmate-db:v1 \ No newline at end of file diff --git a/docker/Dockerfile_db b/docker/Dockerfile_db index 262e3ca..b6a246c 100644 --- a/docker/Dockerfile_db +++ b/docker/Dockerfile_db @@ -1,4 +1,4 @@ -FROM postgres:13.3-alpine +FROM postgres:14.1-alpine ARG POSTGRES_USER ARG POSTGRES_PASSWORD diff --git a/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java new file mode 100644 index 0000000..48fd059 --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java @@ -0,0 +1,34 @@ +package ru.serega6531.packmate; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.service.StreamService; + +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +@Component +@Slf4j +@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true") +public class OldStreamsCleanupTask { + + private final StreamService service; + private final int oldStreamsThreshold; + + public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) { + this.service = service; + this.oldStreamsThreshold = oldStreamsThreshold; + } + + @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M") + public void cleanup() { + ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES); + log.info("Cleaning up old streams (before {})", before); + long deleted = service.cleanupOldStreams(before); + log.info("Deleted {} rows", deleted); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index 7b0788e..a20a8f4 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -1,18 +1,23 @@ package ru.serega6531.packmate.model; -import lombok.Data; +import lombok.*; +import org.hibernate.Hibernate; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import java.util.Objects; -@Data +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @Table(name = "service") public class CtfService { @Id - private int port; + private Integer port; private String name; @@ -28,4 +33,16 @@ public class CtfService { private boolean parseWebSockets; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + CtfService that = (CtfService) o; + return port != null && Objects.equals(port, that.port); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } \ No newline at end of file diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index 4d54e33..1bb1e02 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -1,9 +1,11 @@ package ru.serega6531.packmate.model; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import javax.persistence.*; +import java.util.Objects; @Entity @GenericGenerator( @@ -20,12 +22,11 @@ import javax.persistence.*; @Builder @Getter @ToString -@EqualsAndHashCode(exclude = "packet") public class FoundPattern { @Id @GeneratedValue(generator = "found_pattern_generator") - private int id; + private Long id; @ManyToOne @JoinColumn(name = "packet_id", nullable = false) @@ -38,6 +39,18 @@ public class FoundPattern { private int endPosition; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + FoundPattern that = (FoundPattern) o; + return id != null && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 8d3c7ee..6bccc94 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -1,12 +1,17 @@ package ru.serega6531.packmate.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import javax.persistence.*; +import java.util.Objects; import java.util.Set; -@Data +@Getter +@Setter +@RequiredArgsConstructor @Entity @GenericGenerator( name = "packet_generator", @@ -17,11 +22,9 @@ import java.util.Set; @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") } ) -@NoArgsConstructor @AllArgsConstructor @Builder @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") }) -@EqualsAndHashCode(exclude = "stream") public class Packet { @Id @@ -34,9 +37,9 @@ public class Packet { @Transient private int ttl; - @ManyToOne - @JoinColumn(name = "stream_id", nullable = false) - private Stream stream; + @Column(name = "stream_id") + @JsonIgnore + private Long streamId; @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER) private Set matches; @@ -62,4 +65,16 @@ public class Packet { return "Packet(id=" + id + ", content=" + getContentString() + ")"; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Packet packet = (Packet) o; + return id != null && Objects.equals(id, packet.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Pattern.java b/src/main/java/ru/serega6531/packmate/model/Pattern.java index f108752..9017934 100644 --- a/src/main/java/ru/serega6531/packmate/model/Pattern.java +++ b/src/main/java/ru/serega6531/packmate/model/Pattern.java @@ -1,17 +1,21 @@ package ru.serega6531.packmate.model; -import lombok.Data; -import lombok.ToString; +import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; import ru.serega6531.packmate.model.enums.PatternSearchType; -import javax.persistence.*; -import java.util.List; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import java.util.Objects; -@Data -@ToString(exclude = "matchedStreams") +@Getter +@Setter +@RequiredArgsConstructor +@ToString @Entity @GenericGenerator( name = "pattern_generator", @@ -26,7 +30,7 @@ public class Pattern { @Id @GeneratedValue(generator = "pattern_generator") - private int id; + private Integer id; private boolean enabled; @@ -46,8 +50,16 @@ public class Pattern { private long searchStartTimestamp; - @ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY) - private List matchedStreams; - + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Pattern pattern = (Pattern) o; + return id != null && Objects.equals(id, pattern.id); + } + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index ad48d1b..ef8585d 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -1,17 +1,20 @@ package ru.serega6531.packmate.model; -import lombok.Data; -import lombok.ToString; +import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; -@Data -@ToString(exclude = "packets") +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @GenericGenerator( name = "stream_generator", @@ -33,8 +36,10 @@ public class Stream { private Protocol protocol; - @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL) + @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true) + @JoinColumn(name = "stream_id") @OrderBy("id") + @ToString.Exclude private List packets; private long startTimestamp; @@ -42,6 +47,12 @@ public class Stream { private long endTimestamp; @ManyToMany + @JoinTable( + name = "stream_found_patterns", + joinColumns = @JoinColumn(name = "stream_id"), + inverseJoinColumns = @JoinColumn(name = "pattern_id") + ) + @ToString.Exclude private Set foundPatterns = new HashSet<>(); private boolean favorite; @@ -52,4 +63,16 @@ public class Stream { @Column(columnDefinition = "char(3)") private String userAgentHash; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Stream stream = (Stream) o; + return id != null && Objects.equals(id, stream.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index be306d0..240ea60 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -12,4 +12,6 @@ public interface StreamRepository extends JpaRepository, JpaSpecif @Modifying void setFavorite(long id, boolean favorite); + long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); + } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 2e6a48f..cebba57 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -20,6 +20,7 @@ import ru.serega6531.packmate.repository.StreamRepository; import ru.serega6531.packmate.service.optimization.RsaKeysHolder; import ru.serega6531.packmate.service.optimization.StreamOptimizer; +import java.time.ZonedDateTime; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -90,6 +91,15 @@ public class StreamService { } } + countingService.countStream(service.getPort(), packets.size()); + + List optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + + if (isStreamIgnored(optimizedPackets, service)) { + log.debug("New stream is ignored"); + return false; + } + Optional firstIncoming = packets.stream() .filter(Packet::isIncoming) .findFirst(); @@ -101,27 +111,15 @@ public class StreamService { stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp()); stream.setService(service.getPort()); - countingService.countStream(service.getPort(), packets.size()); + String userAgentHash = getUserAgentHash(optimizedPackets); + stream.setUserAgentHash(userAgentHash); - packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + Set foundPatterns = matchPatterns(optimizedPackets, service); + stream.setFoundPatterns(foundPatterns); + stream.setPackets(optimizedPackets); - if (isStreamIgnored(packets, service)) { - log.debug("New stream is ignored"); - return false; - } - - processUserAgent(packets, stream); Stream savedStream = save(stream); - for (Packet packet : packets) { - packet.setStream(savedStream); - } - - Set foundPatterns = matchPatterns(packets, service); - savedStream.setFoundPatterns(foundPatterns); - savedStream.setPackets(packets); - savedStream = save(savedStream); - subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream))); return true; } @@ -143,7 +141,7 @@ public class StreamService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId())); } - private void processUserAgent(List packets, Stream stream) { + private String getUserAgentHash(List packets) { String ua = null; for (Packet packet : packets) { String content = packet.getContentString(); @@ -155,7 +153,9 @@ public class StreamService { } if (ua != null) { - stream.setUserAgentHash(calculateUserAgentHash(ua)); + return calculateUserAgentHash(ua); + } else { + return null; } } @@ -242,6 +242,14 @@ public class StreamService { return repository.findById(id); } + /** + * @return Number of deleted rows + */ + @Transactional + public long cleanupOldStreams(ZonedDateTime before) { + return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000); + } + @Transactional public void setFavorite(long id, boolean favorite) { repository.setFavorite(id, favorite); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 4abd2b1..37067a5 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -15,13 +15,16 @@ spring: enable-capture: true -capture-mode: LIVE # LIVE, FILE +capture-mode: LIVE # LIVE, FILE, VIEW interface-name: enp0s31f6 pcap-file: file.pcap local-ip: "192.168.0.125" account-login: BinaryBears account-password: 123456 -udp-stream-timeout: 20 # секунд -tcp-stream-timeout: 40 # секунд -timeout-stream-check-interval: 10 # секунд +udp-stream-timeout: 20 # seconds +tcp-stream-timeout: 40 # seconds +timeout-stream-check-interval: 10 # seconds +old-streams-cleanup-enabled: true +old-streams-threshold: 240 # minutes +cleanup-interval: 5 # minutes ignore-empty-packets: true \ No newline at end of file