Возможность удалять старые данные

docker-compose.yml

@@ -11,6 +11,9 @@ services:
       PCAP_FILE: ${PACKMATE_PCAP_FILE}
       WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears}
       WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456}
+      OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false}
+      OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5}
+      OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240}
     env_file:
       - .env
     container_name: packmate-app
@@ -25,11 +28,12 @@ services:
       "--spring.datasource.username=$${DB_USER}", "--spring.datasource.password=$${DB_PASSWORD}",
       "--capture-mode=$${MODE}", "--pcap-file=$${PCAP_FILE}",
       "--interface-name=$${INTERFACE}", "--local-ip=$${LOCAL_IP}", "--account-login=$${WEB_LOGIN}",
+      "--old-streams-cleanup-enabled=$${OLD_STREAMS_CLEANUP_ENABLED}", "--cleanup-interval=$${OLD_STREAMS_CLEANUP_INTERVAL}",
+      "--old-streams-threshold=$${OLD_STREAMS_CLEANUP_THRESHOLD}",
       "--account-password=$${WEB_PASSWORD}", "--server.port=65000", "--server.address=0.0.0.0"
     ]
     depends_on:
       - db
-    restart: unless-stopped
   db:
     container_name: packmate-db
     build:
@@ -45,4 +49,3 @@ services:
     - "./data/postgres_data:/var/lib/postgresql/data"
     network_mode: "host"
     image: packmate-db:v1
-    restart: unless-stopped

docker/Dockerfile_db

@@ -1,4 +1,4 @@
-FROM postgres:13.3-alpine
+FROM postgres:14.1-alpine
 
 ARG POSTGRES_USER
 ARG POSTGRES_PASSWORD

src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java

@@ -0,0 +1,34 @@
+package ru.serega6531.packmate;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import ru.serega6531.packmate.service.StreamService;
+
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+
+@Component
+@Slf4j
+@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true")
+public class OldStreamsCleanupTask {
+
+    private final StreamService service;
+    private final int oldStreamsThreshold;
+
+    public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) {
+        this.service = service;
+        this.oldStreamsThreshold = oldStreamsThreshold;
+    }
+
+    @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M")
+    public void cleanup() {
+        ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES);
+        log.info("Cleaning up old streams (before {})", before);
+        long deleted = service.cleanupOldStreams(before);
+        log.info("Deleted {} rows", deleted);
+    }
+
+}

src/main/java/ru/serega6531/packmate/model/CtfService.java

@@ -1,18 +1,23 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.*;
+import org.hibernate.Hibernate;
 
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
+import java.util.Objects;
 
-@Data
+@Getter
+@Setter
+@ToString
+@RequiredArgsConstructor
 @Entity
 @Table(name = "service")
 public class CtfService {
 
     @Id
-    private int port;
+    private Integer port;
 
     private String name;
 
@@ -28,4 +33,16 @@ public class CtfService {
 
     private boolean parseWebSockets;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        CtfService that = (CtfService) o;
+        return port != null && Objects.equals(port, that.port);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/FoundPattern.java

@@ -1,9 +1,11 @@
 package ru.serega6531.packmate.model;
 
 import lombok.*;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
 
 import javax.persistence.*;
+import java.util.Objects;
 
 @Entity
 @GenericGenerator(
@@ -20,12 +22,11 @@ import javax.persistence.*;
 @Builder
 @Getter
 @ToString
-@EqualsAndHashCode(exclude = "packet")
 public class FoundPattern {
 
     @Id
     @GeneratedValue(generator = "found_pattern_generator")
-    private int id;
+    private Long id;
 
     @ManyToOne
     @JoinColumn(name = "packet_id", nullable = false)
@@ -38,6 +39,18 @@ public class FoundPattern {
 
     private int endPosition;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        FoundPattern that = (FoundPattern) o;
+        return id != null && Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }
 
 

src/main/java/ru/serega6531/packmate/model/Packet.java

@@ -1,12 +1,17 @@
 package ru.serega6531.packmate.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.*;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
 
 import javax.persistence.*;
+import java.util.Objects;
 import java.util.Set;
 
-@Data
+@Getter
+@Setter
+@RequiredArgsConstructor
 @Entity
 @GenericGenerator(
         name = "packet_generator",
@@ -17,11 +22,9 @@ import java.util.Set;
                 @org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
         }
 )
-@NoArgsConstructor
 @AllArgsConstructor
 @Builder
 @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") })
-@EqualsAndHashCode(exclude = "stream")
 public class Packet {
 
     @Id
@@ -34,9 +37,9 @@ public class Packet {
     @Transient
     private int ttl;
 
-    @ManyToOne
+    @Column(name = "stream_id")
-    @JoinColumn(name = "stream_id", nullable = false)
+    @JsonIgnore
-    private Stream stream;
+    private Long streamId;
 
     @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER)
     private Set<FoundPattern> matches;
@@ -62,4 +65,16 @@ public class Packet {
         return "Packet(id=" + id + ", content=" + getContentString() + ")";
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Packet packet = (Packet) o;
+        return id != null && Objects.equals(id, packet.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/Pattern.java

@@ -1,17 +1,21 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.*;
-import lombok.ToString;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
 import ru.serega6531.packmate.model.enums.PatternActionType;
 import ru.serega6531.packmate.model.enums.PatternDirectionType;
 import ru.serega6531.packmate.model.enums.PatternSearchType;
 
-import javax.persistence.*;
+import javax.persistence.Entity;
-import java.util.List;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+import java.util.Objects;
 
-@Data
+@Getter
-@ToString(exclude = "matchedStreams")
+@Setter
+@RequiredArgsConstructor
+@ToString
 @Entity
 @GenericGenerator(
         name = "pattern_generator",
@@ -26,7 +30,7 @@ public class Pattern {
 
     @Id
     @GeneratedValue(generator = "pattern_generator")
-    private int id;
+    private Integer id;
 
     private boolean enabled;
 
@@ -46,8 +50,16 @@ public class Pattern {
 
     private long searchStartTimestamp;
 
-    @ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY)
+    @Override
-    private List<Stream> matchedStreams;
+    public boolean equals(Object o) {
-
+        if (this == o) return true;
-
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Pattern pattern = (Pattern) o;
+        return id != null && Objects.equals(id, pattern.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/Stream.java

@@ -1,17 +1,20 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.*;
-import lombok.ToString;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
 import ru.serega6531.packmate.model.enums.Protocol;
 
 import javax.persistence.*;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
-@Data
+@Getter
-@ToString(exclude = "packets")
+@Setter
+@ToString
+@RequiredArgsConstructor
 @Entity
 @GenericGenerator(
         name = "stream_generator",
@@ -33,8 +36,10 @@ public class Stream {
 
     private Protocol protocol;
 
-    @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL)
+    @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true)
+    @JoinColumn(name = "stream_id")
     @OrderBy("id")
+    @ToString.Exclude
     private List<Packet> packets;
 
     private long startTimestamp;
@@ -42,6 +47,12 @@ public class Stream {
     private long endTimestamp;
 
     @ManyToMany
+    @JoinTable(
+            name = "stream_found_patterns",
+            joinColumns = @JoinColumn(name = "stream_id"),
+            inverseJoinColumns = @JoinColumn(name = "pattern_id")
+    )
+    @ToString.Exclude
     private Set<Pattern> foundPatterns = new HashSet<>();
 
     private boolean favorite;
@@ -52,4 +63,16 @@ public class Stream {
     @Column(columnDefinition = "char(3)")
     private String userAgentHash;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Stream stream = (Stream) o;
+        return id != null && Objects.equals(id, stream.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/repository/StreamRepository.java

@@ -12,4 +12,6 @@ public interface StreamRepository extends JpaRepository<Stream, Long>, JpaSpecif
     @Modifying
     void setFavorite(long id, boolean favorite);
 
+    long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold);
+
 }

src/main/java/ru/serega6531/packmate/service/StreamService.java

@@ -20,6 +20,7 @@ import ru.serega6531.packmate.repository.StreamRepository;
 import ru.serega6531.packmate.service.optimization.RsaKeysHolder;
 import ru.serega6531.packmate.service.optimization.StreamOptimizer;
 
+import java.time.ZonedDateTime;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
@@ -90,6 +91,15 @@ public class StreamService {
             }
         }
 
+        countingService.countStream(service.getPort(), packets.size());
+
+        List<Packet> optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
+
+        if (isStreamIgnored(optimizedPackets, service)) {
+            log.debug("New stream is ignored");
+            return false;
+        }
+
         Optional<Packet> firstIncoming = packets.stream()
                 .filter(Packet::isIncoming)
                 .findFirst();
@@ -101,27 +111,15 @@ public class StreamService {
         stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp());
         stream.setService(service.getPort());
 
-        countingService.countStream(service.getPort(), packets.size());
+        String userAgentHash = getUserAgentHash(optimizedPackets);
+        stream.setUserAgentHash(userAgentHash);
 
-        packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
+        Set<Pattern> foundPatterns = matchPatterns(optimizedPackets, service);
+        stream.setFoundPatterns(foundPatterns);
+        stream.setPackets(optimizedPackets);
 
-        if (isStreamIgnored(packets, service)) {
-            log.debug("New stream is ignored");
-            return false;
-        }
-
-        processUserAgent(packets, stream);
         Stream savedStream = save(stream);
 
-        for (Packet packet : packets) {
-            packet.setStream(savedStream);
-        }
-
-        Set<Pattern> foundPatterns = matchPatterns(packets, service);
-        savedStream.setFoundPatterns(foundPatterns);
-        savedStream.setPackets(packets);
-        savedStream = save(savedStream);
-
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream)));
         return true;
     }
@@ -143,7 +141,7 @@ public class StreamService {
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId()));
     }
 
-    private void processUserAgent(List<Packet> packets, Stream stream) {
+    private String getUserAgentHash(List<Packet> packets) {
         String ua = null;
         for (Packet packet : packets) {
             String content = packet.getContentString();
@@ -155,7 +153,9 @@ public class StreamService {
         }
 
         if (ua != null) {
-            stream.setUserAgentHash(calculateUserAgentHash(ua));
+            return calculateUserAgentHash(ua);
+        } else {
+            return null;
         }
     }
 
@@ -242,6 +242,14 @@ public class StreamService {
         return repository.findById(id);
     }
 
+    /**
+     * @return Number of deleted rows
+     */
+    @Transactional
+    public long cleanupOldStreams(ZonedDateTime before) {
+        return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000);
+    }
+
     @Transactional
     public void setFavorite(long id, boolean favorite) {
         repository.setFavorite(id, favorite);

src/main/resources/application.yml

@@ -15,13 +15,16 @@ spring:
 
 
 enable-capture: true
-capture-mode: LIVE  # LIVE, FILE
+capture-mode: LIVE  # LIVE, FILE, VIEW
 interface-name: enp0s31f6
 pcap-file: file.pcap
 local-ip: "192.168.0.125"
 account-login: BinaryBears
 account-password: 123456
-udp-stream-timeout: 20 # секунд
+udp-stream-timeout: 20 # seconds
-tcp-stream-timeout: 40 # секунд
+tcp-stream-timeout: 40 # seconds
-timeout-stream-check-interval: 10 # секунд
+timeout-stream-check-interval: 10 # seconds
+old-streams-cleanup-enabled: true
+old-streams-threshold: 240 # minutes
+cleanup-interval: 5 # minutes
 ignore-empty-packets: true