Merge branch 'cleanup-old-data' into 'master'

Cleanup old streams, optimize db stuff

See merge request packmate/Packmate!12

README.md

@@ -82,6 +82,17 @@ PACKMATE_PCAP_FILE=dump.pcap
 PACKMATE_MODE=VIEW
 ```
 
+При захвате живого трафика рекомендуется включать удаление старых стримов, иначе ближе к концу
+соревнования анализатор будет медленнее работать.
+```dotenv
+PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true
+# Интервал удаления старых стримов (в минутах).
+# Лучше ставить маленькое число, чтобы стримы удалялись маленькими кусками, и это не нагружало систему
+PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1
+# Насколько старым стрим должен быть для удаления (в минутах от текущего времени)
+PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240
+```
+
 Чтобы использовать расшифровку TLS, нужно положить соответствующий приватный ключ, который
 использовался для генерации сертификата, в папку `rsa_keys`.
 

README_EN.md

@@ -76,6 +76,17 @@ PACKMATE_MODE=FILE
 PACKMATE_PCAP_FILE=dump.pcap
 ```
 
+When capturing live traffic it's better to turn on old streams removal. Otherwise, after some time Packmate 
+will start working slower.
+```dotenv
+PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true
+# Old streams removal interval (in minutes).
+# It's better to use small numbers so the streams are removed in small chunks and don't overload the server.
+PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1
+# How old the stream must be to be removed (in minutes before current time)
+PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240
+```
+
 To decrypt TLS, put the private key used to generate a certificate into the `rsa_keys` folder.
 
 ### Launch

build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id 'org.springframework.boot' version '2.4.1'
+    id 'org.springframework.boot' version '2.6.3'
     id 'java'
 }
 
@@ -27,19 +27,20 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-security"
     implementation "org.springframework.boot:spring-boot-starter-websocket"
     implementation 'org.springframework.session:spring-session-core'
-    compile 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0'
+    implementation 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0'
-    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
+    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
-    compile group: 'commons-io', name: 'commons-io', version: '2.7'
+    implementation group: 'commons-io', name: 'commons-io', version: '2.11.0'
-    compile 'org.pcap4j:pcap4j-core:1.8.2'
+    implementation 'org.pcap4j:pcap4j-core:1.8.2'
-    compile 'org.pcap4j:pcap4j-packetfactory-static:1.8.2'
+    implementation 'org.pcap4j:pcap4j-packetfactory-static:1.8.2'
-    compile group: 'com.google.guava', name: 'guava', version: '30.1-jre'
+    implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
-    compile group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1'
+    implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1'
-    compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.68'
+    implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69'
-    compile group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.68'
+    implementation group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.70'
-    compile group: 'org.modelmapper', name: 'modelmapper', version: '2.3.0'
+    implementation group: 'org.modelmapper', name: 'modelmapper', version: '2.4.5'
+    compileOnly 'org.jetbrains:annotations:22.0.0'
     compileOnly 'org.projectlombok:lombok'
     runtimeOnly 'org.springframework.boot:spring-boot-devtools'
     runtimeOnly 'org.postgresql:postgresql'
     annotationProcessor 'org.projectlombok:lombok'
-    testCompile 'org.junit.jupiter:junit-jupiter:5.6.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2'
 }

docker-compose.yml

@@ -5,12 +5,15 @@ services:
       DB_USER: ${PACKMATE_DB_USER:-packmate}
       DB_PASSWORD: ${PACKMATE_DB_PASSWORD:-K604YnL3G1hp2RDkCZNjGpxbyNpNHTRb}
       DB_NAME: ${PACKMATE_DB_NAME:-packmate}
-      INTERFACE: ${PACKMATE_INTERFACE}
+      INTERFACE: ${PACKMATE_INTERFACE:-}
       LOCAL_IP: ${PACKMATE_LOCAL_IP}
       MODE: ${PACKMATE_MODE:-LIVE}
-      PCAP_FILE: ${PACKMATE_PCAP_FILE}
+      PCAP_FILE: ${PACKMATE_PCAP_FILE:-}
       WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears}
       WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456}
+      OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false}
+      OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5}
+      OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240}
     env_file:
       - .env
     container_name: packmate-app
@@ -25,11 +28,12 @@ services:
       "--spring.datasource.username=$${DB_USER}", "--spring.datasource.password=$${DB_PASSWORD}",
       "--capture-mode=$${MODE}", "--pcap-file=$${PCAP_FILE}",
       "--interface-name=$${INTERFACE}", "--local-ip=$${LOCAL_IP}", "--account-login=$${WEB_LOGIN}",
+      "--old-streams-cleanup-enabled=$${OLD_STREAMS_CLEANUP_ENABLED}", "--cleanup-interval=$${OLD_STREAMS_CLEANUP_INTERVAL}",
+      "--old-streams-threshold=$${OLD_STREAMS_CLEANUP_THRESHOLD}",
       "--account-password=$${WEB_PASSWORD}", "--server.port=65000", "--server.address=0.0.0.0"
     ]
     depends_on:
       - db
-    restart: unless-stopped
   db:
     container_name: packmate-db
     build:
@@ -44,5 +48,4 @@ services:
     volumes:
     - "./data/postgres_data:/var/lib/postgresql/data"
     network_mode: "host"
-    image: packmate-db:v1
+    image: packmate-db:v1
-    restart: unless-stopped

docker/Dockerfile_app

@@ -12,5 +12,5 @@ RUN ./gradlew --no-daemon build -x test
 FROM adoptopenjdk/openjdk15:alpine-jre
 WORKDIR /app
 RUN apk --no-cache add libpcap
-COPY --from=1 /tmp/compile/build/libs/packmate-*.jar app.jar
+COPY --from=1 /tmp/compile/build/libs/packmate-*-SNAPSHOT.jar app.jar
 EXPOSE 65000:65000

docker/Dockerfile_db

@@ -1,4 +1,4 @@
-FROM postgres:13.3-alpine
+FROM postgres:14.1-alpine
 
 ARG POSTGRES_USER
 ARG POSTGRES_PASSWORD

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/ru/serega6531/packmate/PackmateApplication.java

@@ -1,33 +1,13 @@
 package ru.serega6531.packmate;
 
-import org.pcap4j.core.PcapNativeException;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.event.EventListener;
-import ru.serega6531.packmate.model.enums.CaptureMode;
-import ru.serega6531.packmate.service.PcapService;
 
 @SpringBootApplication
 public class PackmateApplication {
 
-    @Value("${enable-capture}")
-    private boolean enableCapture;
-
-    @Value("${capture-mode}")
-    private CaptureMode captureMode;
-
     public static void main(String[] args) {
         SpringApplication.run(PackmateApplication.class, args);
     }
 
-    @EventListener(ApplicationReadyEvent.class)
-    public void afterStartup(ApplicationReadyEvent event) throws PcapNativeException {
-        if (enableCapture && captureMode == CaptureMode.LIVE) {
-            final PcapService pcapService = event.getApplicationContext().getBean(PcapService.class);
-            pcapService.start();
-        }
-    }
-
 }

src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java

@@ -1,25 +1,14 @@
 package ru.serega6531.packmate.configuration;
 
-import lombok.extern.slf4j.Slf4j;
 import org.pcap4j.core.PcapNativeException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.context.event.EventListener;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent;
-import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
-import org.springframework.security.config.annotation.web.builders.HttpSecurity;
-import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
-import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 import org.springframework.security.crypto.password.PasswordEncoder;
-import org.springframework.web.socket.config.annotation.EnableWebSocket;
-import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
-import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
-import ru.serega6531.packmate.WebSocketHandler;
 import ru.serega6531.packmate.model.enums.CaptureMode;
 import ru.serega6531.packmate.pcap.FilePcapWorker;
 import ru.serega6531.packmate.pcap.LivePcapWorker;
@@ -32,25 +21,9 @@ import ru.serega6531.packmate.service.SubscriptionService;
 import java.net.UnknownHostException;
 
 @Configuration
-@EnableWebSecurity
 @EnableScheduling
-@EnableWebSocket
 @EnableAsync
-@Slf4j
+public class ApplicationConfiguration {
-public class ApplicationConfiguration extends WebSecurityConfigurerAdapter implements WebSocketConfigurer {
-
-    @Value("${account-login}")
-    private String login;
-
-    @Value("${account-password}")
-    private String password;
-
-    private final WebSocketHandler webSocketHandler;
-
-    @Autowired
-    public ApplicationConfiguration(WebSocketHandler webSocketHandler) {
-        this.webSocketHandler = webSocketHandler;
-    }
 
     @Bean(destroyMethod = "stop")
     @Autowired
@@ -64,48 +37,13 @@ public class ApplicationConfiguration extends WebSecurityConfigurerAdapter imple
         return switch (captureMode) {
             case LIVE -> new LivePcapWorker(servicesService, streamService, localIpString, interfaceName);
             case FILE -> new FilePcapWorker(servicesService, streamService, subscriptionService, localIpString, filename);
-            default -> new NoOpPcapWorker();
+            case VIEW -> new NoOpPcapWorker();
         };
     }
 
-    @Autowired
-    public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception {
-        auth.inMemoryAuthentication()
-                .withUser(login)
-                .password(passwordEncoder().encode(password))
-                .authorities("ROLE_USER");
-    }
-
-    @Override
-    protected void configure(HttpSecurity http) throws Exception {
-        http.csrf()
-                .disable()
-                .authorizeRequests()
-                .antMatchers("/site.webmanifest")
-                .permitAll()
-                .anyRequest().authenticated()
-                .and()
-                .httpBasic()
-                .and()
-                .headers()
-                .frameOptions()
-                .sameOrigin();
-    }
-
     @Bean
     public PasswordEncoder passwordEncoder() {
         return new BCryptPasswordEncoder();
     }
 
-    @EventListener
-    public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) {
-        log.info("Login failed for user {}, password {}",
-                e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials());
-    }
-
-    @Override
-    public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
-        registry.addHandler(webSocketHandler, "/api/ws")
-                .withSockJS();
-    }
 }

src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java

@@ -0,0 +1,63 @@
+package ru.serega6531.packmate.configuration;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.event.EventListener;
+import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.crypto.password.PasswordEncoder;
+
+@Configuration
+@EnableWebSecurity
+@Slf4j
+public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
+
+    @Value("${account-login}")
+    private String login;
+
+    @Value("${account-password}")
+    private String password;
+
+    private final PasswordEncoder passwordEncoder;
+
+    @Autowired
+    public SecurityConfiguration(PasswordEncoder passwordEncoder) {
+        this.passwordEncoder = passwordEncoder;
+    }
+
+    @Autowired
+    public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception {
+        auth.inMemoryAuthentication()
+                .withUser(login)
+                .password(passwordEncoder.encode(password))
+                .authorities("ROLE_USER");
+    }
+
+    @Override
+    protected void configure(HttpSecurity http) throws Exception {
+        http.csrf()
+                .disable()
+                .authorizeRequests()
+                .antMatchers("/site.webmanifest")
+                .permitAll()
+                .anyRequest().authenticated()
+                .and()
+                .httpBasic()
+                .and()
+                .headers()
+                .frameOptions()
+                .sameOrigin();
+    }
+
+    @EventListener
+    public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) {
+        log.info("Login failed for user {}, password {}",
+                e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials());
+    }
+
+}

src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java

@@ -0,0 +1,27 @@
+package ru.serega6531.packmate.configuration;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.socket.config.annotation.EnableWebSocket;
+import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
+import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
+import ru.serega6531.packmate.controller.WebSocketHandler;
+
+@EnableWebSocket
+@Configuration
+public class WebSocketConfiguration implements WebSocketConfigurer {
+
+    private final WebSocketHandler webSocketHandler;
+
+    @Autowired
+    public WebSocketConfiguration(WebSocketHandler webSocketHandler) {
+        this.webSocketHandler = webSocketHandler;
+    }
+
+    @Override
+    public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
+        registry.addHandler(webSocketHandler, "/api/ws")
+                .withSockJS();
+    }
+
+}

src/main/java/ru/serega6531/packmate/controller/PacketController.java

@@ -1,17 +1,13 @@
 package ru.serega6531.packmate.controller;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.PostMapping;
+import ru.serega6531.packmate.model.Packet;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-import ru.serega6531.packmate.model.Stream;
 import ru.serega6531.packmate.model.pojo.PacketDto;
+import ru.serega6531.packmate.model.pojo.PacketPagination;
 import ru.serega6531.packmate.service.StreamService;
 
-import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 @RestController
@@ -26,15 +22,11 @@ public class PacketController {
     }
 
     @PostMapping("/{streamId}")
-    public List<PacketDto> getPacketsForStream(@PathVariable long streamId) {
+    public List<PacketDto> getPacketsForStream(@PathVariable long streamId, @RequestBody PacketPagination pagination) {
-        final Optional<Stream> stream = streamService.find(streamId);
+        List<Packet> packets = streamService.getPackets(streamId, pagination.getStartingFrom(), pagination.getPageSize());
-        if (stream.isPresent()) {
+        return packets.stream()
-            return stream.get().getPackets().stream()
+                .map(streamService::packetToDto)
-                    .map(streamService::packetToDto)
+                .collect(Collectors.toList());
-                    .collect(Collectors.toList());
-        } else {
-            return Collections.emptyList();
-        }
     }
 
 }

src/main/java/ru/serega6531/packmate/controller/StreamController.java

@@ -2,7 +2,7 @@ package ru.serega6531.packmate.controller;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
-import ru.serega6531.packmate.model.pojo.Pagination;
+import ru.serega6531.packmate.model.pojo.StreamPagination;
 import ru.serega6531.packmate.model.pojo.StreamDto;
 import ru.serega6531.packmate.service.StreamService;
 
@@ -22,14 +22,14 @@ public class StreamController {
     }
 
     @PostMapping("/all")
-    public List<StreamDto> getStreams(@RequestBody Pagination pagination) {
+    public List<StreamDto> getStreams(@RequestBody StreamPagination pagination) {
         return service.findAll(pagination, Optional.empty(), pagination.isFavorites()).stream()
                 .map(service::streamToDto)
                 .collect(Collectors.toList());
     }
 
     @PostMapping("/{port}")
-    public List<StreamDto> getStreams(@PathVariable int port, @RequestBody Pagination pagination) {
+    public List<StreamDto> getStreams(@PathVariable int port, @RequestBody StreamPagination pagination) {
         return service.findAll(pagination, Optional.of(port), pagination.isFavorites()).stream()
                 .map(service::streamToDto)
                 .collect(Collectors.toList());

src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java

@@ -1,4 +1,4 @@
-package ru.serega6531.packmate;
+package ru.serega6531.packmate.controller;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;

src/main/java/ru/serega6531/packmate/model/CtfService.java

@@ -1,19 +1,26 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.*;
+import org.hibernate.Hibernate;
 
+import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
+import java.util.Objects;
 
-@Data
+@Getter
+@Setter
+@ToString
+@RequiredArgsConstructor
 @Entity
 @Table(name = "service")
 public class CtfService {
 
     @Id
-    private int port;
+    private Integer port;
 
+    @Column(nullable = false)
     private String name;
 
     private boolean decryptTls;
@@ -28,4 +35,16 @@ public class CtfService {
 
     private boolean parseWebSockets;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        CtfService that = (CtfService) o;
+        return port != null && Objects.equals(port, that.port);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/FoundPattern.java

@@ -1,18 +1,22 @@
 package ru.serega6531.packmate.model;
 
 import lombok.*;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
+import org.hibernate.annotations.Parameter;
 
 import javax.persistence.*;
+import java.util.Objects;
 
 @Entity
 @GenericGenerator(
         name = "found_pattern_generator",
         strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator",
         parameters = {
-                @org.hibernate.annotations.Parameter(name = "sequence_name", value = "found_pattern_seq"),
+                @Parameter(name = "sequence_name", value = "found_pattern_seq"),
-                @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"),
+                @Parameter(name = "initial_value", value = "1"),
-                @org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
+                @Parameter(name = "increment_size", value = "2000"),
+                @Parameter(name = "optimizer", value = "hilo")
         }
 )
 @NoArgsConstructor
@@ -20,14 +24,14 @@ import javax.persistence.*;
 @Builder
 @Getter
 @ToString
-@EqualsAndHashCode(exclude = "packet")
+@Table(indexes = { @Index(name = "found_pattern_packet_id_index", columnList = "packet_id DESC") })
 public class FoundPattern {
 
     @Id
-    @GeneratedValue(generator = "found_pattern_generator")
+    @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "found_pattern_generator")
-    private int id;
+    private Long id;
 
-    @ManyToOne
+    @ManyToOne(optional = false)
     @JoinColumn(name = "packet_id", nullable = false)
     @Setter
     private Packet packet;
@@ -38,6 +42,18 @@ public class FoundPattern {
 
     private int endPosition;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        FoundPattern that = (FoundPattern) o;
+        return id != null && Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }
 
 

src/main/java/ru/serega6531/packmate/model/Packet.java

@@ -1,27 +1,31 @@
 package ru.serega6531.packmate.model;
 
 import lombok.*;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
+import org.hibernate.annotations.Parameter;
 
 import javax.persistence.*;
+import java.util.Objects;
 import java.util.Set;
 
-@Data
+@Getter
+@Setter
+@RequiredArgsConstructor
 @Entity
 @GenericGenerator(
         name = "packet_generator",
         strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator",
         parameters = {
-                @org.hibernate.annotations.Parameter(name = "sequence_name", value = "packet_seq"),
+                @Parameter(name = "sequence_name", value = "packet_seq"),
-                @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"),
+                @Parameter(name = "initial_value", value = "1"),
-                @org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
+                @Parameter(name = "increment_size", value = "20000"),
+                @Parameter(name = "optimizer", value = "hilo")
         }
 )
-@NoArgsConstructor
 @AllArgsConstructor
 @Builder
 @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") })
-@EqualsAndHashCode(exclude = "stream")
 public class Packet {
 
     @Id
@@ -34,11 +38,11 @@ public class Packet {
     @Transient
     private int ttl;
 
-    @ManyToOne
+    @ManyToOne(fetch = FetchType.LAZY, optional = false)
     @JoinColumn(name = "stream_id", nullable = false)
     private Stream stream;
 
-    @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER)
+    @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, orphanRemoval = true)
     private Set<FoundPattern> matches;
 
     private long timestamp;
@@ -51,6 +55,7 @@ public class Packet {
 
     private boolean tlsDecrypted;
 
+    @Column(nullable = false)
     private byte[] content;
 
     @Transient
@@ -62,4 +67,16 @@ public class Packet {
         return "Packet(id=" + id + ", content=" + getContentString() + ")";
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Packet packet = (Packet) o;
+        return id != null && Objects.equals(id, packet.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/Pattern.java

@@ -1,53 +1,76 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
 import lombok.ToString;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
+import org.hibernate.annotations.Parameter;
 import ru.serega6531.packmate.model.enums.PatternActionType;
 import ru.serega6531.packmate.model.enums.PatternDirectionType;
 import ru.serega6531.packmate.model.enums.PatternSearchType;
 
 import javax.persistence.*;
-import java.util.List;
+import java.util.Objects;
 
-@Data
+@Getter
-@ToString(exclude = "matchedStreams")
+@Setter
+@RequiredArgsConstructor
+@ToString
 @Entity
 @GenericGenerator(
         name = "pattern_generator",
         strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator",
         parameters = {
-                @org.hibernate.annotations.Parameter(name = "sequence_name", value = "pattern_seq"),
+                @Parameter(name = "sequence_name", value = "pattern_seq"),
-                @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"),
+                @Parameter(name = "initial_value", value = "1"),
-                @org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
+                @Parameter(name = "increment_size", value = "1")
         }
 )
 public class Pattern {
 
     @Id
     @GeneratedValue(generator = "pattern_generator")
-    private int id;
+    private Integer id;
 
     private boolean enabled;
 
+    @Column(nullable = false)
     private String name;
 
+    @Column(nullable = false)
     private String value;
 
+    @Column(nullable = false)
     private String color;  // для вставки в css
 
+    @Enumerated
+    @Column(nullable = false)
     private PatternSearchType searchType;
 
+    @Enumerated
+    @Column(nullable = false)
     private PatternDirectionType directionType;
 
+    @Enumerated
+    @Column(nullable = false)
     private PatternActionType actionType;
 
     private Integer serviceId;
 
     private long searchStartTimestamp;
 
-    @ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY)
+    @Override
-    private List<Stream> matchedStreams;
+    public boolean equals(Object o) {
-
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Pattern pattern = (Pattern) o;
+        return id != null && Objects.equals(id, pattern.id);
+    }
 
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/Stream.java

@@ -1,55 +1,85 @@
 package ru.serega6531.packmate.model;
 
-import lombok.Data;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
 import lombok.ToString;
+import org.hibernate.Hibernate;
 import org.hibernate.annotations.GenericGenerator;
+import org.hibernate.annotations.Parameter;
 import ru.serega6531.packmate.model.enums.Protocol;
 
 import javax.persistence.*;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
-@Data
+@Getter
-@ToString(exclude = "packets")
+@Setter
+@ToString
+@RequiredArgsConstructor
 @Entity
 @GenericGenerator(
         name = "stream_generator",
         strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator",
         parameters = {
-                @org.hibernate.annotations.Parameter(name = "sequence_name", value = "stream_seq"),
+                @Parameter(name = "sequence_name", value = "stream_seq"),
-                @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"),
+                @Parameter(name = "initial_value", value = "1"),
-                @org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
+                @Parameter(name = "increment_size", value = "1000"),
+                @Parameter(name = "optimizer", value = "hilo")
         }
 )
+@Table(indexes = {@Index(name = "stream_id_desc_index", columnList = "id DESC")})
 public class Stream {
 
     @Id
     @GeneratedValue(generator = "stream_generator")
     private Long id;
 
-    @Column(name = "service_id")
+    @Column(name = "service_id", nullable = false)
     private int service;
 
+    @Enumerated
+    @Column(nullable = false)
     private Protocol protocol;
 
-    @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL)
+    @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL, orphanRemoval = true)
     @OrderBy("id")
+    @ToString.Exclude
     private List<Packet> packets;
 
     private long startTimestamp;
 
     private long endTimestamp;
 
-    @ManyToMany
+    @ManyToMany(fetch = FetchType.EAGER)
+    @JoinTable(
+            name = "stream_found_patterns",
+            joinColumns = @JoinColumn(name = "stream_id"),
+            inverseJoinColumns = @JoinColumn(name = "pattern_id")
+    )
+    @ToString.Exclude
     private Set<Pattern> foundPatterns = new HashSet<>();
 
     private boolean favorite;
 
-    @Column(columnDefinition = "smallint")
+    @Column(nullable = false, columnDefinition = "smallint")
     private int ttl;
 
     @Column(columnDefinition = "char(3)")
     private String userAgentHash;
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
+        Stream stream = (Stream) o;
+        return id != null && Objects.equals(id, stream.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
 }

src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java

@@ -0,0 +1,14 @@
+package ru.serega6531.packmate.model.pojo;
+
+import lombok.Data;
+import org.jetbrains.annotations.Nullable;
+
+@Data
+public class PacketPagination {
+
+    @Nullable
+    private Long startingFrom;
+
+    private int pageSize;
+
+}

src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java

@@ -1,20 +0,0 @@
-package ru.serega6531.packmate.model.pojo;
-
-import lombok.Data;
-import org.springframework.data.domain.Sort;
-import ru.serega6531.packmate.model.Pattern;
-
-@Data
-public class Pagination {
-
-    private Sort.Direction direction;
-
-    private long startingFrom;
-
-    private int pageSize;
-
-    private boolean favorites; // только для стримов, определяет, искать только избранные стримы или все
-
-    private Pattern pattern; // только для стримов, если не null, ищем стримы с этим паттерном
-
-}

src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java

@@ -0,0 +1,20 @@
+package ru.serega6531.packmate.model.pojo;
+
+import lombok.Data;
+import org.jetbrains.annotations.Nullable;
+import ru.serega6531.packmate.model.Pattern;
+
+@Data
+public class StreamPagination {
+
+    @Nullable
+    private Long startingFrom;
+
+    private int pageSize;
+
+    private boolean favorites; // определяет, искать только избранные стримы или все
+
+    @Nullable
+    private Pattern pattern; // если не null, ищем стримы с этим паттерном
+
+}

src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java

@@ -5,6 +5,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
 import org.pcap4j.core.BpfProgram;
 import org.pcap4j.core.PacketListener;
 import org.pcap4j.core.PcapHandle;
@@ -46,8 +47,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
     private final ListMultimap<UnfinishedStream, ru.serega6531.packmate.model.Packet> unfinishedUdpStreams = ArrayListMultimap.create();
 
     // в следующих мапах в значениях находится srcIp соответствующего пакета
-    private final SetMultimap<UnfinishedStream, ImmutablePair<InetAddress, Integer>> fins = HashMultimap.create();
+    private final SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> fins = HashMultimap.create();
-    private final SetMultimap<UnfinishedStream, ImmutablePair<InetAddress, Integer>> acks = HashMultimap.create();
+    private final SetMultimap<UnfinishedStream, Pair<InetAddress, Integer>> acks = HashMultimap.create();
 
     protected AbstractPcapWorker(ServicesService servicesService,
                               StreamService streamService,
@@ -182,8 +183,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
      * Udp не имеет фазы закрытия, поэтому закрывается только по таймауту
      */
     private void checkTcpTermination(boolean ack, boolean fin, boolean rst,
-                                     ImmutablePair<InetAddress, Integer> sourceIpAndPort,
+                                     Pair<InetAddress, Integer> sourceIpAndPort,
-                                     ImmutablePair<InetAddress, Integer> destIpAndPort,
+                                     Pair<InetAddress, Integer> destIpAndPort,
                                      UnfinishedStream stream) {
 
         if (fin) {
@@ -256,7 +257,6 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
     }
 
     @Override
-    @SneakyThrows
     public void setFilter(String filter) {
         this.filter = filter;
         applyFilter();

src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java

@@ -79,4 +79,9 @@ public class FilePcapWorker extends AbstractPcapWorker {
 
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.PCAP_STOPPED, null));
     }
+
+    @Override
+    public String getExecutorState() {
+        return "inline";
+    }
 }

src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java

@@ -10,7 +10,9 @@ import ru.serega6531.packmate.service.ServicesService;
 import ru.serega6531.packmate.service.StreamService;
 
 import java.net.UnknownHostException;
-import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 @Slf4j
 public class LivePcapWorker extends AbstractPcapWorker {
@@ -30,7 +32,7 @@ public class LivePcapWorker extends AbstractPcapWorker {
 
         BasicThreadFactory factory = new BasicThreadFactory.Builder()
                 .namingPattern("pcap-processor").build();
-        processorExecutorService = Executors.newSingleThreadExecutor(factory);
+        processorExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), factory);
     }
 
     public void start() throws PcapNativeException {
@@ -63,4 +65,8 @@ public class LivePcapWorker extends AbstractPcapWorker {
         log.info("Intercept stopped");
     }
 
+    @Override
+    public String getExecutorState() {
+        return processorExecutorService.toString();
+    }
 }

src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java

@@ -24,4 +24,9 @@ public class NoOpPcapWorker implements PcapWorker {
     @Override
     public void setFilter(String filter) {
     }
+
+    @Override
+    public String getExecutorState() {
+        return "none";
+    }
 }

src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java

@@ -20,4 +20,5 @@ public interface PcapWorker {
 
     void setFilter(String filter);
 
+    String getExecutorState();
 }

src/main/java/ru/serega6531/packmate/repository/StreamRepository.java

@@ -1,15 +1,28 @@
 package ru.serega6531.packmate.repository;
 
-import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.domain.Pageable;
-import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.*;
-import org.springframework.data.jpa.repository.Modifying;
+import ru.serega6531.packmate.model.Packet;
-import org.springframework.data.jpa.repository.Query;
 import ru.serega6531.packmate.model.Stream;
 
+import javax.persistence.QueryHint;
+import java.util.List;
+
 public interface StreamRepository extends JpaRepository<Stream, Long>, JpaSpecificationExecutor<Stream> {
 
     @Query("UPDATE Stream SET favorite = :favorite WHERE id = :id")
     @Modifying
     void setFavorite(long id, boolean favorite);
 
+    long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold);
+
+    @Query("SELECT DISTINCT p FROM Packet p " +
+            "LEFT JOIN FETCH p.matches " +
+            "WHERE p.stream.id = :streamId " +
+            "AND (:startingFrom IS NULL OR p.id > :startingFrom) " +
+            "ORDER BY p.id"
+    )
+    @QueryHints(@QueryHint(name = org.hibernate.jpa.QueryHints.HINT_PASS_DISTINCT_THROUGH, value = "false"))
+    List<Packet> getPackets(long streamId, Long startingFrom, Pageable pageable);
+
 }

src/main/java/ru/serega6531/packmate/service/PatternService.java

@@ -15,6 +15,7 @@ import ru.serega6531.packmate.model.pojo.PatternDto;
 import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
 import ru.serega6531.packmate.repository.PatternRepository;
 
+import javax.annotation.PostConstruct;
 import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -40,7 +41,10 @@ public class PatternService {
         this.streamService = streamService;
         this.subscriptionService = subscriptionService;
         this.modelMapper = modelMapper;
+    }
 
+    @PostConstruct
+    public void init() {
         repository.findAll().forEach(p -> patterns.put(p.getId(), p));
         log.info("Loaded {} patterns", patterns.size());
     }

src/main/java/ru/serega6531/packmate/service/PcapService.java

@@ -41,17 +41,27 @@ public class PcapService {
     }
 
     public void updateFilter(Collection<CtfService> services) {
-        final String ports = services.stream()
+        String filter;
-                .map(CtfService::getPort)
-                .map(p -> "port " + p)
-                .collect(Collectors.joining(" or "));
 
-        final String format = "(tcp or udp) and (%s)";
+        if (services.isEmpty()) {
-        String filter = String.format(format, ports);
+            filter = "tcp or udp";
+        } else {
+            final String ports = services.stream()
+                    .map(CtfService::getPort)
+                    .map(p -> "port " + p)
+                    .collect(Collectors.joining(" or "));
+
+            final String format = "(tcp or udp) and (%s)";
+            filter = String.format(format, ports);
+        }
 
         log.debug("New filter: " + filter);
 
         worker.setFilter(filter);
     }
 
+    public String getExecutorState() {
+        return worker.getExecutorState();
+    }
+
 }

src/main/java/ru/serega6531/packmate/service/ServicesService.java

@@ -12,12 +12,10 @@ import ru.serega6531.packmate.model.pojo.ServiceDto;
 import ru.serega6531.packmate.model.pojo.SubscriptionMessage;
 import ru.serega6531.packmate.repository.ServiceRepository;
 
+import javax.annotation.PostConstruct;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.Collection;
+import java.util.*;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
 
 @Service
 @Slf4j
@@ -43,7 +41,10 @@ public class ServicesService {
         this.pcapService = pcapService;
         this.modelMapper = modelMapper;
         this.localIp = InetAddress.getByName(localIpString);
+    }
 
+    @PostConstruct
+    public void init() {
         repository.findAll().forEach(s -> services.put(s.getPort(), s));
         log.info("Loaded {} services", services.size());
     }
@@ -78,7 +79,7 @@ public class ServicesService {
 
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.DELETE_SERVICE, port));
 
-        pcapService.updateFilter(findAll());
+        updateFilter();
     }
 
     public CtfService save(CtfService service) {
@@ -89,11 +90,15 @@ public class ServicesService {
 
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.SAVE_SERVICE, toDto(saved)));
 
-        pcapService.updateFilter(findAll());
+        updateFilter();
 
         return saved;
     }
 
+    public void updateFilter() {
+        pcapService.updateFilter(findAll());
+    }
+
     public ServiceDto toDto(CtfService service) {
         return modelMapper.map(service, ServiceDto.class);
     }

src/main/java/ru/serega6531/packmate/service/StreamService.java

@@ -1,10 +1,12 @@
 package ru.serega6531.packmate.service;
 
 import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
 import org.modelmapper.ModelMapper;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.domain.Specification;
 import org.springframework.scheduling.annotation.Async;
@@ -20,6 +22,7 @@ import ru.serega6531.packmate.repository.StreamRepository;
 import ru.serega6531.packmate.service.optimization.RsaKeysHolder;
 import ru.serega6531.packmate.service.optimization.StreamOptimizer;
 
+import java.time.ZonedDateTime;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
@@ -90,6 +93,15 @@ public class StreamService {
             }
         }
 
+        countingService.countStream(service.getPort(), packets.size());
+
+        List<Packet> optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
+
+        if (isStreamIgnored(optimizedPackets, service)) {
+            log.debug("New stream is ignored");
+            return false;
+        }
+
         Optional<Packet> firstIncoming = packets.stream()
                 .filter(Packet::isIncoming)
                 .findFirst();
@@ -101,27 +113,19 @@ public class StreamService {
         stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp());
         stream.setService(service.getPort());
 
-        countingService.countStream(service.getPort(), packets.size());
+        String userAgentHash = getUserAgentHash(optimizedPackets);
+        stream.setUserAgentHash(userAgentHash);
 
-        packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
+        Set<Pattern> foundPatterns = matchPatterns(optimizedPackets, service);
+        stream.setFoundPatterns(foundPatterns);
+        stream.setPackets(optimizedPackets);
 
-        if (isStreamIgnored(packets, service)) {
+        for (Packet packet : optimizedPackets) {
-            log.debug("New stream is ignored");
+            packet.setStream(stream);
-            return false;
         }
 
-        processUserAgent(packets, stream);
         Stream savedStream = save(stream);
 
-        for (Packet packet : packets) {
-            packet.setStream(savedStream);
-        }
-
-        Set<Pattern> foundPatterns = matchPatterns(packets, service);
-        savedStream.setFoundPatterns(foundPatterns);
-        savedStream.setPackets(packets);
-        savedStream = save(savedStream);
-
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream)));
         return true;
     }
@@ -143,7 +147,7 @@ public class StreamService {
         subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId()));
     }
 
-    private void processUserAgent(List<Packet> packets, Stream stream) {
+    private String getUserAgentHash(List<Packet> packets) {
         String ua = null;
         for (Packet packet : packets) {
             String content = packet.getContentString();
@@ -155,7 +159,9 @@ public class StreamService {
         }
 
         if (ua != null) {
-            stream.setUserAgentHash(calculateUserAgentHash(ua));
+            return calculateUserAgentHash(ua);
+        } else {
+            return null;
         }
     }
 
@@ -238,8 +244,17 @@ public class StreamService {
         return saved;
     }
 
-    public Optional<Stream> find(long id) {
+    public List<Packet> getPackets(long streamId, @Nullable Long startingFrom, int pageSize) {
-        return repository.findById(id);
+//        long safeStartingFrom = startingFrom != null ? startingFrom : 0;
+        return repository.getPackets(streamId, startingFrom, Pageable.ofSize(pageSize));
+    }
+
+    /**
+     * @return Number of deleted rows
+     */
+    @Transactional
+    public long cleanupOldStreams(ZonedDateTime before) {
+        return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000);
     }
 
     @Transactional
@@ -247,14 +262,13 @@ public class StreamService {
         repository.setFavorite(id, favorite);
     }
 
-    public List<Stream> findAll(Pagination pagination, Optional<Integer> service, boolean onlyFavorites) {
+    public List<Stream> findAll(StreamPagination pagination, Optional<Integer> service, boolean onlyFavorites) {
-        PageRequest page = PageRequest.of(0, pagination.getPageSize(), pagination.getDirection(), "id");
+        PageRequest page = PageRequest.of(0, pagination.getPageSize(), Sort.Direction.DESC, "id");
 
-        Specification<Stream> spec;
+        Specification<Stream> spec = Specification.where(null);
-        if (pagination.getDirection() == Sort.Direction.ASC) {
+
-            spec = streamIdGreaterThan(pagination.getStartingFrom());
+        if (pagination.getStartingFrom() != null) {
-        } else {
+            spec = spec.and(streamIdLessThan(pagination.getStartingFrom()));
-            spec = streamIdLessThan(pagination.getStartingFrom());
         }
 
         if (service.isPresent()) {
@@ -293,10 +307,6 @@ public class StreamService {
         return (root, query, cb) -> cb.equal(root.get("favorite"), true);
     }
 
-    private Specification<Stream> streamIdGreaterThan(long id) {
-        return (root, query, cb) -> cb.greaterThan(root.get("id"), id);
-    }
-
     private Specification<Stream> streamIdLessThan(long id) {
         return (root, query, cb) -> cb.lessThan(root.get("id"), id);
     }

src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java

@@ -0,0 +1,23 @@
+package ru.serega6531.packmate.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import ru.serega6531.packmate.service.PcapService;
+
+@Component
+@Slf4j
+public class ExecutorStateLoggerTask {
+
+    private final PcapService service;
+
+    public ExecutorStateLoggerTask(PcapService service) {
+        this.service = service;
+    }
+
+    @Scheduled(fixedDelayString = "PT1M", initialDelayString = "PT1M")
+    public void cleanup() {
+        log.info("Executor state: {}", service.getExecutorState());
+    }
+
+}

src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java

@@ -0,0 +1,34 @@
+package ru.serega6531.packmate.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import ru.serega6531.packmate.service.StreamService;
+
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+
+@Component
+@Slf4j
+@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true")
+public class OldStreamsCleanupTask {
+
+    private final StreamService service;
+    private final int oldStreamsThreshold;
+
+    public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) {
+        this.service = service;
+        this.oldStreamsThreshold = oldStreamsThreshold;
+    }
+
+    @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M")
+    public void cleanup() {
+        ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES);
+        log.info("Cleaning up old non-favorite streams (before {})", before);
+        long deleted = service.cleanupOldStreams(before);
+        log.info("Deleted {} rows", deleted);
+    }
+
+}

src/main/java/ru/serega6531/packmate/tasks/StartupListener.java

@@ -0,0 +1,40 @@
+package ru.serega6531.packmate.tasks;
+
+import org.pcap4j.core.PcapNativeException;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import ru.serega6531.packmate.model.enums.CaptureMode;
+import ru.serega6531.packmate.service.PcapService;
+import ru.serega6531.packmate.service.ServicesService;
+
+@Component
+public class StartupListener {
+
+    @Value("${enable-capture}")
+    private boolean enableCapture;
+
+    @Value("${capture-mode}")
+    private CaptureMode captureMode;
+
+    private final PcapService pcapService;
+    private final ServicesService servicesService;
+
+    public StartupListener(PcapService pcapService, ServicesService servicesService) {
+        this.pcapService = pcapService;
+        this.servicesService = servicesService;
+    }
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void afterStartup() throws PcapNativeException {
+        if (enableCapture) {
+            servicesService.updateFilter();
+
+            if (captureMode == CaptureMode.LIVE) {
+                pcapService.start();
+            }
+        }
+    }
+
+}

src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java

@@ -1,4 +1,4 @@
-package ru.serega6531.packmate;
+package ru.serega6531.packmate.tasks;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;

src/main/resources/application.yml

@@ -9,19 +9,25 @@ spring:
       ddl-auto: update
     properties:
       hibernate:
+        jdbc:
+          batch_size: 20
+        order_inserts: true
         temp:
           use_jdbc_metadata_defaults: false
     database-platform: org.hibernate.dialect.PostgreSQLDialect
 
 
 enable-capture: true
-capture-mode: LIVE  # LIVE, FILE
+capture-mode: LIVE  # LIVE, FILE, VIEW
 interface-name: enp0s31f6
 pcap-file: file.pcap
 local-ip: "192.168.0.125"
 account-login: BinaryBears
 account-password: 123456
-udp-stream-timeout: 20 # секунд
+udp-stream-timeout: 20 # seconds
-tcp-stream-timeout: 40 # секунд
+tcp-stream-timeout: 40 # seconds
-timeout-stream-check-interval: 10 # секунд
+timeout-stream-check-interval: 10 # seconds
+old-streams-cleanup-enabled: true
+old-streams-threshold: 240 # minutes
+cleanup-interval: 5 # minutes
 ignore-empty-packets: true

src/test/java/ru/serega6531/packmate/PatternMatcherTest.java

@@ -1,7 +1,7 @@
 package ru.serega6531.packmate;
 
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
-import org.springframework.util.Assert;
 import ru.serega6531.packmate.model.FoundPattern;
 import ru.serega6531.packmate.model.Pattern;
 import ru.serega6531.packmate.model.enums.PatternSearchType;
@@ -27,13 +27,14 @@ public class PatternMatcherTest {
                         .build());
 
         final Pattern pattern = new Pattern();
+        pattern.setId(1);
         pattern.setValue("[a-f]{3}");
         pattern.setSearchType(PatternSearchType.REGEX);
 
         final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern));
         final Set<FoundPattern> matches = matcher.findMatches();
 
-        Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString());
+        assertMatchesAreCorrect(correctMatches, matches);
     }
 
     @Test
@@ -50,13 +51,14 @@ public class PatternMatcherTest {
                         .build());
 
         final Pattern pattern = new Pattern();
+        pattern.setId(1);
         pattern.setValue("bbb");
         pattern.setSearchType(PatternSearchType.SUBSTRING);
 
         final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern));
         final Set<FoundPattern> matches = matcher.findMatches();
 
-        Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString());
+        assertMatchesAreCorrect(correctMatches, matches);
     }
 
     @Test
@@ -73,13 +75,27 @@ public class PatternMatcherTest {
                         .build());
 
         final Pattern pattern = new Pattern();
+        pattern.setId(1);
         pattern.setValue("AAaa");
         pattern.setSearchType(PatternSearchType.SUBBYTES);
 
         final PatternMatcher matcher = new PatternMatcher(content, List.of(pattern));
         final Set<FoundPattern> matches = matcher.findMatches();
 
-        Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString());
+        assertMatchesAreCorrect(correctMatches, matches);
+    }
+
+    private void assertMatchesAreCorrect(Set<FoundPattern> correctMatches, Set<FoundPattern> foundMatches) {
+        Assertions.assertEquals(correctMatches.size(), foundMatches.size());
+
+        Assertions.assertTrue(correctMatches.stream().allMatch(correct ->
+                foundMatches.stream().anyMatch(found -> matchesEqual(correct, found))
+        ));
+    }
+
+    private boolean matchesEqual(FoundPattern one, FoundPattern two) {
+        return one.getStartPosition() == two.getStartPosition() &&
+                one.getEndPosition() == two.getEndPosition();
     }
 
 }