diff --git a/README.md b/README.md index 2b3a478..2b5fc40 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,17 @@ PACKMATE_PCAP_FILE=dump.pcap PACKMATE_MODE=VIEW ``` +При захвате живого трафика рекомендуется включать удаление старых стримов, иначе ближе к концу +соревнования анализатор будет медленнее работать. +```dotenv +PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true +# Интервал удаления старых стримов (в минутах). +# Лучше ставить маленькое число, чтобы стримы удалялись маленькими кусками, и это не нагружало систему +PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1 +# Насколько старым стрим должен быть для удаления (в минутах от текущего времени) +PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240 +``` + Чтобы использовать расшифровку TLS, нужно положить соответствующий приватный ключ, который использовался для генерации сертификата, в папку `rsa_keys`. diff --git a/README_EN.md b/README_EN.md index 098ce0c..abec29e 100644 --- a/README_EN.md +++ b/README_EN.md @@ -76,6 +76,17 @@ PACKMATE_MODE=FILE PACKMATE_PCAP_FILE=dump.pcap ``` +When capturing live traffic it's better to turn on old streams removal. Otherwise, after some time Packmate +will start working slower. +```dotenv +PACKMATE_OLD_STREAMS_CLEANUP_ENABLED=true +# Old streams removal interval (in minutes). +# It's better to use small numbers so the streams are removed in small chunks and don't overload the server. +PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL=1 +# How old the stream must be to be removed (in minutes before current time) +PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD=240 +``` + To decrypt TLS, put the private key used to generate a certificate into the `rsa_keys` folder. ### Launch diff --git a/build.gradle b/build.gradle index 5cb5758..01658eb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'org.springframework.boot' version '2.4.1' + id 'org.springframework.boot' version '2.6.3' id 'java' } @@ -27,19 +27,20 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-security" implementation "org.springframework.boot:spring-boot-starter-websocket" implementation 'org.springframework.session:spring-session-core' - compile 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' - compile group: 'commons-io', name: 'commons-io', version: '2.7' - compile 'org.pcap4j:pcap4j-core:1.8.2' - compile 'org.pcap4j:pcap4j-packetfactory-static:1.8.2' - compile group: 'com.google.guava', name: 'guava', version: '30.1-jre' - compile group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.68' - compile group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.68' - compile group: 'org.modelmapper', name: 'modelmapper', version: '2.3.0' + implementation 'com.github.jmnarloch:modelmapper-spring-boot-starter:1.1.0' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.11.0' + implementation 'org.pcap4j:pcap4j-core:1.8.2' + implementation 'org.pcap4j:pcap4j-packetfactory-static:1.8.2' + implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + implementation group: 'org.java-websocket', name: 'Java-WebSocket', version: '1.5.1' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + implementation group: 'org.bouncycastle', name: 'bctls-jdk15on', version: '1.70' + implementation group: 'org.modelmapper', name: 'modelmapper', version: '2.4.5' + compileOnly 'org.jetbrains:annotations:22.0.0' compileOnly 'org.projectlombok:lombok' runtimeOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'org.postgresql:postgresql' annotationProcessor 'org.projectlombok:lombok' - testCompile 'org.junit.jupiter:junit-jupiter:5.6.2' + testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2' } diff --git a/docker-compose.yml b/docker-compose.yml index d0c4ab7..26db4b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,15 @@ services: DB_USER: ${PACKMATE_DB_USER:-packmate} DB_PASSWORD: ${PACKMATE_DB_PASSWORD:-K604YnL3G1hp2RDkCZNjGpxbyNpNHTRb} DB_NAME: ${PACKMATE_DB_NAME:-packmate} - INTERFACE: ${PACKMATE_INTERFACE} + INTERFACE: ${PACKMATE_INTERFACE:-} LOCAL_IP: ${PACKMATE_LOCAL_IP} MODE: ${PACKMATE_MODE:-LIVE} - PCAP_FILE: ${PACKMATE_PCAP_FILE} + PCAP_FILE: ${PACKMATE_PCAP_FILE:-} WEB_LOGIN: ${PACKMATE_WEB_LOGIN:-BinaryBears} WEB_PASSWORD: ${PACKMATE_WEB_PASSWORD:-123456} + OLD_STREAMS_CLEANUP_ENABLED: ${PACKMATE_OLD_STREAMS_CLEANUP_ENABLED:-false} + OLD_STREAMS_CLEANUP_INTERVAL: ${PACKMATE_OLD_STREAMS_CLEANUP_INTERVAL:-5} + OLD_STREAMS_CLEANUP_THRESHOLD: ${PACKMATE_OLD_STREAMS_CLEANUP_THRESHOLD:-240} env_file: - .env container_name: packmate-app @@ -25,11 +28,12 @@ services: "--spring.datasource.username=$${DB_USER}", "--spring.datasource.password=$${DB_PASSWORD}", "--capture-mode=$${MODE}", "--pcap-file=$${PCAP_FILE}", "--interface-name=$${INTERFACE}", "--local-ip=$${LOCAL_IP}", "--account-login=$${WEB_LOGIN}", + "--old-streams-cleanup-enabled=$${OLD_STREAMS_CLEANUP_ENABLED}", "--cleanup-interval=$${OLD_STREAMS_CLEANUP_INTERVAL}", + "--old-streams-threshold=$${OLD_STREAMS_CLEANUP_THRESHOLD}", "--account-password=$${WEB_PASSWORD}", "--server.port=65000", "--server.address=0.0.0.0" ] depends_on: - db - restart: unless-stopped db: container_name: packmate-db build: @@ -44,5 +48,4 @@ services: volumes: - "./data/postgres_data:/var/lib/postgresql/data" network_mode: "host" - image: packmate-db:v1 - restart: unless-stopped + image: packmate-db:v1 \ No newline at end of file diff --git a/docker/Dockerfile_app b/docker/Dockerfile_app index 1d44a46..f7ebd65 100644 --- a/docker/Dockerfile_app +++ b/docker/Dockerfile_app @@ -12,5 +12,5 @@ RUN ./gradlew --no-daemon build -x test FROM adoptopenjdk/openjdk15:alpine-jre WORKDIR /app RUN apk --no-cache add libpcap -COPY --from=1 /tmp/compile/build/libs/packmate-*.jar app.jar +COPY --from=1 /tmp/compile/build/libs/packmate-*-SNAPSHOT.jar app.jar EXPOSE 65000:65000 \ No newline at end of file diff --git a/docker/Dockerfile_db b/docker/Dockerfile_db index 262e3ca..b6a246c 100644 --- a/docker/Dockerfile_db +++ b/docker/Dockerfile_db @@ -1,4 +1,4 @@ -FROM postgres:13.3-alpine +FROM postgres:14.1-alpine ARG POSTGRES_USER ARG POSTGRES_PASSWORD diff --git a/frontend b/frontend index 9ff0cef..b5d3074 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit 9ff0cefd57a40d515d1b729b014a6a887b5bce04 +Subproject commit b5d3074f7694276cca1c98a251935a88c0bdbd96 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 4d9ca16..2e6e589 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/ru/serega6531/packmate/PackmateApplication.java b/src/main/java/ru/serega6531/packmate/PackmateApplication.java index ddad525..a1ae6f3 100644 --- a/src/main/java/ru/serega6531/packmate/PackmateApplication.java +++ b/src/main/java/ru/serega6531/packmate/PackmateApplication.java @@ -1,33 +1,13 @@ package ru.serega6531.packmate; -import org.pcap4j.core.PcapNativeException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import ru.serega6531.packmate.model.enums.CaptureMode; -import ru.serega6531.packmate.service.PcapService; @SpringBootApplication public class PackmateApplication { - @Value("${enable-capture}") - private boolean enableCapture; - - @Value("${capture-mode}") - private CaptureMode captureMode; - public static void main(String[] args) { SpringApplication.run(PackmateApplication.class, args); } - @EventListener(ApplicationReadyEvent.class) - public void afterStartup(ApplicationReadyEvent event) throws PcapNativeException { - if (enableCapture && captureMode == CaptureMode.LIVE) { - final PcapService pcapService = event.getApplicationContext().getBean(PcapService.class); - pcapService.start(); - } - } - } diff --git a/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java index 3dbf80a..f86522c 100644 --- a/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java +++ b/src/main/java/ru/serega6531/packmate/configuration/ApplicationConfiguration.java @@ -1,25 +1,14 @@ package ru.serega6531.packmate.configuration; -import lombok.extern.slf4j.Slf4j; import org.pcap4j.core.PcapNativeException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent; -import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.web.socket.config.annotation.EnableWebSocket; -import org.springframework.web.socket.config.annotation.WebSocketConfigurer; -import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; -import ru.serega6531.packmate.WebSocketHandler; import ru.serega6531.packmate.model.enums.CaptureMode; import ru.serega6531.packmate.pcap.FilePcapWorker; import ru.serega6531.packmate.pcap.LivePcapWorker; @@ -32,25 +21,9 @@ import ru.serega6531.packmate.service.SubscriptionService; import java.net.UnknownHostException; @Configuration -@EnableWebSecurity @EnableScheduling -@EnableWebSocket @EnableAsync -@Slf4j -public class ApplicationConfiguration extends WebSecurityConfigurerAdapter implements WebSocketConfigurer { - - @Value("${account-login}") - private String login; - - @Value("${account-password}") - private String password; - - private final WebSocketHandler webSocketHandler; - - @Autowired - public ApplicationConfiguration(WebSocketHandler webSocketHandler) { - this.webSocketHandler = webSocketHandler; - } +public class ApplicationConfiguration { @Bean(destroyMethod = "stop") @Autowired @@ -64,48 +37,13 @@ public class ApplicationConfiguration extends WebSecurityConfigurerAdapter imple return switch (captureMode) { case LIVE -> new LivePcapWorker(servicesService, streamService, localIpString, interfaceName); case FILE -> new FilePcapWorker(servicesService, streamService, subscriptionService, localIpString, filename); - default -> new NoOpPcapWorker(); + case VIEW -> new NoOpPcapWorker(); }; } - @Autowired - public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception { - auth.inMemoryAuthentication() - .withUser(login) - .password(passwordEncoder().encode(password)) - .authorities("ROLE_USER"); - } - - @Override - protected void configure(HttpSecurity http) throws Exception { - http.csrf() - .disable() - .authorizeRequests() - .antMatchers("/site.webmanifest") - .permitAll() - .anyRequest().authenticated() - .and() - .httpBasic() - .and() - .headers() - .frameOptions() - .sameOrigin(); - } - @Bean public PasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); } - @EventListener - public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) { - log.info("Login failed for user {}, password {}", - e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials()); - } - - @Override - public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(webSocketHandler, "/api/ws") - .withSockJS(); - } } diff --git a/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java new file mode 100644 index 0000000..0d8a2bf --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/configuration/SecurityConfiguration.java @@ -0,0 +1,63 @@ +package ru.serega6531.packmate.configuration; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.EventListener; +import org.springframework.security.authentication.event.AuthenticationFailureBadCredentialsEvent; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +@EnableWebSecurity +@Slf4j +public class SecurityConfiguration extends WebSecurityConfigurerAdapter { + + @Value("${account-login}") + private String login; + + @Value("${account-password}") + private String password; + + private final PasswordEncoder passwordEncoder; + + @Autowired + public SecurityConfiguration(PasswordEncoder passwordEncoder) { + this.passwordEncoder = passwordEncoder; + } + + @Autowired + public void configureGlobal(AuthenticationManagerBuilder auth) throws Exception { + auth.inMemoryAuthentication() + .withUser(login) + .password(passwordEncoder.encode(password)) + .authorities("ROLE_USER"); + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + http.csrf() + .disable() + .authorizeRequests() + .antMatchers("/site.webmanifest") + .permitAll() + .anyRequest().authenticated() + .and() + .httpBasic() + .and() + .headers() + .frameOptions() + .sameOrigin(); + } + + @EventListener + public void authenticationFailed(AuthenticationFailureBadCredentialsEvent e) { + log.info("Login failed for user {}, password {}", + e.getAuthentication().getPrincipal(), e.getAuthentication().getCredentials()); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java new file mode 100644 index 0000000..1e2f55f --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/configuration/WebSocketConfiguration.java @@ -0,0 +1,27 @@ +package ru.serega6531.packmate.configuration; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +import ru.serega6531.packmate.controller.WebSocketHandler; + +@EnableWebSocket +@Configuration +public class WebSocketConfiguration implements WebSocketConfigurer { + + private final WebSocketHandler webSocketHandler; + + @Autowired + public WebSocketConfiguration(WebSocketHandler webSocketHandler) { + this.webSocketHandler = webSocketHandler; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(webSocketHandler, "/api/ws") + .withSockJS(); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/controller/PacketController.java b/src/main/java/ru/serega6531/packmate/controller/PacketController.java index 4d9db1b..e70a38f 100644 --- a/src/main/java/ru/serega6531/packmate/controller/PacketController.java +++ b/src/main/java/ru/serega6531/packmate/controller/PacketController.java @@ -1,17 +1,13 @@ package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import ru.serega6531.packmate.model.Stream; +import org.springframework.web.bind.annotation.*; +import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.pojo.PacketDto; +import ru.serega6531.packmate.model.pojo.PacketPagination; import ru.serega6531.packmate.service.StreamService; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; @RestController @@ -26,15 +22,11 @@ public class PacketController { } @PostMapping("/{streamId}") - public List getPacketsForStream(@PathVariable long streamId) { - final Optional stream = streamService.find(streamId); - if (stream.isPresent()) { - return stream.get().getPackets().stream() - .map(streamService::packetToDto) - .collect(Collectors.toList()); - } else { - return Collections.emptyList(); - } + public List getPacketsForStream(@PathVariable long streamId, @RequestBody PacketPagination pagination) { + List packets = streamService.getPackets(streamId, pagination.getStartingFrom(), pagination.getPageSize()); + return packets.stream() + .map(streamService::packetToDto) + .collect(Collectors.toList()); } } diff --git a/src/main/java/ru/serega6531/packmate/controller/StreamController.java b/src/main/java/ru/serega6531/packmate/controller/StreamController.java index e8697b8..e0a77fa 100644 --- a/src/main/java/ru/serega6531/packmate/controller/StreamController.java +++ b/src/main/java/ru/serega6531/packmate/controller/StreamController.java @@ -2,7 +2,7 @@ package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import ru.serega6531.packmate.model.pojo.Pagination; +import ru.serega6531.packmate.model.pojo.StreamPagination; import ru.serega6531.packmate.model.pojo.StreamDto; import ru.serega6531.packmate.service.StreamService; @@ -22,14 +22,14 @@ public class StreamController { } @PostMapping("/all") - public List getStreams(@RequestBody Pagination pagination) { + public List getStreams(@RequestBody StreamPagination pagination) { return service.findAll(pagination, Optional.empty(), pagination.isFavorites()).stream() .map(service::streamToDto) .collect(Collectors.toList()); } @PostMapping("/{port}") - public List getStreams(@PathVariable int port, @RequestBody Pagination pagination) { + public List getStreams(@PathVariable int port, @RequestBody StreamPagination pagination) { return service.findAll(pagination, Optional.of(port), pagination.isFavorites()).stream() .map(service::streamToDto) .collect(Collectors.toList()); diff --git a/src/main/java/ru/serega6531/packmate/WebSocketHandler.java b/src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java similarity index 95% rename from src/main/java/ru/serega6531/packmate/WebSocketHandler.java rename to src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java index c0075a0..2c77404 100644 --- a/src/main/java/ru/serega6531/packmate/WebSocketHandler.java +++ b/src/main/java/ru/serega6531/packmate/controller/WebSocketHandler.java @@ -1,4 +1,4 @@ -package ru.serega6531.packmate; +package ru.serega6531.packmate.controller; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/src/main/java/ru/serega6531/packmate/model/CtfService.java b/src/main/java/ru/serega6531/packmate/model/CtfService.java index 7b0788e..737f2fb 100644 --- a/src/main/java/ru/serega6531/packmate/model/CtfService.java +++ b/src/main/java/ru/serega6531/packmate/model/CtfService.java @@ -1,19 +1,26 @@ package ru.serega6531.packmate.model; -import lombok.Data; +import lombok.*; +import org.hibernate.Hibernate; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import java.util.Objects; -@Data +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @Table(name = "service") public class CtfService { @Id - private int port; + private Integer port; + @Column(nullable = false) private String name; private boolean decryptTls; @@ -28,4 +35,16 @@ public class CtfService { private boolean parseWebSockets; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + CtfService that = (CtfService) o; + return port != null && Objects.equals(port, that.port); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } \ No newline at end of file diff --git a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java index 4d54e33..27b4500 100644 --- a/src/main/java/ru/serega6531/packmate/model/FoundPattern.java +++ b/src/main/java/ru/serega6531/packmate/model/FoundPattern.java @@ -1,18 +1,22 @@ package ru.serega6531.packmate.model; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import javax.persistence.*; +import java.util.Objects; @Entity @GenericGenerator( name = "found_pattern_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "found_pattern_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "found_pattern_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "2000"), + @Parameter(name = "optimizer", value = "hilo") } ) @NoArgsConstructor @@ -20,14 +24,14 @@ import javax.persistence.*; @Builder @Getter @ToString -@EqualsAndHashCode(exclude = "packet") +@Table(indexes = { @Index(name = "found_pattern_packet_id_index", columnList = "packet_id DESC") }) public class FoundPattern { @Id - @GeneratedValue(generator = "found_pattern_generator") - private int id; + @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "found_pattern_generator") + private Long id; - @ManyToOne + @ManyToOne(optional = false) @JoinColumn(name = "packet_id", nullable = false) @Setter private Packet packet; @@ -38,6 +42,18 @@ public class FoundPattern { private int endPosition; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + FoundPattern that = (FoundPattern) o; + return id != null && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Packet.java b/src/main/java/ru/serega6531/packmate/model/Packet.java index 8d3c7ee..c35af64 100644 --- a/src/main/java/ru/serega6531/packmate/model/Packet.java +++ b/src/main/java/ru/serega6531/packmate/model/Packet.java @@ -1,27 +1,31 @@ package ru.serega6531.packmate.model; import lombok.*; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import javax.persistence.*; +import java.util.Objects; import java.util.Set; -@Data +@Getter +@Setter +@RequiredArgsConstructor @Entity @GenericGenerator( name = "packet_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "packet_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "packet_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "20000"), + @Parameter(name = "optimizer", value = "hilo") } ) -@NoArgsConstructor @AllArgsConstructor @Builder @Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") }) -@EqualsAndHashCode(exclude = "stream") public class Packet { @Id @@ -34,11 +38,11 @@ public class Packet { @Transient private int ttl; - @ManyToOne + @ManyToOne(fetch = FetchType.LAZY, optional = false) @JoinColumn(name = "stream_id", nullable = false) private Stream stream; - @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, orphanRemoval = true) private Set matches; private long timestamp; @@ -51,6 +55,7 @@ public class Packet { private boolean tlsDecrypted; + @Column(nullable = false) private byte[] content; @Transient @@ -62,4 +67,16 @@ public class Packet { return "Packet(id=" + id + ", content=" + getContentString() + ")"; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Packet packet = (Packet) o; + return id != null && Objects.equals(id, packet.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Pattern.java b/src/main/java/ru/serega6531/packmate/model/Pattern.java index f108752..6d5acad 100644 --- a/src/main/java/ru/serega6531/packmate/model/Pattern.java +++ b/src/main/java/ru/serega6531/packmate/model/Pattern.java @@ -1,53 +1,76 @@ package ru.serega6531.packmate.model; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.ToString; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import ru.serega6531.packmate.model.enums.PatternActionType; import ru.serega6531.packmate.model.enums.PatternDirectionType; import ru.serega6531.packmate.model.enums.PatternSearchType; import javax.persistence.*; -import java.util.List; +import java.util.Objects; -@Data -@ToString(exclude = "matchedStreams") +@Getter +@Setter +@RequiredArgsConstructor +@ToString @Entity @GenericGenerator( name = "pattern_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "pattern_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "pattern_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "1") } ) public class Pattern { @Id @GeneratedValue(generator = "pattern_generator") - private int id; + private Integer id; private boolean enabled; + @Column(nullable = false) private String name; + @Column(nullable = false) private String value; + @Column(nullable = false) private String color; // для вставки в css + @Enumerated + @Column(nullable = false) private PatternSearchType searchType; + @Enumerated + @Column(nullable = false) private PatternDirectionType directionType; + @Enumerated + @Column(nullable = false) private PatternActionType actionType; private Integer serviceId; private long searchStartTimestamp; - @ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY) - private List matchedStreams; - + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Pattern pattern = (Pattern) o; + return id != null && Objects.equals(id, pattern.id); + } + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/Stream.java b/src/main/java/ru/serega6531/packmate/model/Stream.java index ad48d1b..4c2eb19 100644 --- a/src/main/java/ru/serega6531/packmate/model/Stream.java +++ b/src/main/java/ru/serega6531/packmate/model/Stream.java @@ -1,55 +1,85 @@ package ru.serega6531.packmate.model; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.ToString; +import org.hibernate.Hibernate; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; import ru.serega6531.packmate.model.enums.Protocol; import javax.persistence.*; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; -@Data -@ToString(exclude = "packets") +@Getter +@Setter +@ToString +@RequiredArgsConstructor @Entity @GenericGenerator( name = "stream_generator", strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", parameters = { - @org.hibernate.annotations.Parameter(name = "sequence_name", value = "stream_seq"), - @org.hibernate.annotations.Parameter(name = "initial_value", value = "1"), - @org.hibernate.annotations.Parameter(name = "increment_size", value = "1") + @Parameter(name = "sequence_name", value = "stream_seq"), + @Parameter(name = "initial_value", value = "1"), + @Parameter(name = "increment_size", value = "1000"), + @Parameter(name = "optimizer", value = "hilo") } ) +@Table(indexes = {@Index(name = "stream_id_desc_index", columnList = "id DESC")}) public class Stream { @Id @GeneratedValue(generator = "stream_generator") private Long id; - @Column(name = "service_id") + @Column(name = "service_id", nullable = false) private int service; + @Enumerated + @Column(nullable = false) private Protocol protocol; - @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL) + @OneToMany(mappedBy = "stream", cascade = CascadeType.ALL, orphanRemoval = true) @OrderBy("id") + @ToString.Exclude private List packets; private long startTimestamp; private long endTimestamp; - @ManyToMany + @ManyToMany(fetch = FetchType.EAGER) + @JoinTable( + name = "stream_found_patterns", + joinColumns = @JoinColumn(name = "stream_id"), + inverseJoinColumns = @JoinColumn(name = "pattern_id") + ) + @ToString.Exclude private Set foundPatterns = new HashSet<>(); private boolean favorite; - @Column(columnDefinition = "smallint") + @Column(nullable = false, columnDefinition = "smallint") private int ttl; @Column(columnDefinition = "char(3)") private String userAgentHash; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false; + Stream stream = (Stream) o; + return id != null && Objects.equals(id, stream.id); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } } diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java new file mode 100644 index 0000000..4f97c7e --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/model/pojo/PacketPagination.java @@ -0,0 +1,14 @@ +package ru.serega6531.packmate.model.pojo; + +import lombok.Data; +import org.jetbrains.annotations.Nullable; + +@Data +public class PacketPagination { + + @Nullable + private Long startingFrom; + + private int pageSize; + +} diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java deleted file mode 100644 index 0b2baec..0000000 --- a/src/main/java/ru/serega6531/packmate/model/pojo/Pagination.java +++ /dev/null @@ -1,20 +0,0 @@ -package ru.serega6531.packmate.model.pojo; - -import lombok.Data; -import org.springframework.data.domain.Sort; -import ru.serega6531.packmate.model.Pattern; - -@Data -public class Pagination { - - private Sort.Direction direction; - - private long startingFrom; - - private int pageSize; - - private boolean favorites; // только для стримов, определяет, искать только избранные стримы или все - - private Pattern pattern; // только для стримов, если не null, ищем стримы с этим паттерном - -} diff --git a/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java b/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java new file mode 100644 index 0000000..d78606f --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/model/pojo/StreamPagination.java @@ -0,0 +1,20 @@ +package ru.serega6531.packmate.model.pojo; + +import lombok.Data; +import org.jetbrains.annotations.Nullable; +import ru.serega6531.packmate.model.Pattern; + +@Data +public class StreamPagination { + + @Nullable + private Long startingFrom; + + private int pageSize; + + private boolean favorites; // определяет, искать только избранные стримы или все + + @Nullable + private Pattern pattern; // если не null, ищем стримы с этим паттерном + +} diff --git a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java index 1f8eaa8..b7d619a 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/AbstractPcapWorker.java @@ -5,6 +5,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.pcap4j.core.BpfProgram; import org.pcap4j.core.PacketListener; import org.pcap4j.core.PcapHandle; @@ -46,8 +47,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { private final ListMultimap unfinishedUdpStreams = ArrayListMultimap.create(); // в следующих мапах в значениях находится srcIp соответствующего пакета - private final SetMultimap> fins = HashMultimap.create(); - private final SetMultimap> acks = HashMultimap.create(); + private final SetMultimap> fins = HashMultimap.create(); + private final SetMultimap> acks = HashMultimap.create(); protected AbstractPcapWorker(ServicesService servicesService, StreamService streamService, @@ -182,8 +183,8 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { * Udp не имеет фазы закрытия, поэтому закрывается только по таймауту */ private void checkTcpTermination(boolean ack, boolean fin, boolean rst, - ImmutablePair sourceIpAndPort, - ImmutablePair destIpAndPort, + Pair sourceIpAndPort, + Pair destIpAndPort, UnfinishedStream stream) { if (fin) { @@ -256,7 +257,6 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener { } @Override - @SneakyThrows public void setFilter(String filter) { this.filter = filter; applyFilter(); diff --git a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java index 730d552..0f9e178 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/FilePcapWorker.java @@ -79,4 +79,9 @@ public class FilePcapWorker extends AbstractPcapWorker { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.PCAP_STOPPED, null)); } + + @Override + public String getExecutorState() { + return "inline"; + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java index 0802e91..f225851 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/LivePcapWorker.java @@ -10,7 +10,9 @@ import ru.serega6531.packmate.service.ServicesService; import ru.serega6531.packmate.service.StreamService; import java.net.UnknownHostException; -import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; @Slf4j public class LivePcapWorker extends AbstractPcapWorker { @@ -30,7 +32,7 @@ public class LivePcapWorker extends AbstractPcapWorker { BasicThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern("pcap-processor").build(); - processorExecutorService = Executors.newSingleThreadExecutor(factory); + processorExecutorService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), factory); } public void start() throws PcapNativeException { @@ -63,4 +65,8 @@ public class LivePcapWorker extends AbstractPcapWorker { log.info("Intercept stopped"); } + @Override + public String getExecutorState() { + return processorExecutorService.toString(); + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java index 0cad155..f26c48b 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/NoOpPcapWorker.java @@ -24,4 +24,9 @@ public class NoOpPcapWorker implements PcapWorker { @Override public void setFilter(String filter) { } + + @Override + public String getExecutorState() { + return "none"; + } } diff --git a/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java b/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java index 91f8fde..eb81c78 100644 --- a/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java +++ b/src/main/java/ru/serega6531/packmate/pcap/PcapWorker.java @@ -20,4 +20,5 @@ public interface PcapWorker { void setFilter(String filter); + String getExecutorState(); } diff --git a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java index be306d0..1af9dce 100644 --- a/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java +++ b/src/main/java/ru/serega6531/packmate/repository/StreamRepository.java @@ -1,15 +1,28 @@ package ru.serega6531.packmate.repository; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.*; +import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.Stream; +import javax.persistence.QueryHint; +import java.util.List; + public interface StreamRepository extends JpaRepository, JpaSpecificationExecutor { @Query("UPDATE Stream SET favorite = :favorite WHERE id = :id") @Modifying void setFavorite(long id, boolean favorite); + long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold); + + @Query("SELECT DISTINCT p FROM Packet p " + + "LEFT JOIN FETCH p.matches " + + "WHERE p.stream.id = :streamId " + + "AND (:startingFrom IS NULL OR p.id > :startingFrom) " + + "ORDER BY p.id" + ) + @QueryHints(@QueryHint(name = org.hibernate.jpa.QueryHints.HINT_PASS_DISTINCT_THROUGH, value = "false")) + List getPackets(long streamId, Long startingFrom, Pageable pageable); + } diff --git a/src/main/java/ru/serega6531/packmate/service/PatternService.java b/src/main/java/ru/serega6531/packmate/service/PatternService.java index 6ff6ab6..3af8832 100644 --- a/src/main/java/ru/serega6531/packmate/service/PatternService.java +++ b/src/main/java/ru/serega6531/packmate/service/PatternService.java @@ -15,6 +15,7 @@ import ru.serega6531.packmate.model.pojo.PatternDto; import ru.serega6531.packmate.model.pojo.SubscriptionMessage; import ru.serega6531.packmate.repository.PatternRepository; +import javax.annotation.PostConstruct; import java.time.Instant; import java.util.*; import java.util.concurrent.TimeUnit; @@ -40,7 +41,10 @@ public class PatternService { this.streamService = streamService; this.subscriptionService = subscriptionService; this.modelMapper = modelMapper; + } + @PostConstruct + public void init() { repository.findAll().forEach(p -> patterns.put(p.getId(), p)); log.info("Loaded {} patterns", patterns.size()); } diff --git a/src/main/java/ru/serega6531/packmate/service/PcapService.java b/src/main/java/ru/serega6531/packmate/service/PcapService.java index 528fc77..b633055 100644 --- a/src/main/java/ru/serega6531/packmate/service/PcapService.java +++ b/src/main/java/ru/serega6531/packmate/service/PcapService.java @@ -41,17 +41,27 @@ public class PcapService { } public void updateFilter(Collection services) { - final String ports = services.stream() - .map(CtfService::getPort) - .map(p -> "port " + p) - .collect(Collectors.joining(" or ")); + String filter; - final String format = "(tcp or udp) and (%s)"; - String filter = String.format(format, ports); + if (services.isEmpty()) { + filter = "tcp or udp"; + } else { + final String ports = services.stream() + .map(CtfService::getPort) + .map(p -> "port " + p) + .collect(Collectors.joining(" or ")); + + final String format = "(tcp or udp) and (%s)"; + filter = String.format(format, ports); + } log.debug("New filter: " + filter); worker.setFilter(filter); } + public String getExecutorState() { + return worker.getExecutorState(); + } + } diff --git a/src/main/java/ru/serega6531/packmate/service/ServicesService.java b/src/main/java/ru/serega6531/packmate/service/ServicesService.java index 014e950..3d8454b 100644 --- a/src/main/java/ru/serega6531/packmate/service/ServicesService.java +++ b/src/main/java/ru/serega6531/packmate/service/ServicesService.java @@ -12,12 +12,10 @@ import ru.serega6531.packmate.model.pojo.ServiceDto; import ru.serega6531.packmate.model.pojo.SubscriptionMessage; import ru.serega6531.packmate.repository.ServiceRepository; +import javax.annotation.PostConstruct; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; @Service @Slf4j @@ -43,7 +41,10 @@ public class ServicesService { this.pcapService = pcapService; this.modelMapper = modelMapper; this.localIp = InetAddress.getByName(localIpString); + } + @PostConstruct + public void init() { repository.findAll().forEach(s -> services.put(s.getPort(), s)); log.info("Loaded {} services", services.size()); } @@ -78,7 +79,7 @@ public class ServicesService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.DELETE_SERVICE, port)); - pcapService.updateFilter(findAll()); + updateFilter(); } public CtfService save(CtfService service) { @@ -89,11 +90,15 @@ public class ServicesService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.SAVE_SERVICE, toDto(saved))); - pcapService.updateFilter(findAll()); + updateFilter(); return saved; } + public void updateFilter() { + pcapService.updateFilter(findAll()); + } + public ServiceDto toDto(CtfService service) { return modelMapper.map(service, ServiceDto.class); } diff --git a/src/main/java/ru/serega6531/packmate/service/StreamService.java b/src/main/java/ru/serega6531/packmate/service/StreamService.java index 2e6a48f..d43101f 100644 --- a/src/main/java/ru/serega6531/packmate/service/StreamService.java +++ b/src/main/java/ru/serega6531/packmate/service/StreamService.java @@ -1,10 +1,12 @@ package ru.serega6531.packmate.service; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; import org.modelmapper.ModelMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.data.jpa.domain.Specification; import org.springframework.scheduling.annotation.Async; @@ -20,6 +22,7 @@ import ru.serega6531.packmate.repository.StreamRepository; import ru.serega6531.packmate.service.optimization.RsaKeysHolder; import ru.serega6531.packmate.service.optimization.StreamOptimizer; +import java.time.ZonedDateTime; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -90,6 +93,15 @@ public class StreamService { } } + countingService.countStream(service.getPort(), packets.size()); + + List optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + + if (isStreamIgnored(optimizedPackets, service)) { + log.debug("New stream is ignored"); + return false; + } + Optional firstIncoming = packets.stream() .filter(Packet::isIncoming) .findFirst(); @@ -101,27 +113,19 @@ public class StreamService { stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp()); stream.setService(service.getPort()); - countingService.countStream(service.getPort(), packets.size()); + String userAgentHash = getUserAgentHash(optimizedPackets); + stream.setUserAgentHash(userAgentHash); - packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream(); + Set foundPatterns = matchPatterns(optimizedPackets, service); + stream.setFoundPatterns(foundPatterns); + stream.setPackets(optimizedPackets); - if (isStreamIgnored(packets, service)) { - log.debug("New stream is ignored"); - return false; + for (Packet packet : optimizedPackets) { + packet.setStream(stream); } - processUserAgent(packets, stream); Stream savedStream = save(stream); - for (Packet packet : packets) { - packet.setStream(savedStream); - } - - Set foundPatterns = matchPatterns(packets, service); - savedStream.setFoundPatterns(foundPatterns); - savedStream.setPackets(packets); - savedStream = save(savedStream); - subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream))); return true; } @@ -143,7 +147,7 @@ public class StreamService { subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId())); } - private void processUserAgent(List packets, Stream stream) { + private String getUserAgentHash(List packets) { String ua = null; for (Packet packet : packets) { String content = packet.getContentString(); @@ -155,7 +159,9 @@ public class StreamService { } if (ua != null) { - stream.setUserAgentHash(calculateUserAgentHash(ua)); + return calculateUserAgentHash(ua); + } else { + return null; } } @@ -238,8 +244,17 @@ public class StreamService { return saved; } - public Optional find(long id) { - return repository.findById(id); + public List getPackets(long streamId, @Nullable Long startingFrom, int pageSize) { +// long safeStartingFrom = startingFrom != null ? startingFrom : 0; + return repository.getPackets(streamId, startingFrom, Pageable.ofSize(pageSize)); + } + + /** + * @return Number of deleted rows + */ + @Transactional + public long cleanupOldStreams(ZonedDateTime before) { + return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000); } @Transactional @@ -247,14 +262,13 @@ public class StreamService { repository.setFavorite(id, favorite); } - public List findAll(Pagination pagination, Optional service, boolean onlyFavorites) { - PageRequest page = PageRequest.of(0, pagination.getPageSize(), pagination.getDirection(), "id"); + public List findAll(StreamPagination pagination, Optional service, boolean onlyFavorites) { + PageRequest page = PageRequest.of(0, pagination.getPageSize(), Sort.Direction.DESC, "id"); - Specification spec; - if (pagination.getDirection() == Sort.Direction.ASC) { - spec = streamIdGreaterThan(pagination.getStartingFrom()); - } else { - spec = streamIdLessThan(pagination.getStartingFrom()); + Specification spec = Specification.where(null); + + if (pagination.getStartingFrom() != null) { + spec = spec.and(streamIdLessThan(pagination.getStartingFrom())); } if (service.isPresent()) { @@ -293,10 +307,6 @@ public class StreamService { return (root, query, cb) -> cb.equal(root.get("favorite"), true); } - private Specification streamIdGreaterThan(long id) { - return (root, query, cb) -> cb.greaterThan(root.get("id"), id); - } - private Specification streamIdLessThan(long id) { return (root, query, cb) -> cb.lessThan(root.get("id"), id); } diff --git a/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java b/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java new file mode 100644 index 0000000..d022c0d --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/tasks/ExecutorStateLoggerTask.java @@ -0,0 +1,23 @@ +package ru.serega6531.packmate.tasks; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.service.PcapService; + +@Component +@Slf4j +public class ExecutorStateLoggerTask { + + private final PcapService service; + + public ExecutorStateLoggerTask(PcapService service) { + this.service = service; + } + + @Scheduled(fixedDelayString = "PT1M", initialDelayString = "PT1M") + public void cleanup() { + log.info("Executor state: {}", service.getExecutorState()); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java b/src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java new file mode 100644 index 0000000..7635a3b --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/tasks/OldStreamsCleanupTask.java @@ -0,0 +1,34 @@ +package ru.serega6531.packmate.tasks; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.service.StreamService; + +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; + +@Component +@Slf4j +@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true") +public class OldStreamsCleanupTask { + + private final StreamService service; + private final int oldStreamsThreshold; + + public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) { + this.service = service; + this.oldStreamsThreshold = oldStreamsThreshold; + } + + @Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M") + public void cleanup() { + ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES); + log.info("Cleaning up old non-favorite streams (before {})", before); + long deleted = service.cleanupOldStreams(before); + log.info("Deleted {} rows", deleted); + } + +} diff --git a/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java b/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java new file mode 100644 index 0000000..16e923c --- /dev/null +++ b/src/main/java/ru/serega6531/packmate/tasks/StartupListener.java @@ -0,0 +1,40 @@ +package ru.serega6531.packmate.tasks; + +import org.pcap4j.core.PcapNativeException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import ru.serega6531.packmate.model.enums.CaptureMode; +import ru.serega6531.packmate.service.PcapService; +import ru.serega6531.packmate.service.ServicesService; + +@Component +public class StartupListener { + + @Value("${enable-capture}") + private boolean enableCapture; + + @Value("${capture-mode}") + private CaptureMode captureMode; + + private final PcapService pcapService; + private final ServicesService servicesService; + + public StartupListener(PcapService pcapService, ServicesService servicesService) { + this.pcapService = pcapService; + this.servicesService = servicesService; + } + + @EventListener(ApplicationReadyEvent.class) + public void afterStartup() throws PcapNativeException { + if (enableCapture) { + servicesService.updateFilter(); + + if (captureMode == CaptureMode.LIVE) { + pcapService.start(); + } + } + } + +} diff --git a/src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java b/src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java similarity index 97% rename from src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java rename to src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java index d251fb7..466074b 100644 --- a/src/main/java/ru/serega6531/packmate/TimeoutStreamsSaver.java +++ b/src/main/java/ru/serega6531/packmate/tasks/TimeoutStreamsSaver.java @@ -1,4 +1,4 @@ -package ru.serega6531.packmate; +package ru.serega6531.packmate.tasks; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 4abd2b1..3272e0b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,19 +9,25 @@ spring: ddl-auto: update properties: hibernate: + jdbc: + batch_size: 20 + order_inserts: true temp: use_jdbc_metadata_defaults: false database-platform: org.hibernate.dialect.PostgreSQLDialect enable-capture: true -capture-mode: LIVE # LIVE, FILE +capture-mode: LIVE # LIVE, FILE, VIEW interface-name: enp0s31f6 pcap-file: file.pcap local-ip: "192.168.0.125" account-login: BinaryBears account-password: 123456 -udp-stream-timeout: 20 # секунд -tcp-stream-timeout: 40 # секунд -timeout-stream-check-interval: 10 # секунд +udp-stream-timeout: 20 # seconds +tcp-stream-timeout: 40 # seconds +timeout-stream-check-interval: 10 # seconds +old-streams-cleanup-enabled: true +old-streams-threshold: 240 # minutes +cleanup-interval: 5 # minutes ignore-empty-packets: true \ No newline at end of file diff --git a/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java b/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java index f550c0f..f4e450f 100644 --- a/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java +++ b/src/test/java/ru/serega6531/packmate/PatternMatcherTest.java @@ -1,7 +1,7 @@ package ru.serega6531.packmate; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.springframework.util.Assert; import ru.serega6531.packmate.model.FoundPattern; import ru.serega6531.packmate.model.Pattern; import ru.serega6531.packmate.model.enums.PatternSearchType; @@ -27,13 +27,14 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("[a-f]{3}"); pattern.setSearchType(PatternSearchType.REGEX); final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); } @Test @@ -50,13 +51,14 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("bbb"); pattern.setSearchType(PatternSearchType.SUBSTRING); final PatternMatcher matcher = new PatternMatcher(content.getBytes(), List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); } @Test @@ -73,13 +75,27 @@ public class PatternMatcherTest { .build()); final Pattern pattern = new Pattern(); + pattern.setId(1); pattern.setValue("AAaa"); pattern.setSearchType(PatternSearchType.SUBBYTES); final PatternMatcher matcher = new PatternMatcher(content, List.of(pattern)); final Set matches = matcher.findMatches(); - Assert.isTrue(matches.equals(correctMatches), "Incorrect search: " + matches.toString()); + assertMatchesAreCorrect(correctMatches, matches); + } + + private void assertMatchesAreCorrect(Set correctMatches, Set foundMatches) { + Assertions.assertEquals(correctMatches.size(), foundMatches.size()); + + Assertions.assertTrue(correctMatches.stream().allMatch(correct -> + foundMatches.stream().anyMatch(found -> matchesEqual(correct, found)) + )); + } + + private boolean matchesEqual(FoundPattern one, FoundPattern two) { + return one.getStartPosition() == two.getStartPosition() && + one.getEndPosition() == two.getEndPosition(); } }