0xb00b5/0xb00b5-packmate
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Возможность удалять старые данные

Browse Source
This commit is contained in:
sshkurov
2022-01-29 23:07:55 +03:00
parent 7cf7233c4b
commit 560e0d444f
11 changed files with 183 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/ru/serega6531/packmate/OldStreamsCleanupTask.java Normal file
View File

@@ -0,0 +1,34 @@
package ru.serega6531.packmate;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import ru.serega6531.packmate.service.StreamService;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
@Component
@Slf4j
@ConditionalOnProperty(name = "old-streams-cleanup-enabled", havingValue = "true")
public class OldStreamsCleanupTask {
private final StreamService service;
private final int oldStreamsThreshold;
public OldStreamsCleanupTask(StreamService service, @Value("${old-streams-threshold}") int oldStreamsThreshold) {
this.service = service;
this.oldStreamsThreshold = oldStreamsThreshold;
}
@Scheduled(fixedDelayString = "PT${cleanup-interval}M", initialDelayString = "PT1M")
public void cleanup() {
ZonedDateTime before = ZonedDateTime.now().minus(oldStreamsThreshold, ChronoUnit.MINUTES);
log.info("Cleaning up old streams (before {})", before);
long deleted = service.cleanupOldStreams(before);
log.info("Deleted {} rows", deleted);
}
}

23
src/main/java/ru/serega6531/packmate/model/CtfService.java
View File

@@ -1,18 +1,23 @@
package ru.serega6531.packmate.model;
import lombok.Data;
import lombok.*;
import org.hibernate.Hibernate;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import java.util.Objects;
@Data
@Getter
@Setter
@ToString
@RequiredArgsConstructor
@Entity
@Table(name = "service")
public class CtfService {
@Id
private int port;
private Integer port;
private String name;
@@ -28,4 +33,16 @@ public class CtfService {
private boolean parseWebSockets;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
CtfService that = (CtfService) o;
return port != null && Objects.equals(port, that.port);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}

17
src/main/java/ru/serega6531/packmate/model/FoundPattern.java
View File

@@ -1,9 +1,11 @@
package ru.serega6531.packmate.model;
import lombok.*;
import org.hibernate.Hibernate;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Objects;
@Entity
@GenericGenerator(
@@ -20,12 +22,11 @@ import javax.persistence.*;
@Builder
@Getter
@ToString
@EqualsAndHashCode(exclude = "packet")
public class FoundPattern {
@Id
@GeneratedValue(generator = "found_pattern_generator")
private int id;
private Long id;
@ManyToOne
@JoinColumn(name = "packet_id", nullable = false)
@@ -38,6 +39,18 @@ public class FoundPattern {
private int endPosition;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
FoundPattern that = (FoundPattern) o;
return id != null && Objects.equals(id, that.id);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}

27
src/main/java/ru/serega6531/packmate/model/Packet.java
View File

@@ -1,12 +1,17 @@
package ru.serega6531.packmate.model;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import org.hibernate.Hibernate;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Objects;
import java.util.Set;
@Data
@Getter
@Setter
@RequiredArgsConstructor
@Entity
@GenericGenerator(
name = "packet_generator",
@@ -17,11 +22,9 @@ import java.util.Set;
@org.hibernate.annotations.Parameter(name = "increment_size", value = "1")
}
)
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Table(indexes = { @Index(name = "stream_id_index", columnList = "stream_id") })
@EqualsAndHashCode(exclude = "stream")
public class Packet {
@Id
@@ -34,9 +37,9 @@ public class Packet {
@Transient
private int ttl;
@ManyToOne
@JoinColumn(name = "stream_id", nullable = false)
private Stream stream;
@Column(name = "stream_id")
@JsonIgnore
private Long streamId;
@OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, fetch = FetchType.EAGER)
private Set<FoundPattern> matches;
@@ -62,4 +65,16 @@ public class Packet {
return "Packet(id=" + id + ", content=" + getContentString() + ")";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Packet packet = (Packet) o;
return id != null && Objects.equals(id, packet.id);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}

32
src/main/java/ru/serega6531/packmate/model/Pattern.java
View File

@@ -1,17 +1,21 @@
package ru.serega6531.packmate.model;
import lombok.Data;
import lombok.ToString;
import lombok.*;
import org.hibernate.Hibernate;
import org.hibernate.annotations.GenericGenerator;
import ru.serega6531.packmate.model.enums.PatternActionType;
import ru.serega6531.packmate.model.enums.PatternDirectionType;
import ru.serega6531.packmate.model.enums.PatternSearchType;
import javax.persistence.*;
import java.util.List;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import java.util.Objects;
@Data
@ToString(exclude = "matchedStreams")
@Getter
@Setter
@RequiredArgsConstructor
@ToString
@Entity
@GenericGenerator(
name = "pattern_generator",
@@ -26,7 +30,7 @@ public class Pattern {
@Id
@GeneratedValue(generator = "pattern_generator")
private int id;
private Integer id;
private boolean enabled;
@@ -46,8 +50,16 @@ public class Pattern {
private long searchStartTimestamp;
@ManyToMany(mappedBy = "foundPatterns", fetch = FetchType.LAZY)
private List<Stream> matchedStreams;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Pattern pattern = (Pattern) o;
return id != null && Objects.equals(id, pattern.id);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}

33
src/main/java/ru/serega6531/packmate/model/Stream.java
View File

@@ -1,17 +1,20 @@
package ru.serega6531.packmate.model;
import lombok.Data;
import lombok.ToString;
import lombok.*;
import org.hibernate.Hibernate;
import org.hibernate.annotations.GenericGenerator;
import ru.serega6531.packmate.model.enums.Protocol;
import javax.persistence.*;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@Data
@ToString(exclude = "packets")
@Getter
@Setter
@ToString
@RequiredArgsConstructor
@Entity
@GenericGenerator(
name = "stream_generator",
@@ -33,8 +36,10 @@ public class Stream {
private Protocol protocol;
@OneToMany(mappedBy = "stream", cascade = CascadeType.ALL)
@OneToMany(cascade = CascadeType.ALL, orphanRemoval = true)
@JoinColumn(name = "stream_id")
@OrderBy("id")
@ToString.Exclude
private List<Packet> packets;
private long startTimestamp;
@@ -42,6 +47,12 @@ public class Stream {
private long endTimestamp;
@ManyToMany
@JoinTable(
name = "stream_found_patterns",
joinColumns = @JoinColumn(name = "stream_id"),
inverseJoinColumns = @JoinColumn(name = "pattern_id")
)
@ToString.Exclude
private Set<Pattern> foundPatterns = new HashSet<>();
private boolean favorite;
@@ -52,4 +63,16 @@ public class Stream {
@Column(columnDefinition = "char(3)")
private String userAgentHash;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || Hibernate.getClass(this) != Hibernate.getClass(o)) return false;
Stream stream = (Stream) o;
return id != null && Objects.equals(id, stream.id);
}
@Override
public int hashCode() {
return getClass().hashCode();
}
}

2
src/main/java/ru/serega6531/packmate/repository/StreamRepository.java
View File

@@ -12,4 +12,6 @@ public interface StreamRepository extends JpaRepository<Stream, Long>, JpaSpecif
@Modifying
void setFavorite(long id, boolean favorite);
long deleteByEndTimestampBeforeAndFavoriteIsFalse(long threshold);
}

46
src/main/java/ru/serega6531/packmate/service/StreamService.java
View File

@@ -20,6 +20,7 @@ import ru.serega6531.packmate.repository.StreamRepository;
import ru.serega6531.packmate.service.optimization.RsaKeysHolder;
import ru.serega6531.packmate.service.optimization.StreamOptimizer;
import java.time.ZonedDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
@@ -90,6 +91,15 @@ public class StreamService {
}
}
countingService.countStream(service.getPort(), packets.size());
List<Packet> optimizedPackets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
if (isStreamIgnored(optimizedPackets, service)) {
log.debug("New stream is ignored");
return false;
}
Optional<Packet> firstIncoming = packets.stream()
.filter(Packet::isIncoming)
.findFirst();
@@ -101,27 +111,15 @@ public class StreamService {
stream.setEndTimestamp(packets.get(packets.size() - 1).getTimestamp());
stream.setService(service.getPort());
countingService.countStream(service.getPort(), packets.size());
String userAgentHash = getUserAgentHash(optimizedPackets);
stream.setUserAgentHash(userAgentHash);
packets = new StreamOptimizer(keysHolder, service, packets).optimizeStream();
Set<Pattern> foundPatterns = matchPatterns(optimizedPackets, service);
stream.setFoundPatterns(foundPatterns);
stream.setPackets(optimizedPackets);
if (isStreamIgnored(packets, service)) {
log.debug("New stream is ignored");
return false;
}
processUserAgent(packets, stream);
Stream savedStream = save(stream);
for (Packet packet : packets) {
packet.setStream(savedStream);
}
Set<Pattern> foundPatterns = matchPatterns(packets, service);
savedStream.setFoundPatterns(foundPatterns);
savedStream.setPackets(packets);
savedStream = save(savedStream);
subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.NEW_STREAM, streamToDto(savedStream)));
return true;
}
@@ -143,7 +141,7 @@ public class StreamService {
subscriptionService.broadcast(new SubscriptionMessage(SubscriptionMessageType.FINISH_LOOKBACK, pattern.getId()));
}
private void processUserAgent(List<Packet> packets, Stream stream) {
private String getUserAgentHash(List<Packet> packets) {
String ua = null;
for (Packet packet : packets) {
String content = packet.getContentString();
@@ -155,7 +153,9 @@ public class StreamService {
}
if (ua != null) {
stream.setUserAgentHash(calculateUserAgentHash(ua));
return calculateUserAgentHash(ua);
} else {
return null;
}
}
@@ -242,6 +242,14 @@ public class StreamService {
return repository.findById(id);
}
/**
* @return Number of deleted rows
*/
@Transactional
public long cleanupOldStreams(ZonedDateTime before) {
return repository.deleteByEndTimestampBeforeAndFavoriteIsFalse(before.toEpochSecond() * 1000);
}
@Transactional
public void setFavorite(long id, boolean favorite) {
repository.setFavorite(id, favorite);

11
src/main/resources/application.yml
View File

@@ -15,13 +15,16 @@ spring:
enable-capture: true
capture-mode: LIVE # LIVE, FILE
capture-mode: LIVE # LIVE, FILE, VIEW
interface-name: enp0s31f6
pcap-file: file.pcap
local-ip: "192.168.0.125"
account-login: BinaryBears
account-password: 123456
udp-stream-timeout: 20 # секунд
tcp-stream-timeout: 40 # секунд
timeout-stream-check-interval: 10 # секунд
udp-stream-timeout: 20 # seconds
tcp-stream-timeout: 40 # seconds
timeout-stream-check-interval: 10 # seconds
old-streams-cleanup-enabled: true
old-streams-threshold: 240 # minutes
cleanup-interval: 5 # minutes
ignore-empty-packets: true