Store dates in Instant

This commit is contained in:
Sergey Shkurov
2023-08-01 21:40:23 +04:00
parent 938031f1de
commit db8ffbfcdd
11 changed files with 45 additions and 26 deletions

View File

@@ -6,6 +6,8 @@ import org.hibernate.annotations.GenericGenerator;
import org.hibernate.annotations.Parameter; import org.hibernate.annotations.Parameter;
import jakarta.persistence.*; import jakarta.persistence.*;
import java.time.Instant;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
@@ -45,7 +47,7 @@ public class Packet {
@OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, orphanRemoval = true) @OneToMany(mappedBy = "packet", cascade = CascadeType.ALL, orphanRemoval = true)
private Set<FoundPattern> matches; private Set<FoundPattern> matches;
private long timestamp; private Instant timestamp;
private boolean incoming; // true если от клиента к серверу, иначе false private boolean incoming; // true если от клиента к серверу, иначе false

View File

@@ -10,6 +10,8 @@ import org.hibernate.annotations.Parameter;
import ru.serega6531.packmate.model.enums.Protocol; import ru.serega6531.packmate.model.enums.Protocol;
import jakarta.persistence.*; import jakarta.persistence.*;
import java.time.Instant;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@@ -49,9 +51,9 @@ public class Stream {
@ToString.Exclude @ToString.Exclude
private List<Packet> packets; private List<Packet> packets;
private long startTimestamp; private Instant startTimestamp;
private long endTimestamp; private Instant endTimestamp;
@ManyToMany @ManyToMany
@JoinTable( @JoinTable(

View File

@@ -1,7 +1,9 @@
package ru.serega6531.packmate.model.pojo; package ru.serega6531.packmate.model.pojo;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data; import lombok.Data;
import java.time.Instant;
import java.util.Set; import java.util.Set;
@Data @Data
@@ -9,7 +11,8 @@ public class PacketDto {
private Long id; private Long id;
private Set<FoundPatternDto> matches; private Set<FoundPatternDto> matches;
private long timestamp; @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS)
private Instant timestamp;
private boolean incoming; private boolean incoming;
private boolean ungzipped; private boolean ungzipped;
private boolean webSocketParsed; private boolean webSocketParsed;

View File

@@ -1,8 +1,10 @@
package ru.serega6531.packmate.model.pojo; package ru.serega6531.packmate.model.pojo;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data; import lombok.Data;
import ru.serega6531.packmate.model.enums.Protocol; import ru.serega6531.packmate.model.enums.Protocol;
import java.time.Instant;
import java.util.Set; import java.util.Set;
@Data @Data
@@ -11,8 +13,10 @@ public class StreamDto {
private Long id; private Long id;
private int service; private int service;
private Protocol protocol; private Protocol protocol;
private long startTimestamp; @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS)
private long endTimestamp; private Instant startTimestamp;
@JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS)
private Instant endTimestamp;
private Set<Integer> foundPatternsIds; private Set<Integer> foundPatternsIds;
private boolean favorite; private boolean favorite;
private int ttl; private int ttl;

View File

@@ -18,6 +18,8 @@ import ru.serega6531.packmate.service.StreamService;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
@@ -82,7 +84,7 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
return; return;
} }
final long time = pcap.getTimestamp().getTime(); final Instant time = pcap.getTimestamp().toInstant();
if (rawPacket.contains(TcpPacket.class)) { if (rawPacket.contains(TcpPacket.class)) {
final TcpPacket packet = rawPacket.get(TcpPacket.class); final TcpPacket packet = rawPacket.get(TcpPacket.class);
@@ -93,7 +95,7 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
} }
} }
private void gotTcpPacket(TcpPacket packet, InetAddress sourceIp, InetAddress destIp, int ttl, long time) { private void gotTcpPacket(TcpPacket packet, InetAddress sourceIp, InetAddress destIp, int ttl, Instant time) {
final TcpPacket.TcpHeader tcpHeader = packet.getHeader(); final TcpPacket.TcpHeader tcpHeader = packet.getHeader();
int sourcePort = tcpHeader.getSrcPort().valueAsInt(); int sourcePort = tcpHeader.getSrcPort().valueAsInt();
int destPort = tcpHeader.getDstPort().valueAsInt(); int destPort = tcpHeader.getDstPort().valueAsInt();
@@ -127,7 +129,7 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
} }
} }
private void gotUdpPacket(UdpPacket packet, InetAddress sourceIp, InetAddress destIp, int ttl, long time) { private void gotUdpPacket(UdpPacket packet, InetAddress sourceIp, InetAddress destIp, int ttl, Instant time) {
final UdpPacket.UdpHeader udpHeader = packet.getHeader(); final UdpPacket.UdpHeader udpHeader = packet.getHeader();
int sourcePort = udpHeader.getSrcPort().valueAsInt(); int sourcePort = udpHeader.getSrcPort().valueAsInt();
int destPort = udpHeader.getDstPort().valueAsInt(); int destPort = udpHeader.getDstPort().valueAsInt();
@@ -156,7 +158,7 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
} }
} }
private UnfinishedStream addNewPacket(InetAddress sourceIp, InetAddress destIp, long time, private UnfinishedStream addNewPacket(InetAddress sourceIp, InetAddress destIp, Instant time,
int sourcePort, int destPort, int ttl, byte[] content, Protocol protocol) { int sourcePort, int destPort, int ttl, byte[] content, Protocol protocol) {
var incoming = destIp.equals(localIp); var incoming = destIp.equals(localIp);
var stream = new UnfinishedStream(sourceIp, destIp, sourcePort, destPort, protocol); var stream = new UnfinishedStream(sourceIp, destIp, sourcePort, destPort, protocol);
@@ -222,17 +224,17 @@ public abstract class AbstractPcapWorker implements PcapWorker, PacketListener {
@Override @Override
@SneakyThrows @SneakyThrows
public int closeTimeoutStreams(Protocol protocol, long timeoutMillis) { public int closeTimeoutStreams(Protocol protocol, Duration timeout) {
return processorExecutorService.submit(() -> { return processorExecutorService.submit(() -> {
int streamsClosed = 0; int streamsClosed = 0;
final long time = System.currentTimeMillis(); final Instant time = Instant.now();
final var streams = (protocol == Protocol.TCP) ? this.unfinishedTcpStreams : this.unfinishedUdpStreams; final var streams = (protocol == Protocol.TCP) ? this.unfinishedTcpStreams : this.unfinishedUdpStreams;
final var oldStreams = Multimaps.asMap(streams).entrySet().stream() final var oldStreams = Multimaps.asMap(streams).entrySet().stream()
.filter(entry -> { .filter(entry -> {
final var packets = entry.getValue(); final var packets = entry.getValue();
return time - packets.get(packets.size() - 1).getTimestamp() > timeoutMillis; return Duration.between(packets.get(packets.size() - 1).getTimestamp(), time).compareTo(timeout) > 0;
}) })
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

View File

@@ -2,6 +2,8 @@ package ru.serega6531.packmate.pcap;
import ru.serega6531.packmate.model.enums.Protocol; import ru.serega6531.packmate.model.enums.Protocol;
import java.time.Duration;
public class NoOpPcapWorker implements PcapWorker { public class NoOpPcapWorker implements PcapWorker {
@Override @Override
public void start() { public void start() {
@@ -16,7 +18,7 @@ public class NoOpPcapWorker implements PcapWorker {
} }
@Override @Override
public int closeTimeoutStreams(Protocol protocol, long timeoutMillis) { public int closeTimeoutStreams(Protocol protocol, Duration timeout) {
return 0; return 0;
} }

View File

@@ -3,6 +3,8 @@ package ru.serega6531.packmate.pcap;
import org.pcap4j.core.PcapNativeException; import org.pcap4j.core.PcapNativeException;
import ru.serega6531.packmate.model.enums.Protocol; import ru.serega6531.packmate.model.enums.Protocol;
import java.time.Duration;
public interface PcapWorker { public interface PcapWorker {
void start() throws PcapNativeException; void start() throws PcapNativeException;
@@ -16,7 +18,7 @@ public interface PcapWorker {
/** /**
* Выполняется в потоке обработчика * Выполняется в потоке обработчика
*/ */
int closeTimeoutStreams(Protocol protocol, long timeoutMillis); int closeTimeoutStreams(Protocol protocol, Duration timeout);
void setFilter(String filter); void setFilter(String filter);

View File

@@ -5,6 +5,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import ru.serega6531.packmate.model.enums.CaptureMode; import ru.serega6531.packmate.model.enums.CaptureMode;
import java.net.InetAddress; import java.net.InetAddress;
import java.time.Duration;
@ConfigurationProperties("packmate") @ConfigurationProperties("packmate")
public record PackmateProperties( public record PackmateProperties(
@@ -24,8 +25,8 @@ public record PackmateProperties(
) {} ) {}
public record TimeoutProperties( public record TimeoutProperties(
int udpStreamTimeout, Duration udpStreamTimeout,
int tcpStreamTimeout, Duration tcpStreamTimeout,
int checkInterval int checkInterval
){} ){}

View File

@@ -3,6 +3,7 @@ package ru.serega6531.packmate.service.optimization;
import ru.serega6531.packmate.model.Packet; import ru.serega6531.packmate.model.Packet;
import ru.serega6531.packmate.utils.PacketUtils; import ru.serega6531.packmate.utils.PacketUtils;
import java.time.Instant;
import java.util.List; import java.util.List;
public class PacketsMerger { public class PacketsMerger {
@@ -42,7 +43,7 @@ public class PacketsMerger {
*/ */
private void compress(List<Packet> packets, int start, int end) { private void compress(List<Packet> packets, int start, int end) {
final List<Packet> cut = packets.subList(start, end); final List<Packet> cut = packets.subList(start, end);
final long timestamp = cut.get(0).getTimestamp(); final Instant timestamp = cut.get(0).getTimestamp();
final boolean httpProcessed = cut.stream().anyMatch(Packet::isHttpProcessed); final boolean httpProcessed = cut.stream().anyMatch(Packet::isHttpProcessed);
final boolean webSocketParsed = cut.stream().anyMatch(Packet::isWebSocketParsed); final boolean webSocketParsed = cut.stream().anyMatch(Packet::isWebSocketParsed);
final boolean tlsDecrypted = cut.get(0).isTlsDecrypted(); final boolean tlsDecrypted = cut.get(0).isTlsDecrypted();

View File

@@ -5,11 +5,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.serega6531.packmate.properties.PackmateProperties;
import ru.serega6531.packmate.model.enums.Protocol; import ru.serega6531.packmate.model.enums.Protocol;
import ru.serega6531.packmate.pcap.PcapWorker; import ru.serega6531.packmate.pcap.PcapWorker;
import ru.serega6531.packmate.properties.PackmateProperties;
import java.util.concurrent.TimeUnit; import java.time.Duration;
@Component @Component
@Slf4j @Slf4j
@@ -17,15 +17,15 @@ import java.util.concurrent.TimeUnit;
public class TimeoutStreamsSaver { public class TimeoutStreamsSaver {
private final PcapWorker pcapWorker; private final PcapWorker pcapWorker;
private final long udpStreamTimeoutMillis; private final Duration udpStreamTimeoutMillis;
private final long tcpStreamTimeoutMillis; private final Duration tcpStreamTimeoutMillis;
@Autowired @Autowired
public TimeoutStreamsSaver(PcapWorker pcapWorker, public TimeoutStreamsSaver(PcapWorker pcapWorker,
PackmateProperties properties) { PackmateProperties properties) {
this.pcapWorker = pcapWorker; this.pcapWorker = pcapWorker;
this.udpStreamTimeoutMillis = TimeUnit.SECONDS.toMillis(properties.timeout().udpStreamTimeout()); this.udpStreamTimeoutMillis = properties.timeout().udpStreamTimeout();
this.tcpStreamTimeoutMillis = TimeUnit.SECONDS.toMillis(properties.timeout().tcpStreamTimeout()); this.tcpStreamTimeoutMillis = properties.timeout().tcpStreamTimeout();
} }
@Scheduled(fixedRateString = "PT${packmate.timeout.check-interval}S", initialDelayString = "PT${packmate.timeout.check-interval}S") @Scheduled(fixedRateString = "PT${packmate.timeout.check-interval}S", initialDelayString = "PT${packmate.timeout.check-interval}S")

View File

@@ -28,8 +28,8 @@ packmate:
account-login: BinaryBears account-login: BinaryBears
account-password: 123456 account-password: 123456
timeout: timeout:
udp-stream-timeout: 20 # seconds udp-stream-timeout: 20S
tcp-stream-timeout: 40 # seconds tcp-stream-timeout: 40S
check-interval: 10 # seconds check-interval: 10 # seconds
cleanup: cleanup:
enabled: true enabled: true