From ee54671930ec52fd6a6ed8858dbb89e9f04c3e28 Mon Sep 17 00:00:00 2001 From: Domingo Dirutigliano Date: Sat, 15 Mar 2025 10:01:33 +0100 Subject: [PATCH] correctly supporting HTTP compression + more compression types supported --- Dockerfile | 6 +- fgex-lib/firegex/nfproxy/models/http.py | 60 ++++++++++++++++--- fgex-lib/requirements.txt | 2 + frontend/src/components/DocsButton.tsx | 22 +++---- .../src/components/NFProxy/NFProxyDocs.tsx | 10 +++- .../src/components/NFRegex/NFRegexDocs.tsx | 4 +- .../components/PortHijack/PortHijackDocs.tsx | 4 +- 7 files changed, 83 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 451aa22..192d503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,13 +16,17 @@ RUN bun run build FROM --platform=$TARGETARCH registry.fedoraproject.org/fedora:latest RUN dnf -y update && dnf install -y python3.13-devel @development-tools gcc-c++ \ libnetfilter_queue-devel libnfnetlink-devel libmnl-devel libcap-ng-utils nftables \ - vectorscan-devel libtins-devel python3-nftables libpcap-devel boost-devel uv + vectorscan-devel libtins-devel python3-nftables libpcap-devel boost-devel uv git RUN mkdir -p /execute/modules WORKDIR /execute ADD ./backend/requirements.txt /execute/requirements.txt RUN uv pip install --no-cache --system -r /execute/requirements.txt + +RUN git clone https://github.com/domysh/brotli && cd brotli && pip install . && cd .. && rm -rf brotli && \ + git clone https://github.com/domysh/python-zstd --recurse && cd python-zstd && pip install . && cd .. && rm -rf python-zstd + COPY ./fgex-lib /execute/fgex-lib RUN uv pip install --no-cache --system ./fgex-lib diff --git a/fgex-lib/firegex/nfproxy/models/http.py b/fgex-lib/firegex/nfproxy/models/http.py index 7dec859..082c93d 100644 --- a/fgex-lib/firegex/nfproxy/models/http.py +++ b/fgex-lib/firegex/nfproxy/models/http.py @@ -6,6 +6,11 @@ from firegex.nfproxy.internals.models import FullStreamAction, ExceptionAction from dataclasses import dataclass, field from collections import deque from typing import Type +from zstd import ZSTD_uncompress +import gzip +import io +import zlib +import brotli @dataclass class InternalHTTPMessage: @@ -14,6 +19,7 @@ class InternalHTTPMessage: headers: dict[str, str] = field(default_factory=dict) lheaders: dict[str, str] = field(default_factory=dict) # lowercase copy of the headers body: bytes|None = field(default=None) + body_decoded: bool = field(default=False) headers_complete: bool = field(default=False) message_complete: bool = field(default=False) status: str|None = field(default=None) @@ -114,14 +120,52 @@ class InternalCallbackHandler(): def on_message_complete(self): self.msg.body = self.buffers._body_buffer self.buffers._body_buffer = b"" - try: - if "gzip" in self.content_encoding.lower(): - import gzip - import io - with gzip.GzipFile(fileobj=io.BytesIO(self.msg.body)) as f: - self.msg.body = f.read() - except Exception as e: - print(f"Error decompressing gzip: {e}: skipping", flush=True) + encodings = [ele.strip() for ele in self.content_encoding.lower().split(",")] + decode_success = True + decoding_body = self.msg.body + for enc in reversed(encodings): + if not enc: + continue + if enc == "deflate": + try: + decompress = zlib.decompressobj(-zlib.MAX_WBITS) + decoding_body = decompress.decompress(decoding_body) + decoding_body += decompress.flush() + except Exception as e: + print(f"Error decompressing deflate: {e}: skipping", flush=True) + decode_success = False + break + elif enc == "br": + try: + decoding_body = brotli.decompress(decoding_body) + except Exception as e: + print(f"Error decompressing brotli: {e}: skipping", flush=True) + decode_success = False + break + elif enc == "gzip": + try: + if "gzip" in self.content_encoding.lower(): + with gzip.GzipFile(fileobj=io.BytesIO(decoding_body)) as f: + decoding_body = f.read() + except Exception as e: + print(f"Error decompressing gzip: {e}: skipping", flush=True) + decode_success = False + break + elif enc == "zstd": + try: + decoding_body = ZSTD_uncompress(decoding_body) + except Exception as e: + print(f"Error decompressing zstd: {e}: skipping", flush=True) + decode_success = False + break + else: + decode_success = False + break + + if decode_success: + self.msg.body = decoding_body + self.msg.body_decoded = True + self.msg.message_complete = True self.has_begun = False if not self._packet_to_stream(): diff --git a/fgex-lib/requirements.txt b/fgex-lib/requirements.txt index 15e2e37..82f6118 100644 --- a/fgex-lib/requirements.txt +++ b/fgex-lib/requirements.txt @@ -1,6 +1,8 @@ typer==0.15.2 pydantic>=2 typing-extensions>=4.7.1 +zstd # waiting for pull request to be merged +brotli # waiting for pull request to be merged watchfiles fgex pyllhttp diff --git a/frontend/src/components/DocsButton.tsx b/frontend/src/components/DocsButton.tsx index 6be3537..0c4821e 100644 --- a/frontend/src/components/DocsButton.tsx +++ b/frontend/src/components/DocsButton.tsx @@ -1,4 +1,4 @@ -import { ActionIcon, ActionIconProps, Box, Modal, ScrollArea, Title, Tooltip } from "@mantine/core"; +import { ActionIcon, ActionIconProps, Box, Container, Modal, ScrollArea, ScrollAreaAutosize, Title, Tooltip } from "@mantine/core"; import { useState } from "react"; import { FaBookBookmark } from "react-icons/fa6"; import { NFRegexDocs } from "./NFRegex/NFRegexDocs"; @@ -23,15 +23,17 @@ export const DocsButton = ({ doc, ...props }: { doc: EnumToPrimitiveUnion setOpen(false)} fullScreen title={ Firegex Docs 📕 } scrollAreaComponent={ScrollArea.Autosize}> - { - doc == DocType.NFREGEX ? - : - doc == DocType.NFPROXY ? - : - doc == DocType.PORTHIJACK ? - : - Docs not found - } + + { + doc == DocType.NFREGEX ? + : + doc == DocType.NFPROXY ? + : + doc == DocType.PORTHIJACK ? + : + Docs not found + } + } diff --git a/frontend/src/components/NFProxy/NFProxyDocs.tsx b/frontend/src/components/NFProxy/NFProxyDocs.tsx index bcddcb0..e5332db 100644 --- a/frontend/src/components/NFProxy/NFProxyDocs.tsx +++ b/frontend/src/components/NFProxy/NFProxyDocs.tsx @@ -82,7 +82,7 @@ const TCPBadge = () => { export const NFProxyDocs = () => { return ( - + <> 🌐 Netfilter Proxy Documentation 📖 Overview @@ -249,6 +249,9 @@ export const NFProxyDocs = () => { body: The body of the request (read only). It's None if the body has not arrived yet. + + body_decoded: By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False. + http_version: The http version of the request (read only) @@ -308,6 +311,9 @@ export const NFProxyDocs = () => { body: The body of the response (read only). It's None if the body has not arrived yet. + + body_decoded: By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False. + http_version: The http version of the response (read only) @@ -430,6 +436,6 @@ export const NFProxyDocs = () => { Here's a pyfilter code commented example: - + ); }; diff --git a/frontend/src/components/NFRegex/NFRegexDocs.tsx b/frontend/src/components/NFRegex/NFRegexDocs.tsx index aee093b..07d724c 100644 --- a/frontend/src/components/NFRegex/NFRegexDocs.tsx +++ b/frontend/src/components/NFRegex/NFRegexDocs.tsx @@ -2,7 +2,7 @@ import { Container, Title, Text, List } from "@mantine/core"; export const NFRegexDocs = () => { return ( - + <> 📡 Netfilter Regex Documentation 📖 Overview @@ -64,6 +64,6 @@ export const NFRegexDocs = () => { - + ); }; diff --git a/frontend/src/components/PortHijack/PortHijackDocs.tsx b/frontend/src/components/PortHijack/PortHijackDocs.tsx index 1ddd48b..07f7640 100644 --- a/frontend/src/components/PortHijack/PortHijackDocs.tsx +++ b/frontend/src/components/PortHijack/PortHijackDocs.tsx @@ -4,7 +4,7 @@ import { HELP_NFPROXY_SIM } from "../NFProxy/NFProxyDocs" export const PortHijackDocs = () => { - return + return <> ⚡️ Hijack port to proxy 📖 Overview @@ -33,5 +33,5 @@ export const PortHijackDocs = () => { but externaly the packets exists as connections to the original service. This mangle is done only for external packet arriving from the external ip indicated, localhost traffic won't be touched. - + } \ No newline at end of file