correctly supporting HTTP compression + more compression types supported

Dockerfile

@@ -16,13 +16,17 @@ RUN bun run build
 FROM --platform=$TARGETARCH registry.fedoraproject.org/fedora:latest
 RUN dnf -y update && dnf install -y python3.13-devel @development-tools gcc-c++ \
     libnetfilter_queue-devel libnfnetlink-devel libmnl-devel libcap-ng-utils nftables \
-    vectorscan-devel libtins-devel python3-nftables libpcap-devel boost-devel uv
+    vectorscan-devel libtins-devel python3-nftables libpcap-devel boost-devel uv git
 
 RUN mkdir -p /execute/modules
 WORKDIR /execute
 
 ADD ./backend/requirements.txt /execute/requirements.txt
 RUN uv pip install --no-cache --system -r /execute/requirements.txt
+
+RUN git clone https://github.com/domysh/brotli && cd brotli && pip install . && cd .. && rm -rf brotli && \
+    git clone https://github.com/domysh/python-zstd --recurse && cd python-zstd && pip install . && cd .. && rm -rf python-zstd
+
 COPY ./fgex-lib /execute/fgex-lib
 RUN uv pip install --no-cache --system ./fgex-lib
 

fgex-lib/firegex/nfproxy/models/http.py

@@ -6,6 +6,11 @@ from firegex.nfproxy.internals.models import FullStreamAction, ExceptionAction
 from dataclasses import dataclass, field
 from collections import deque
 from typing import Type
+from zstd import ZSTD_uncompress
+import gzip
+import io
+import zlib
+import brotli
 
 @dataclass
 class InternalHTTPMessage:
@@ -14,6 +19,7 @@ class InternalHTTPMessage:
     headers: dict[str, str] = field(default_factory=dict)
     lheaders: dict[str, str] = field(default_factory=dict) # lowercase copy of the headers
     body: bytes|None = field(default=None)
+    body_decoded: bool = field(default=False)
     headers_complete: bool = field(default=False)
     message_complete: bool = field(default=False)
     status: str|None = field(default=None)
@@ -114,14 +120,52 @@ class InternalCallbackHandler():
     def on_message_complete(self):
         self.msg.body = self.buffers._body_buffer
         self.buffers._body_buffer = b""
+        encodings = [ele.strip() for ele in self.content_encoding.lower().split(",")]
+        decode_success = True
+        decoding_body = self.msg.body
+        for enc in reversed(encodings):
+            if not enc:
+                continue
+            if enc == "deflate":
+                try:
+                    decompress = zlib.decompressobj(-zlib.MAX_WBITS)
+                    decoding_body = decompress.decompress(decoding_body)
+                    decoding_body += decompress.flush()
+                except Exception as e:
+                    print(f"Error decompressing deflate: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "br":
+                try:
+                    decoding_body = brotli.decompress(decoding_body)
+                except Exception as e:
+                    print(f"Error decompressing brotli: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "gzip":      
                 try:
                     if "gzip" in self.content_encoding.lower():
-                import gzip
+                        with gzip.GzipFile(fileobj=io.BytesIO(decoding_body)) as f:
-                import io
+                            decoding_body = f.read()
-                with gzip.GzipFile(fileobj=io.BytesIO(self.msg.body)) as f:
-                    self.msg.body = f.read()
                 except Exception as e:
                     print(f"Error decompressing gzip: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            elif enc == "zstd":
+                try:
+                    decoding_body = ZSTD_uncompress(decoding_body)
+                except Exception as e:
+                    print(f"Error decompressing zstd: {e}: skipping", flush=True)
+                    decode_success = False
+                    break
+            else:
+                decode_success = False
+                break
+        
+        if decode_success:
+            self.msg.body = decoding_body
+            self.msg.body_decoded = True
+
         self.msg.message_complete = True
         self.has_begun = False
         if not self._packet_to_stream():

fgex-lib/requirements.txt

@@ -1,6 +1,8 @@
 typer==0.15.2
 pydantic>=2
 typing-extensions>=4.7.1
+zstd    # waiting for pull request to be merged
+brotli  # waiting for pull request to be merged
 watchfiles
 fgex
 pyllhttp

frontend/src/components/DocsButton.tsx

@@ -1,4 +1,4 @@
-import { ActionIcon, ActionIconProps, Box, Modal, ScrollArea, Title, Tooltip } from "@mantine/core";
+import { ActionIcon, ActionIconProps, Box, Container, Modal, ScrollArea, ScrollAreaAutosize, Title, Tooltip } from "@mantine/core";
 import { useState } from "react";
 import { FaBookBookmark } from "react-icons/fa6";
 import { NFRegexDocs } from "./NFRegex/NFRegexDocs";
@@ -23,6 +23,7 @@ export const DocsButton = ({ doc, ...props }: { doc: EnumToPrimitiveUnion<DocTyp
         <Modal opened={open} onClose={() => setOpen(false)} fullScreen title={
             <Title order={2}>Firegex Docs 📕</Title>
         } scrollAreaComponent={ScrollArea.Autosize}>
+            <Container style={{padding: "1rem", maxWidth:"90vw"}}>
                 {
                     doc == DocType.NFREGEX ? 
                         <NFRegexDocs />:
@@ -32,6 +33,7 @@ export const DocsButton = ({ doc, ...props }: { doc: EnumToPrimitiveUnion<DocTyp
                         <PortHijackDocs />:
                     <Title order={3}>Docs not found</Title>
                 }
+            </Container>
         </Modal>
     </Box>
 }

frontend/src/components/NFProxy/NFProxyDocs.tsx

@@ -82,7 +82,7 @@ const TCPBadge = () => {
 
 export const NFProxyDocs = () => {
   return (
-    <Container size="xl">
+    <>
       <Title order={1}>🌐 Netfilter Proxy Documentation</Title>
 
       <Title order={2} mt="xl" mb="sm">📖 Overview</Title>
@@ -249,6 +249,9 @@ export const NFProxyDocs = () => {
           <List.Item>
             <strong>body: </strong> The body of the request (read only). It's None if the body has not arrived yet.
           </List.Item>
+          <List.Item>
+            <strong>body_decoded: </strong> By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False.
+          </List.Item>
           <List.Item>
             <strong>http_version: </strong> The http version of the request (read only)
           </List.Item>
@@ -308,6 +311,9 @@ export const NFProxyDocs = () => {
           <List.Item>
             <strong>body: </strong> The body of the response (read only). It's None if the body has not arrived yet.
           </List.Item>
+          <List.Item>
+            <strong>body_decoded: </strong> By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False.
+          </List.Item>
           <List.Item>
             <strong>http_version: </strong> The http version of the response (read only)
           </List.Item>
@@ -430,6 +436,6 @@ export const NFProxyDocs = () => {
         Here's a pyfilter code commented example:
         <CodeHighlight code={EXAMPLE_PYFILTER} language="python" my="sm"/>
       </Text>
-    </Container>
+    </>
   );
 };

frontend/src/components/NFRegex/NFRegexDocs.tsx

@@ -2,7 +2,7 @@ import { Container, Title, Text, List } from "@mantine/core";
 
 export const NFRegexDocs = () => {
   return (
-    <Container size="xl">
+    <>
         <Title order={1}>📡 Netfilter Regex Documentation</Title>
 
         <Title order={2} mt="xl" mb="sm">📖 Overview</Title>
@@ -64,6 +64,6 @@ export const NFRegexDocs = () => {
                 </Text>
             </List.Item>
         </List>
-    </Container>
+    </>
   );
 };

frontend/src/components/PortHijack/PortHijackDocs.tsx

@@ -4,7 +4,7 @@ import { HELP_NFPROXY_SIM } from "../NFProxy/NFProxyDocs"
 
 
 export const PortHijackDocs = () => {
-    return <Container size="xl">
+    return <>
           <Title order={1}>⚡️ Hijack port to proxy</Title>
     
           <Title order={2} mt="xl" mb="sm">📖 Overview</Title>
@@ -33,5 +33,5 @@ export const PortHijackDocs = () => {
             but externaly the packets exists as connections to the original service. This mangle is done only for external packet arriving from the external ip indicated, localhost traffic won't be touched.
           </Text>
           <Space h="xl" />
-        </Container>
+        </>
 }