From 6853960b6da56b3ec18965dbe045dfc377a4022b Mon Sep 17 00:00:00 2001 From: Domingo Dirutigliano Date: Sun, 22 Jun 2025 17:42:21 +0200 Subject: [PATCH] feature: HttpFullRequest and HttpFullResponse implementation --- fgex-lib/README.md | 13 +- fgex-lib/firegex/nfproxy/models/__init__.py | 37 +- fgex-lib/firegex/nfproxy/models/http.py | 44 +- .../src/components/NFProxy/NFProxyDocs.tsx | 1146 +++++++++++------ tests/nfproxy_test.py | 487 +++++-- 5 files changed, 1233 insertions(+), 494 deletions(-) diff --git a/fgex-lib/README.md b/fgex-lib/README.md index 1d35fe6..45241f8 100644 --- a/fgex-lib/README.md +++ b/fgex-lib/README.md @@ -50,7 +50,7 @@ def my_filter(raw_packet: RawPacket): #Logging filter ## Data handlers -### RawPacket +### RawPacket ```python from firegex.nfproxy import RawPacket ``` @@ -111,6 +111,12 @@ from firegex.nfproxy import HttpRequestHeader ``` This handler will be called only when the request headers are complete. It will receive a HttpRequestHeader object with the same properties as HttpRequest. +### HttpFullRequest +```python +from firegex.nfproxy import HttpFullRequest +``` +This handler will be called only when the request is complete. It will receive a HttpFullRequest object with the same properties as HttpRequest. + ### HttpResponse ```python from firegex.nfproxy import HttpResponse @@ -138,3 +144,8 @@ from firegex.nfproxy import HttpResponseHeader ``` This handler will be called only when the response headers are complete. It will receive a HttpResponseHeader object with the same properties as HttpResponse. +### HttpFullResponse +```python +from firegex.nfproxy import HttpFullResponse +``` +This handler will be called only when the response is complete. It will receive a HttpFullResponse object with the same properties as HttpResponse. diff --git a/fgex-lib/firegex/nfproxy/models/__init__.py b/fgex-lib/firegex/nfproxy/models/__init__.py index 9da4c6a..c990fcf 100644 --- a/fgex-lib/firegex/nfproxy/models/__init__.py +++ b/fgex-lib/firegex/nfproxy/models/__init__.py @@ -1,5 +1,17 @@ -from firegex.nfproxy.models.tcp import TCPInputStream, TCPOutputStream, TCPClientStream, TCPServerStream -from firegex.nfproxy.models.http import HttpRequest, HttpResponse, HttpRequestHeader, HttpResponseHeader +from firegex.nfproxy.models.tcp import ( + TCPInputStream, + TCPOutputStream, + TCPClientStream, + TCPServerStream, +) +from firegex.nfproxy.models.http import ( + HttpRequest, + HttpResponse, + HttpRequestHeader, + HttpResponseHeader, + HttpFullRequest, + HttpFullResponse, +) from firegex.nfproxy.internals.data import RawPacket from enum import Enum @@ -17,15 +29,28 @@ type_annotations_associations = { HttpResponse: HttpResponse._fetch_packet, HttpRequestHeader: HttpRequestHeader._fetch_packet, HttpResponseHeader: HttpResponseHeader._fetch_packet, - } + HttpFullRequest: HttpFullRequest._fetch_packet, + HttpFullResponse: HttpFullResponse._fetch_packet, + }, } + class Protocols(Enum): TCP = "tcp" HTTP = "http" + __all__ = [ "RawPacket", - "TCPInputStream", "TCPOutputStream", "TCPClientStream", "TCPServerStream", - "HttpRequest", "HttpResponse", "HttpRequestHeader", "HttpResponseHeader", "Protocols" -] \ No newline at end of file + "TCPInputStream", + "TCPOutputStream", + "TCPClientStream", + "TCPServerStream", + "HttpRequest", + "HttpResponse", + "HttpRequestHeader", + "HttpResponseHeader", + "HttpFullRequest", + "HttpFullResponse", + "Protocols", +] diff --git a/fgex-lib/firegex/nfproxy/models/http.py b/fgex-lib/firegex/nfproxy/models/http.py index c450d61..ea08f1b 100644 --- a/fgex-lib/firegex/nfproxy/models/http.py +++ b/fgex-lib/firegex/nfproxy/models/http.py @@ -73,6 +73,7 @@ class InternalCallbackHandler: messages: deque[InternalHTTPMessage] = deque() _ws_extentions = None _ws_raised_error = False + release_message_headers = True def reset_data(self): self.msg = InternalHTTPMessage() @@ -604,6 +605,7 @@ class InternalBasicHttpMetaClass: if ( not internal_data.call_mem["headers_were_set"] and parser.msg.headers_complete + and parser.release_message_headers ): messages_tosend.append( parser.msg @@ -641,7 +643,7 @@ class HttpRequest(InternalBasicHttpMetaClass): @staticmethod def _parser_class() -> str: - return "full_http" + return "http_module" def __repr__(self): return f"" @@ -664,12 +666,46 @@ class HttpResponse(InternalBasicHttpMetaClass): @staticmethod def _parser_class() -> str: - return "full_http" + return "http_module" def __repr__(self): return f"" +class HttpFullRequest(HttpRequest): + """ + HTTP Request handler + This data handler will be called when the request data is complete + """ + + def _contructor_hook(self): + self._parser.release_message_headers = False + + @staticmethod + def _parser_class() -> str: + return "http_full" + + def __repr__(self): + return f"" + + +class HttpFullResponse(HttpResponse): + """ + HTTP Response handler + This data handler will be called when the response data is complete + """ + + def _contructor_hook(self): + self._parser.release_message_headers = False + + @staticmethod + def _parser_class() -> str: + return "http_full" + + def __repr__(self): + return f"" + + class HttpRequestHeader(HttpRequest): """ HTTP Request Header handler @@ -681,7 +717,7 @@ class HttpRequestHeader(HttpRequest): @staticmethod def _parser_class() -> str: - return "header_http" + return "http_header" class HttpResponseHeader(HttpResponse): @@ -695,4 +731,4 @@ class HttpResponseHeader(HttpResponse): @staticmethod def _parser_class() -> str: - return "header_http" + return "http_header" diff --git a/frontend/src/components/NFProxy/NFProxyDocs.tsx b/frontend/src/components/NFProxy/NFProxyDocs.tsx index 838a215..a12ee86 100644 --- a/frontend/src/components/NFProxy/NFProxyDocs.tsx +++ b/frontend/src/components/NFProxy/NFProxyDocs.tsx @@ -1,9 +1,18 @@ import { CodeHighlight } from "@mantine/code-highlight"; -import { Container, Title, Text, List, Code, Space, Badge, Box } from "@mantine/core"; +import { + Container, + Title, + Text, + List, + Code, + Space, + Badge, + Box, +} from "@mantine/core"; import { CgEditBlackPoint } from "react-icons/cg"; import { EXAMPLE_PYFILTER } from "./utils"; -const IMPORT_CODE_EXAMPLE = `from firegex.nfproxy import pyfilter, ACCEPT, REJECT` +const IMPORT_CODE_EXAMPLE = `from firegex.nfproxy import pyfilter, ACCEPT, REJECT`; const FOO_FILTER_CODE = `from firegex.nfproxy import pyfilter, ACCEPT, REJECT @@ -17,8 +26,7 @@ def none_filter(): # This is a filter that does nothing useless_function() return ACCEPT -` - +`; const TYPING_ARGS_EXAMPLE = `from firegex.nfproxy import pyfilter, ACCEPT, REJECT from firegex.nfproxy.models import HttpRequest @@ -28,7 +36,7 @@ def filter_with_args(http_request: HttpRequest) -> int: if http_request.body: if b"ILLEGAL" in http_request.body: return REJECT -` +`; const IMPORT_FULL_ACTION_STREAM = `from firegex.nfproxy import FullStreamAction @@ -39,7 +47,7 @@ class FullStreamAction(Enum): ACCEPT = 1 REJECT = 2 DROP = 3 -` +`; const ENUM_IMPORT_AND_DEFINITION = `from firegex.nfproxy import ExceptionAction @@ -50,7 +58,7 @@ class ExceptionAction(Enum): DROP = 1 # Drop the connection that caused the exception REJECT = 2 # Reject the connection that caused the exception NOACTION = 3 # Do nothing, the excpetion will be signaled and the stream will be accepted without calling anymore the pyfilters (for the current stream) -` +`; export const HELP_NFPROXY_SIM = `➤ fgex nfproxy -h @@ -69,391 +77,757 @@ export const HELP_NFPROXY_SIM = `➤ fgex nfproxy -h │ --from-port INTEGER The port of the local server [default: 7474] │ │ -6 Use IPv6 for the connection │ │ --help -h Show this message and exit. │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯` +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯`; const HttpBadge = () => { - return HTTP -} + return ( + + HTTP + + ); +}; const TCPBadge = () => { - return TCP -} - + return ( + + TCP + + ); +}; export const NFProxyDocs = () => { - return ( - <> - 🌐 Netfilter Proxy Documentation + return ( + <> + 🌐 Netfilter Proxy Documentation - 📖 Overview - - Netfilter Proxy is a simulated proxy that leverages nfqueue to intercept network packets. - It follows a similar workflow to NFRegex but introduces Python-based filtering capabilities, - providing users with the flexibility to upload custom filters. - + + 📖 Overview + + + Netfilter Proxy is a simulated proxy that leverages{" "} + + nfqueue + {" "} + to intercept network packets. It follows a similar workflow to + NFRegex but introduces Python-based filtering capabilities, + providing users with the flexibility to upload custom filters. + - ⚙️ How to use Netfilter Proxy - - To use Netfilter Proxy, simply create and upload a Python filter. The filter is passed to the C++ binary, - which then processes packets using the provided logic. This allows you to tailor the filtering behavior - to your needs. - - 💡 How to write pyfilters? - - First of all install the firegex lib and update it running pip install -U fgex. - After that you can use firegex module. - - With this code we imported the pyfilter decorator and the ACCEPT and REJECT statements.
- Let's create a first (useless) filter to see the syntax: - - You see that the filter must be decorated with the pyfilter decorator and must return a statement about how to manage that packet. -
- You can save every data about the current flow in the global variables, the code you write will be executed only once for flow. The globals variables are isolated between flows. - For each packet the filter functions will be called with the required paramethers and using the same globals as before. -
- Saving data in globals of other modules is not recommended, because that memory is shared by the flows managed by the same thread and lead to unexpected behaviors. -
- Global variables that starts with '__firegex' are reserved for internal use, don't use them. -
- You can manage when the function is called and also getting some data specifying some paramethers, using type decorators. - Default values of the paramethers will be ignored, also kvargs values will be ignored. -
- Functions with no type decorator are considered invalid pyfilters! -
- - In this code we are filtering all the http requests that contains the word "ILLEGAL" in the body. All the other packets will be accepted (default behavior). - The function will be called only if at least internally teh HTTP request header has been parsed, and also when the body will be parsed. -
- If we have multiple paramether, the function will be called only if with the packet arrived is possible to build all the paramethers. - - 🔧 How can I test the filter? - - You can test your filter by using fgex command installed by firegex lib: This will run a local proxy to a remote destination with the filter you specified. -
- This can be done by running for instance: fgex nfproxy test_http.py 127.0.0.1 8080 --proto http - - You don't need to restart the proxy every time you change the filter, the filter will be reloaded automatically. - - 📦 Packet Statements - - Here there are all the statments you can return from a filter: - - ACCEPT: The packet will be accepted and forwarded to the destination. (default if None is returned) - REJECT: The connection will be closed and all the packets will be dropped. - DROP: This packet and all the following will be dropped. (This not simulate a connection closure) - UNSTABLE_MANGLE: The packet will be modified and forwarded. You can edit the packet only with RawPacket data handler. (This is an unstable statement, use it carefully) - - - ⚙️ Data Structures - - Here there are all the data structure you can use for your filters: - - - <CgEditBlackPoint style={{marginBottom: -3}}/> RawPacket - - This data is the raw packet processed by nfqueue. It contains: - - - - - data: The raw packet data assembled by libtins (read only). - - - is_input: It's true if the packet is incoming, false if it's outgoing. (read only) - - - is_ipv6: It's true if the packet is IPv6, false if it's IPv4. (read only) - - - is_tcp: It's true if the packet is TCP, false if it's UDP. (read only) - - - l4_size: The size of l4 payload (read only) - - - raw_packet_header_len: The size of the raw packet header (read only) - - - raw_packet: The raw packet data with ip and TCP header. You can edit all the packet content and it will be modified if you send - the UNSTABLE_MANGLE statement. Be careful, beacause the associated layer 4 data can be different from 'data' filed that instead arrives from libtins. - When you edit this field, l4_size and l4_data will be updated automatically. - - - l4_data: The l4 payload data, directly taken by the raw packet. You can edit all the packet content and it will be modified if you send - the UNSTABLE_MANGLE statement. Be careful, beacause the associated layer 4 data can be different from 'data' filed that instead arrives from libtins. When you edit this field, l4_size and raw_packet will be updated automatically. - - - - - <CgEditBlackPoint style={{marginBottom: -3}}/> TCPInputStream (alias: TCPClientStream) - - This data is the TCP input stream: this handler is called only on is_input=True packets. The filters that handles this data will be called only in this case. - - - - - data: The entire stream in input direction. (read only) - - - total_stream_size: The size of the entire stream in input direction. (read only) - - - is_ipv6: It's true if the stream is IPv6, false if it's IPv4. (read only) - - - - - <CgEditBlackPoint style={{marginBottom: -3}}/> TCPOutputStream (alias TCPServerStream) - - This data is the TCP output stream: this handler is called only on is_input=False packets. The filters that handles this data will be called only in this case. - - - - - data: The entire stream in output direction. (read only) - - - total_stream_size: The size of the entire stream in output direction. (read only) - - - is_ipv6: It's true if the stream is IPv6, false if it's IPv4. (read only) - - - - - <CgEditBlackPoint style={{marginBottom: -3}}/> HttpRequest - - This data is the Http request processed by nfqueue. This handler can be called twice per request: once when the http headers are complete, and once when the body is complete. - If the http data arrives in 1 single TCP packet, this handler will be called once - - - - - url: The url of the request (read only) - - - headers: The headers of the request (read only). The keys and values are exactly the same as the original request (case sensitive). (values can be list in case the same header field is repeated) - - - get_header(key:str, default = None): A function that returns the value of a header: it matches the key without case sensitivity. If the header is not found, it returns the default value. (if the same header field is repeated, its value is concatenated with a comma, this function will never return a list) - - - user_agent: The user agent of the request (read only) - - - content_encoding: The content encoding of the request (read only) - - - content_length: The content length of the request (read only) - - - body: The body of the request (read only). It's None if the body has not arrived yet. - - - body_decoded: By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False. - - - http_version: The http version of the request (read only) - - - keep_alive: It's true if the connection was marked for keep alive, false if it's not. (read only) - - - should_upgrade: It's true if the connection should be upgraded, false if it's not. (read only) - - - upgrading_to_h2: It's true if the connection is upgrading to h2, false if it's not. (read only) - - - ws_stream: It's a list of websockets.frames.Frame decoded (permessage-deflate is supported). (read only) [docs] - - - upgrading_to_ws: It's true if the connection is upgrading to ws, false if it's not. (read only) - - - method: The method of the request (read only) - - - headers_complete: It's true if the headers are complete, false if they are not. (read only) - - - message_complete: It's true if the message is complete, false if it's not. (read only) - - - total_size: The size of the entire http request (read only) - - - stream: It's the buffer that contains the stream of the websocket traffic in input. This is used only if should_upgrade is True. (read only) - - - - - <CgEditBlackPoint style={{marginBottom: -3}}/> HttpRequestHeader - - Same as HttpRequest, but this handler is called only when the headers are complete and body is not buffered. Body will always be None - - <CgEditBlackPoint style={{marginBottom: -3}}/> HttpResponse - - This data is the Http response processed by nfqueue. This handler can be called twice per response: once when the http headers are complete, and once when the body is complete. - If the http data arrives in 1 single TCP packet, this handler will be called once - - - - - url: The url of the response (read only) - - - headers: The headers of the response (read only). The keys and values are exactly the same as the original response (case sensitive). (values can be list in case the same header field is repeated) - - - get_header(key:str, default = None): A function that returns the value of a header: it matches the key without case sensitivity. If the header is not found, it returns the default value. (if the same header field is repeated, its value is concatenated with a comma, this function will never return a list) - - - user_agent: The user agent of the response (read only) - - - content_encoding: The content encoding of the response (read only) - - - content_length: The content length of the response (read only) - - - body: The body of the response (read only). It's None if the body has not arrived yet. - - - body_decoded: By default the body will be decoded following the content encoding. gzip, br, deflate and zstd are supported. If the decoding fails and body is not None this paramether will be False. - - - http_version: The http version of the response (read only) - - - keep_alive: It's true if the connection was marked for keep alive, false if it's not. (read only) - - - should_upgrade: It's true if the connection should be upgraded, false if it's not. (read only) - - - upgrading_to_h2: It's true if the connection is upgrading to h2, false if it's not. (read only) - - - ws_stream: It's a list of websockets.frames.Frame decoded (permessage-deflate is supported). (read only) [docs] - - - upgrading_to_ws: It's true if the connection is upgrading to ws, false if it's not. (read only) - - - status_code: The status code of the response (read only) (int) - - - headers_complete: It's true if the headers are complete, false if they are not. (read only) - - - message_complete: It's true if the message is complete, false if it's not. (read only) - - - total_size: The size of the entire http response (read only) - - - stream: It's the buffer that contains the stream of the websocket traffic in output. This is used only if should_upgrade is True. (read only) - - - - - <CgEditBlackPoint style={{marginBottom: -3}}/> HttpResponseHeader - - Same as HttpResponse, but this handler is called only when the headers are complete and body is not buffered. Body will always be None - ⚠️ Stream Limiter - - What happen if in a specific TCP stream you have a lot of data? The stream limiter will be activated and some action will be taken. - You can configure the action performed by setting some option in the globals: -
- First import the FullStreamAction enum: - - Then you can set in the globals these options: - - - FGEX_STREAM_MAX_SIZE: Sets the maximum size of the stream. If the stream exceeds this size, the FGEX_FULL_STREAM_ACTION will be performed. (this limit is applyed at the single stream related to the single data handler). - For example if TCPInputStream has reached the limit but HttpResponse has not, the action will be performed only on the TCPInputStream. The default is 1MB. - - - FGEX_FULL_STREAM_ACTION: Sets the action performed when the stream exceeds the FGEX_STREAM_MAX_SIZE. The default is FullStreamAction.FLUSH. - - - Heres will be explained every type of action you can set: - - - FLUSH: Flush the stream and continue to acquire new packets (default) - - - DROP: Drop the next stream packets - like a DROP action by filter - - - REJECT: Reject the stream and close the connection - like a REJECT action by filter - - - ACCEPT: Stops to call pyfilters and accept the traffic - - - - ⚠️ Other Options - - Here's other enums that you could need to use: - - Then you can set in the globals these options: - - - FGEX_INVALID_ENCODING_ACTION: Sets the action performed when the stream has an invalid encoding (due to a parser crash). The default is ExceptionAction.REJECT. - - - - 🚀 How It Works - - The proxy is built on a multi-threaded architecture and integrates Python for dynamic filtering: - - - - - Packet Interception: - The nfqueue kernel module intercepts network packets(a netfilter module) 🔍
- The rules for attach the nfqueue on the network traffic is done by the nftables lib with json APIs by the python manager. - - - - - Packet Reading: - A dedicated thread reads packets from nfqueue. 🧵 - - - - - Multi-threaded Analysis: - The C++ binary launches multiple threads, each starting its own Python interpreter. - Thanks to Python 3.12’s support for a per-interpeter GIL, real multithreading is achieved. - Traffic is distributed among threads based on IP addresses and port hashing, ensuring that - packets belonging to the same flow are processed by the same thread. ⚡️ - - - - - Python Filter Integration: - Users can upload custom Python filters which are then executed by the interpreter, - allowing for dynamic and flexible packet handling. 🐍 - - - - - HTTP Parsing: - A Python wrapper for llhttp (forked and adapted for working with multi-interpeters) is used to parse HTTP connections, making it easier to handle - and analyze HTTP traffic. 📡 - - - - - 📚 Additional Resources - - Here's a pyfilter code commented example: - - - - ); + + ⚙️ How to use Netfilter Proxy + + + To use Netfilter Proxy, simply create and upload a Python + filter. The filter is passed to the C++ binary, which then + processes packets using the provided logic. This allows you to + tailor the filtering behavior to your needs. + + + 💡 How to write pyfilters? + + + First of all install the firegex lib and update it running{" "} + pip install -U fgex. After that you can use{" "} + firegex module. + + With this code we imported the pyfilter decorator + and the ACCEPT and REJECT statements. +
+ Let's create a first (useless) filter to see the syntax: + + You see that the filter must be decorated with the{" "} + pyfilter decorator and must return a statement + about how to manage that packet. +
+ + You can save every data about the current flow in the global + variables, the code you write will be executed only once for + flow. The globals variables are isolated between flows. For each + packet the filter functions will be called with the required + paramethers and using the same globals as before. +
+ + + Saving data in globals of other modules is not recommended, + because that memory is shared by the flows managed by the + same thread and lead to unexpected behaviors. + +
+ + + Global variables that starts with '__firegex' are reserved + for internal use, don't use them. + +
+ + You can manage when the function is called and also getting some + data specifying some paramethers, using type decorators. Default + values of the paramethers will be ignored, also kvargs values + will be ignored. +
+ + + Functions with no type decorator are considered invalid + pyfilters! + +
+ + + In this code we are filtering all the http requests that + contains the word "ILLEGAL" in the body. All the other packets + will be accepted (default behavior). The function will be called + only if at least internally teh HTTP request header has been + parsed, and also when the body will be parsed. +
+ + If we have multiple paramether, the function will be called only + if with the packet arrived is possible to build all the + paramethers. + + + 🔧 How can I test the filter? + + + You can test your filter by using fgex command + installed by firegex lib: This will run a local proxy to a + remote destination with the filter you specified. +
+ + This can be done by running for instance:{" "} + + fgex nfproxy test_http.py 127.0.0.1 8080 --proto http + + + You don't need to restart the proxy every time you change the + filter, the filter will be reloaded automatically. + + + 📦 Packet Statements + + + Here there are all the statments you can return from a filter: + + + ACCEPT: The packet will be accepted + and forwarded to the destination. (default if None is + returned) + + + REJECT: The connection will be closed + and all the packets will be dropped. + + + DROP: This packet and all the + following will be dropped. (This not simulate a + connection closure) + + + UNSTABLE_MANGLE: The packet will be + modified and forwarded. You can edit the packet only + with RawPacket data handler. (This is an unstable + statement, use it carefully) + + + + + ⚙️ Data Structures + + + Here there are all the data structure you can use for your + filters: + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} /> RawPacket + + + + + + + This data is the raw packet processed by nfqueue. It contains: + + + + + + data: The raw packet data assembled by + libtins (read only). + + + is_input: It's true if the packet is + incoming, false if it's outgoing. (read only) + + + is_ipv6: It's true if the packet is + IPv6, false if it's IPv4. (read only) + + + is_tcp: It's true if the packet is + TCP, false if it's UDP. (read only) + + + l4_size: The size of l4 payload (read + only) + + + raw_packet_header_len: The size of the + raw packet header (read only) + + + raw_packet: The raw packet data with + ip and TCP header. You can edit all the packet content + and it will be modified if you send the UNSTABLE_MANGLE + statement.{" "} + + Be careful, beacause the associated layer 4 data can + be different from 'data' filed that instead arrives + from libtins. + + When you edit this field, l4_size and l4_data will be + updated automatically. + + + l4_data: The l4 payload data, directly + taken by the raw packet. You can edit all the packet + content and it will be modified if you send the + UNSTABLE_MANGLE statement.{" "} + + Be careful, beacause the associated layer 4 data can + be different from 'data' filed that instead arrives + from libtins. + {" "} + When you edit this field, l4_size and raw_packet will be + updated automatically. + + + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + TCPInputStream (alias: TCPClientStream) + + + + + + + This data is the TCP input stream: this handler is called only + on is_input=True packets. The filters that handles this data + will be called only in this case. + + + + + + data: The entire stream in input + direction. (read only) + + + total_stream_size: The size of the + entire stream in input direction. (read only) + + + is_ipv6: It's true if the stream is + IPv6, false if it's IPv4. (read only) + + + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + TCPOutputStream (alias TCPServerStream) + + + + + + + This data is the TCP output stream: this handler is called only + on is_input=False packets. The filters that handles this data + will be called only in this case. + + + + + + data: The entire stream in output + direction. (read only) + + + total_stream_size: The size of the + entire stream in output direction. (read only) + + + is_ipv6: It's true if the stream is + IPv6, false if it's IPv4. (read only) + + + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpRequest + + + + + + This data is the Http request processed by nfqueue. This handler + can be called twice per request: once when the http headers are + complete, and once when the body is complete. + + + If the http data arrives in 1 single TCP packet, this handler + will be called once + + + + + + url: The url of the request (read + only) + + + headers: The headers of the request + (read only). The keys and values are exactly the same as + the original request (case sensitive). (values can be + list in case the same header field is repeated) + + + get_header(key:str, default = None): A + function that returns the value of a header: it matches + the key without case sensitivity. If the header is not + found, it returns the default value. (if the same header + field is repeated, its value is concatenated with a + comma, this function will never return a list) + + + user_agent: The user agent of the + request (read only) + + + content_encoding: The content encoding + of the request (read only) + + + content_length: The content length of + the request (read only) + + + body: The body of the request (read + only). It's None if the body has not arrived yet. + + + body_decoded: By default the body will + be decoded following the content encoding. gzip, br, + deflate and zstd are supported. If the decoding fails + and body is not None this paramether will be False. + + + http_version: The http version of the + request (read only) + + + keep_alive: It's true if the + connection was marked for keep alive, false if it's not. + (read only) + + + should_upgrade: It's true if the + connection should be upgraded, false if it's not. (read + only) + + + upgrading_to_h2: It's true if the + connection is upgrading to h2, false if it's not. (read + only) + + + ws_stream: It's a list of + websockets.frames.Frame decoded (permessage-deflate is + supported). (read only) [ + + docs + + ] + + + upgrading_to_ws: It's true if the + connection is upgrading to ws, false if it's not. (read + only) + + + method: The method of the request + (read only) + + + headers_complete: It's true if the + headers are complete, false if they are not. (read only) + + + message_complete: It's true if the + message is complete, false if it's not. (read only) + + + total_size: The size of the entire + http request (read only) + + + stream: It's the buffer that contains + the stream of the websocket traffic in input. This is + used only if should_upgrade is True. (read only) + + + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpRequestHeader + + + + + + Same as HttpRequest, but this handler is called only when the + headers are complete and body is not buffered. Body will always + be None + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpFullRequest + + + + + + Same as HttpRequest, but this handler is called only when the + request data is complete + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpResponse + + + + + + This data is the Http response processed by nfqueue. This + handler can be called twice per response: once when the http + headers are complete, and once when the body is complete. + + + If the http data arrives in 1 single TCP packet, this handler + will be called once + + + + + + url: The url of the response (read + only) + + + headers: The headers of the response + (read only). The keys and values are exactly the same as + the original response (case sensitive). (values can be + list in case the same header field is repeated) + + + get_header(key:str, default = None): A + function that returns the value of a header: it matches + the key without case sensitivity. If the header is not + found, it returns the default value. (if the same header + field is repeated, its value is concatenated with a + comma, this function will never return a list) + + + user_agent: The user agent of the + response (read only) + + + content_encoding: The content encoding + of the response (read only) + + + content_length: The content length of + the response (read only) + + + body: The body of the response (read + only). It's None if the body has not arrived yet. + + + body_decoded: By default the body will + be decoded following the content encoding. gzip, br, + deflate and zstd are supported. If the decoding fails + and body is not None this paramether will be False. + + + http_version: The http version of the + response (read only) + + + keep_alive: It's true if the + connection was marked for keep alive, false if it's not. + (read only) + + + should_upgrade: It's true if the + connection should be upgraded, false if it's not. (read + only) + + + upgrading_to_h2: It's true if the + connection is upgrading to h2, false if it's not. (read + only) + + + ws_stream: It's a list of + websockets.frames.Frame decoded (permessage-deflate is + supported). (read only) [ + + docs + + ] + + + upgrading_to_ws: It's true if the + connection is upgrading to ws, false if it's not. (read + only) + + + status_code: The status code of the + response (read only) (int) + + + headers_complete: It's true if the + headers are complete, false if they are not. (read only) + + + message_complete: It's true if the + message is complete, false if it's not. (read only) + + + total_size: The size of the entire + http response (read only) + + + stream: It's the buffer that contains + the stream of the websocket traffic in output. This is + used only if should_upgrade is True. (read only) + + + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpResponseHeader + + + + + + Same as HttpResponse, but this handler is called only when the + headers are complete and body is not buffered. Body will always + be None + + + + <CgEditBlackPoint style={{ marginBottom: -3 }} />{" "} + HttpFullResponse + + + + + + Same as HttpResponse, but this handler is called only when the + response data is complete + + + ⚠️ Stream Limiter + + + What happen if in a specific TCP stream you have a lot of data? + The stream limiter will be activated and some action will be + taken. You can configure the action performed by setting some + option in the globals: +
+ + First import the FullStreamAction enum: + + Then you can set in the globals these options: + + + FGEX_STREAM_MAX_SIZE: Sets the maximum + size of the stream. If the stream exceeds this size, the + FGEX_FULL_STREAM_ACTION will be performed. (this limit + is applyed at the single stream related to the single + data handler). For example if TCPInputStream has reached + the limit but HttpResponse has not, the action will be + performed only on the TCPInputStream. The default is + 1MB. + + + FGEX_FULL_STREAM_ACTION: Sets the + action performed when the stream exceeds the + FGEX_STREAM_MAX_SIZE. The default is + FullStreamAction.FLUSH. + + + Heres will be explained every type of action you can set: + + + FLUSH: Flush the stream and continue + to acquire new packets (default) + + + DROP: Drop the next stream packets - + like a DROP action by filter + + + REJECT: Reject the stream and close + the connection - like a REJECT action by filter + + + ACCEPT: Stops to call pyfilters and + accept the traffic + + + + + ⚠️ Other Options + + + Here's other enums that you could need to use: + + Then you can set in the globals these options: + + + FGEX_INVALID_ENCODING_ACTION: Sets the + action performed when the stream has an invalid encoding + (due to a parser crash). The default is + ExceptionAction.REJECT. + + + + + 🚀 How It Works + + + The proxy is built on a multi-threaded architecture and + integrates Python for dynamic filtering: + + + + + Packet Interception: + The{" "} + + nfqueue + {" "} + kernel module intercepts network packets(a{" "} + netfilter module) + 🔍 +
+ The rules for attach the nfqueue on the network traffic + is done by the nftables lib with json APIs by the python + manager. + + + + + Packet Reading: A dedicated thread + reads packets from{" "} + + nfqueue + + . 🧵 + + + + + Multi-threaded Analysis: + The C++ binary launches multiple threads, each starting + its own Python interpreter. Thanks to Python 3.12’s + support for{" "} + + a per-interpeter GIL + + , real multithreading is achieved. Traffic is + distributed among threads based on IP addresses and port + hashing, ensuring that packets belonging to the same + flow are processed by the same thread. ⚡️ + + + + + Python Filter Integration: + Users can upload custom Python filters which are then + executed by the interpreter, allowing for dynamic and + flexible packet handling. 🐍 + + + + + HTTP Parsing: + + A Python wrapper for llhttp + {" "} + (forked and adapted for working with multi-interpeters) + is used to parse HTTP connections, making it easier to + handle and analyze HTTP traffic. 📡 + + + + + + 📚 Additional Resources + + + Here's a pyfilter code commented example: + + + + ); }; diff --git a/tests/nfproxy_test.py b/tests/nfproxy_test.py index 290704a..f9c4175 100644 --- a/tests/nfproxy_test.py +++ b/tests/nfproxy_test.py @@ -7,12 +7,35 @@ import secrets import time parser = argparse.ArgumentParser() -parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:4444/") -parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password') -parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the test service', default="Test Service") -parser.add_argument("--port", "-P", type=int , required=False, help='Port of the test service', default=1337) -parser.add_argument("--ipv6", "-6" , action="store_true", help='Test Ipv6', default=False) -parser.add_argument("--verbose", "-V" , action="store_true", help='Verbose output', default=False) +parser.add_argument( + "--address", + "-a", + type=str, + required=False, + help="Address of firegex backend", + default="http://127.0.0.1:4444/", +) +parser.add_argument("--password", "-p", type=str, required=True, help="Firegex password") +parser.add_argument( + "--service_name", + "-n", + type=str, + required=False, + help="Name of the test service", + default="Test Service", +) +parser.add_argument( + "--port", + "-P", + type=int, + required=False, + help="Port of the test service", + default=1337, +) +parser.add_argument("--ipv6", "-6", action="store_true", help="Test Ipv6", default=False) +parser.add_argument( + "--verbose", "-V", action="store_true", help="Verbose output", default=False +) args = parser.parse_args() sep() @@ -21,28 +44,31 @@ puts(f"{args.address}", color=colors.yellow) firegex = FiregexAPI(args.address) -#Login -if (firegex.login(args.password)): +# Login +if firegex.login(args.password): puts("Sucessfully logged in ✔", color=colors.green) else: puts("Test Failed: Unknown response or wrong passowrd ✗", color=colors.red) exit(1) -#Create server -server = TcpServer(args.port,ipv6=args.ipv6, verbose=args.verbose) +# Create server +server = TcpServer(args.port, ipv6=args.ipv6, verbose=args.verbose) srvs = firegex.nfproxy_get_services() for ele in srvs: - if ele['name'] == args.service_name: - firegex.nfproxy_delete_service(ele['service_id']) + if ele["name"] == args.service_name: + firegex.nfproxy_delete_service(ele["service_id"]) -service_id = firegex.nfproxy_add_service(args.service_name, args.port, "http" , "::1" if args.ipv6 else "127.0.0.1" ) +service_id = firegex.nfproxy_add_service( + args.service_name, args.port, "http", "::1" if args.ipv6 else "127.0.0.1" +) if service_id: puts(f"Sucessfully created service {service_id} ✔", color=colors.green) else: puts("Test Failed: Failed to create service ✗", color=colors.red) exit(1) + def exit_test(code): if service_id: server.stop() @@ -50,11 +76,12 @@ def exit_test(code): if firegex.nfproxy_delete_service(service_id): puts("Sucessfully deleted service ✔", color=colors.green) else: - puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) - """ + puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) + """ exit(code) -if(firegex.nfproxy_start_service(service_id)): + +if firegex.nfproxy_start_service(service_id): puts("Sucessfully started service ✔", color=colors.green) else: puts("Test Failed: Failed to start service ✗", color=colors.red) @@ -85,21 +112,27 @@ def verdict_test(packet:RawPacket): BASE_FILTER_VERDICT_NAME = "verdict_test" -def get_vedict_test(to_match:str, action:str, mangle_to:str="REDACTED"): - return BASE_FILTER_VERDICT_TEST.replace("%%TEST%%", to_match).replace("%%ACTION%%", action).replace("%%MANGLE%%", mangle_to) + +def get_vedict_test(to_match: str, action: str, mangle_to: str = "REDACTED"): + return ( + BASE_FILTER_VERDICT_TEST.replace("%%TEST%%", to_match) + .replace("%%ACTION%%", action) + .replace("%%MANGLE%%", mangle_to) + ) -#Check if filter is present in the service +# Check if filter is present in the service n_blocked = 0 n_mangled = 0 + def checkFilter(match_bytes, filter_name, should_work=True, mangle_with=None): if mangle_with: if should_work: global n_mangled for r in firegex.nfproxy_get_service_pyfilters(service_id): if r["name"] == filter_name: - #Test the filter + # Test the filter pre_packet = secrets.token_bytes(40) post_packet = secrets.token_bytes(40) server.connect_client() @@ -107,82 +140,136 @@ def checkFilter(match_bytes, filter_name, should_work=True, mangle_with=None): real_response = server.recv_packet() expected_response = pre_packet + mangle_with + post_packet if real_response == expected_response: - puts("The malicious request was successfully mangled ✔", color=colors.green) + puts( + "The malicious request was successfully mangled ✔", + color=colors.green, + ) n_mangled += 1 time.sleep(1) - if firegex.nfproxy_get_pyfilter(service_id, filter_name)["edited_packets"] == n_mangled: - puts("The packet was reported as mangled in the API ✔", color=colors.green) + if ( + firegex.nfproxy_get_pyfilter(service_id, filter_name)[ + "edited_packets" + ] + == n_mangled + ): + puts( + "The packet was reported as mangled in the API ✔", + color=colors.green, + ) else: - puts("Test Failed: The packet wasn't reported as mangled in the API ✗", color=colors.red) + puts( + "Test Failed: The packet wasn't reported as mangled in the API ✗", + color=colors.red, + ) exit_test(1) server.send_packet(pre_packet) if server.recv_packet() == pre_packet: - puts("Is able to communicate after mangle ✔", color=colors.green) + puts( + "Is able to communicate after mangle ✔", + color=colors.green, + ) else: - puts("Test Failed: Couldn't communicate after mangle ✗", color=colors.red) + puts( + "Test Failed: Couldn't communicate after mangle ✗", + color=colors.red, + ) exit_test(1) else: - puts("Test Failed: The request wasn't mangled ✗", color=colors.red) + puts( + "Test Failed: The request wasn't mangled ✗", color=colors.red + ) exit_test(1) server.close_client() return puts("Test Failed: The filter wasn't found ✗", color=colors.red) else: - if server.sendCheckData(secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40)): + if server.sendCheckData( + secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40) + ): puts("The request wasn't mangled ✔", color=colors.green) else: - puts("Test Failed: The request was mangled when it shouldn't have", color=colors.red) + puts( + "Test Failed: The request was mangled when it shouldn't have", + color=colors.red, + ) exit_test(1) else: if should_work: global n_blocked for r in firegex.nfproxy_get_service_pyfilters(service_id): if r["name"] == filter_name: - #Test the filter - if not server.sendCheckData(secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40)): - puts("The malicious request was successfully blocked ✔", color=colors.green) + # Test the filter + if not server.sendCheckData( + secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40) + ): + puts( + "The malicious request was successfully blocked ✔", + color=colors.green, + ) n_blocked += 1 time.sleep(1) - if firegex.nfproxy_get_pyfilter(service_id, filter_name)["blocked_packets"] == n_blocked: - puts("The packet was reported as blocked in the API ✔", color=colors.green) + if ( + firegex.nfproxy_get_pyfilter(service_id, filter_name)[ + "blocked_packets" + ] + == n_blocked + ): + puts( + "The packet was reported as blocked in the API ✔", + color=colors.green, + ) else: - puts("Test Failed: The packet wasn't reported as blocked in the API ✗", color=colors.red) + puts( + "Test Failed: The packet wasn't reported as blocked in the API ✗", + color=colors.red, + ) exit_test(1) else: - puts("Test Failed: The request wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The request wasn't blocked ✗", color=colors.red + ) exit_test(1) return puts("Test Failed: The filter wasn't found ✗", color=colors.red) exit_test(1) else: - if server.sendCheckData(secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40)): + if server.sendCheckData( + secrets.token_bytes(40) + match_bytes + secrets.token_bytes(40) + ): puts("The request wasn't blocked ✔", color=colors.green) else: - puts("Test Failed: The request was blocked when it shouldn't have", color=colors.red) + puts( + "Test Failed: The request was blocked when it shouldn't have", + color=colors.red, + ) exit_test(1) -#Add new filter + +# Add new filter secret = bytes(secrets.token_hex(16).encode()) -if firegex.nfproxy_set_code(service_id,get_vedict_test(secret.decode(), "REJECT")): - puts(f"Sucessfully added filter for {str(secret)} in REJECT mode ✔", color=colors.green) +if firegex.nfproxy_set_code(service_id, get_vedict_test(secret.decode(), "REJECT")): + puts( + f"Sucessfully added filter for {str(secret)} in REJECT mode ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) checkFilter(secret, BASE_FILTER_VERDICT_NAME) -#Pause the proxy +# Pause the proxy if firegex.nfproxy_stop_service(service_id): puts(f"Sucessfully paused service with id {service_id} ✔", color=colors.green) else: puts("Test Failed: Coulnd't pause the service ✗", color=colors.red) exit_test(1) -#Check if it's actually paused +# Check if it's actually paused checkFilter(secret, BASE_FILTER_VERDICT_NAME, should_work=False) -#Start firewall +# Start firewall if firegex.nfproxy_start_service(service_id): puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green) else: @@ -191,25 +278,26 @@ else: checkFilter(secret, BASE_FILTER_VERDICT_NAME) -#Disable filter +# Disable filter if firegex.nfproxy_disable_pyfilter(service_id, BASE_FILTER_VERDICT_NAME): puts(f"Sucessfully disabled filter {BASE_FILTER_VERDICT_NAME} ✔", color=colors.green) -else: +else: puts("Test Failed: Coulnd't disable the filter ✗", color=colors.red) exit_test(1) -#Check if it's actually disabled +# Check if it's actually disabled checkFilter(secret, BASE_FILTER_VERDICT_NAME, should_work=False) -#Enable filter +# Enable filter if firegex.nfproxy_enable_pyfilter(service_id, BASE_FILTER_VERDICT_NAME): puts(f"Sucessfully enabled filter {BASE_FILTER_VERDICT_NAME} ✔", color=colors.green) -else: +else: puts("Test Failed: Coulnd't enable the regex ✗", color=colors.red) exit_test(1) checkFilter(secret, BASE_FILTER_VERDICT_NAME) + def remove_filters(): global n_blocked, n_mangled server.stop() @@ -220,14 +308,17 @@ def remove_filters(): n_blocked = 0 n_mangled = 0 + remove_filters() -#Check if it's actually deleted +# Check if it's actually deleted checkFilter(secret, BASE_FILTER_VERDICT_NAME, should_work=False) -#Check if DROP works -if firegex.nfproxy_set_code(service_id,get_vedict_test(secret.decode(), "DROP")): - puts(f"Sucessfully added filter for {str(secret)} in DROP mode ✔", color=colors.green) +# Check if DROP works +if firegex.nfproxy_set_code(service_id, get_vedict_test(secret.decode(), "DROP")): + puts( + f"Sucessfully added filter for {str(secret)} in DROP mode ✔", color=colors.green + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -236,10 +327,16 @@ checkFilter(secret, BASE_FILTER_VERDICT_NAME) remove_filters() -#Check if UNSTABLE_MANGLE works -mangle_result = secrets.token_hex(4).encode() # Mangle to a smaller packet -if firegex.nfproxy_set_code(service_id, get_vedict_test(secret.decode(), "UNSTABLE_MANGLE", mangle_result.decode())): - puts(f"Sucessfully added filter for {str(secret)} in UNSTABLE_MANGLE mode to a smaller packet size ✔", color=colors.green) +# Check if UNSTABLE_MANGLE works +mangle_result = secrets.token_hex(4).encode() # Mangle to a smaller packet +if firegex.nfproxy_set_code( + service_id, + get_vedict_test(secret.decode(), "UNSTABLE_MANGLE", mangle_result.decode()), +): + puts( + f"Sucessfully added filter for {str(secret)} in UNSTABLE_MANGLE mode to a smaller packet size ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -248,10 +345,16 @@ checkFilter(secret, BASE_FILTER_VERDICT_NAME, mangle_with=mangle_result) remove_filters() -#Check if UNSTABLE_MANGLE works -mangle_result = secrets.token_hex(60).encode() # Mangle to a bigger packet -if firegex.nfproxy_set_code(service_id, get_vedict_test(secret.decode(), "UNSTABLE_MANGLE", mangle_result.decode())): - puts(f"Sucessfully added filter for {str(secret)} in UNSTABLE_MANGLE mode to a bigger packet size ✔", color=colors.green) +# Check if UNSTABLE_MANGLE works +mangle_result = secrets.token_hex(60).encode() # Mangle to a bigger packet +if firegex.nfproxy_set_code( + service_id, + get_vedict_test(secret.decode(), "UNSTABLE_MANGLE", mangle_result.decode()), +): + puts( + f"Sucessfully added filter for {str(secret)} in UNSTABLE_MANGLE mode to a bigger packet size ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -273,12 +376,15 @@ def data_type_test(packet:TCPInputStream): """ if firegex.nfproxy_set_code(service_id, TCP_INPUT_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for TCPInputStream ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for TCPInputStream ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) -data_split = len(secret)//2 +data_split = len(secret) // 2 server.connect_client() server.send_packet(secret[:data_split]) if server.recv_packet() == secret[:data_split]: @@ -309,12 +415,15 @@ def data_type_test(packet:TCPOutputStream): """ if firegex.nfproxy_set_code(service_id, TCP_OUTPUT_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for TCPOutputStream ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for TCPOutputStream ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) -data_split = len(secret)//2 +data_split = len(secret) // 2 server.connect_client() server.send_packet(secret[:data_split]) if server.recv_packet() == secret[:data_split]: @@ -363,7 +472,10 @@ def data_type_test(req:HttpRequest): """ if firegex.nfproxy_set_code(service_id, HTTP_REQUEST_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for HttpRequest ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for HttpRequest ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -371,23 +483,94 @@ else: server.connect_client() server.send_packet(REQUEST_HEADER_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious header was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious header wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() server.connect_client() server.send_packet(REQUEST_BODY_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious body was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious body was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious body wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious body wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() remove_filters() +HTTP_FULL_REQUEST_STREAM_TEST = f""" +from firegex.nfproxy.models import HttpFullRequest +from firegex.nfproxy import pyfilter, ACCEPT, UNSTABLE_MANGLE, DROP, REJECT + +@pyfilter +def data_type_test(req:HttpFullRequest): + if not req.body: + return ACCEPT + if not req.get_header("x-test"): + return ACCEPT + if {repr(secret.decode())} in req.get_header("x-test"): + return REJECT + if {repr(secret)} in req.body: + return REJECT + +""" + +if firegex.nfproxy_set_code(service_id, HTTP_FULL_REQUEST_STREAM_TEST): + puts( + f"Sucessfully added filter for {str(secret)} for HttpFullRequest ✔", + color=colors.green, + ) +else: + puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) + exit_test(1) + +server.connect_client() +server.send_packet(REQUEST_HEADER_TEST.encode()) +if not server.recv_packet(): + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) +else: + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) + exit_test(1) +server.close_client() + +server.connect_client() +server.send_packet(REQUEST_BODY_TEST.encode()) +if not server.recv_packet(): + puts( + "The malicious HTTP request with the malicious body was successfully blocked ✔", + color=colors.green, + ) +else: + puts( + "Test Failed: The HTTP request with the malicious body wasn't blocked ✗", + color=colors.red, + ) + exit_test(1) +server.close_client() + +remove_filters() + + HTTP_REQUEST_HEADER_STREAM_TEST = f""" from firegex.nfproxy.models import HttpRequestHeader from firegex.nfproxy import pyfilter, ACCEPT, UNSTABLE_MANGLE, DROP, REJECT @@ -400,7 +583,10 @@ def data_type_test(req:HttpRequestHeader): """ if firegex.nfproxy_set_code(service_id, HTTP_REQUEST_HEADER_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for HttpRequestHeader ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for HttpRequestHeader ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -408,9 +594,15 @@ else: server.connect_client() server.send_packet(REQUEST_HEADER_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious header was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious header wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() @@ -447,7 +639,10 @@ def data_type_test(req:HttpResponse): """ if firegex.nfproxy_set_code(service_id, HTTP_RESPONSE_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for HttpResponse ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for HttpResponse ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -455,23 +650,95 @@ else: server.connect_client() server.send_packet(RESPONSE_HEADER_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious header was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious header wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() server.connect_client() server.send_packet(RESPONSE_BODY_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious body was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious body was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious body wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious body wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() remove_filters() + +HTTP_FULL_RESPONSE_STREAM_TEST = f""" +from firegex.nfproxy.models import HttpFullResponse +from firegex.nfproxy import pyfilter, ACCEPT, UNSTABLE_MANGLE, DROP, REJECT + +@pyfilter +def data_type_test(req:HttpFullResponse): + if not req.body: + return ACCEPT + if not req.get_header("x-test"): + return ACCEPT + if {repr(secret.decode())} in req.get_header("x-test"): + return REJECT + if {repr(secret)} in req.body: + return REJECT + +""" + +if firegex.nfproxy_set_code(service_id, HTTP_FULL_RESPONSE_STREAM_TEST): + puts( + f"Sucessfully added filter for {str(secret)} for HttpFullResponse ✔", + color=colors.green, + ) +else: + puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) + exit_test(1) + +server.connect_client() +server.send_packet(RESPONSE_HEADER_TEST.encode()) +if not server.recv_packet(): + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) +else: + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) + exit_test(1) +server.close_client() + +server.connect_client() +server.send_packet(RESPONSE_BODY_TEST.encode()) +if not server.recv_packet(): + puts( + "The malicious HTTP request with the malicious body was successfully blocked ✔", + color=colors.green, + ) +else: + puts( + "Test Failed: The HTTP request with the malicious body wasn't blocked ✗", + color=colors.red, + ) + exit_test(1) +server.close_client() + +remove_filters() + + HTTP_RESPONSE_HEADER_STREAM_TEST = f""" from firegex.nfproxy.models import HttpResponseHeader from firegex.nfproxy import pyfilter, ACCEPT, UNSTABLE_MANGLE, DROP, REJECT @@ -484,7 +751,10 @@ def data_type_test(req:HttpResponseHeader): """ if firegex.nfproxy_set_code(service_id, HTTP_RESPONSE_HEADER_STREAM_TEST): - puts(f"Sucessfully added filter for {str(secret)} for HttpResponseHeader ✔", color=colors.green) + puts( + f"Sucessfully added filter for {str(secret)} for HttpResponseHeader ✔", + color=colors.green, + ) else: puts(f"Test Failed: Couldn't add the filter {str(secret)} ✗", color=colors.red) exit_test(1) @@ -492,18 +762,24 @@ else: server.connect_client() server.send_packet(RESPONSE_HEADER_TEST.encode()) if not server.recv_packet(): - puts("The malicious HTTP request with the malicious header was successfully blocked ✔", color=colors.green) + puts( + "The malicious HTTP request with the malicious header was successfully blocked ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP request with the malicious header wasn't blocked ✗", color=colors.red) + puts( + "Test Failed: The HTTP request with the malicious header wasn't blocked ✗", + color=colors.red, + ) exit_test(1) server.close_client() remove_filters() -#Simulating requests is more complex due to websocket extensions handshake +# Simulating requests is more complex due to websocket extensions handshake -WS_REQUEST_PARSING_TEST = b'GET /sock/?EIO=4&transport=websocket HTTP/1.1\r\nHost: localhost:8080\r\nConnection: Upgrade\r\nPragma: no-cache\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)\xac AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36\r\nUpgrade: websocket\r\nOrigin: http://localhost:8080\r\nSec-WebSocket-Version: 13\r\nAccept-Encoding: gzip, deflate, br, zstd\r\nAccept-Language: it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6,zh;q=0.5\r\nCookie: cookie-consent=true; _iub_cs-86405163=%7B%22timestamp%22%3A%222024-09-12T18%3A20%3A18.627Z%22%2C%22version%22%3A%221.65.1%22%2C%22purposes%22%3A%7B%221%22%3Atrue%2C%224%22%3Atrue%7D%2C%22id%22%3A86405163%2C%22cons%22%3A%7B%22rand%22%3A%222b09e6%22%7D%7D\r\nSec-WebSocket-Key: eE01O3/ZShPKsrykACLAaA==\r\nSec-WebSocket-Extensions: permessage-deflate; client_max_window_bits\r\n\r\n\xc1\x84#\x8a\xb2\xbb\x11\xbb\xb2\xbb' -WS_RESPONSE_PARSING_TEST = b'HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: eGnJqUSoSKE3wOfKD2M3G82RsS8=\r\nSec-WebSocket-Extensions: permessage-deflate\r\ndate: Sat, 15 Mar 2025 12:04:19 GMT\r\nserver: uvicorn\r\n\r\n\xc1_2\xa8V*\xceLQ\xb2Rr1\xb4\xc8\xf6r\x0c\xf3\xaf\xd25\xf7\x8e\xf4\xb3LsttrW\xd2Q*-H/JLI-V\xb2\x8a\x8e\xd5Q*\xc8\xccK\x0f\xc9\xccM\xcd/-Q\xb222\x00\x02\x88\x98g^IjQYb\x0eP\xd0\x14,\x98\x9bX\x11\x90X\x99\x93\x9f\x084\xda\xd0\x00\x0cj\x01\x00\xc1\x1b21\x80\xd9e\xe1n\x19\x9e\xe3RP\x9a[Z\x99\x93j\xea\x15\x00\xb4\xcbC\xa9\x16\x00' +WS_REQUEST_PARSING_TEST = b"GET /sock/?EIO=4&transport=websocket HTTP/1.1\r\nHost: localhost:8080\r\nConnection: Upgrade\r\nPragma: no-cache\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)\xac AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36\r\nUpgrade: websocket\r\nOrigin: http://localhost:8080\r\nSec-WebSocket-Version: 13\r\nAccept-Encoding: gzip, deflate, br, zstd\r\nAccept-Language: it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6,zh;q=0.5\r\nCookie: cookie-consent=true; _iub_cs-86405163=%7B%22timestamp%22%3A%222024-09-12T18%3A20%3A18.627Z%22%2C%22version%22%3A%221.65.1%22%2C%22purposes%22%3A%7B%221%22%3Atrue%2C%224%22%3Atrue%7D%2C%22id%22%3A86405163%2C%22cons%22%3A%7B%22rand%22%3A%222b09e6%22%7D%7D\r\nSec-WebSocket-Key: eE01O3/ZShPKsrykACLAaA==\r\nSec-WebSocket-Extensions: permessage-deflate; client_max_window_bits\r\n\r\n\xc1\x84#\x8a\xb2\xbb\x11\xbb\xb2\xbb" +WS_RESPONSE_PARSING_TEST = b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: eGnJqUSoSKE3wOfKD2M3G82RsS8=\r\nSec-WebSocket-Extensions: permessage-deflate\r\ndate: Sat, 15 Mar 2025 12:04:19 GMT\r\nserver: uvicorn\r\n\r\n\xc1_2\xa8V*\xceLQ\xb2Rr1\xb4\xc8\xf6r\x0c\xf3\xaf\xd25\xf7\x8e\xf4\xb3LsttrW\xd2Q*-H/JLI-V\xb2\x8a\x8e\xd5Q*\xc8\xccK\x0f\xc9\xccM\xcd/-Q\xb222\x00\x02\x88\x98g^IjQYb\x0eP\xd0\x14,\x98\x9bX\x11\x90X\x99\x93\x9f\x084\xda\xd0\x00\x0cj\x01\x00\xc1\x1b21\x80\xd9e\xe1n\x19\x9e\xe3RP\x9a[Z\x99\x93j\xea\x15\x00\xb4\xcbC\xa9\x16\x00" HTTP_REQUEST_WS_PARSING_TEST = """ from firegex.nfproxy.models import HttpRequest, HttpResponse @@ -520,7 +796,10 @@ def data_type_test(req:HttpResponse): """ if firegex.nfproxy_set_code(service_id, HTTP_REQUEST_WS_PARSING_TEST): - puts("Sucessfully added filter websocket parsing with HttpRequest and HttpResponse ✔", color=colors.green) + puts( + "Sucessfully added filter websocket parsing with HttpRequest and HttpResponse ✔", + color=colors.green, + ) else: puts("Test Failed: Couldn't add the websocket parsing filter ✗", color=colors.red) exit_test(1) @@ -528,43 +807,57 @@ else: server.connect_client() server.send_packet(WS_REQUEST_PARSING_TEST, server_reply=WS_RESPONSE_PARSING_TEST) if server.recv_packet(): - puts("The HTTP websocket upgrade request was successfully parsed ✔", color=colors.green) + puts( + "The HTTP websocket upgrade request was successfully parsed ✔", + color=colors.green, + ) else: - puts("Test Failed: The HTTP websocket upgrade request wasn't parsed (an error occurred) ✗", color=colors.red) + puts( + "Test Failed: The HTTP websocket upgrade request wasn't parsed (an error occurred) ✗", + color=colors.red, + ) exit_test(1) server.close_client() remove_filters() -#Rename service -if firegex.nfproxy_rename_service(service_id,f"{args.service_name}2"): +# Rename service +if firegex.nfproxy_rename_service(service_id, f"{args.service_name}2"): puts(f"Sucessfully renamed service to {args.service_name}2 ✔", color=colors.green) else: puts("Test Failed: Coulnd't rename service ✗", color=colors.red) exit_test(1) -#Check if service was renamed correctly +# Check if service was renamed correctly service = firegex.nfproxy_get_service(service_id) if service["name"] == f"{args.service_name}2": puts("Checked that service was renamed correctly ✔", color=colors.green) else: puts("Test Failed: Service wasn't renamed correctly ✗", color=colors.red) exit_test(1) - -#Rename back service -if(firegex.nfproxy_rename_service(service_id,f"{args.service_name}")): + +# Rename back service +if firegex.nfproxy_rename_service(service_id, f"{args.service_name}"): puts(f"Sucessfully renamed service to {args.service_name} ✔", color=colors.green) else: puts("Test Failed: Coulnd't rename service ✗", color=colors.red) exit_test(1) -#Change settings -if(firegex.nfproxy_settings_service(service_id, 1338, "::dead:beef" if args.ipv6 else "123.123.123.123", True)): +# Change settings +if firegex.nfproxy_settings_service( + service_id, 1338, "::dead:beef" if args.ipv6 else "123.123.123.123", True +): srv_updated = firegex.nfproxy_get_service(service_id) - if srv_updated["port"] == 1338 and ("::dead:beef" if args.ipv6 else "123.123.123.123") in srv_updated["ip_int"] and srv_updated["fail_open"]: + if ( + srv_updated["port"] == 1338 + and ("::dead:beef" if args.ipv6 else "123.123.123.123") in srv_updated["ip_int"] + and srv_updated["fail_open"] + ): puts("Sucessfully changed service settings ✔", color=colors.green) else: - puts("Test Failed: Service settings weren't updated correctly ✗", color=colors.red) + puts( + "Test Failed: Service settings weren't updated correctly ✗", color=colors.red + ) exit_test(1) else: puts("Test Failed: Coulnd't change service settings ✗", color=colors.red)