diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 8588195..0f03adc 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -36,10 +36,10 @@ jobs: run: echo TAG_NAME=$(echo $GITHUB_REF | cut -d / -f 3) >> $GITHUB_OUTPUT - name: Update version in setup.py run: >- - sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" proxy-client/setup.py; - sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" proxy-client/firegex/__init__.py; + sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/setup.py; + sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" fgex-lib/firegex/__init__.py; - name: Build package - run: cd proxy-client && python -m build && mv ./dist ../ + run: cd fgex-lib && python -m build && mv ./dist ../ - name: Publish package uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 with: diff --git a/fgex-lib/README.md b/fgex-lib/README.md index d441b8b..c7e1afa 100644 --- a/fgex-lib/README.md +++ b/fgex-lib/README.md @@ -1,3 +1,142 @@ # Firegex Python Library and CLI -It's a work in progress! \ No newline at end of file +This is the Python library for Firegex. It is used to get additional features of Firegex and use the feature of the command `fgex`. + +## Installation + +```bash +pip install -U firegex +``` + +fgex is an alias of firegex. You can use `fgex` instead of `firegex`. + +## Command line usage: + +```text +➤ fgex nfproxy -h + + Usage: fgex nfproxy [OPTIONS] FILTER_FILE ADDRESS PORT + + Run an nfproxy simulation + +╭─ Arguments ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ * filter_file TEXT The path to the filter file [default: None] [required] │ +│ * address TEXT The address of the target to proxy [default: None] [required] │ +│ * port INTEGER The port of the target to proxy [default: None] [required] │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --proto [tcp|http] The protocol to proxy [default: tcp] │ +│ --from-address TEXT The address of the local server [default: None] │ +│ --from-port INTEGER The port of the local server [default: 7474] │ +│ -6 Use IPv6 for the connection │ +│ --help -h Show this message and exit. │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +## Library usage: + +## NfProxy decorator + +```python +from firegex.nfproxy import pyfilter +``` +This decorator is used to create a filter for the nfproxy. +Example: +```python +@pyfilter +def my_filter(raw_packet: RawPacket): #Logging filter + print(raw_packet.data) +``` + +## Data handlers + +### RawPacket +```python +from firegex.nfproxy import RawPacket +``` +This handler will be called every time arrives a packet from the network. It will receive a RawPacket object with the following properties: +- is_input: bool - It's true if the packet is an input packet, false if it's an output packet +- is_ipv6: bool - It's true if the packet is an ipv6 packet, false if it's an ipv4 packet +- is_tcp: bool - It's true if the packet is a tcp packet, false if it's an udp packet +- data: bytes - The data of the packet assembled and sorted from TCP +- l4_size: int - The size of the layer 4 data +- raw_packet_header_len: int - The size of the original packet header +- l4_data: bytes - The layer 4 payload of the packet +- raw_packet: bytes - The raw packet with IP and TCP headers + +### TCPInputStream +Alias: TCPClientStream +```python +from firegex.nfproxy import TCPInputStream +``` +This handler will be called every time a TCP stream is assembled in input. It will receive a TCPInputStream object with the following properties: +- data: bytes - The data of the packets assembled and sorted from TCP +- is_ipv6: bool - It's true if the packet is an ipv6 packet, false if it's an ipv4 packet +- total_stream_size: int - The size of the stream + +### TCPOutputStream +Alias: TCPServerStream +```python +from firegex.nfproxy import TCPOutputStream +``` +This handler will be called every time a TCP stream is assembled in output. It will receive a TCPOutputStream object with the following properties: +- data: bytes - The data of the packets assembled and sorted from TCP +- is_ipv6: bool - It's true if the packet is an ipv6 packet, false if it's an ipv4 packet +- total_stream_size: int - The size of the stream + +### HttpRequest +```python +from firegex.nfproxy import HttpRequest +``` +This handler will be called twice: one for the request headers and one for the request body. It will receive a HttpRequest object with the following properties: +- method: bytes - The method of the request +- url: str - The url of the request +- headers: dict - The headers of the request +- user_agent: str - The user agent of the request +- content_encoding: str - The content encoding of the request +- has_begun: bool - It's true if the request has begun +- body: bytes - The body of the request +- headers_complete: bool - It's true if the headers are complete +- message_complete: bool - It's true if the message is complete +- http_version: str - The http version of the request +- keep_alive: bool - It's true if the request should keep alive +- should_upgrade: bool - It's true if the request should upgrade +- content_length: int - The content length of the request +- get_header(header: str, default=None): str - Get a header from the request without caring about the case +- total_size: int - The total size of the stream +- stream: bytes - The stream of the request + +### HttpRequestHeader +```python +from firegex.nfproxy import HttpRequestHeader +``` +This handler will be called only when the request headers are complete. It will receive a HttpRequestHeader object with the same properties as HttpRequest. + +### HttpResponse +```python +from firegex.nfproxy import HttpResponse +``` +This handler will be called twice: one for the response headers and one for the response body. It will receive a HttpResponse object with the following properties: +- status_code: int - The status code of the response +- url: str - The url of the response +- headers: dict - The headers of the response +- user_agent: str - The user agent of the response +- content_encoding: str - The content encoding of the response +- has_begun: bool - It's true if the response has begun +- body: bytes - The body of the response +- headers_complete: bool - It's true if the headers are complete +- message_complete: bool - It's true if the message is complete +- http_version: str - The http version of the response +- keep_alive: bool - It's true if the response should keep alive +- should_upgrade: bool - It's true if the response should upgrade +- content_length: int - The content length of the response +- get_header(header: str, default=None): str - Get a header from the response without caring about the case +- total_size: int - The total size of the stream +- stream: bytes - The stream of the response + +### HttpResponseHeader +```python +from firegex.nfproxy import HttpResponseHeader +``` +This handler will be called only when the response headers are complete. It will receive a HttpResponseHeader object with the same properties as HttpResponse. + diff --git a/fgex-lib/fgex-pip/README.md b/fgex-lib/fgex-pip/README.md index c31e729..5590ce2 100644 --- a/fgex-lib/fgex-pip/README.md +++ b/fgex-lib/fgex-pip/README.md @@ -1,5 +1,3 @@ # Firegex python library -Alias of 'firegex' libaray - -It's a work in progress! \ No newline at end of file +Alias of 'firegex' libaray. This library is used to get additional feature of firegex and use the feature of the command 'fgex'. \ No newline at end of file diff --git a/fgex-lib/firegex/nfproxy/internals/data.py b/fgex-lib/firegex/nfproxy/internals/data.py index ce5062c..1542318 100644 --- a/fgex-lib/firegex/nfproxy/internals/data.py +++ b/fgex-lib/firegex/nfproxy/internals/data.py @@ -2,9 +2,7 @@ from firegex.nfproxy.internals.models import FilterHandler from firegex.nfproxy.internals.models import FullStreamAction class RawPacket: - """ - class rapresentation of the nfqueue packet sent in this context by the c++ core - """ + "class rapresentation of the nfqueue packet sent in python context by the c++ core" def __init__(self, data: bytes, @@ -24,30 +22,37 @@ class RawPacket: @property def is_input(self) -> bool: + "It's true if the packet is an input packet, false if it's an output packet" return self.__is_input @property def is_ipv6(self) -> bool: + "It's true if the packet is an ipv6 packet, false if it's an ipv4 packet" return self.__is_ipv6 @property def is_tcp(self) -> bool: + "It's true if the packet is a tcp packet, false if it's an udp packet" return self.__is_tcp @property def data(self) -> bytes: + "The data of the packet assembled and sorted from TCP" return self.__data @property def l4_size(self) -> int: + "The size of the layer 4 data" return self.__l4_size @property def raw_packet_header_len(self) -> int: + "The size of the original packet header" return self.__raw_packet_header_size @property def l4_data(self) -> bytes: + "The layer 4 payload of the packet" return self.__raw_packet[self.raw_packet_header_len:] @l4_data.setter @@ -60,6 +65,7 @@ class RawPacket: @property def raw_packet(self) -> bytes: + "The raw packet with IP and TCP headers" return self.__raw_packet @raw_packet.setter @@ -92,6 +98,7 @@ class RawPacket: class DataStreamCtx: + "class to store the context of the data handler" def __init__(self, glob: dict, init_pkt: bool = True): if "__firegex_pyfilter_ctx" not in glob.keys(): diff --git a/fgex-lib/firegex/nfproxy/internals/models.py b/fgex-lib/firegex/nfproxy/internals/models.py index 86c1819..89cff79 100644 --- a/fgex-lib/firegex/nfproxy/internals/models.py +++ b/fgex-lib/firegex/nfproxy/internals/models.py @@ -2,12 +2,14 @@ from dataclasses import dataclass, field from enum import Enum class Action(Enum): + """Action to be taken by the filter""" ACCEPT = 0 DROP = 1 REJECT = 2 MANGLE = 3 class FullStreamAction(Enum): + """Action to be taken by the filter when the stream is full""" FLUSH = 0 ACCEPT = 1 REJECT = 2 @@ -15,6 +17,7 @@ class FullStreamAction(Enum): @dataclass class FilterHandler: + """Filter handler""" func: callable name: str params: dict[type, callable] @@ -22,6 +25,7 @@ class FilterHandler: @dataclass class PacketHandlerResult: + """Packet handler result""" glob: dict = field(repr=False) action: Action = Action.ACCEPT matched_by: str = None diff --git a/fgex-lib/firegex/nfproxy/models/http.py b/fgex-lib/firegex/nfproxy/models/http.py index 9cbc5f0..6a3aac7 100644 --- a/fgex-lib/firegex/nfproxy/models/http.py +++ b/fgex-lib/firegex/nfproxy/models/http.py @@ -129,6 +129,7 @@ class InternalHttpResponse(InternalCallbackHandler, pyllhttp.Response): super(pyllhttp.Response, self).__init__() class InternalBasicHttpMetaClass: + """Internal class to handle HTTP requests and responses""" def __init__(self): self._parser: InternalHttpRequest|InternalHttpResponse @@ -138,57 +139,71 @@ class InternalBasicHttpMetaClass: @property def total_size(self) -> int: + """Total size of the stream""" return self._parser.total_size @property def url(self) -> str|None: + """URL of the message""" return self._parser.url @property def headers(self) -> dict[str, str]: + """Headers of the message""" return self._parser.headers @property def user_agent(self) -> str: + """User agent of the message""" return self._parser.user_agent @property def content_encoding(self) -> str: + """Content encoding of the message""" return self._parser.content_encoding @property def has_begun(self) -> bool: + """If the message has begun""" return self._parser.has_begun @property def body(self) -> bytes: + """Body of the message""" return self._parser.body @property def headers_complete(self) -> bool: + """If the headers are complete""" return self._parser.headers_complete @property def message_complete(self) -> bool: + """If the message is complete""" return self._parser.message_complete @property def http_version(self) -> str: + """HTTP version of the message""" return self._parser.http_version @property def keep_alive(self) -> bool: + """If the message should keep alive""" return self._parser.keep_alive @property def should_upgrade(self) -> bool: + """If the message should upgrade""" return self._parser.should_upgrade @property def content_length(self) -> int|None: + """Content length of the message""" return self._parser.content_length_parsed def get_header(self, header: str, default=None) -> str: + """Get a header from the message without caring about the case""" return self._parser.lheaders.get(header.lower(), default) def _packet_to_stream(self, internal_data: DataStreamCtx): @@ -259,6 +274,11 @@ class InternalBasicHttpMetaClass: return datahandler class HttpRequest(InternalBasicHttpMetaClass): + """ + HTTP Request handler + This data handler will be called twice, first with the headers complete, and second with the body complete + """ + def __init__(self): super().__init__() # These will be used in the metaclass @@ -267,6 +287,7 @@ class HttpRequest(InternalBasicHttpMetaClass): @property def method(self) -> bytes: + """Method of the request""" return self._parser.method_parsed def _before_fetch_callable_checks(self, internal_data: DataStreamCtx): @@ -276,6 +297,11 @@ class HttpRequest(InternalBasicHttpMetaClass): return f"" class HttpResponse(InternalBasicHttpMetaClass): + """ + HTTP Response handler + This data handler will be called twice, first with the headers complete, and second with the body complete + """ + def __init__(self): super().__init__() self._parser: InternalHttpResponse = InternalHttpResponse() @@ -283,6 +309,7 @@ class HttpResponse(InternalBasicHttpMetaClass): @property def status_code(self) -> int: + """Status code of the response""" return self._parser.status def _before_fetch_callable_checks(self, internal_data: DataStreamCtx): @@ -292,6 +319,11 @@ class HttpResponse(InternalBasicHttpMetaClass): return f"" class HttpRequestHeader(HttpRequest): + """ + HTTP Request Header handler + This data handler will be called only once, the headers are complete, the body will be empty and not buffered + """ + def __init__(self): super().__init__() self._parser._save_body = False @@ -306,6 +338,11 @@ class HttpRequestHeader(HttpRequest): return False class HttpResponseHeader(HttpResponse): + """ + HTTP Response Header handler + This data handler will be called only once, the headers are complete, the body will be empty and not buffered + """ + def __init__(self): super().__init__() self._parser._save_body = False diff --git a/fgex-lib/firegex/nfproxy/models/tcp.py b/fgex-lib/firegex/nfproxy/models/tcp.py index fc46431..7e12166 100644 --- a/fgex-lib/firegex/nfproxy/models/tcp.py +++ b/fgex-lib/firegex/nfproxy/models/tcp.py @@ -7,20 +7,27 @@ class InternalTCPStream: data: bytes, is_ipv6: bool, ): - self.data = bytes(data) + self.__data = bytes(data) self.__is_ipv6 = bool(is_ipv6) self.__total_stream_size = len(data) + @property + def data(self) -> bytes: + """The data of the packets assembled and sorted from TCP""" + return self.__data + @property def is_ipv6(self) -> bool: + """It's true if the packet is an ipv6 packet, false if it's an ipv4 packet""" return self.__is_ipv6 @property def total_stream_size(self) -> int: + """The size of the stream""" return self.__total_stream_size def _push_new_data(self, data: bytes): - self.data += data + self.__data += data self.__total_stream_size += len(data) @classmethod diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 8812f44..a4d4766 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash cd "$(dirname "$0")" @@ -31,6 +31,5 @@ if [[ "$ERROR" == "0" ]] then python3 benchmark.py -p $PASSWORD -r 5 -d 1 -s 10 || ERROR=1 fi - exit $ERROR