0xb00b5/firegex-traffic-viewer
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nfproxy module writing: written part of the firegex lib, frontend refactored and improved, c++ improves

Browse Source
This commit is contained in:
Domingo Dirutigliano
2025-02-20 19:51:28 +01:00
parent d6e7cab353
commit 8652f40235
51 changed files with 1864 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/binsrc/classes/nfqueue.cpp
View File

@@ -131,6 +131,15 @@ class PktRequest {
}
}
void mangle_custom_pkt(const uint8_t* pkt, size_t pkt_size){
if (action == FilterAction::NOACTION){
action = FilterAction::MANGLE;
perfrom_action(pkt, pkt_size);
}else{
throw invalid_argument("Cannot mangle a packet that has already been accepted or dropped");
}
}
FilterAction get_action(){
return action;
}
@@ -141,7 +150,7 @@ class PktRequest {
}
private:
void perfrom_action(){
void perfrom_action(const uint8_t* custom_data = nullptr, size_t custom_data_size = 0){
char buf[MNL_SOCKET_BUFFER_SIZE];
struct nlmsghdr *nlh_verdict = nfq_nlmsg_put(buf, NFQNL_MSG_VERDICT, ntohs(res_id));
switch (action)
@@ -153,7 +162,9 @@ class PktRequest {
nfq_nlmsg_verdict_put(nlh_verdict, ntohl(packet_id), NF_DROP );
break;
case FilterAction::MANGLE:{
if (is_ipv6){
if (custom_data != nullptr){
nfq_nlmsg_verdict_put_pkt(nlh_verdict, custom_data, custom_data_size);
}else if (is_ipv6){
nfq_nlmsg_verdict_put_pkt(nlh_verdict, ipv6->serialize().data(), ipv6->size());
}else{
nfq_nlmsg_verdict_put_pkt(nlh_verdict, ipv4->serialize().data(), ipv4->size());

80
backend/binsrc/nfproxy.cpp
View File

@@ -1,18 +1,87 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "proxytun/settings.cpp"
#include "proxytun/proxytun.cpp"
#include "pyproxy/settings.cpp"
#include "pyproxy/pyproxy.cpp"
#include "classes/netfilter.cpp"
#include <syncstream>
#include <iostream>
#include <stdexcept>
#include <cstdlib>
#include <endian.h>
using namespace std;
using namespace Firegex::PyProxy;
using Firegex::NfQueue::MultiThreadQueue;
/*
How python code is handles:
User code example:
```python
from firegex.nfproxy import DROP, ACCEPT, pyfilter
@pyfilter
def invalid_curl_agent(http):
if "curl" in http.headers.get("User-Agent", ""):
return DROP
return ACCEPT
```
The code is now edited adding an intestation and a end statement:
```python
global __firegex_pyfilter_enabled, __firegex_proto
__firegex_pyfilter_enabled = ["invalid_curl_agent", "func3"] # This list is dynamically generated by firegex backend
__firegex_proto = "http"
import firegex.nfproxy.internals
<user_code>
firegex.nfproxy.internals.compile() # This function can save other global variables, to use by the packet handler and is used generally to check and optimize the code
````
This code will be executed only once, and is needed to build the global and local context to use
The globals and locals generated here are copied for each connection, and are used to handle the packets
Using C API will be injected in global context the following informations:
__firegex_packet_info = {
"data" = b"raw data found on L4",
"raw_packet" = b"raw packet",
"is_input" = True, # If the packet is incoming from a client
"is_ipv6" = False, # If the packet is ipv6
"is_tcp" = True, # If the packet is tcp
}
As result the packet handler is responsible to return a dictionary in the global context with the following dictionary:
__firegex_pyfilter_result = {
"action": REJECT, # One of PyFilterResponse
"matched_by": "invalid_curl_agent", # The function that matched the packet (used if action = DROP or REJECT or MANGLE)
"mangled_packet": b"new packet" # The new packet to send to the kernel (used if action = MANGLE)
}
PyFilterResponse {
ACCEPT = 0,
DROP = 1,
REJECT = 2,
MANGLE = 3,
EXCEPTION = 4,
INVALID = 5
};
Every time a packet is received, the packet handler will execute the following code:
```python
firegex.nfproxy.internals.handle_packet()
````
The TCP stream is sorted by libtins using c++ code, but the c++ code is not responsabile di buffer the stream, but only to sort those
So firegex handle_packet has to implement a way to limit memory usage, this dipends on what methods you choose to use to filter packets
firegex lib will give you all the needed possibilities to do this is many ways
Final note: is not raccomanded to use variables that starts with __firegex_ in your code, because they may break the nfproxy
*/
ssize_t read_check(int __fd, void *__buf, size_t __nbytes){
ssize_t bytes = read(__fd, __buf, __nbytes);
if (bytes == 0){
@@ -30,7 +99,10 @@ void config_updater (){
while (true){
uint32_t code_size;
read_check(STDIN_FILENO, &code_size, 4);
vector<uint8_t> code(code_size);
//Python will send number always in little endian
code_size = le32toh(code_size);
string code;
code.resize(code_size);
read_check(STDIN_FILENO, code.data(), code_size);
cerr << "[info] [updater] Updating configuration" << endl;
try{
@@ -44,10 +116,12 @@ void config_updater (){
}
}
int main(int argc, char *argv[]){
Py_Initialize();
atexit(Py_Finalize);
init_handle_packet_code(); //Compile the static code used to handle packets
if (freopen(nullptr, "rb", stdin) == nullptr){ // We need to read from stdin binary data
cerr << "[fatal] [main] Failed to reopen stdin in binary mode" << endl;

165
backend/binsrc/proxytun/proxytun.cpp
View File

@@ -1,165 +0,0 @@
#ifndef PROXY_TUNNEL_CLASS_CPP
#define PROXY_TUNNEL_CLASS_CPP
#include <linux/netfilter/nfnetlink_queue.h>
#include <libnetfilter_queue/libnetfilter_queue.h>
#include <linux/netfilter/nfnetlink_conntrack.h>
#include <tins/tins.h>
#include <tins/tcp_ip/stream_follower.h>
#include <tins/tcp_ip/stream_identifier.h>
#include <libmnl/libmnl.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nfnetlink.h>
#include <linux/types.h>
#include <stdexcept>
#include <thread>
#include <syncstream>
#include <iostream>
#include "../classes/netfilter.cpp"
#include "stream_ctx.cpp"
#include "settings.cpp"
using Tins::TCPIP::Stream;
using Tins::TCPIP::StreamFollower;
using namespace std;
namespace Firegex {
namespace PyProxy {
class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
public:
stream_ctx sctx;
StreamFollower follower;
struct {
bool matching_has_been_called = false;
bool already_closed = false;
bool result;
NfQueue::PktRequest<PyProxyQueue>* pkt;
} match_ctx;
void before_loop() override {
follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this));
follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this));
}
bool filter_action(NfQueue::PktRequest<PyProxyQueue>* pkt){
shared_ptr<PyCodeConfig> conf = config;
auto stream_search = sctx.streams_ctx.find(pkt->sid);
pyfilter_ctx* stream_match;
if (stream_search == sctx.streams_ctx.end()){
// TODO: New pyfilter_ctx
}else{
stream_match = stream_search->second;
}
bool has_matched = false;
//TODO exec filtering action
if (has_matched){
// Say to firegex what filter has matched
//osyncstream(cout) << "BLOCKED " << rules_vector[match_res.matched] << endl;
return false;
}
return true;
}
//If the stream has already been matched, drop all data, and try to close the connection
static void keep_fin_packet(PyProxyQueue* pkt){
pkt->match_ctx.matching_has_been_called = true;
pkt->match_ctx.already_closed = true;
}
static void on_data_recv(Stream& stream, PyProxyQueue* pkt, string data) {
pkt->match_ctx.matching_has_been_called = true;
pkt->match_ctx.already_closed = false;
bool result = pkt->filter_action(pkt->match_ctx.pkt);
if (!result){
pkt->sctx.clean_stream_by_id(pkt->match_ctx.pkt->sid);
stream.client_data_callback(bind(keep_fin_packet, pkt));
stream.server_data_callback(bind(keep_fin_packet, pkt));
}
pkt->match_ctx.result = result;
}
//Input data filtering
static void on_client_data(Stream& stream, PyProxyQueue* pkt) {
on_data_recv(stream, pkt, string(stream.client_payload().begin(), stream.client_payload().end()));
}
//Server data filtering
static void on_server_data(Stream& stream, PyProxyQueue* pkt) {
on_data_recv(stream, pkt, string(stream.server_payload().begin(), stream.server_payload().end()));
}
// A stream was terminated. The second argument is the reason why it was terminated
static void on_stream_close(Stream& stream, PyProxyQueue* pkt) {
stream_id stream_id = stream_id::make_identifier(stream);
pkt->sctx.clean_stream_by_id(stream_id);
}
static void on_new_stream(Stream& stream, PyProxyQueue* pkt) {
stream.auto_cleanup_payloads(true);
if (stream.is_partial_stream()) {
//TODO take a decision about this...
stream.enable_recovery_mode(10 * 1024);
}
stream.client_data_callback(bind(on_client_data, placeholders::_1, pkt));
stream.server_data_callback(bind(on_server_data, placeholders::_1, pkt));
stream.stream_closed_callback(bind(on_stream_close, placeholders::_1, pkt));
}
void handle_next_packet(NfQueue::PktRequest<PyProxyQueue>* pkt) override{
if (pkt->l4_proto != NfQueue::L4Proto::TCP){
throw invalid_argument("Only TCP and UDP are supported");
}
Tins::PDU* application_layer = pkt->tcp->inner_pdu();
u_int16_t payload_size = 0;
if (application_layer != nullptr){
payload_size = application_layer->size();
}
match_ctx.matching_has_been_called = false;
match_ctx.pkt = pkt;
if (pkt->is_ipv6){
follower.process_packet(*pkt->ipv6);
}else{
follower.process_packet(*pkt->ipv4);
}
// Do an action only is an ordered packet has been received
if (match_ctx.matching_has_been_called){
bool empty_payload = payload_size == 0;
//In this 2 cases we have to remove all data about the stream
if (!match_ctx.result || match_ctx.already_closed){
sctx.clean_stream_by_id(pkt->sid);
//If the packet has data, we have to remove it
if (!empty_payload){
Tins::PDU* data_layer = pkt->tcp->release_inner_pdu();
if (data_layer != nullptr){
delete data_layer;
}
}
//For the first matched data or only for data packets, we set FIN bit
//This only for client packets, because this will trigger server to close the connection
//Packets will be filtered anyway also if client don't send packets
if ((!match_ctx.result || !empty_payload) && pkt->is_input){
pkt->tcp->set_flag(Tins::TCP::FIN,1);
pkt->tcp->set_flag(Tins::TCP::ACK,1);
pkt->tcp->set_flag(Tins::TCP::SYN,0);
}
//Send the edited packet to the kernel
return pkt->mangle();
}
}
return pkt->accept();
}
~PyProxyQueue() {
sctx.clean();
}
};
}}
#endif // PROXY_TUNNEL_CLASS_CPP

22
backend/binsrc/proxytun/settings.cpp
View File

@@ -1,22 +0,0 @@
#ifndef PROXY_TUNNEL_SETTINGS_CPP
#define PROXY_TUNNEL_SETTINGS_CPP
#include <vector>
#include <memory>
using namespace std;
class PyCodeConfig{
public:
const vector<uint8_t> code;
public:
PyCodeConfig(vector<uint8_t> pycode): code(pycode){}
PyCodeConfig(): code(vector<uint8_t>()){}
~PyCodeConfig(){}
};
shared_ptr<PyCodeConfig> config;
#endif // PROXY_TUNNEL_SETTINGS_CPP

39
backend/binsrc/proxytun/stream_ctx.cpp
View File

@@ -1,39 +0,0 @@
#ifndef STREAM_CTX_CPP
#define STREAM_CTX_CPP
#include <iostream>
#include <tins/tcp_ip/stream_identifier.h>
#include <map>
using namespace std;
typedef Tins::TCPIP::StreamIdentifier stream_id;
struct pyfilter_ctx {
void * pyglob; // TODO python glob???
string pycode;
};
typedef map<stream_id, pyfilter_ctx*> matching_map;
struct stream_ctx {
matching_map streams_ctx;
void clean_stream_by_id(stream_id sid){
auto stream_search = streams_ctx.find(sid);
if (stream_search != streams_ctx.end()){
auto stream_match = stream_search->second;
//DEALLOC PY GLOB TODO
delete stream_match;
}
}
void clean(){
for (auto ele: streams_ctx){
//TODO dealloc ele.second.pyglob
delete ele.second;
}
}
};
#endif // STREAM_CTX_CPP

232
backend/binsrc/pyproxy/pyproxy.cpp Normal file
View File

@@ -0,0 +1,232 @@
#ifndef PROXY_TUNNEL_CLASS_CPP
#define PROXY_TUNNEL_CLASS_CPP
#include <linux/netfilter/nfnetlink_queue.h>
#include <libnetfilter_queue/libnetfilter_queue.h>
#include <linux/netfilter/nfnetlink_conntrack.h>
#include <tins/tins.h>
#include <tins/tcp_ip/stream_follower.h>
#include <tins/tcp_ip/stream_identifier.h>
#include <libmnl/libmnl.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nfnetlink.h>
#include <linux/types.h>
#include <stdexcept>
#include <thread>
#include <syncstream>
#include <iostream>
#include "../classes/netfilter.cpp"
#include "stream_ctx.cpp"
#include "settings.cpp"
#include <Python.h>
using Tins::TCPIP::Stream;
using Tins::TCPIP::StreamFollower;
using namespace std;
namespace Firegex {
namespace PyProxy {
class PyProxyQueue: public NfQueue::ThreadNfQueue<PyProxyQueue> {
private:
u_int16_t latest_config_ver = 0;
public:
stream_ctx sctx;
StreamFollower follower;
PyGILState_STATE gstate;
PyInterpreterConfig py_thread_config = {
.use_main_obmalloc = 0,
.allow_fork = 0,
.allow_exec = 0,
.allow_threads = 0,
.allow_daemon_threads = 0,
.check_multi_interp_extensions = 1,
.gil = PyInterpreterConfig_OWN_GIL,
};
PyThreadState *tstate = NULL;
PyStatus pystatus;
struct {
bool matching_has_been_called = false;
bool already_closed = false;
bool rejected = true;
NfQueue::PktRequest<PyProxyQueue>* pkt;
} match_ctx;
void before_loop() override {
// Create thred structure for python
gstate = PyGILState_Ensure();
// Create a new interpreter for the thread
pystatus = Py_NewInterpreterFromConfig(&tstate, &py_thread_config);
if (PyStatus_Exception(pystatus)) {
Py_ExitStatusException(pystatus);
cerr << "[fatal] [main] Failed to create new interpreter" << endl;
exit(EXIT_FAILURE);
}
// Setting callbacks for the stream follower
follower.new_stream_callback(bind(on_new_stream, placeholders::_1, this));
follower.stream_termination_callback(bind(on_stream_close, placeholders::_1, this));
}
inline void print_blocked_reason(const string& func_name){
osyncstream(cout) << "BLOCKED " << func_name << endl;
}
inline void print_mangle_reason(const string& func_name){
osyncstream(cout) << "MANGLED " << func_name << endl;
}
inline void print_exception_reason(){
osyncstream(cout) << "EXCEPTION" << endl;
}
//If the stream has already been matched, drop all data, and try to close the connection
static void keep_fin_packet(PyProxyQueue* proxy_info){
proxy_info->match_ctx.matching_has_been_called = true;
proxy_info->match_ctx.already_closed = true;
}
void filter_action(NfQueue::PktRequest<PyProxyQueue>* pkt, Stream& stream){
auto stream_search = sctx.streams_ctx.find(pkt->sid);
pyfilter_ctx* stream_match;
if (stream_search == sctx.streams_ctx.end()){
shared_ptr<PyCodeConfig> conf = config;
//If config is not set, ignore the stream
if (conf->glob == nullptr || conf->local == nullptr){
stream.client_data_callback(nullptr);
stream.server_data_callback(nullptr);
return pkt->accept();
}
stream_match = new pyfilter_ctx(conf->glob, conf->local);
sctx.streams_ctx.insert_or_assign(pkt->sid, stream_match);
}else{
stream_match = stream_search->second;
}
auto result = stream_match->handle_packet(pkt);
switch(result.action){
case PyFilterResponse::ACCEPT:
pkt->accept();
case PyFilterResponse::DROP:
print_blocked_reason(*result.filter_match_by);
sctx.clean_stream_by_id(pkt->sid);
stream.client_data_callback(nullptr);
stream.server_data_callback(nullptr);
break;
case PyFilterResponse::REJECT:
sctx.clean_stream_by_id(pkt->sid);
stream.client_data_callback(bind(keep_fin_packet, this));
stream.server_data_callback(bind(keep_fin_packet, this));
pkt->ctx->match_ctx.rejected = true; //Handler will take care of the rest
break;
case PyFilterResponse::MANGLE:
print_mangle_reason(*result.filter_match_by);
pkt->mangle_custom_pkt((uint8_t*)result.mangled_packet->c_str(), result.mangled_packet->size());
break;
case PyFilterResponse::EXCEPTION:
case PyFilterResponse::INVALID:
print_exception_reason();
sctx.clean_stream_by_id(pkt->sid);
//Free the packet data
stream.client_data_callback(nullptr);
stream.server_data_callback(nullptr);
pkt->accept();
break;
}
}
static void on_data_recv(Stream& stream, PyProxyQueue* proxy_info, string data) {
proxy_info->match_ctx.matching_has_been_called = true;
proxy_info->match_ctx.already_closed = false;
proxy_info->filter_action(proxy_info->match_ctx.pkt, stream);
}
//Input data filtering
static void on_client_data(Stream& stream, PyProxyQueue* proxy_info) {
on_data_recv(stream, proxy_info, string(stream.client_payload().begin(), stream.client_payload().end()));
}
//Server data filtering
static void on_server_data(Stream& stream, PyProxyQueue* proxy_info) {
on_data_recv(stream, proxy_info, string(stream.server_payload().begin(), stream.server_payload().end()));
}
// A stream was terminated. The second argument is the reason why it was terminated
static void on_stream_close(Stream& stream, PyProxyQueue* proxy_info) {
stream_id stream_id = stream_id::make_identifier(stream);
proxy_info->sctx.clean_stream_by_id(stream_id);
}
static void on_new_stream(Stream& stream, PyProxyQueue* proxy_info) {
stream.auto_cleanup_payloads(true);
if (stream.is_partial_stream()) {
stream.enable_recovery_mode(10 * 1024);
}
stream.client_data_callback(bind(on_client_data, placeholders::_1, proxy_info));
stream.server_data_callback(bind(on_server_data, placeholders::_1, proxy_info));
stream.stream_closed_callback(bind(on_stream_close, placeholders::_1, proxy_info));
}
void handle_next_packet(NfQueue::PktRequest<PyProxyQueue>* pkt) override{
if (pkt->l4_proto != NfQueue::L4Proto::TCP){
throw invalid_argument("Only TCP and UDP are supported");
}
Tins::PDU* application_layer = pkt->tcp->inner_pdu();
u_int16_t payload_size = 0;
if (application_layer != nullptr){
payload_size = application_layer->size();
}
match_ctx.matching_has_been_called = false;
match_ctx.pkt = pkt;
if (pkt->is_ipv6){
follower.process_packet(*pkt->ipv6);
}else{
follower.process_packet(*pkt->ipv4);
}
// Do an action only is an ordered packet has been received
if (match_ctx.matching_has_been_called){
bool empty_payload = payload_size == 0;
//In this 2 cases we have to remove all data about the stream
if (!match_ctx.rejected || match_ctx.already_closed){
sctx.clean_stream_by_id(pkt->sid);
//If the packet has data, we have to remove it
if (!empty_payload){
Tins::PDU* data_layer = pkt->tcp->release_inner_pdu();
if (data_layer != nullptr){
delete data_layer;
}
}
//For the first matched data or only for data packets, we set FIN bit
//This only for client packets, because this will trigger server to close the connection
//Packets will be filtered anyway also if client don't send packets
if ((!match_ctx.rejected || !empty_payload) && pkt->is_input){
pkt->tcp->set_flag(Tins::TCP::FIN,1);
pkt->tcp->set_flag(Tins::TCP::ACK,1);
pkt->tcp->set_flag(Tins::TCP::SYN,0);
}
//Send the edited packet to the kernel
return pkt->mangle();
}else{
//Fallback to the default action
if (pkt->get_action() == NfQueue::FilterAction::NOACTION){
return pkt->accept();
}
}
}else{
return pkt->accept();
}
}
~PyProxyQueue() {
// Closing first the interpreter
Py_EndInterpreter(tstate);
// Releasing the GIL and the thread data structure
PyGILState_Release(gstate);
sctx.clean();
}
};
}}
#endif // PROXY_TUNNEL_CLASS_CPP

71
backend/binsrc/pyproxy/settings.cpp Normal file
View File

@@ -0,0 +1,71 @@
#ifndef PROXY_TUNNEL_SETTINGS_CPP
#define PROXY_TUNNEL_SETTINGS_CPP
#include <Python.h>
#include <vector>
#include <memory>
#include <iostream>
using namespace std;
namespace Firegex {
namespace PyProxy {
class PyCodeConfig{
public:
PyObject* glob = nullptr;
PyObject* local = nullptr;
private:
void _clean(){
Py_XDECREF(glob);
Py_XDECREF(local);
}
public:
PyCodeConfig(const string& pycode){
PyObject* compiled_code = Py_CompileStringExFlags(pycode.c_str(), "<pyfilter>", Py_file_input, NULL, 2);
if (compiled_code == nullptr){
std::cerr << "[fatal] [main] Failed to compile the code" << endl;
_clean();
throw invalid_argument("Failed to compile the code");
}
glob = PyDict_New();
local = PyDict_New();
PyObject* result = PyEval_EvalCode(compiled_code, glob, local);
Py_XDECREF(compiled_code);
if (!result){
PyErr_Print();
_clean();
std::cerr << "[fatal] [main] Failed to execute the code" << endl;
throw invalid_argument("Failed to execute the code, maybe an invalid filter code has been provided");
}
Py_DECREF(result);
}
PyCodeConfig(){}
~PyCodeConfig(){
_clean();
}
};
shared_ptr<PyCodeConfig> config;
PyObject* py_handle_packet_code = nullptr;
void init_handle_packet_code(){
py_handle_packet_code = Py_CompileStringExFlags(
"firegex.nfproxy.internals.handle_packet()\n", "<pyfilter>",
Py_file_input, NULL, 2);
if (py_handle_packet_code == nullptr){
std::cerr << "[fatal] [main] Failed to compile the utility python code (strange behaviour, probably a bug)" << endl;
throw invalid_argument("Failed to compile the code");
}
}
}}
#endif // PROXY_TUNNEL_SETTINGS_CPP

202
backend/binsrc/pyproxy/stream_ctx.cpp Normal file
View File

@@ -0,0 +1,202 @@
#ifndef STREAM_CTX_CPP
#define STREAM_CTX_CPP
#include <iostream>
#include <tins/tcp_ip/stream_identifier.h>
#include <map>
#include <Python.h>
#include "../classes/netfilter.cpp"
#include "settings.cpp"
using namespace std;
namespace Firegex {
namespace PyProxy {
class PyCodeConfig;
class PyProxyQueue;
enum PyFilterResponse {
ACCEPT = 0,
DROP = 1,
REJECT = 2,
MANGLE = 3,
EXCEPTION = 4,
INVALID = 5
};
struct py_filter_response {
PyFilterResponse action;
string* filter_match_by = nullptr;
string* mangled_packet = nullptr;
~py_filter_response(){
delete mangled_packet;
delete filter_match_by;
}
};
typedef Tins::TCPIP::StreamIdentifier stream_id;
struct pyfilter_ctx {
PyObject * glob = nullptr;
PyObject * local = nullptr;
pyfilter_ctx(PyObject * original_glob, PyObject * original_local){
PyObject *copy = PyImport_ImportModule("copy");
if (copy == nullptr){
PyErr_Print();
throw invalid_argument("Failed to import copy module");
}
PyObject *deepcopy = PyObject_GetAttrString(copy, "deepcopy");
glob = PyObject_CallFunctionObjArgs(deepcopy, original_glob, NULL);
if (glob == nullptr){
PyErr_Print();
throw invalid_argument("Failed to deepcopy the global dict");
}
local = PyObject_CallFunctionObjArgs(deepcopy, original_local, NULL);
if (local == nullptr){
PyErr_Print();
throw invalid_argument("Failed to deepcopy the local dict");
}
Py_DECREF(copy);
}
~pyfilter_ctx(){
Py_XDECREF(glob);
Py_XDECREF(local);
}
inline void set_item_to_glob(const char* key, PyObject* value){
set_item_to_dict(glob, key, value);
}
inline PyObject* get_item_from_glob(const char* key){
return PyDict_GetItemString(glob, key);
}
void del_item_from_glob(const char* key){
if (PyDict_DelItemString(glob, key) != 0){
PyErr_Print();
throw invalid_argument("Failed to delete item from dict");
}
}
inline void set_item_to_local(const char* key, PyObject* value){
set_item_to_dict(local, key, value);
}
inline void set_item_to_dict(PyObject* dict, const char* key, PyObject* value){
if (PyDict_SetItemString(dict, key, value) != 0){
PyErr_Print();
throw invalid_argument("Failed to set item to dict");
}
}
py_filter_response handle_packet(
NfQueue::PktRequest<PyProxyQueue>* pkt
){
PyObject * packet_info = PyDict_New();
set_item_to_dict(packet_info, "data", PyBytes_FromStringAndSize(pkt->data, pkt->data_size));
set_item_to_dict(packet_info, "raw_packet", PyBytes_FromStringAndSize(pkt->packet.c_str(), pkt->packet.size()));
set_item_to_dict(packet_info, "is_input", PyBool_FromLong(pkt->is_input));
set_item_to_dict(packet_info, "is_ipv6", PyBool_FromLong(pkt->is_ipv6));
set_item_to_dict(packet_info, "is_tcp", PyBool_FromLong(pkt->l4_proto == NfQueue::L4Proto::TCP));
// Set packet info to the global context
set_item_to_glob("__firegex_packet_info", packet_info);
PyObject * result = PyEval_EvalCode(py_handle_packet_code, glob, local);
del_item_from_glob("__firegex_packet_info");
Py_DECREF(packet_info);
if (!result){
PyErr_Print();
return py_filter_response{PyFilterResponse::EXCEPTION, nullptr};
}
Py_DECREF(result);
result = get_item_from_glob("__firegex_pyfilter_result");
if (result == nullptr){
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
if (!PyDict_Check(result)){
PyErr_Print();
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
PyObject* action = PyDict_GetItemString(result, "action");
if (action == nullptr){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
if (!PyLong_Check(action)){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
PyFilterResponse action_enum = (PyFilterResponse)PyLong_AsLong(action);
if (action_enum == PyFilterResponse::ACCEPT || action_enum == PyFilterResponse::EXCEPTION || action_enum == PyFilterResponse::INVALID){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{action_enum, nullptr, nullptr};
}else{
PyObject *func_name_py = PyDict_GetItemString(result, "matched_by");
if (func_name_py == nullptr){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
if (!PyUnicode_Check(func_name_py)){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
string* func_name = new string(PyUnicode_AsUTF8(func_name_py));
if (action_enum == PyFilterResponse::DROP || action_enum == PyFilterResponse::REJECT){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{action_enum, func_name, nullptr};
}
if (action_enum != PyFilterResponse::MANGLE){
PyObject* mangled_packet = PyDict_GetItemString(result, "mangled_packet");
if (mangled_packet == nullptr){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
if (!PyBytes_Check(mangled_packet)){
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
string* pkt_str = new string(PyBytes_AsString(mangled_packet), PyBytes_Size(mangled_packet));
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::MANGLE, func_name, pkt_str};
}
}
del_item_from_glob("__firegex_pyfilter_result");
return py_filter_response{PyFilterResponse::INVALID, nullptr, nullptr};
}
};
typedef map<stream_id, pyfilter_ctx*> matching_map;
struct stream_ctx {
matching_map streams_ctx;
void clean_stream_by_id(stream_id sid){
auto stream_search = streams_ctx.find(sid);
if (stream_search != streams_ctx.end()){
auto stream_match = stream_search->second;
delete stream_match;
}
}
void clean(){
for (auto ele: streams_ctx){
delete ele.second;
}
}
};
}}
#endif // STREAM_CTX_CPP

11
backend/binsrc/regex/regex_rules.cpp
View File

@@ -76,12 +76,11 @@ class RegexRules{
}else{
hs_free_database(db);
}
}
private:
static inline u_int16_t glob_seq = 0;
u_int16_t version;
static inline uint16_t glob_seq = 0;
uint16_t version;
vector<pair<string, decoded_regex>> decoded_input_rules;
vector<pair<string, decoded_regex>> decoded_output_rules;
bool is_stream = true;
@@ -96,9 +95,7 @@ class RegexRules{
input_ruleset.hs_db = nullptr;
}
}
void fill_ruleset(vector<pair<string, decoded_regex>> & decoded, regex_ruleset & ruleset){
size_t n_of_regex = decoded.size();
if (n_of_regex == 0){
@@ -150,7 +147,6 @@ class RegexRules{
public:
RegexRules(vector<string> raw_rules, bool is_stream){
this->is_stream = is_stream;
this->version = ++glob_seq; // 0 version is a invalid version (useful for some logics)
for(string ele : raw_rules){
try{
decoded_regex rule = decode_regex(ele);
@@ -170,6 +166,7 @@ class RegexRules{
free_dbs();
throw current_exception();
}
this->version = ++glob_seq; // 0 version is the null version
}
u_int16_t ver(){

28
backend/modules/nfproxy/firegex.py
View File

@@ -22,7 +22,7 @@ class FiregexInterceptor:
self.update_task: asyncio.Task
self.ack_arrived = False
self.ack_status = None
self.ack_fail_what = ""
self.ack_fail_what = "Unknown"
self.ack_lock = asyncio.Lock()
async def _call_stats_updater_callback(self, filter: PyFilter):
@@ -79,12 +79,14 @@ class FiregexInterceptor:
if filter_id in self.filter_map:
self.filter_map[filter_id].blocked_packets+=1
await self.filter_map[filter_id].update()
if line.startswith("EDITED "):
if line.startswith("MANGLED "):
filter_id = line.split()[1]
async with self.filter_map_lock:
if filter_id in self.filter_map:
self.filter_map[filter_id].edited_packets+=1
await self.filter_map[filter_id].update()
if line.startswith("EXCEPTION"):
print("TODO EXCEPTION HANDLING") # TODO
if line.startswith("ACK "):
self.ack_arrived = True
self.ack_status = line.split()[1].upper() == "OK"
@@ -103,10 +105,9 @@ class FiregexInterceptor:
if self.process and self.process.returncode is None:
self.process.kill()
async def _update_config(self, filters_codes):
async def _update_config(self, code):
async with self.update_config_lock:
# TODO write compiled code correctly
# self.process.stdin.write((" ".join(filters_codes)+"\n").encode())
self.process.stdin.write(len(code).to_bytes(4, byteorder='big')+code.encode())
await self.process.stdin.drain()
try:
async with asyncio.timeout(3):
@@ -114,11 +115,22 @@ class FiregexInterceptor:
except TimeoutError:
pass
if not self.ack_arrived or not self.ack_status:
await self.stop()
raise HTTPException(status_code=500, detail=f"NFQ error: {self.ack_fail_what}")
async def reload(self, filters:list[PyFilter]):
async with self.filter_map_lock:
self.filter_map = self.compile_filters(filters)
# TODO COMPILE CODE
#await self._update_config(filters_codes) TODO pass the compiled code
if os.path.exists(f"db/nfproxy_filters/{self.srv.id}.py"):
with open(f"db/nfproxy_filters/{self.srv.id}.py") as f:
filter_file = f.read()
else:
filter_file = ""
await self._update_config(
"global __firegex_pyfilter_enabled\n" +
"__firegex_pyfilter_enabled = [" + ", ".join([repr(f.name) for f in filters]) + "]\n" +
"__firegex_proto = " + repr(self.srv.proto) + "\n" +
"import firegex.nfproxy.internals\n\n" +
filter_file + "\n\n" +
"firegex.nfproxy.internals.compile()"
)

3
backend/modules/nfregex/firegex.py
View File

@@ -79,7 +79,7 @@ class FiregexInterceptor:
self.update_task: asyncio.Task
self.ack_arrived = False
self.ack_status = None
self.ack_fail_what = ""
self.ack_fail_what = "Unknown"
self.ack_lock = asyncio.Lock()
@classmethod
@@ -160,6 +160,7 @@ class FiregexInterceptor:
except TimeoutError:
pass
if not self.ack_arrived or not self.ack_status:
await self.stop()
raise HTTPException(status_code=500, detail=f"NFQ error: {self.ack_fail_what}")

66
backend/routers/nfproxy.py
View File

@@ -7,6 +7,9 @@ from modules.nfproxy.firewall import STATUS, FirewallManager
from utils.sqlite import SQLite
from utils import ip_parse, refactor_name, socketio_emit, PortType
from utils.models import ResetRequest, StatusMessageModel
import os
from firegex.nfproxy.internals import get_filter_names
from fastapi.responses import PlainTextResponse
class ServiceModel(BaseModel):
service_id: str
@@ -47,6 +50,9 @@ class ServiceAddResponse(BaseModel):
status:str
service_id: str|None = None
class SetPyFilterForm(BaseModel):
code: str
app = APIRouter()
db = SQLite('db/nft-pyfilters.db', {
@@ -70,7 +76,7 @@ db = SQLite('db/nft-pyfilters.db', {
},
'QUERY':[
"CREATE UNIQUE INDEX IF NOT EXISTS unique_services ON services (port, ip_int, proto);",
"CREATE UNIQUE INDEX IF NOT EXISTS unique_pyfilter_service ON pyfilter (name, service_id);"
"CREATE UNIQUE INDEX IF NOT EXISTS unique_pyfilter_service ON pyfilter (name, service_id);"
]
})
@@ -174,6 +180,8 @@ async def service_delete(service_id: str):
"""Request the deletion of a specific service"""
db.query('DELETE FROM services WHERE service_id = ?;', service_id)
db.query('DELETE FROM pyfilter WHERE service_id = ?;', service_id)
if os.path.exists(f"db/nfproxy_filters/{service_id}.py"):
os.remove(f"db/nfproxy_filters/{service_id}.py")
await firewall.remove(service_id)
await refresh_frontend()
return {'status': 'ok'}
@@ -253,17 +261,6 @@ async def get_pyfilter_by_id(filter_id: int):
raise HTTPException(status_code=400, detail="This filter does not exists!")
return res[0]
@app.delete('/pyfilters/{filter_id}', response_model=StatusMessageModel)
async def pyfilter_delete(filter_id: int):
"""Delete a pyfilter using his id"""
res = db.query('SELECT * FROM pyfilter WHERE filter_id = ?;', filter_id)
if len(res) != 0:
db.query('DELETE FROM pyfilter WHERE filter_id = ?;', filter_id)
await firewall.get(res[0]["service_id"]).update_filters()
await refresh_frontend()
return {'status': 'ok'}
@app.post('/pyfilters/{filter_id}/enable', response_model=StatusMessageModel)
async def pyfilter_enable(filter_id: int):
"""Request the enabling of a pyfilter"""
@@ -304,6 +301,49 @@ async def add_new_service(form: ServiceAddForm):
await refresh_frontend()
return {'status': 'ok', 'service_id': srv_id}
@app.put('/services/{service_id}/pyfilters/code', response_model=StatusMessageModel)
async def set_pyfilters(service_id: str, form: SetPyFilterForm):
"""Set the python filter for a service"""
service = db.query("SELECT service_id, proto FROM services WHERE service_id = ?;", service_id)
if len(service) == 0:
raise HTTPException(status_code=400, detail="This service does not exists!")
service = service[0]
srv_proto = service["proto"]
try:
found_filters = get_filter_names(form.code, srv_proto)
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
# Remove filters that are not in the new code
existing_filters = db.query("SELECT filter_id FROM pyfilter WHERE service_id = ?;", service_id)
for filter in existing_filters:
if filter["name"] not in found_filters:
db.query("DELETE FROM pyfilter WHERE filter_id = ?;", filter["filter_id"])
# Add filters that are in the new code but not in the database
for filter in found_filters:
if not db.query("SELECT 1 FROM pyfilter WHERE service_id = ? AND name = ?;", service_id, filter):
db.query("INSERT INTO pyfilter (name, service_id) VALUES (?, ?);", filter, service["service_id"])
# Eventually edited filters will be reloaded
os.makedirs("db/nfproxy_filters", exist_ok=True)
with open(f"db/nfproxy_filters/{service_id}.py", "w") as f:
f.write(form.code)
await firewall.get(service_id).update_filters()
await refresh_frontend()
return {'status': 'ok'}
@app.get('/services/{service_id}/pyfilters/code', response_class=PlainTextResponse)
async def get_pyfilters(service_id: str):
"""Get the python filter for a service"""
if not db.query("SELECT 1 FROM services s WHERE s.service_id = ?;", service_id):
raise HTTPException(status_code=400, detail="This service does not exists!")
try:
with open(f"db/nfproxy_filters/{service_id}.py") as f:
return f.read()
except FileNotFoundError:
return ""
#TODO check all the APIs and add
# 1. API to change the python filter file
# 1. API to change the python filter file (DONE)
# 2. a socketio mechanism to lock the previous feature