0xb00b5/firegex-traffic-viewer
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improves on the nfregex binary

Browse Source
This commit is contained in:
Domingo Dirutigliano
2025-02-03 02:04:10 +01:00
parent ec279a4fc5
commit 3ffccdddfb
9 changed files with 116 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@@ -36,13 +36,3 @@ firegex.py
**/npm-debug.log*
**/yarn-debug.log*
**/yarn-error.log*
#rust
/backend/binsrc/nfqueue_regex/debug/
/backend/binsrc/nfqueue_regex/target/
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
**/*.pdb

38
backend/binsrc/classes/netfilter.cpp
View File

@@ -98,12 +98,21 @@ struct packet_info {
typedef bool NetFilterQueueCallback(packet_info &);
Tins::PDU * find_transport_layer(Tins::PDU* pkt){
Tins::PDU * find_transport_layer(Tins::PDU* pkt, Tins::PDU::PDUType* type = nullptr, size_t* payload_size = nullptr){
Tins::PDU::PDUType fetched_type;
while(pkt != nullptr){
if (pkt->pdu_type() == Tins::PDU::TCP || pkt->pdu_type() == Tins::PDU::UDP) {
fetched_type = pkt->pdu_type();
if (fetched_type == Tins::PDU::TCP || fetched_type == Tins::PDU::UDP) {
if (type != nullptr){
*type = fetched_type;
}
if (payload_size != nullptr){
*payload_size = pkt->inner_pdu()->size();
}
return pkt;
}
pkt = pkt->inner_pdu();
}
return nullptr;
}
@@ -358,32 +367,40 @@ class NetfilterQueue {
nlh_verdict = nfq_nlmsg_put(buf, NFQNL_MSG_VERDICT, ntohs(nfg->res_id));
// Check IP protocol version
Tins::PDU *packet;
Tins::PDU *packet, *transport_layer;
Tins::PDU::PDUType transport_layer_type;
size_t payload_size;
if ( ((payload)[0] & 0xf0) == 0x40 ){
Tins::IP parsed = Tins::IP(payload, plen);
packet = &parsed;
transport_layer = find_transport_layer(packet, &transport_layer_type, &payload_size);
}else{
Tins::IPv6 parsed = Tins::IPv6(payload, plen);
packet = &parsed;
transport_layer = find_transport_layer(packet, &transport_layer_type, &payload_size);
}
Tins::PDU *transport_layer = find_transport_layer(packet);
if(transport_layer == nullptr || transport_layer->inner_pdu() == nullptr){
nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT );
}else{
bool is_tcp = transport_layer->pdu_type() == Tins::PDU::TCP;
int size = transport_layer->inner_pdu()->size();
bool is_tcp = transport_layer_type == Tins::PDU::TCP;
packet_info pktinfo{
packet: string(payload, payload+plen),
payload: string(payload+plen - size, payload+plen),
stream_id: "", // TODO We need to calculate this
is_input: true, // TODO We need to detect this
payload: string(payload+plen - payload_size, payload+plen),
stream_id: "udp", // For udp we don't need to track the stream
is_input: false, // TODO We need to detect this
is_tcp: is_tcp,
sctx: sctx,
};
cerr << "PKT " << endl;
if (is_tcp){
cerr << "PORCO8888" << endl;
sctx->tcp_match_util.matching_has_been_called = false;
sctx->tcp_match_util.pkt_info = &pktinfo;
sctx->follower.process_packet(*packet);
nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT );
//sctx->follower.process_packet(*packet);
cerr << "NOT BLOCKED BUT YES 0_0 " << (sctx->tcp_match_util.matching_has_been_called && !sctx->tcp_match_util.result) << endl;
/*
if (sctx->tcp_match_util.matching_has_been_called && !sctx->tcp_match_util.result){
auto tcp_layer = (Tins::TCP *)transport_layer;
tcp_layer->release_inner_pdu();
@@ -394,6 +411,7 @@ class NetfilterQueue {
nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT );
delete tcp_layer;
}
*/
}else if(callback_func(pktinfo)){
nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT );
} else{

6
backend/binsrc/classes/regex_rules.cpp
View File

@@ -20,7 +20,7 @@ struct decoded_regex {
struct regex_ruleset {
hs_database_t* hs_db = nullptr;
char** regexes = nullptr;
vector<string> regexes;
};
decoded_regex decode_regex(string regex){
@@ -107,6 +107,10 @@ class RegexRules{
throw runtime_error( "Failed to compile hyperscan db" );
}
ruleset.hs_db = rebuilt_db;
ruleset.regexes = vector<string>(n_of_regex);
for(int i = 0; i < n_of_regex; i++){
ruleset.regexes[i] = decoded[i].first;
}
}
public:

24
backend/binsrc/nfqueue.cpp
View File

@@ -32,7 +32,7 @@ void config_updater (){
}
try{
regex_config.reset(new RegexRules(raw_rules, regex_config->stream_mode()));
cerr << "[info] [updater] Config update done" << endl;
cerr << "[info] [updater] Config update done to ver "<< regex_config->ver() << endl;
}catch(...){
cerr << "[error] [updater] Failed to build new configuration!" << endl;
// TODO send a row on stdout for this error
@@ -42,7 +42,7 @@ void config_updater (){
}
void inline scratch_setup(regex_ruleset &conf, hs_scratch_t* & scratch){
if (scratch == nullptr){
if (scratch == nullptr && conf.hs_db != nullptr){
if (hs_alloc_scratch(conf.hs_db, &scratch) != HS_SUCCESS) {
throw invalid_argument("Cannot alloc scratch");
}
@@ -73,18 +73,21 @@ bool filter_callback(packet_info & info){
auto res = (matched_data*)ctx;
res->has_matched = true;
res->matched = id;
return 1; // Stop matching
return -1; // Stop matching
};
hs_stream_t* stream_match;
if (conf->stream_mode()){
matching_map match_map = info.is_input ? info.sctx->in_hs_streams : info.sctx->out_hs_streams;
auto stream_search = match_map.find(info.stream_id);
hs_stream_t* stream_match;
if (stream_search == match_map.end()){
if (hs_open_stream(regex_matcher, 0, &stream_match) != HS_SUCCESS) {
cerr << "[error] [filter_callback] Error opening the stream matcher (hs)" << endl;
throw invalid_argument("Cannot open stream match on hyperscan");
}
match_map[info.stream_id] = stream_match;
if (info.is_tcp){
match_map[info.stream_id] = stream_match;
}
}else{
stream_match = stream_search->second;
}
@@ -98,10 +101,17 @@ bool filter_callback(packet_info & info){
0, scratch_space, match_func, &match_res
);
}
if (err != HS_SUCCESS) {
if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
cerr << "[error] [filter_callback] Error while matching the stream (hs)" << endl;
throw invalid_argument("Error while matching the stream with hyperscan");
}
if (
!info.is_tcp && conf->stream_mode() &&
hs_close_stream(stream_match, scratch_space, nullptr, nullptr) != HS_SUCCESS
){
cerr << "[error] [filter_callback] Error closing the stream matcher (hs)" << endl;
throw invalid_argument("Cannot close stream match on hyperscan");
}
if (match_res.has_matched){
auto rules_vector = info.is_input ? conf->input_ruleset.regexes : conf->output_ruleset.regexes;
stringstream msg;
@@ -123,7 +133,7 @@ int main(int argc, char *argv[]){
if (matchmode != nullptr && strcmp(matchmode, "block") == 0){
stream_mode = false;
}
cerr << "[info] [main] Using " << n_of_threads << " threads" << endl;
cerr << "[info] [main] Using " << n_of_threads << " threads, stream mode: " << stream_mode << endl;
regex_config.reset(new RegexRules(stream_mode));
NFQueueSequence<filter_callback> queues(n_of_threads);

3
backend/modules/nfregex/firegex.py
View File

@@ -5,6 +5,7 @@ import re
import os
import asyncio
import traceback
from utils import DEBUG
nft = FiregexTables()
@@ -99,6 +100,8 @@ class FiregexInterceptor:
try:
while True:
line = (await self.process.stdout.readuntil()).decode()
if DEBUG:
print(line)
if line.startswith("BLOCKED"):
regex_id = line.split()[1]
async with self.filter_map_lock:

2
tests/benchmark.py
View File

@@ -99,7 +99,7 @@ print(f"{results[0]} MB/s")
for i in range(1,args.num_of_regexes+1):
regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode()
if not firegex.nf_add_regex(service_id,regex,"B",active=True,is_case_sensitive=False):
puts("Benchmark Failed: Coulnd't add the regex ✗", color=colors.red)
puts("Benchmark Failed: Couldn't add the regex ✗", color=colors.red)
exit_test(1)
puts(f"Performance with {i} regex(s): ", color=colors.red, end='')
results.append(getReading(args.port))

92
tests/nf_test.py
View File

@@ -45,7 +45,7 @@ def exit_test(code):
service_id = firegex.nf_add_service(args.service_name, args.port, args.proto , "::1" if args.ipv6 else "127.0.0.1" )
if service_id:
puts("Sucessfully created service {service_id} ✔", color=colors.green)
puts(f"Sucessfully created service {service_id} ✔", color=colors.green)
else:
puts("Test Failed: Failed to create service ✗", color=colors.red)
exit(1)
@@ -58,21 +58,26 @@ else:
server.start()
time.sleep(0.5)
if server.sendCheckData(secrets.token_bytes(432)):
puts("Successfully tested first proxy with no regex ✔", color=colors.green)
else:
puts("Test Failed: Data was corrupted ", color=colors.red)
try:
if server.sendCheckData(secrets.token_bytes(432)):
puts("Successfully tested first proxy with no regex ✔", color=colors.green)
else:
puts("Test Failed: Data was corrupted ", color=colors.red)
exit_test(1)
except Exception as e:
puts("Test Failed: Couldn't send data to the server ", color=colors.red)
exit_test(1)
#Add new regex
secret = bytes(secrets.token_hex(16).encode())
regex = base64.b64encode(secret).decode()
if firegex.nf_add_regex(service_id,regex,"B",active=True,is_case_sensitive=True):
puts(f"Sucessfully added regex {str(secret)} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't add the regex {str(secret)} ✗", color=colors.red)
puts(f"Test Failed: Couldn't add the regex {str(secret)} ✗", color=colors.red)
exit_test(1)
#Check if regex is present in the service
n_blocked = 0
@@ -105,6 +110,18 @@ def checkRegex(regex, should_work=True, upper=False):
puts("Test Failed: The request was blocked when it shouldn't have", color=colors.red)
exit_test(1)
def clear_regexes():
global n_blocked
n_blocked = 0
for r in firegex.nf_get_service_regexes(service_id):
if r["regex"] == regex:
if(firegex.nf_delete_regex(r["id"])):
puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red)
exit_test(1)
break
checkRegex(regex)
#Pause the proxy
@@ -128,39 +145,31 @@ checkRegex(regex)
#Disable regex
for r in firegex.nf_get_service_regexes(service_id):
if r["regex"] == regex:
if(firegex.nf_disable_regex(r["id"])):
puts(f"Sucessfully disabled regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't disable the regex ✗", color=colors.red)
exit_test(1)
break
if r["regex"] == regex:
if(firegex.nf_disable_regex(r["id"])):
puts(f"Sucessfully disabled regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't disable the regex ✗", color=colors.red)
exit_test(1)
break
#Check if it's actually disabled
checkRegex(regex,should_work=False)
#Enable regex
for r in firegex.nf_get_service_regexes(service_id):
if r["regex"] == regex:
if(firegex.nf_enable_regex(r["id"])):
puts(f"Sucessfully enabled regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't enable the regex ✗", color=colors.red)
exit_test(1)
break
if r["regex"] == regex:
if(firegex.nf_enable_regex(r["id"])):
puts(f"Sucessfully enabled regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't enable the regex ✗", color=colors.red)
exit_test(1)
break
checkRegex(regex)
#Delete regex
n_blocked = 0
for r in firegex.nf_get_service_regexes(service_id):
if r["regex"] == regex:
if(firegex.nf_delete_regex(r["id"])):
puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red)
exit_test(1)
break
clear_regexes()
#Check if it's actually deleted
checkRegex(regex,should_work=False)
@@ -172,19 +181,22 @@ else:
puts(f"Test Failed: Coulnd't add the case insensitive regex {str(secret)} ✗", color=colors.red)
exit_test(1)
checkRegex(regex,upper=True)
checkRegex(regex, upper=True)
checkRegex(regex)
clear_regexes()
#Create Server regex and verify that should not matches
if(firegex.nf_add_regex(service_id,regex,"S",active=True, is_case_sensitive=True)):
puts(f"Sucessfully added server to client regex {str(secret)} ✔", color=colors.green)
else:
puts(f"Test Failed: Coulnd't add server to client regex {str(secret)} ✗", color=colors.red)
exit_test(1)
checkRegex(regex, should_work=False)
#Delete regex
n_blocked = 0
for r in firegex.nf_get_service_regexes(service_id):
if r["regex"] == regex:
if(firegex.nf_delete_regex(r["id"])):
puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green)
else:
puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red)
exit_test(1)
break
clear_regexes()
#Rename service
if(firegex.nf_rename_service(service_id,f"{args.service_name}2")):

2
tests/ph_test.py
View File

@@ -45,7 +45,7 @@ def exit_test(code):
#Create and start serivce
service_id = firegex.ph_add_service(args.service_name, args.port, args.port+1, args.proto , "::1" if args.ipv6 else "127.0.0.1", "::1" if args.ipv6 else "127.0.0.1")
if service_id:
puts("Sucessfully created service {service_id} ✔", color=colors.green)
puts(f"Sucessfully created service {service_id} ✔", color=colors.green)
else:
puts("Test Failed: Failed to create service ✗", color=colors.red)
exit(1)

16
tests/run_tests.sh
View File

@@ -9,23 +9,23 @@ ERROR=0
pip3 install -r requirements.txt
echo "Running standard API test"
python3 api_test.py -p $PASSWORD || ERROR=1
#python3 api_test.py -p $PASSWORD || ERROR=1
echo "Running Netfilter Regex TCP ipv4"
python3 nf_test.py -p $PASSWORD -m tcp || ERROR=1
echo "Running Netfilter Regex TCP ipv6"
python3 nf_test.py -p $PASSWORD -m tcp -6 || ERROR=1
#python3 nf_test.py -p $PASSWORD -m tcp -6 || ERROR=1
echo "Running Netfilter Regex UDP ipv4"
python3 nf_test.py -p $PASSWORD -m udp || ERROR=1
#python3 nf_test.py -p $PASSWORD -m udp || ERROR=1
echo "Running Netfilter Regex UDP ipv6"
python3 nf_test.py -p $PASSWORD -m udp -6 || ERROR=1
#python3 nf_test.py -p $PASSWORD -m udp -6 || ERROR=1
echo "Running Port Hijack TCP ipv4"
python3 ph_test.py -p $PASSWORD -m tcp || ERROR=1
#python3 ph_test.py -p $PASSWORD -m tcp || ERROR=1
echo "Running Port Hijack TCP ipv6"
python3 ph_test.py -p $PASSWORD -m tcp -6 || ERROR=1
#python3 ph_test.py -p $PASSWORD -m tcp -6 || ERROR=1
echo "Running Port Hijack UDP ipv4"
python3 ph_test.py -p $PASSWORD -m udp || ERROR=1
#python3 ph_test.py -p $PASSWORD -m udp || ERROR=1
echo "Running Port Hijack UDP ipv6"
python3 ph_test.py -p $PASSWORD -m udp -6 || ERROR=1
#python3 ph_test.py -p $PASSWORD -m udp -6 || ERROR=1
exit $ERROR