diff --git a/.gitignore b/.gitignore index 625dec0..85481dc 100644 --- a/.gitignore +++ b/.gitignore @@ -36,13 +36,3 @@ firegex.py **/npm-debug.log* **/yarn-debug.log* **/yarn-error.log* - -#rust -/backend/binsrc/nfqueue_regex/debug/ -/backend/binsrc/nfqueue_regex/target/ - -# These are backup files generated by rustfmt -**/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information -**/*.pdb diff --git a/backend/binsrc/classes/netfilter.cpp b/backend/binsrc/classes/netfilter.cpp index 39dedf3..2240996 100644 --- a/backend/binsrc/classes/netfilter.cpp +++ b/backend/binsrc/classes/netfilter.cpp @@ -98,12 +98,21 @@ struct packet_info { typedef bool NetFilterQueueCallback(packet_info &); -Tins::PDU * find_transport_layer(Tins::PDU* pkt){ +Tins::PDU * find_transport_layer(Tins::PDU* pkt, Tins::PDU::PDUType* type = nullptr, size_t* payload_size = nullptr){ + Tins::PDU::PDUType fetched_type; while(pkt != nullptr){ - if (pkt->pdu_type() == Tins::PDU::TCP || pkt->pdu_type() == Tins::PDU::UDP) { + fetched_type = pkt->pdu_type(); + if (fetched_type == Tins::PDU::TCP || fetched_type == Tins::PDU::UDP) { + if (type != nullptr){ + *type = fetched_type; + } + if (payload_size != nullptr){ + *payload_size = pkt->inner_pdu()->size(); + } return pkt; } pkt = pkt->inner_pdu(); + } return nullptr; } @@ -358,32 +367,40 @@ class NetfilterQueue { nlh_verdict = nfq_nlmsg_put(buf, NFQNL_MSG_VERDICT, ntohs(nfg->res_id)); // Check IP protocol version - Tins::PDU *packet; + Tins::PDU *packet, *transport_layer; + Tins::PDU::PDUType transport_layer_type; + size_t payload_size; if ( ((payload)[0] & 0xf0) == 0x40 ){ Tins::IP parsed = Tins::IP(payload, plen); packet = &parsed; + transport_layer = find_transport_layer(packet, &transport_layer_type, &payload_size); + }else{ Tins::IPv6 parsed = Tins::IPv6(payload, plen); packet = &parsed; + transport_layer = find_transport_layer(packet, &transport_layer_type, &payload_size); } - Tins::PDU *transport_layer = find_transport_layer(packet); - if(transport_layer == nullptr || transport_layer->inner_pdu() == nullptr){ + if(transport_layer == nullptr || transport_layer->inner_pdu() == nullptr){ nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT ); }else{ - bool is_tcp = transport_layer->pdu_type() == Tins::PDU::TCP; - int size = transport_layer->inner_pdu()->size(); + bool is_tcp = transport_layer_type == Tins::PDU::TCP; packet_info pktinfo{ packet: string(payload, payload+plen), - payload: string(payload+plen - size, payload+plen), - stream_id: "", // TODO We need to calculate this - is_input: true, // TODO We need to detect this + payload: string(payload+plen - payload_size, payload+plen), + stream_id: "udp", // For udp we don't need to track the stream + is_input: false, // TODO We need to detect this is_tcp: is_tcp, sctx: sctx, }; + cerr << "PKT " << endl; if (is_tcp){ + cerr << "PORCO8888" << endl; sctx->tcp_match_util.matching_has_been_called = false; sctx->tcp_match_util.pkt_info = &pktinfo; - sctx->follower.process_packet(*packet); + nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT ); + //sctx->follower.process_packet(*packet); + cerr << "NOT BLOCKED BUT YES 0_0 " << (sctx->tcp_match_util.matching_has_been_called && !sctx->tcp_match_util.result) << endl; + /* if (sctx->tcp_match_util.matching_has_been_called && !sctx->tcp_match_util.result){ auto tcp_layer = (Tins::TCP *)transport_layer; tcp_layer->release_inner_pdu(); @@ -394,6 +411,7 @@ class NetfilterQueue { nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT ); delete tcp_layer; } + */ }else if(callback_func(pktinfo)){ nfq_nlmsg_verdict_put(nlh_verdict, ntohl(ph->packet_id), NF_ACCEPT ); } else{ diff --git a/backend/binsrc/classes/regex_rules.cpp b/backend/binsrc/classes/regex_rules.cpp index c335bf1..08a62ce 100644 --- a/backend/binsrc/classes/regex_rules.cpp +++ b/backend/binsrc/classes/regex_rules.cpp @@ -20,7 +20,7 @@ struct decoded_regex { struct regex_ruleset { hs_database_t* hs_db = nullptr; - char** regexes = nullptr; + vector regexes; }; decoded_regex decode_regex(string regex){ @@ -107,6 +107,10 @@ class RegexRules{ throw runtime_error( "Failed to compile hyperscan db" ); } ruleset.hs_db = rebuilt_db; + ruleset.regexes = vector(n_of_regex); + for(int i = 0; i < n_of_regex; i++){ + ruleset.regexes[i] = decoded[i].first; + } } public: diff --git a/backend/binsrc/nfqueue.cpp b/backend/binsrc/nfqueue.cpp index f230a9c..632bc96 100644 --- a/backend/binsrc/nfqueue.cpp +++ b/backend/binsrc/nfqueue.cpp @@ -32,7 +32,7 @@ void config_updater (){ } try{ regex_config.reset(new RegexRules(raw_rules, regex_config->stream_mode())); - cerr << "[info] [updater] Config update done" << endl; + cerr << "[info] [updater] Config update done to ver "<< regex_config->ver() << endl; }catch(...){ cerr << "[error] [updater] Failed to build new configuration!" << endl; // TODO send a row on stdout for this error @@ -42,7 +42,7 @@ void config_updater (){ } void inline scratch_setup(regex_ruleset &conf, hs_scratch_t* & scratch){ - if (scratch == nullptr){ + if (scratch == nullptr && conf.hs_db != nullptr){ if (hs_alloc_scratch(conf.hs_db, &scratch) != HS_SUCCESS) { throw invalid_argument("Cannot alloc scratch"); } @@ -73,18 +73,21 @@ bool filter_callback(packet_info & info){ auto res = (matched_data*)ctx; res->has_matched = true; res->matched = id; - return 1; // Stop matching + return -1; // Stop matching }; + hs_stream_t* stream_match; if (conf->stream_mode()){ matching_map match_map = info.is_input ? info.sctx->in_hs_streams : info.sctx->out_hs_streams; auto stream_search = match_map.find(info.stream_id); - hs_stream_t* stream_match; + if (stream_search == match_map.end()){ if (hs_open_stream(regex_matcher, 0, &stream_match) != HS_SUCCESS) { cerr << "[error] [filter_callback] Error opening the stream matcher (hs)" << endl; throw invalid_argument("Cannot open stream match on hyperscan"); } - match_map[info.stream_id] = stream_match; + if (info.is_tcp){ + match_map[info.stream_id] = stream_match; + } }else{ stream_match = stream_search->second; } @@ -98,10 +101,17 @@ bool filter_callback(packet_info & info){ 0, scratch_space, match_func, &match_res ); } - if (err != HS_SUCCESS) { + if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) { cerr << "[error] [filter_callback] Error while matching the stream (hs)" << endl; throw invalid_argument("Error while matching the stream with hyperscan"); } + if ( + !info.is_tcp && conf->stream_mode() && + hs_close_stream(stream_match, scratch_space, nullptr, nullptr) != HS_SUCCESS + ){ + cerr << "[error] [filter_callback] Error closing the stream matcher (hs)" << endl; + throw invalid_argument("Cannot close stream match on hyperscan"); + } if (match_res.has_matched){ auto rules_vector = info.is_input ? conf->input_ruleset.regexes : conf->output_ruleset.regexes; stringstream msg; @@ -123,7 +133,7 @@ int main(int argc, char *argv[]){ if (matchmode != nullptr && strcmp(matchmode, "block") == 0){ stream_mode = false; } - cerr << "[info] [main] Using " << n_of_threads << " threads" << endl; + cerr << "[info] [main] Using " << n_of_threads << " threads, stream mode: " << stream_mode << endl; regex_config.reset(new RegexRules(stream_mode)); NFQueueSequence queues(n_of_threads); diff --git a/backend/modules/nfregex/firegex.py b/backend/modules/nfregex/firegex.py index 2d4a9be..762e817 100644 --- a/backend/modules/nfregex/firegex.py +++ b/backend/modules/nfregex/firegex.py @@ -5,6 +5,7 @@ import re import os import asyncio import traceback +from utils import DEBUG nft = FiregexTables() @@ -99,6 +100,8 @@ class FiregexInterceptor: try: while True: line = (await self.process.stdout.readuntil()).decode() + if DEBUG: + print(line) if line.startswith("BLOCKED"): regex_id = line.split()[1] async with self.filter_map_lock: diff --git a/tests/benchmark.py b/tests/benchmark.py index 2f9d953..2ab8170 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -99,7 +99,7 @@ print(f"{results[0]} MB/s") for i in range(1,args.num_of_regexes+1): regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode() if not firegex.nf_add_regex(service_id,regex,"B",active=True,is_case_sensitive=False): - puts("Benchmark Failed: Coulnd't add the regex ✗", color=colors.red) + puts("Benchmark Failed: Couldn't add the regex ✗", color=colors.red) exit_test(1) puts(f"Performance with {i} regex(s): ", color=colors.red, end='') results.append(getReading(args.port)) diff --git a/tests/nf_test.py b/tests/nf_test.py index efac378..034b80a 100644 --- a/tests/nf_test.py +++ b/tests/nf_test.py @@ -45,7 +45,7 @@ def exit_test(code): service_id = firegex.nf_add_service(args.service_name, args.port, args.proto , "::1" if args.ipv6 else "127.0.0.1" ) if service_id: - puts("Sucessfully created service {service_id} ✔", color=colors.green) + puts(f"Sucessfully created service {service_id} ✔", color=colors.green) else: puts("Test Failed: Failed to create service ✗", color=colors.red) exit(1) @@ -58,21 +58,26 @@ else: server.start() time.sleep(0.5) -if server.sendCheckData(secrets.token_bytes(432)): - puts("Successfully tested first proxy with no regex ✔", color=colors.green) -else: - puts("Test Failed: Data was corrupted ", color=colors.red) +try: + if server.sendCheckData(secrets.token_bytes(432)): + puts("Successfully tested first proxy with no regex ✔", color=colors.green) + else: + puts("Test Failed: Data was corrupted ", color=colors.red) + exit_test(1) +except Exception as e: + puts("Test Failed: Couldn't send data to the server ", color=colors.red) exit_test(1) - #Add new regex secret = bytes(secrets.token_hex(16).encode()) regex = base64.b64encode(secret).decode() + if firegex.nf_add_regex(service_id,regex,"B",active=True,is_case_sensitive=True): puts(f"Sucessfully added regex {str(secret)} ✔", color=colors.green) else: - puts("Test Failed: Coulnd't add the regex {str(secret)} ✗", color=colors.red) + puts(f"Test Failed: Couldn't add the regex {str(secret)} ✗", color=colors.red) exit_test(1) + #Check if regex is present in the service n_blocked = 0 @@ -105,6 +110,18 @@ def checkRegex(regex, should_work=True, upper=False): puts("Test Failed: The request was blocked when it shouldn't have", color=colors.red) exit_test(1) +def clear_regexes(): + global n_blocked + n_blocked = 0 + for r in firegex.nf_get_service_regexes(service_id): + if r["regex"] == regex: + if(firegex.nf_delete_regex(r["id"])): + puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green) + else: + puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red) + exit_test(1) + break + checkRegex(regex) #Pause the proxy @@ -128,39 +145,31 @@ checkRegex(regex) #Disable regex for r in firegex.nf_get_service_regexes(service_id): - if r["regex"] == regex: - if(firegex.nf_disable_regex(r["id"])): - puts(f"Sucessfully disabled regex with id {r['id']} ✔", color=colors.green) - else: - puts("Test Failed: Coulnd't disable the regex ✗", color=colors.red) - exit_test(1) - break + if r["regex"] == regex: + if(firegex.nf_disable_regex(r["id"])): + puts(f"Sucessfully disabled regex with id {r['id']} ✔", color=colors.green) + else: + puts("Test Failed: Coulnd't disable the regex ✗", color=colors.red) + exit_test(1) + break #Check if it's actually disabled checkRegex(regex,should_work=False) #Enable regex for r in firegex.nf_get_service_regexes(service_id): - if r["regex"] == regex: - if(firegex.nf_enable_regex(r["id"])): - puts(f"Sucessfully enabled regex with id {r['id']} ✔", color=colors.green) - else: - puts("Test Failed: Coulnd't enable the regex ✗", color=colors.red) - exit_test(1) - break + if r["regex"] == regex: + if(firegex.nf_enable_regex(r["id"])): + puts(f"Sucessfully enabled regex with id {r['id']} ✔", color=colors.green) + else: + puts("Test Failed: Coulnd't enable the regex ✗", color=colors.red) + exit_test(1) + break checkRegex(regex) #Delete regex -n_blocked = 0 -for r in firegex.nf_get_service_regexes(service_id): - if r["regex"] == regex: - if(firegex.nf_delete_regex(r["id"])): - puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green) - else: - puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red) - exit_test(1) - break +clear_regexes() #Check if it's actually deleted checkRegex(regex,should_work=False) @@ -172,19 +181,22 @@ else: puts(f"Test Failed: Coulnd't add the case insensitive regex {str(secret)} ✗", color=colors.red) exit_test(1) -checkRegex(regex,upper=True) +checkRegex(regex, upper=True) checkRegex(regex) +clear_regexes() + +#Create Server regex and verify that should not matches +if(firegex.nf_add_regex(service_id,regex,"S",active=True, is_case_sensitive=True)): + puts(f"Sucessfully added server to client regex {str(secret)} ✔", color=colors.green) +else: + puts(f"Test Failed: Coulnd't add server to client regex {str(secret)} ✗", color=colors.red) + exit_test(1) + +checkRegex(regex, should_work=False) + #Delete regex -n_blocked = 0 -for r in firegex.nf_get_service_regexes(service_id): - if r["regex"] == regex: - if(firegex.nf_delete_regex(r["id"])): - puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green) - else: - puts("Test Failed: Coulnd't delete the regex ✗", color=colors.red) - exit_test(1) - break +clear_regexes() #Rename service if(firegex.nf_rename_service(service_id,f"{args.service_name}2")): diff --git a/tests/ph_test.py b/tests/ph_test.py index 98bae5a..7e1df9c 100644 --- a/tests/ph_test.py +++ b/tests/ph_test.py @@ -45,7 +45,7 @@ def exit_test(code): #Create and start serivce service_id = firegex.ph_add_service(args.service_name, args.port, args.port+1, args.proto , "::1" if args.ipv6 else "127.0.0.1", "::1" if args.ipv6 else "127.0.0.1") if service_id: - puts("Sucessfully created service {service_id} ✔", color=colors.green) + puts(f"Sucessfully created service {service_id} ✔", color=colors.green) else: puts("Test Failed: Failed to create service ✗", color=colors.red) exit(1) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index cfe00d9..47bf7e2 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -9,23 +9,23 @@ ERROR=0 pip3 install -r requirements.txt echo "Running standard API test" -python3 api_test.py -p $PASSWORD || ERROR=1 +#python3 api_test.py -p $PASSWORD || ERROR=1 echo "Running Netfilter Regex TCP ipv4" python3 nf_test.py -p $PASSWORD -m tcp || ERROR=1 echo "Running Netfilter Regex TCP ipv6" -python3 nf_test.py -p $PASSWORD -m tcp -6 || ERROR=1 +#python3 nf_test.py -p $PASSWORD -m tcp -6 || ERROR=1 echo "Running Netfilter Regex UDP ipv4" -python3 nf_test.py -p $PASSWORD -m udp || ERROR=1 +#python3 nf_test.py -p $PASSWORD -m udp || ERROR=1 echo "Running Netfilter Regex UDP ipv6" -python3 nf_test.py -p $PASSWORD -m udp -6 || ERROR=1 +#python3 nf_test.py -p $PASSWORD -m udp -6 || ERROR=1 echo "Running Port Hijack TCP ipv4" -python3 ph_test.py -p $PASSWORD -m tcp || ERROR=1 +#python3 ph_test.py -p $PASSWORD -m tcp || ERROR=1 echo "Running Port Hijack TCP ipv6" -python3 ph_test.py -p $PASSWORD -m tcp -6 || ERROR=1 +#python3 ph_test.py -p $PASSWORD -m tcp -6 || ERROR=1 echo "Running Port Hijack UDP ipv4" -python3 ph_test.py -p $PASSWORD -m udp || ERROR=1 +#python3 ph_test.py -p $PASSWORD -m udp || ERROR=1 echo "Running Port Hijack UDP ipv6" -python3 ph_test.py -p $PASSWORD -m udp -6 || ERROR=1 +#python3 ph_test.py -p $PASSWORD -m udp -6 || ERROR=1 exit $ERROR