pyhton and c related fixes on empty filters and mangling

backend/binsrc/classes/nfqueue.cpp

@@ -132,7 +132,7 @@ class PktRequest {
 			_original_size = ipv4->size();
 		}
 		l4_proto = fill_l4_info();
-		#ifdef DEBUG
+		#ifdef PKTDEBUG
 		if (tcp){			
 			cerr << "[DEBUG] NEW_PACKET " << (is_input?"-> IN ":"<- OUT")  << " [SIZE: " << data_size() << "] FLAGS: " << (tcp->get_flag(Tins::TCP::FIN)?"FIN ":"") << (tcp->get_flag(Tins::TCP::SYN)?"SYN ":"") << (tcp->get_flag(Tins::TCP::RST)?"RST ":"") << (tcp->get_flag(Tins::TCP::ACK)?"ACK ":"") << (tcp->get_flag(Tins::TCP::PSH)?"PSH ":"") << endl;
 			cerr << "[SEQ: " << tcp->seq() << "] [ACK: " << tcp->ack_seq() << "]" << " [WIN: " << tcp->window() << "] [FLAGS: " << tcp->flags() << "]\n" << endl;
@@ -243,7 +243,7 @@ class PktRequest {
 			tcp->ack_seq(tcp->ack_seq() - ack_seq_offset->in);
 			tcp->seq(tcp->seq() + ack_seq_offset->out);
 		}
-		#ifdef DEBUG
+		#ifdef PKTDEBUG
 		size_t new_size = inner_data_size(tcp);
 		cerr << "[DEBUG] FIXED PKT  " << (is_input?"-> IN ":"<- OUT")  << " [SIZE: " << data_size() << "] FLAGS: " << (tcp->get_flag(Tins::TCP::FIN)?"FIN ":"") << (tcp->get_flag(Tins::TCP::SYN)?"SYN ":"") << (tcp->get_flag(Tins::TCP::RST)?"RST ":"") << (tcp->get_flag(Tins::TCP::ACK)?"ACK ":"") << (tcp->get_flag(Tins::TCP::PSH)?"PSH ":"") << endl;
 		cerr << "[SEQ: " << tcp->seq() << "] [ACK: " << tcp->ack_seq() << "]" << " [WIN: " << tcp->window() << "] [FLAGS: " << tcp->flags() << "]\n" << endl;
@@ -372,13 +372,13 @@ class PktRequest {
 					reserialize();
 				}
 				nfq_nlmsg_verdict_put_pkt(nlh_verdict, packet.data(), packet.size());
-				#ifdef DEBUG
+				#ifdef PKTDEBUG
 				if (tcp){
 					cerr << "[DEBUG] MANGLEDPKT " << (is_input?"-> IN ":"<- OUT")  << " [SIZE: " << data_size() << "] FLAGS: " << (tcp->get_flag(Tins::TCP::FIN)?"FIN ":"") << (tcp->get_flag(Tins::TCP::SYN)?"SYN ":"") << (tcp->get_flag(Tins::TCP::RST)?"RST ":"") << (tcp->get_flag(Tins::TCP::ACK)?"ACK ":"") << (tcp->get_flag(Tins::TCP::PSH)?"PSH ":"") << endl;
 					cerr << "[SEQ: " << tcp->seq() << "] [ACK: " << tcp->ack_seq() << "]" << " [WIN: " << tcp->window() << "] [FLAGS: " << tcp->flags() << "]\n" << endl;
 				}
 				#endif
-				size_t payload_offset = data_size() != _data_original_size;
+				size_t payload_offset = data_size() - _data_original_size;
 				if (tcp && ack_seq_offset && payload_offset != 0){
 					if (is_input){
 						ack_seq_offset->in += payload_offset;

fgex-lib/firegex/nfproxy/__init__.py

@@ -25,7 +25,9 @@ def pyfilter(func):
 
 def get_pyfilters():
     """Returns the list of functions marked with @pyfilter."""
+    if hasattr(pyfilter, "registry"):
         return list(pyfilter.registry)
+    return []
 
 def clear_pyfilter_registry():
     """Clears the pyfilter registry."""

tests/benchmark.py

@@ -6,7 +6,6 @@ from time import sleep
 import iperf3
 import csv
 import argparse
-import base64
 import secrets
 
 parser = argparse.ArgumentParser()
@@ -96,8 +95,8 @@ def gen_regex():
     else:
         regex = custom_regex.pop()
     """
-    regex = secrets.token_hex(20)
-    return base64.b64encode(bytes(regex.encode())).decode()
+    return secrets.token_hex(20)
+
 
 #Get baseline reading 
 puts("Baseline without proxy: ", color=colors.blue, end='')

tests/comparemark.py

@@ -16,7 +16,7 @@ parser.add_argument("--password", "-p", type=str, required=True, help='Firegex p
 parser.add_argument("--duration", "-d", type=int, required=False, help='Duration of the Benchmark in seconds', default=5)
 parser.add_argument("--output-file", "-o", type=str, required=False, help='Output results csv file', default="comparemark.csv")
 parser.add_argument("--num-of-streams", "-s", type=int, required=False, help='Number of concurrent streams', default=1)
-parser.add_argument("--number-of-values", "-V", type=int, required=False, help='Number of values to generate', default=100)
+parser.add_argument("--number-of-values", "-V", type=int, required=False, help='Number of values to generate', default=10)
 
 args = parser.parse_args()
 sep()
@@ -133,7 +133,9 @@ for _ in range(args.number_of_values):
     no_regex_nfregex.append(data)
     print(f"{data} MB/s")
 
-if not firegex.nfregex_add_regex(service_id,text_filter_key,"B",active=True,is_case_sensitive=False): 
+if firegex.nfregex_add_regex(service_id,text_filter_key,"B",active=True,is_case_sensitive=False):
+    puts(f"Sucessfully added regex for {str(text_filter_key)} ✔", color=colors.green)
+else:
     puts("Benchmark Failed: Couldn't add the regex ✗", color=colors.red)
     exit_test(1)
 
@@ -191,7 +193,7 @@ from firegex.nfproxy import pyfilter, REJECT
 
 @pyfilter
 def verdict_test(packet:RawPacket):
-    if {repr(text_filter_key)} in packet.data:
+    if {repr(text_filter_key.encode())} in packet.data:
         return REJECT
 """
 

tests/nfproxy_test.py

@@ -12,6 +12,7 @@ parser.add_argument("--password", "-p", type=str, required=True, help='Firegex p
 parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the test service', default="Test Service")
 parser.add_argument("--port", "-P", type=int , required=False, help='Port of the test service', default=1337)
 parser.add_argument("--ipv6", "-6" , action="store_true", help='Test Ipv6', default=False)
+parser.add_argument("--verbose", "-V" , action="store_true", help='Verbose output', default=False)
 
 args = parser.parse_args()
 sep()
@@ -28,7 +29,7 @@ else:
     exit(1)
 
 #Create server
-server = TcpServer(args.port,ipv6=args.ipv6)
+server = TcpServer(args.port,ipv6=args.ipv6, verbose=args.verbose)
 
 srvs = firegex.nfproxy_get_services()
 for ele in srvs:
@@ -45,10 +46,12 @@ else:
 def exit_test(code):
     if service_id:
         server.stop()
+        """
         if firegex.nfproxy_delete_service(service_id):
             puts("Sucessfully deleted service ✔", color=colors.green)
         else:
             puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red)  
+        """    
     exit(code)
 
 if(firegex.nfproxy_start_service(service_id)):

tests/nfregex_test.py

@@ -74,9 +74,8 @@ except Exception:
     exit_test(1)
 #Add new regex
 secret = bytes(secrets.token_hex(16).encode())
-regex = base64.b64encode(secret).decode()
 
-if firegex.nfregex_add_regex(service_id,regex,"B",active=True,is_case_sensitive=True): 
+if firegex.nfregex_add_regex(service_id,secret,"B",active=True,is_case_sensitive=True): 
     puts(f"Sucessfully added regex {str(secret)} ✔", color=colors.green)
 else:
     puts(f"Test Failed: Couldn't add the regex {str(secret)} ✗", color=colors.red)
@@ -95,9 +94,9 @@ def checkRegex(regex, should_work=True, upper=False, deleted=False):
     if should_work:
         global n_blocked
         for r in firegex.nfregex_get_service_regexes(service_id):
-            if r["regex"] == regex:
+            if r["regex"] == secret:
                 #Test the regex
-                s = base64.b64decode(regex).upper() if upper else base64.b64decode(regex)
+                s = regex.upper() if upper else regex
                 if not server.sendCheckData(secrets.token_bytes(40) + s +  secrets.token_bytes(40)):
                     puts("The malicious request was successfully blocked ✔", color=colors.green)
                     n_blocked += 1
@@ -140,7 +139,7 @@ def clear_regexes():
     global n_blocked
     n_blocked = 0
     for r in firegex.nfregex_get_service_regexes(service_id):
-        if r["regex"] == regex:
+        if r["regex"] == secret:
             if(firegex.nfregex_delete_regex(r["id"])): 
                 puts(f"Sucessfully deleted regex with id {r['id']} ✔", color=colors.green)
             else: 
@@ -153,7 +152,7 @@ def clear_regexes():
         puts("Test Failed: Metrics found after deleting the regex ✗", color=colors.red)
         exit_test(1)
 
-checkRegex(regex)
+checkRegex(secret)
 
 #Pause the proxy
 if(firegex.nfregex_stop_service(service_id)):
@@ -163,7 +162,7 @@ else:
     exit_test(1)
 
 #Check if it's actually paused
-checkRegex(regex,should_work=False)
+checkRegex(secret,should_work=False)
 
 #Start firewall
 if(firegex.nfregex_start_service(service_id)):
@@ -172,11 +171,11 @@ else:
     puts("Test Failed: Coulnd't start the service ✗", color=colors.red)
     exit_test(1)
 
-checkRegex(regex)
+checkRegex(secret)
 
 #Disable regex 
 for r in firegex.nfregex_get_service_regexes(service_id):
-    if r["regex"] == regex:
+    if r["regex"] == secret:
         if(firegex.nfregex_disable_regex(r["id"])): 
             puts(f"Sucessfully disabled regex with id {r['id']} ✔", color=colors.green)
         else: 
@@ -185,11 +184,11 @@ for r in firegex.nfregex_get_service_regexes(service_id):
         break
 
 #Check if it's actually disabled
-checkRegex(regex,should_work=False)
+checkRegex(secret,should_work=False)
 
 #Enable regex
 for r in firegex.nfregex_get_service_regexes(service_id):
-    if r["regex"] == regex:
+    if r["regex"] == secret:
         if(firegex.nfregex_enable_regex(r["id"])): 
             puts(f"Sucessfully enabled regex with id {r['id']} ✔", color=colors.green)
         else: 
@@ -197,23 +196,23 @@ for r in firegex.nfregex_get_service_regexes(service_id):
             exit_test(1)
         break
 
-checkRegex(regex)
+checkRegex(secret)
 
 #Delete regex
 clear_regexes()
 
 #Check if it's actually deleted
-checkRegex(regex,should_work=False,deleted=True)
+checkRegex(secret,should_work=False,deleted=True)
 
 #Add case insensitive regex
-if(firegex.nfregex_add_regex(service_id,regex,"B",active=True, is_case_sensitive=False)): 
+if(firegex.nfregex_add_regex(service_id,secret,"B",active=True, is_case_sensitive=False)): 
     puts(f"Sucessfully added case insensitive regex {str(secret)} ✔", color=colors.green)
 else:
     puts(f"Test Failed: Coulnd't add the case insensitive regex {str(secret)} ✗", color=colors.red)
     exit_test(1)
 
-checkRegex(regex, upper=True)
-checkRegex(regex)
+checkRegex(secret, upper=True)
+checkRegex(secret)
 
 clear_regexes()
 

tests/utils/firegexapi.py

@@ -1,5 +1,6 @@
 import string
 from requests import Session
+import base64
 
 def verify(req):
     try:
@@ -108,7 +109,11 @@ class FiregexAPI:
 
     def nfregex_get_service_regexes(self,service_id: str):
         req = self.s.get(f"{self.address}api/nfregex/services/{service_id}/regexes")
-        return req.json()
+        data = req.json()
+        for ele in data:
+            if "regex" in ele:
+                ele["regex"] = base64.b64decode(ele["regex"])
+        return data
 
     def nfregex_get_regex(self,regex_id: str):
         req = self.s.get(f"{self.address}api/nfregex/regexes/{regex_id}")
@@ -127,8 +132,10 @@ class FiregexAPI:
         return verify(req)
 
     def nfregex_add_regex(self, service_id: str, regex: str, mode: str, active: bool, is_case_sensitive: bool):
+        if isinstance(regex, str):
+            regex = regex.encode()
         req = self.s.post(f"{self.address}api/nfregex/regexes", 
-            json={"service_id": service_id, "regex": regex, "mode": mode, "active": active, "is_case_sensitive": is_case_sensitive})
+            json={"service_id": service_id, "regex": base64.b64encode(regex).decode(), "mode": mode, "active": active, "is_case_sensitive": is_case_sensitive})
         return verify(req)
 
     def nfregex_add_service(self, name: str, port: int, proto: str, ip_int: str, fail_open: bool = False):