Updated Benchmarking

tests/README.md

@@ -0,0 +1,84 @@
+# Firegex tests
+
+## [GO BACK](../README.md)
+
+Tests are a quick and dirty way to check if your modification to the backend code dind't break anything.
+
+# Running all the tests
+If you are working on the same machine firegex is running on, you can just run run_tests.sh
+    $ ./run_tests.sh
+It will automatically perform a general API test, Netfilter and Proxy Regex test. 
+You can also run tests manually:
+    $ ./api_test.py -h
+    usage: api_test.py [-h] [--address ADDRESS] --password PASSWORD
+
+    $ ./nf_test.py -h
+    usage: nf_test.py [-h] [--address ADDRESS] --password PASSWORD [--service_name SERVICE_NAME] [--port PORT]
+                    [--ipv6] [--proto {tcp,udp}]
+
+    optional arguments:
+    -h, --help            show this help message and exit
+    --address ADDRESS, -a ADDRESS
+                            Address of firegex backend
+    --password PASSWORD, -p PASSWORD
+                            Firegex password
+    --service_name SERVICE_NAME, -n SERVICE_NAME
+                            Name of the test service
+    --port PORT, -P PORT  Port of the test service
+    --ipv6, -6            Test Ipv6
+    --proto {tcp,udp}, -m {tcp,udp}
+                        Select the protocol
+
+    $ ./px_test.py -h
+    usage: px_test.py [-h] [--address ADDRESS] --password PASSWORD [--service_name SERVICE_NAME] [--port PORT]
+
+    optional arguments:
+    -h, --help            show this help message and exit
+    --address ADDRESS, -a ADDRESS
+                            Address of firegex backend
+    --password PASSWORD, -p PASSWORD
+                            Firegex password
+    --service_name SERVICE_NAME, -n SERVICE_NAME
+                            Name of the test service
+    --port PORT, -P PORT  Port of the test service
+
+# Running a Benchmark
+    ./benchmark.py
+    options:
+    -h, --help            show this help message and exit
+    --address ADDRESS, -a ADDRESS
+                            Address of firegex backend
+    --port PORT, -P PORT  Port of the Benchmark service
+    --internal-port INTERNAL_PORT, -I INTERNAL_PORT
+                            Internal port of the Benchmark service
+    --service-name SERVICE_NAME, -n SERVICE_NAME
+                            Name of the Benchmark service
+    --password PASSWORD, -p PASSWORD
+                            Firegex password
+    --num-of-regexes NUM_OF_REGEXES, -r NUM_OF_REGEXES
+                            Number of regexes to benchmark with
+    --duration DURATION, -d DURATION
+                            Duration of the Benchmark in seconds
+    --output-file OUTPUT_FILE, -o OUTPUT_FILE
+                            Output results csv file
+    --num-of-streams NUM_OF_STREAMS, -s NUM_OF_STREAMS
+                            Output results csv file
+    --mode {netfilter,proxy}, -m {netfilter,proxy}
+                            Type of filtering
+
+Benchmarks let you evaluate the performance of the filters. You can run one by typing in a shell  ```test.py -p FIREGEX_PASSWORD -r NUM_OF_REGEX -d BENCHMARK_DURATION -m proxy``` to benchmark the Proxy based regex filter, or ``` -m netfilter ``` to benchmark the Netfilter based regex filtering.
+It uses iperf3 to benchmark the throughput in MB/s of the server, both with filters, without filters, and for each new added regex. It will automatically add a new random regex untill it has reached NUM_OF_REGEX specified in the arguments. 
+
+You will find a new benchmark.csv file containg the results.
+
+# Firegex Performance Results
+TODO: Update with new graphs
+
+The test was performed on:
+- AMD Ryzen 7 3700X (16 thread) @ 3.600GHz
+- RAM Speed: 3200 MT/s (Dual Channel)
+- Kernel: 5.18.5-arch1-1
+
+Command: `python3 benchmark.py -r 100 -d 1 -s 50`
+
+![Firegex Benchmark](/docs/FiregexBenchmark.png)

tests/benchmark.py

@@ -11,12 +11,14 @@ import iperf3, csv, argparse, base64, secrets
 parser = argparse.ArgumentParser()
 parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:4444/")
 parser.add_argument("--port", "-P", type=int , required=False, help='Port of the Benchmark service', default=1337)
-parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service")
+parser.add_argument("--internal-port", "-I", type=int , required=False, help='Internal port of the Benchmark service', default=1338)
+parser.add_argument("--service-name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service")
 parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password')
-parser.add_argument("--num_of_regexes", "-r", type=int, required=True, help='Number of regexes to benchmark with')
+parser.add_argument("--num-of-regexes", "-r", type=int, required=True, help='Number of regexes to benchmark with')
 parser.add_argument("--duration", "-d", type=int, required=False, help='Duration of the Benchmark in seconds', default=5)
-parser.add_argument("--output_file", "-o", type=str, required=False, help='Output results csv file', default="benchmark.csv")
+parser.add_argument("--output-file", "-o", type=str, required=False, help='Output results csv file', default="benchmark.csv")
-parser.add_argument("--num_of_streams", "-s", type=int, required=False, help='Output results csv file', default=1)
+parser.add_argument("--num-of-streams", "-s", type=int, required=False, help='Output results csv file', default=1)
+parser.add_argument("--mode", "-m" , type=str, required=True, choices=["netfilter","proxy"], help='Type of filtering')
 
 args = parser.parse_args()
 sep()
@@ -30,7 +32,10 @@ if (firegex.login(args.password)): puts(f"Sucessfully logged in ✔", color=colo
 else: puts(f"Benchmark Failed: Unknown response or wrong passowrd ✗", color=colors.red); exit(1)
 
 #Create new Service
-service_id = firegex.nf_add_service(args.service_name, args.port, "tcp", "127.0.0.1/24")
+if args.mode == "netfilter":
+    service_id = firegex.nf_add_service(args.service_name, args.port, "tcp", "127.0.0.1/24")
+else:
+    service_id = firegex.px_add_service(args.service_name, args.port, internalPort=args.internal_port)
 if service_id: puts(f"Sucessfully created service {service_id} ✔", color=colors.green)
 else: puts(f"Test Failed: Failed to create service ✗", color=colors.red); exit(1)
 
@@ -38,7 +43,7 @@ else: puts(f"Test Failed: Failed to create service ✗", color=colors.red); exit
 def startServer():
     server = iperf3.Server()
     server.bind_address = '127.0.0.1'
-    server.port = args.port
+    server.port = args.port if args.mode == "netfilter" else args.internal_port
     server.verbose = False
     while True:
         server.run()
@@ -59,11 +64,14 @@ sleep(1)
 
 #Get baseline reading 
 puts(f"Baseline without proxy: ", color=colors.blue, end='')
-print(f"{getReading(args.port)} MB/s")
+print(f"{getReading(args.port if args.mode == 'netfilter' else args.internal_port)} MB/s")
 
 #Start firewall
-if(firegex.nf_start_service(service_id)): puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green)
+
-else: puts(f"Benchmark Failed: Coulnd't start the service ✗", color=colors.red); exit_test(1)
+if(firegex.nf_start_service(service_id) if args.mode == "netfilter" else firegex.px_start_service(service_id)):
+     puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green)
+else:
+     puts(f"Benchmark Failed: Coulnd't start the service ✗", color=colors.red); exit_test(1)
 
 #Get no regexs reading 
 results = []
@@ -74,7 +82,8 @@ print(f"{results[0]} MB/s")
 #Add all the regexs
 for i in range(1,args.num_of_regexes+1):
     regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode()
-    if(not firegex.nf_add_regex(service_id,regex,"B",active=True,is_blacklist=True,is_case_sensitive=False)): puts(f"Benchmark Failed: Coulnd't add the regex ✗", color=colors.red); exit_test(1)
+    if(not (firegex.nf_add_regex if args.mode == "netfilter" else firegex.px_add_regex)(service_id,regex,"B",active=True,is_blacklist=True,is_case_sensitive=False) ): 
+        puts(f"Benchmark Failed: Coulnd't add the regex ✗", color=colors.red); exit_test(1)
     puts(f"Performance with {i} regex(s): ", color=colors.red, end='')
     results.append(getReading(args.port))
     print(f"{results[i]} MB/s")
@@ -87,9 +96,9 @@ with open(args.output_file,'w') as f:
 puts(f"Sucessfully written results to {args.output_file} ✔", color=colors.magenta)
 
 #Delete the Service 
-if(firegex.nf_delete_service(service_id)):
+if(firegex.nf_delete_service(service_id) if args.mode == "netfilter" else firegex.px_delete_service(service_id)):
     puts(f"Sucessfully delete service with id {service_id} ✔", color=colors.green)
 else:
     puts(f"Test Failed: Couldn't delete service ✗", color=colors.red); exit(1)
 
-server.terminate()
+server.terminate()

tests/run_tests.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+pip3 install -r requirements.txt
 echo "Running standard API test"
 python3 api_test.py -p testpassword
 echo "Running Netfilter Regex TCP ipv4"