diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..25b27b6 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,84 @@ +# Firegex tests + +## [GO BACK](../README.md) + +Tests are a quick and dirty way to check if your modification to the backend code dind't break anything. + +# Running all the tests +If you are working on the same machine firegex is running on, you can just run run_tests.sh + $ ./run_tests.sh +It will automatically perform a general API test, Netfilter and Proxy Regex test. +You can also run tests manually: + $ ./api_test.py -h + usage: api_test.py [-h] [--address ADDRESS] --password PASSWORD + + $ ./nf_test.py -h + usage: nf_test.py [-h] [--address ADDRESS] --password PASSWORD [--service_name SERVICE_NAME] [--port PORT] + [--ipv6] [--proto {tcp,udp}] + + optional arguments: + -h, --help show this help message and exit + --address ADDRESS, -a ADDRESS + Address of firegex backend + --password PASSWORD, -p PASSWORD + Firegex password + --service_name SERVICE_NAME, -n SERVICE_NAME + Name of the test service + --port PORT, -P PORT Port of the test service + --ipv6, -6 Test Ipv6 + --proto {tcp,udp}, -m {tcp,udp} + Select the protocol + + $ ./px_test.py -h + usage: px_test.py [-h] [--address ADDRESS] --password PASSWORD [--service_name SERVICE_NAME] [--port PORT] + + optional arguments: + -h, --help show this help message and exit + --address ADDRESS, -a ADDRESS + Address of firegex backend + --password PASSWORD, -p PASSWORD + Firegex password + --service_name SERVICE_NAME, -n SERVICE_NAME + Name of the test service + --port PORT, -P PORT Port of the test service + +# Running a Benchmark + ./benchmark.py + options: + -h, --help show this help message and exit + --address ADDRESS, -a ADDRESS + Address of firegex backend + --port PORT, -P PORT Port of the Benchmark service + --internal-port INTERNAL_PORT, -I INTERNAL_PORT + Internal port of the Benchmark service + --service-name SERVICE_NAME, -n SERVICE_NAME + Name of the Benchmark service + --password PASSWORD, -p PASSWORD + Firegex password + --num-of-regexes NUM_OF_REGEXES, -r NUM_OF_REGEXES + Number of regexes to benchmark with + --duration DURATION, -d DURATION + Duration of the Benchmark in seconds + --output-file OUTPUT_FILE, -o OUTPUT_FILE + Output results csv file + --num-of-streams NUM_OF_STREAMS, -s NUM_OF_STREAMS + Output results csv file + --mode {netfilter,proxy}, -m {netfilter,proxy} + Type of filtering + +Benchmarks let you evaluate the performance of the filters. You can run one by typing in a shell ```test.py -p FIREGEX_PASSWORD -r NUM_OF_REGEX -d BENCHMARK_DURATION -m proxy``` to benchmark the Proxy based regex filter, or ``` -m netfilter ``` to benchmark the Netfilter based regex filtering. +It uses iperf3 to benchmark the throughput in MB/s of the server, both with filters, without filters, and for each new added regex. It will automatically add a new random regex untill it has reached NUM_OF_REGEX specified in the arguments. + +You will find a new benchmark.csv file containg the results. + +# Firegex Performance Results +TODO: Update with new graphs + +The test was performed on: +- AMD Ryzen 7 3700X (16 thread) @ 3.600GHz +- RAM Speed: 3200 MT/s (Dual Channel) +- Kernel: 5.18.5-arch1-1 + +Command: `python3 benchmark.py -r 100 -d 1 -s 50` + +![Firegex Benchmark](/docs/FiregexBenchmark.png) diff --git a/tests/benchmark.py b/tests/benchmark.py index cb60c8c..6625b53 100755 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -11,12 +11,14 @@ import iperf3, csv, argparse, base64, secrets parser = argparse.ArgumentParser() parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:4444/") parser.add_argument("--port", "-P", type=int , required=False, help='Port of the Benchmark service', default=1337) -parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service") +parser.add_argument("--internal-port", "-I", type=int , required=False, help='Internal port of the Benchmark service', default=1338) +parser.add_argument("--service-name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service") parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password') -parser.add_argument("--num_of_regexes", "-r", type=int, required=True, help='Number of regexes to benchmark with') +parser.add_argument("--num-of-regexes", "-r", type=int, required=True, help='Number of regexes to benchmark with') parser.add_argument("--duration", "-d", type=int, required=False, help='Duration of the Benchmark in seconds', default=5) -parser.add_argument("--output_file", "-o", type=str, required=False, help='Output results csv file', default="benchmark.csv") -parser.add_argument("--num_of_streams", "-s", type=int, required=False, help='Output results csv file', default=1) +parser.add_argument("--output-file", "-o", type=str, required=False, help='Output results csv file', default="benchmark.csv") +parser.add_argument("--num-of-streams", "-s", type=int, required=False, help='Output results csv file', default=1) +parser.add_argument("--mode", "-m" , type=str, required=True, choices=["netfilter","proxy"], help='Type of filtering') args = parser.parse_args() sep() @@ -30,7 +32,10 @@ if (firegex.login(args.password)): puts(f"Sucessfully logged in ✔", color=colo else: puts(f"Benchmark Failed: Unknown response or wrong passowrd ✗", color=colors.red); exit(1) #Create new Service -service_id = firegex.nf_add_service(args.service_name, args.port, "tcp", "127.0.0.1/24") +if args.mode == "netfilter": + service_id = firegex.nf_add_service(args.service_name, args.port, "tcp", "127.0.0.1/24") +else: + service_id = firegex.px_add_service(args.service_name, args.port, internalPort=args.internal_port) if service_id: puts(f"Sucessfully created service {service_id} ✔", color=colors.green) else: puts(f"Test Failed: Failed to create service ✗", color=colors.red); exit(1) @@ -38,7 +43,7 @@ else: puts(f"Test Failed: Failed to create service ✗", color=colors.red); exit def startServer(): server = iperf3.Server() server.bind_address = '127.0.0.1' - server.port = args.port + server.port = args.port if args.mode == "netfilter" else args.internal_port server.verbose = False while True: server.run() @@ -59,11 +64,14 @@ sleep(1) #Get baseline reading puts(f"Baseline without proxy: ", color=colors.blue, end='') -print(f"{getReading(args.port)} MB/s") +print(f"{getReading(args.port if args.mode == 'netfilter' else args.internal_port)} MB/s") #Start firewall -if(firegex.nf_start_service(service_id)): puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green) -else: puts(f"Benchmark Failed: Coulnd't start the service ✗", color=colors.red); exit_test(1) + +if(firegex.nf_start_service(service_id) if args.mode == "netfilter" else firegex.px_start_service(service_id)): + puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green) +else: + puts(f"Benchmark Failed: Coulnd't start the service ✗", color=colors.red); exit_test(1) #Get no regexs reading results = [] @@ -74,7 +82,8 @@ print(f"{results[0]} MB/s") #Add all the regexs for i in range(1,args.num_of_regexes+1): regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode() - if(not firegex.nf_add_regex(service_id,regex,"B",active=True,is_blacklist=True,is_case_sensitive=False)): puts(f"Benchmark Failed: Coulnd't add the regex ✗", color=colors.red); exit_test(1) + if(not (firegex.nf_add_regex if args.mode == "netfilter" else firegex.px_add_regex)(service_id,regex,"B",active=True,is_blacklist=True,is_case_sensitive=False) ): + puts(f"Benchmark Failed: Coulnd't add the regex ✗", color=colors.red); exit_test(1) puts(f"Performance with {i} regex(s): ", color=colors.red, end='') results.append(getReading(args.port)) print(f"{results[i]} MB/s") @@ -87,9 +96,9 @@ with open(args.output_file,'w') as f: puts(f"Sucessfully written results to {args.output_file} ✔", color=colors.magenta) #Delete the Service -if(firegex.nf_delete_service(service_id)): +if(firegex.nf_delete_service(service_id) if args.mode == "netfilter" else firegex.px_delete_service(service_id)): puts(f"Sucessfully delete service with id {service_id} ✔", color=colors.green) else: puts(f"Test Failed: Couldn't delete service ✗", color=colors.red); exit(1) -server.terminate() +server.terminate() \ No newline at end of file diff --git a/tests/run_tests.sh b/tests/run_tests.sh index f210baf..2f67aec 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,4 +1,5 @@ #!/bin/sh +pip3 install -r requirements.txt echo "Running standard API test" python3 api_test.py -p testpassword echo "Running Netfilter Regex TCP ipv4"