From 1f21365fa41141a34f5ea9314a601f5559b4fcd9 Mon Sep 17 00:00:00 2001 From: nik012003 Date: Sat, 25 Jun 2022 17:37:21 +0200 Subject: [PATCH] Added benchmarking --- tests/README.md | 41 +++++++++++++- tests/benchmark.py | 120 +++++++++++++++++++++++++++++++++++++++++ tests/requirements.txt | 2 + tests/test.py | 3 +- 4 files changed, 164 insertions(+), 2 deletions(-) create mode 100755 tests/benchmark.py create mode 100644 tests/requirements.txt diff --git a/tests/README.md b/tests/README.md index fa80ccf..16b60a2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -24,4 +24,43 @@ Output of the tests: The request wasn't blocked ✔ Sucessfully delete service with id test-service ✔ -The testing methodology will soon be updated with more edge-cases and a benchmarking tool to evauate the speed and effecitveness of our proxy. \ No newline at end of file +The testing methodology will soon be updated with more edge-cases. + +# Running a Benchmark + ./benchmark.py + --address ADDRESS, -a ADDRESS + Address of firegex backend + --service_port SERVICE_PORT, -P SERVICE_PORT + Port of the Benchmark service + --service_name SERVICE_NAME, -n SERVICE_NAME + Name of the Benchmark service + --password PASSWORD, -p PASSWORD + Firegex password + --num_of_regexes NUM_OF_REGEXES, -r NUM_OF_REGEXES + Number of regexes to benchmark with + --duration DURATION, -d DURATION + Duration of the Benchmark in seconds + +Benchmarks let you evaluate the performance of the proxy. You can run one by typing in a shell ```test.py -p FIREGEX_PASSWORD -r NUM_OF_REGEX -d BENCHMARK_DURATION```. + +It uses iperf3 to benchmark the throuput in MB/s of the server, both with proxy, without proxy, and for each new added regex. It will automatically add a new random regex untill it has reached NUM_OF_REGEX specified in the arguments. + +Example output: + + Benchmarking with 30 will start on http://127.0.0.1:5000/ + Sucessfully logged in ✔ + Sucessfully created service Benchmark Service with public port 1337 ✔ + Sucessfully received the internal port 38249 ✔ + Baseline without proxy: 7145.402353788159MB/s + Sucessfully started service with id benchmark-service ✔ + Performance with no regexes: 2255.4573361742887MB/s + Performance with 1 regex(s): 76.51810976542541MB/s + Performance with 2 regex(s): 38.769568516424684MB/s + Performance with 3 regex(s): 25.976997107893663MB/s + Performance with 4 regex(s): 19.539058399917625MB/s + Performance with 5 regex(s): 14.720692718915746MB/s + Performance with 6 regex(s): 13.101487751340413MB/s + Performance with 7 regex(s): 11.237772047509017MB/s + Performance with 8 regex(s): 9.851833265188406MB/s + Performance with 9 regex(s): 8.725255532797124MB/s + Performance with 10 regex(s): 7.891516589287963MB/s \ No newline at end of file diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100755 index 0000000..d07cc27 --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +import argparse, socket,secrets, base64, iperf3 +from time import sleep +from requests import Session +from multiprocessing import Process + +pref = "\033[" +reset = f"{pref}0m" + +class colors: + black = "30m" + red = "31m" + green = "32m" + yellow = "33m" + blue = "34m" + magenta = "35m" + cyan = "36m" + white = "37m" + +def puts(text, *args, color=colors.white, is_bold=False, **kwargs): + print(f'{pref}{1 if is_bold else 0};{color}' + text + reset, *args, **kwargs) + +def sep(): puts("-----------------------------------", is_bold=True) +parser = argparse.ArgumentParser() +parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:5000/") +parser.add_argument("--service_port", "-P", type=int , required=False, help='Port of the Benchmark service', default=1337) +parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service") +parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password') +parser.add_argument("--num_of_regexes", "-r", type=int, required=True, help='Number of regexes to benchmark with') +parser.add_argument("--duration", "-d", type=int, required=False, help='Duration of the Benchmark in seconds', default=5) + +args = parser.parse_args() +sep() +puts(f"Benchmarking with {args.num_of_regexes} will start on ", color=colors.cyan, end="") +puts(f"{args.address}", color=colors.yellow) + +s = Session() +#Connect to Firegex +req = s.post(f"{args.address}api/login", json={"password":args.password}) +assert req.json()["status"] == "ok", f"Benchmark Failed: Unknown response or wrong passowrd {req.text}" +puts(f"Sucessfully logged in ✔", color=colors.green) + +#Create new Service +req = s.post(f"{args.address}api/services/add" , json={"name":args.service_name,"port":args.service_port}) +assert req.json()["status"] == "ok", f"Benchmark Failed: Couldn't create service {req.text} ✔" +puts(f"Sucessfully created service {args.service_name} with public port {args.service_port} ✔", color=colors.green) + +#Find the Service +req = s.get(f"{args.address}api/services") +internal_port = service_id = None +try: + for service in req.json(): + if service["name"] == args.service_name: + service_id = service["id"] + internal_port = service["internal_port"] + puts(f"Sucessfully received the internal port {internal_port} ✔", color=colors.green) + break +except Exception: + puts(f"Benchmark Failed: Coulnd't get the service internal port {req.text}", color=colors.red) + exit(1) + +#Start iperf3 +def startServer(): + server = iperf3.Server() + server.bind_address = '127.0.0.1' + server.port = internal_port + server.verbose = False + while True: + server.run() + +def getReading(port): + client = iperf3.Client() + client.duration = args.duration + client.server_hostname = '127.0.0.1' + client.port = port + client.protocol = 'tcp' + return client.run().json['end']['sum_received']['bits_per_second']/8e+6 + +server = Process(target=startServer) +server.start() +sleep(1) + + +#Get baseline reading +puts(f"Baseline without proxy: ", color=colors.blue, end='') +print(f"{getReading(internal_port)}MB/s") + +#Start firewall +req = s.get(f"{args.address}api/service/{service_id}/start") +assert req.json()["status"] == "ok", f"Benchmark Failed: Couldn't start the service {req.text}" +puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green) + +#Hacky solution - wait a bit for the server to start +sleep(1) + +#Get no regexs reading +results = [] +puts(f"Performance with no regexes: ", color=colors.yellow , end='') +results.append(getReading(args.service_port)) +print(f"{results[0]}MB/s") + +#Add all the regexs +for i in range(1,args.num_of_regexes+1): + regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode() + req = s.post(f"{args.address}api/regexes/add", + json={"is_blacklist":True,"is_case_sensitive":True,"service_id":service_id,"mode":"B","regex":regex}) + assert req.json()["status"] == "ok", f"Test Failed: Couldn't add regex {req.text}" + puts(f"Performance with {i} regex(s): ", color=colors.red, end='') + results.append(getReading(args.service_port)) + print(f"{results[i]}MB/s") + +print("Results:") +print(results) + +#Delete the Service +req = s.get(f"{args.address}api/service/{service_id}/delete") +assert req.json()["status"] == "ok", f"Benchmark Failed: Couldn't delete service {req.text}" +puts(f"Sucessfully delete service with id {service_id} ✔", color=colors.green) + +server.terminate() \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..4f04d14 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +iperf3 +requests \ No newline at end of file diff --git a/tests/test.py b/tests/test.py index a21257b..35ce542 100755 --- a/tests/test.py +++ b/tests/test.py @@ -25,7 +25,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:5000/") parser.add_argument("--service_port", "-P", type=int , required=False, help='Port of the test service', default=1337) parser.add_argument("--service_name", "-n", type=str , required=False, help='Name of the test service', default="Test Service") -parser.add_argument("--password", "-p", type=str, required=True, help='Firegex passowrd') +parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password') args = parser.parse_args() sep() puts(f"Testing will start on ", color=colors.cyan, end="") @@ -97,6 +97,7 @@ secret = bytes(secrets.token_hex(16).encode()) regex = base64.b64encode(secret).decode() req = s.post(f"{args.address}api/regexes/add", json={"is_blacklist":True,"is_case_sensitive":True,"service_id":service_id,"mode":"B","regex":regex}) +assert req.json()["status"] == "ok", f"Test Failed: Couldn't add regex {req.text}" puts(f"Sucessfully added regex to service with id {service_id} ✔", color=colors.green) #Test the regex