diff --git a/.gitignore b/.gitignore index 88bc9d8..2ccae68 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ /firegex-compose-tmp-file.yml /firegex.py /tests/benchmark.csv +/tests/comparemark.csv # misc **/.DS_Store **/.env.local diff --git a/tests/comparemark.py b/tests/comparemark.py new file mode 100644 index 0000000..259c28d --- /dev/null +++ b/tests/comparemark.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +from utils.colors import colors, puts, sep +from utils.firegexapi import FiregexAPI +from multiprocessing import Process +from time import sleep +import iperf3 +import csv +import argparse +import secrets + +parser = argparse.ArgumentParser() +parser.add_argument("--address", "-a", type=str , required=False, help='Address of firegex backend', default="http://127.0.0.1:4444/") +parser.add_argument("--port", "-P", type=int , required=False, help='Port of the Benchmark service', default=1337) +parser.add_argument("--service-name", "-n", type=str , required=False, help='Name of the Benchmark service', default="Benchmark Service") +parser.add_argument("--password", "-p", type=str, required=True, help='Firegex password') +parser.add_argument("--duration", "-d", type=int, required=False, help='Duration of the Benchmark in seconds', default=5) +parser.add_argument("--output-file", "-o", type=str, required=False, help='Output results csv file', default="comparemark.csv") +parser.add_argument("--num-of-streams", "-s", type=int, required=False, help='Number of concurrent streams', default=1) +parser.add_argument("--number-of-values", "-V", type=int, required=False, help='Number of values to generate', default=100) + +args = parser.parse_args() +sep() +puts("Benchmark for compare firegex features ", color=colors.cyan, end="") +puts(f"{args.address}", color=colors.yellow) + +firegex = FiregexAPI(args.address) + +#Connect to Firegex +if (firegex.login(args.password)): + puts("Sucessfully logged in ✔", color=colors.green) +else: + puts("Benchmark Failed: Unknown response or wrong passowrd ✗", color=colors.red) + exit(1) + +def exit_test(code): + if service_id: + server.kill() + if(firegex.nfregex_delete_service(service_id)): + puts("Sucessfully deleted service ✔", color=colors.green) + else: + puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) + exit_test(1) + exit(code) + +#Cleaning services +srvs = firegex.nfregex_get_services() +for ele in srvs: + if ele['name'] == args.service_name: + firegex.nfregex_delete_service(ele['service_id']) + +srvs = firegex.nfproxy_get_services() +for ele in srvs: + if ele['name'] == args.service_name: + firegex.nfproxy_delete_service(ele['service_id']) + +#Create new Service +service_id = firegex.nfregex_add_service(args.service_name, args.port, "tcp", "127.0.0.1/32") +if service_id: + puts(f"Sucessfully created service {service_id} ✔", color=colors.green) +else: + puts("Test Failed: Failed to create service ✗", color=colors.red) + exit(1) + +args.port = int(args.port) +args.duration = int(args.duration) +args.num_of_streams = int(args.num_of_streams) +args.number_of_values = int(args.number_of_values) + +#Start iperf3 +def startServer(): + server = iperf3.Server() + server.bind_address = '127.0.0.1' + server.port = args.port + server.verbose = False + while True: + server.run() + +global server +server = Process(target=startServer) +server.start() +sleep(1) + +def getReading(port): + global server + attempt = 1 + while True: + try: + client = iperf3.Client() + client.duration = args.duration + client.server_hostname = '127.0.0.1' + client.port = port + client.zerocopy = True + client.verbose = False + client.protocol = 'tcp' + client.num_streams = args.num_of_streams + return round(client.run().json['end']['sum_received']['bits_per_second']/8e+6 , 3) + except Exception as e: + if attempt >= 3: + raise e + else: + server.kill() + server = Process(target=startServer) + server.start() + sleep(1) + puts(f"Faild to run test on attempt: {attempt}, retrying...", color=colors.red) + + +text_filter_key = secrets.token_hex(16) + +baseline_data = [] + +for _ in range(args.number_of_values): + #Get baseline reading + puts("Baseline without any filter: ", color=colors.blue, end='') + data = getReading(args.port) + baseline_data.append(data) + print(f"{data} MB/s") + +#Start nfregex service +if firegex.nfregex_start_service(service_id): + puts(f"Sucessfully started service with id {service_id} ✔", color=colors.green) +else: + puts("Benchmark Failed: Coulnd't start the service ✗", color=colors.red) + exit_test(1) + +#Get no regexes reading +no_regex_nfregex = [] + +for _ in range(args.number_of_values): + #Get baseline reading + puts("Baseline nfregex with no filter: ", color=colors.blue, end='') + data = getReading(args.port) + no_regex_nfregex.append(data) + print(f"{data} MB/s") + +if not firegex.nfregex_add_regex(service_id,text_filter_key,"B",active=True,is_case_sensitive=False): + puts("Benchmark Failed: Couldn't add the regex ✗", color=colors.red) + exit_test(1) + +nfregex_test = [] + +for _ in range(args.number_of_values): + #Get baseline reading + puts("Baseline nfregex with 1 filter: ", color=colors.blue, end='') + data = getReading(args.port) + nfregex_test.append(data) + print(f"{data} MB/s") + +if(firegex.nfregex_delete_service(service_id)): + puts("Sucessfully deleted service ✔", color=colors.green) +else: + puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) + exit_test(1) + +service_id = firegex.nfproxy_add_service(args.service_name, args.port, "http" , "127.0.0.1" ) +if service_id: + puts(f"Sucessfully created service {service_id} ✔", color=colors.green) +else: + puts("Test Failed: Failed to create service ✗", color=colors.red) + exit(1) + +def exit_test(code): + if service_id: + server.stop() + if firegex.nfproxy_delete_service(service_id): + puts("Sucessfully deleted service ✔", color=colors.green) + else: + puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) + exit(1) + exit(code) + +if(firegex.nfproxy_start_service(service_id)): + puts("Sucessfully started service ✔", color=colors.green) +else: + puts("Test Failed: Failed to start service ✗", color=colors.red) + exit_test(1) + +#Get no filters +no_nfproxy_filter = [] + +for _ in range(args.number_of_values): + #Get baseline reading + puts("Baseline nfregex with no filter: ", color=colors.blue, end='') + data = getReading(args.port) + no_nfproxy_filter.append(data) + print(f"{data} MB/s") + +BASE_FILTER = f""" +from firegex.nfproxy.models import RawPacket +from firegex.nfproxy import pyfilter, REJECT + +@pyfilter +def verdict_test(packet:RawPacket): + if {repr(text_filter_key)} in packet.data: + return REJECT +""" + +if firegex.nfproxy_set_code(service_id, BASE_FILTER): + puts(f"Sucessfully added filter for {str(text_filter_key)} in REJECT mode ✔", color=colors.green) +else: + puts(f"Test Failed: Couldn't add the filter {str(text_filter_key)} ✗", color=colors.red) + exit_test(1) + +nfproxy_test = [] + +for _ in range(args.number_of_values): + #Get baseline reading + puts("Baseline nfproxy with 1 filter: ", color=colors.blue, end='') + data = getReading(args.port) + nfproxy_test.append(data) + print(f"{data} MB/s") + +if(firegex.nfproxy_delete_service(service_id)): + puts("Sucessfully deleted service ✔", color=colors.green) +else: + puts("Test Failed: Coulnd't delete serivce ✗", color=colors.red) + exit_test(1) + +with open(args.output_file,'w') as f: + writer = csv.writer(f) + writer.writerow(["Baseline","No NFRegex","NFRegex test","No NFProxy","NFProxy test"]) + for data in zip(baseline_data,no_regex_nfregex,nfregex_test,no_nfproxy_filter,nfproxy_test): + writer.writerow(data) + +puts(f"Sucessfully written results to {args.output_file} ✔", color=colors.magenta) + +server.terminate() diff --git a/tests/results/comparemark_1T.csv b/tests/results/comparemark_1T.csv new file mode 100644 index 0000000..fbc9721 --- /dev/null +++ b/tests/results/comparemark_1T.csv @@ -0,0 +1,101 @@ +Baseline,No NFRegex,NFRegex test,No NFProxy,NFProxy test +81981.648,4263.853,3704.36,20.324,157.027 +83671.333,3679.633,3581.755,80.941,170.117 +83242.024,3768.277,3707.167,17.573,140.528 +80634.406,3391.853,3342.294,18.147,155.149 +80803.554,3782.011,3351.532,91.588,151.125 +81848.118,3369.093,3402.829,16.265,172.256 +83366.402,3829.742,3450.839,74.587,154.558 +80090.458,3705.665,3115.652,59.879,176.729 +82941.674,3402.431,2647.587,82.505,190.973 +82408.062,3771.519,3263.238,80.928,181.133 +81496.751,3488.678,3071.435,61.689,178.78 +80521.896,3329.207,3337.016,72.019,145.631 +77602.53,3451.091,3341.707,79.177,135.22 +78114.748,3034.854,2729.197,86.442,138.892 +78612.458,3903.77,3150.868,86.439,162.732 +78557.432,3519.262,2813.746,92.765,174.977 +78423.457,3676.049,3028.26,90.599,175.308 +77691.075,3700.414,3156.115,94.947,149.884 +78724.55,3620.881,3662.628,96.25,147.07 +77152.12,3753.867,2703.596,74.999,146.374 +78207.892,3436.931,3235.716,84.462,147.718 +78661.125,2809.496,3583.936,71.432,136.834 +79295.427,2900.885,2751.027,110.69,140.917 +78615.012,3426.316,2646.876,84.154,166.676 +78834.792,3099.674,3481.813,75.686,141.639 +78780.089,3956.542,3277.297,68.036,159.049 +78750.782,3619.753,2673.595,88.572,158.745 +78785.339,3706.021,3260.037,63.04,144.391 +79191.743,3796.98,3348.318,68.508,187.599 +79301.11,3744.202,2923.248,85.889,186.882 +78782.096,3596.838,2794.827,81.711,181.645 +79200.126,3808.307,2780.723,68.687,173.319 +78967.339,3379.858,3006.826,60.934,148.293 +78906.393,3249.612,2184.191,71.001,180.972 +78630.399,2985.902,3101.626,69.764,165.321 +79704.089,2960.876,2045.497,78.482,142.952 +78513.421,3392.818,3663.237,74.347,167.331 +78307.521,3042.279,3346.579,83.166,158.617 +79063.707,3035.778,3201.974,58.416,166.345 +78055.727,3225.088,2751.922,73.507,171.018 +78113.799,3446.309,3584.41,55.904,175.959 +78001.812,3494.696,3688.211,69.357,179.091 +78847.427,3535.993,2802.299,65.209,168.031 +78962.324,3490.7,3419.473,88.078,139.561 +78893.894,3233.785,3333.907,92.643,191.195 +79071.778,3583.699,2876.964,80.127,154.041 +77700.317,3642.345,3495.471,92.363,187.342 +78127.515,3717.57,3042.643,83.957,179.093 +77753.945,3250.24,2855.264,81.543,173.467 +78559.121,3273.616,3052.387,68.41,173.04 +75467.465,3285.917,3539.079,75.405,172.916 +78966.94,3515.517,3244.459,67.649,179.084 +78538.167,3213.653,3161.304,73.459,177.122 +78155.567,3620.075,3210.943,78.529,130.819 +78061.614,3540.306,3309.794,65.073,156.299 +78916.707,3792.476,3153.15,66.458,131.09 +78578.739,3545.169,3460.338,64.179,102.586 +77840.068,3391.105,2740.68,68.811,133.416 +77825.185,3518.013,3053.849,79.3,162.133 +78220.815,3460.805,3160.229,81.66,164.07 +77909.209,3678.952,3455.795,68.016,183.943 +78316.362,3391.574,2611.739,61.389,173.824 +77700.766,3428.426,2401.865,75.203,167.738 +77713.844,3564.779,2681.721,67.446,164.205 +78264.528,3338.45,2538.924,80.965,189.552 +77386.644,3240.004,2828.404,82.711,161.537 +77676.799,3329.217,3271.386,68.774,181.252 +77271.864,3337.526,3642.672,76.863,149.401 +77874.371,3727.061,3105.705,64.616,142.859 +77795.355,3321.723,3146.809,68.02,143.332 +76708.922,3174.007,2894.463,58.767,180.303 +73727.748,3326.291,3051.806,92.562,164.844 +77808.963,3131.696,2981.847,71.922,166.886 +77428.354,3034.657,3691.02,66.082,174.526 +77876.376,3376.396,3350.672,77.43,175.058 +77435.002,3771.959,3119.208,89.324,127.042 +77416.511,3692.412,3317.191,65.009,138.299 +77273.918,3143.066,3272.427,69.21,185.665 +77616.307,3439.88,3063.84,69.328,145.758 +77144.846,3478.204,3434.2,70.067,149.344 +77240.102,3569.772,3074.049,78.498,171.137 +76611.234,3188.296,2741.399,81.359,144.142 +77280.077,2354.642,2677.068,88.723,171.205 +77256.012,3186.388,2955.016,86.743,187.69 +76840.739,3929.309,2830.187,78.168,177.322 +76915.179,3097.976,2776.124,75.442,179.338 +77008.17,3206.919,3505.537,65.674,154.079 +76677.356,3283.904,3375.318,73.653,149.791 +72887.202,3613.949,3085.52,61.979,170.373 +76474.928,3521.26,3098.529,69.757,125.84 +76298.561,3345.147,2816.803,65.52,156.367 +75809.858,3270.318,3199.017,65.439,168.195 +75861.394,3549.166,3290.382,64.87,152.735 +75370.778,3285.569,2516.305,66.684,173.87 +75065.778,3476.664,3330.229,63.598,203.248 +75341.359,3766.527,2919.126,66.489,166.597 +75460.105,3209.785,2062.287,85.557,148.266 +75241.869,3190.988,2815.198,83.028,168.218 +75136.126,3527.738,3807.216,86.68,139.161 +72137.416,3585.357,3679.954,87.557,173.405 diff --git a/tests/results/comparemark_8T.csv b/tests/results/comparemark_8T.csv new file mode 100644 index 0000000..67709c4 --- /dev/null +++ b/tests/results/comparemark_8T.csv @@ -0,0 +1,101 @@ +Baseline,No NFRegex,NFRegex test,No NFProxy,NFProxy test +82319.646,4209.583,3712.353,121.857,306.182 +81780.087,4420.337,3496.811,316.474,284.114 +81983.759,3751.124,3196.505,117.156,328.351 +78695.424,3757.432,3319.581,294.987,297.596 +76872.768,3750.539,3343.623,155.728,252.456 +76171.142,3838.93,3200.847,182.992,250.711 +77200.509,3772.995,3233.105,170.134,233.559 +76997.957,3969.131,3583.1,302.452,261.381 +79326.614,3739.169,3206.741,176.701,273.866 +79401.918,3441.961,3544.878,225.435,272.085 +78583.12,3496.333,3541.256,173.373,252.579 +76658.796,3476.902,3608.929,294.946,295.917 +77702.031,3521.065,3505.573,208.097,325.948 +73480.157,3391.395,3539.955,304.048,298.033 +79424.158,3416.78,3703.274,289.852,269.109 +71356.757,3375.566,3108.571,201.685,251.175 +77088.894,3601.237,3145.279,292.816,251.284 +77414.973,3508.696,3486.202,148.038,304.0 +78532.003,3653.737,3257.618,186.423,285.043 +77273.162,3813.761,3257.043,243.815,275.678 +77987.119,3396.437,3218.513,305.975,263.495 +78333.673,3337.983,3151.148,285.922,247.077 +78017.426,3318.777,3603.665,301.569,334.689 +77768.816,3685.071,3856.182,227.686,263.019 +76637.681,3863.354,3228.283,297.693,247.063 +77131.97,3844.799,3130.89,231.325,243.194 +77026.275,3609.298,3290.869,292.635,225.888 +76822.985,3975.261,3295.475,288.454,260.24 +77056.641,3287.228,3158.037,188.899,304.252 +76989.034,3401.383,3921.283,314.093,274.042 +77329.65,3661.154,3574.831,298.921,255.275 +77224.898,3393.809,3729.188,285.657,274.504 +77718.375,3529.669,3698.233,310.755,245.919 +77305.079,3527.914,3944.879,290.005,257.56 +76968.251,4016.634,3136.247,213.477,321.712 +78307.289,3395.383,3664.349,300.439,240.162 +77963.682,4039.967,3612.592,202.378,283.008 +78445.615,3650.661,3720.51,230.353,245.722 +77657.435,3207.357,3722.607,210.455,237.539 +77433.212,3984.341,3668.394,290.21,223.748 +78838.553,3601.481,3774.207,226.082,231.46 +78465.323,3605.593,3607.373,287.807,228.475 +79409.105,3924.074,3501.094,295.633,215.127 +78772.314,3528.259,3587.289,315.391,235.149 +78771.636,3887.227,3604.21,300.321,230.153 +78380.529,3721.388,3359.047,276.379,217.366 +78735.007,3715.385,3550.428,304.04,232.071 +79239.305,3727.222,3694.185,230.126,229.113 +79335.595,3829.679,3586.249,311.71,223.944 +78907.995,3416.885,3717.573,209.111,206.589 +78185.685,3442.233,3791.86,241.424,237.809 +78976.545,3632.26,3644.855,286.019,221.146 +79193.796,3345.46,3999.078,272.678,235.079 +78351.693,4125.214,3514.847,205.092,234.174 +79148.828,3746.693,3287.509,324.361,224.794 +79355.479,3654.642,3380.708,302.979,235.002 +78920.941,3435.363,3476.654,302.36,216.57 +79493.838,3422.457,3523.082,220.413,226.403 +78297.394,3332.065,3795.737,277.971,214.589 +79667.375,3511.727,3479.304,303.529,225.433 +78668.734,3642.407,3764.156,268.119,232.946 +79117.232,3511.716,3784.833,294.041,228.407 +78402.7,3429.197,3729.148,306.63,235.854 +78754.739,3476.241,3731.898,312.728,231.963 +78982.655,3670.246,3885.078,311.042,234.749 +78483.895,3788.399,3312.405,324.085,216.434 +78320.766,3881.708,3199.036,286.672,233.455 +78864.93,3425.428,3468.592,303.478,225.887 +78744.033,3392.627,3770.027,293.394,231.645 +78719.715,3461.113,3643.086,290.809,232.447 +78941.26,3674.411,3569.715,294.524,236.337 +78773.18,3633.675,4036.615,304.011,235.168 +78437.387,3391.726,3958.928,211.65,219.911 +77755.947,3344.37,3203.519,227.663,235.904 +78683.945,2878.771,3507.118,275.101,241.625 +78896.549,3367.711,3679.883,292.634,218.73 +78599.156,3354.189,3882.883,319.451,236.044 +78432.671,3410.264,3231.651,288.116,231.701 +79121.517,3406.463,4386.806,199.473,233.099 +78050.314,3331.028,4252.679,216.699,237.085 +79722.745,3360.358,4028.573,183.874,234.384 +79184.678,3411.561,3738.687,296.739,234.144 +78278.258,3428.837,3916.885,302.304,247.577 +78928.194,3399.923,3826.319,312.149,259.928 +79125.362,3370.74,3683.865,305.887,260.047 +78079.743,3393.67,3615.402,279.919,235.796 +78278.656,3351.376,3351.466,299.925,273.069 +78260.282,3404.873,3394.362,295.447,241.677 +78821.77,3520.697,3575.348,280.468,240.324 +77991.981,3372.504,3274.764,310.425,242.719 +78586.978,3408.246,3696.578,289.962,225.991 +78885.946,3588.335,3127.125,304.318,230.989 +77935.164,3342.819,3185.714,302.692,227.837 +78468.744,4094.828,3199.352,302.191,237.358 +77853.086,3460.435,3743.672,293.124,234.734 +78376.452,3585.199,3395.543,303.307,230.797 +78056.037,3446.037,3514.161,144.436,250.035 +76966.546,3340.455,3788.475,304.016,238.665 +72905.353,3633.641,3809.559,270.187,252.518 +78095.119,3666.076,3668.808,287.138,232.618 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 48ee293..be769a7 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -36,9 +36,5 @@ python3 nfproxy_test.py -p $PASSWORD || ERROR=1 echo "Running Netfilter Proxy ipv6" python3 nfproxy_test.py -p $PASSWORD -6 || ERROR=1 -if [[ "$ERROR" == "0" ]] then - python3 benchmark.py -p $PASSWORD -r 5 -d 1 -s 10 || ERROR=1 -fi - exit $ERROR