diff --git a/start.py b/start.py index 138bd7b..f6dfcb0 100755 --- a/start.py +++ b/start.py @@ -104,7 +104,7 @@ def gen_args(args_to_parse: list[str]|None = None): parser_start.add_argument('--psw-on-web', required=False, help='Setup firegex password on the web interface', action="store_true", default=False) parser_start.add_argument('--port', "-p", type=int, required=False, help='Port where open the web service of the firewall', default=4444) parser_start.add_argument('--logs', required=False, action="store_true", help='Show firegex logs', default=False) - parser_start.add_argument('--version', '-v', required=False, type=str , help='Version of the firegex image to use', default="latest") + parser_start.add_argument('--version', '-v', required=False, type=str , help='Version of the firegex image to use', default=None) #Stop Command parser_stop = subcommands.add_parser('stop', help='Stop the firewall') diff --git a/tests/README.md b/tests/README.md index 8e47c47..e430b6b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -79,14 +79,37 @@ You will find a new benchmark.csv file containg the results. The test was performed on: - Macbook Air M2 16GB RAM -- On a VM powered by OrbStack with Ubuntu 24.04.1 LTS aarch64 -- 6.12.10-orbstack-00297-gf8f6e015b993 +- On a VM powered by OrbStack with Fedora Linux 41 (Container Image) aarch64 +- Linux 6.12.13-orbstack-00304-gede1cf3337c4 -Command: `./benchmark.py -p testpassword -r 50 -d 1 -s 60` +Command: `./benchmark.py -p testpassword -r 50 -d 1 -s 50` -### NOTE: 8 threads performance do not change due to the fact that the source and destination ip is always the same, so the packets are sent to the same thread by the kernel. +NOTE: 8 threads performance before 2.5.0 do not change due to the fact that the source and destination ip is always the same, so the packets are sent to the same thread by the kernel. [https://netfilter.vger.kernel.narkive.com/sTP7613Y/meaning-of-nfqueue-s-queue-balance-option](https://netfilter.vger.kernel.narkive.com/sTP7613Y/meaning-of-nfqueue-s-queue-balance-option) Internally the kernel hashes the source and dest ip and choose the target thread based on the hash. If the source and dest ip are the same, the hash will be the same and the packets will be sent to the same thread. +This is a problem in a CTF, where we usually have a NAT to hide real IPs. + +Firegex 2.5.0 changes the way the threads are assigned to the packets, this is done userland, so we can have a better distribution of the packets between the threads. + +The charts are labeled as follows: `[version]-[n_thread]T` eg. `2.5.0-8T` means Firegex version 2.5.0 with 8 threads. ![Firegex Benchmark](results/Benchmark-chart.png) + + +From the benchmark above we can't see the real advantage of multithreading in 2.5.1, we can better see the advantage of multithreading in the chart below where a fake load in filtering is done. + +The load is simulated by this code: +```cpp +volatile int x = 0; +for (int i=0; i<50000; i++){ + x+=1; +} +``` + +![Firegex Benchmark](results/Benchmark-chart-with-load.png) + +In the chart above we can see that the 2.5.1 version with 8 threads has a better performance than the 2.5.1 version with 1 threads, and we can see it as much as the load increases. + +This particular advantage will be more noticeable with nfproxy module that is not implemented yet. + diff --git a/tests/benchmark.py b/tests/benchmark.py index b69e605..02b5aaa 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -85,8 +85,18 @@ server = Process(target=startServer) server.start() sleep(1) +custom_regex = [ + '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])' +] + def gen_regex(): - regex = secrets.token_hex(8) + """ + if len(custom_regex) == 0: + regex = secrets.token_hex(8) + else: + regex = custom_regex.pop() + """ + regex = secrets.token_hex(20) return base64.b64encode(bytes(regex.encode())).decode() #Get baseline reading diff --git a/tests/results/2.3.3-1T.csv b/tests/results/2.3.3-1T.csv new file mode 100644 index 0000000..e571d8e --- /dev/null +++ b/tests/results/2.3.3-1T.csv @@ -0,0 +1,51 @@ +0,4090.616 +1,2211.62 +2,1165.45 +3,849.39 +4,828.635 +5,741.537 +6,632.721 +7,624.772 +8,529.234 +9,469.688 +10,336.33 +11,427.783 +12,400.662 +13,335.086 +14,342.042 +15,307.283 +16,239.694 +17,295.163 +18,285.787 +19,254.402 +20,250.553 +21,227.146 +22,238.747 +23,234.718 +24,210.484 +25,210.697 +26,205.943 +27,202.568 +28,194.341 +29,189.916 +30,154.228 +31,168.922 +32,173.623 +33,125.431 +34,162.154 +35,149.865 +36,150.088 +37,146.085 +38,137.182 +39,138.686 +40,136.302 +41,132.707 +42,100.928 +43,126.414 +44,125.271 +45,117.839 +46,89.494 +47,116.939 +48,112.517 +49,111.369 +50,108.568 diff --git a/tests/results/2.3.3-8T.csv b/tests/results/2.3.3-8T.csv new file mode 100644 index 0000000..e4d3831 --- /dev/null +++ b/tests/results/2.3.3-8T.csv @@ -0,0 +1,51 @@ +0,3789.988 +1,2069.487 +2,1484.554 +3,956.972 +4,1052.873 +5,739.658 +6,534.722 +7,638.524 +8,573.833 +9,531.658 +10,476.167 +11,443.746 +12,406.027 +13,385.739 +14,341.563 +15,318.699 +16,303.722 +17,284.924 +18,284.336 +19,267.32 +20,202.74 +21,243.849 +22,226.082 +23,214.348 +24,216.8 +25,188.98 +26,158.68 +27,166.556 +28,148.287 +29,149.681 +30,177.043 +31,175.321 +32,165.312 +33,166.943 +34,159.026 +35,156.759 +36,150.216 +37,144.932 +38,146.088 +39,135.897 +40,136.99 +41,128.557 +42,100.307 +43,103.249 +44,123.49 +45,120.39 +46,118.055 +47,115.0 +48,112.593 +49,109.55 +50,109.512 diff --git a/tests/results/2.4.0-1T.csv b/tests/results/2.4.0-1T.csv new file mode 100644 index 0000000..b12df26 --- /dev/null +++ b/tests/results/2.4.0-1T.csv @@ -0,0 +1,51 @@ +0,4216.05 +1,4239.598 +2,2418.527 +3,2227.8 +4,2045.351 +5,2066.161 +6,2214.416 +7,2052.845 +8,2195.199 +9,2186.867 +10,2147.534 +11,2186.652 +12,2178.036 +13,2182.151 +14,2185.324 +15,1812.911 +16,2144.689 +17,2163.525 +18,2073.89 +19,2071.682 +20,2153.502 +21,2144.04 +22,2118.517 +23,2141.19 +24,2167.103 +25,2168.631 +26,2165.555 +27,2158.424 +28,2188.376 +29,2165.311 +30,2168.158 +31,2108.045 +32,2121.414 +33,2022.533 +34,1888.759 +35,2022.837 +36,2015.042 +37,1920.401 +38,2005.037 +39,2028.856 +40,2010.43 +41,1522.342 +42,1525.635 +43,1912.05 +44,1920.256 +45,1753.645 +46,1476.977 +47,1888.645 +48,1949.103 +49,1684.633 +50,1493.935 diff --git a/tests/results/2.4.0-8T.csv b/tests/results/2.4.0-8T.csv new file mode 100644 index 0000000..e894d72 --- /dev/null +++ b/tests/results/2.4.0-8T.csv @@ -0,0 +1,51 @@ +0,4203.31 +1,4283.392 +2,2383.415 +3,2419.701 +4,2038.823 +5,2038.0 +6,2160.869 +7,2192.641 +8,2216.766 +9,2762.56 +10,2160.398 +11,2147.886 +12,2146.47 +13,2158.101 +14,2154.025 +15,1997.694 +16,2028.288 +17,2005.373 +18,2153.945 +19,2190.799 +20,2169.302 +21,2139.842 +22,2155.307 +23,2152.223 +24,2124.155 +25,2103.135 +26,2148.053 +27,2163.366 +28,2122.339 +29,2064.701 +30,2134.748 +31,1632.533 +32,2082.309 +33,1878.795 +34,2009.28 +35,1987.424 +36,1748.364 +37,1725.66 +38,1967.877 +39,1854.637 +40,1903.963 +41,1987.138 +42,1532.547 +43,1569.27 +44,1535.941 +45,1941.715 +46,2014.504 +47,2005.794 +48,2022.972 +49,1740.836 +50,1726.444 diff --git a/tests/results/2.5.1-1T-withload.csv b/tests/results/2.5.1-1T-withload.csv new file mode 100644 index 0000000..fadffaa --- /dev/null +++ b/tests/results/2.5.1-1T-withload.csv @@ -0,0 +1,51 @@ +0,710.619 +1,887.877 +2,981.431 +3,1081.412 +4,1038.514 +5,1029.805 +6,928.317 +7,1130.938 +8,1165.42 +9,925.632 +10,949.483 +11,1021.973 +12,903.878 +13,1001.53 +14,895.351 +15,1026.722 +16,634.727 +17,744.758 +18,978.59 +19,962.375 +20,997.471 +21,929.785 +22,1200.83 +23,1257.741 +24,772.729 +25,683.913 +26,1188.17 +27,919.961 +28,922.225 +29,1066.286 +30,979.399 +31,978.917 +32,988.415 +33,1061.523 +34,942.85 +35,1045.949 +36,883.941 +37,958.41 +38,989.523 +39,1001.121 +40,1080.079 +41,1151.938 +42,1221.644 +43,991.855 +44,1088.344 +45,973.641 +46,952.35 +47,1089.644 +48,939.615 +49,1258.419 +50,949.414 diff --git a/tests/results/2.5.1-1T.csv b/tests/results/2.5.1-1T.csv new file mode 100644 index 0000000..0f7535b --- /dev/null +++ b/tests/results/2.5.1-1T.csv @@ -0,0 +1,51 @@ +0,3245.763 +1,3283.646 +2,3741.157 +3,3691.206 +4,3365.134 +5,3691.457 +6,3354.807 +7,3526.728 +8,3252.62 +9,3551.086 +10,3561.506 +11,3525.577 +12,2776.064 +13,3541.86 +14,3501.34 +15,3692.092 +16,3637.166 +17,3617.031 +18,3700.092 +19,3176.831 +20,3368.038 +21,3716.577 +22,3452.917 +23,3617.604 +24,3651.796 +25,3552.053 +26,3843.18 +27,3720.406 +28,3431.1 +29,3578.973 +30,3561.994 +31,3524.566 +32,3567.537 +33,3626.767 +34,3498.361 +35,3621.396 +36,3297.839 +37,3541.207 +38,3560.364 +39,3589.746 +40,3686.673 +41,3463.811 +42,3428.408 +43,3753.139 +44,3368.89 +45,3324.876 +46,3614.895 +47,3245.942 +48,3257.925 +49,3200.585 +50,3321.55 diff --git a/tests/results/2.5.1-8T-withload.csv b/tests/results/2.5.1-8T-withload.csv new file mode 100644 index 0000000..5ff7b8f --- /dev/null +++ b/tests/results/2.5.1-8T-withload.csv @@ -0,0 +1,51 @@ +0,1790.382 +1,1933.881 +2,1941.564 +3,1926.518 +4,1945.295 +5,1734.462 +6,2009.994 +7,2007.538 +8,2004.825 +9,1848.551 +10,1836.558 +11,1977.19 +12,1987.207 +13,2007.422 +14,1994.914 +15,1982.997 +16,1955.828 +17,1705.883 +18,1983.501 +19,1951.311 +20,1921.772 +21,1956.908 +22,1948.865 +23,1929.387 +24,1814.539 +25,2084.284 +26,1830.901 +27,1946.713 +28,1958.238 +29,1906.573 +30,1895.341 +31,1986.09 +32,1943.785 +33,1879.917 +34,1946.029 +35,1858.958 +36,2009.44 +37,1876.749 +38,1967.254 +39,1968.595 +40,1846.438 +41,1955.897 +42,1986.446 +43,1965.143 +44,1963.016 +45,1890.88 +46,1998.801 +47,1682.048 +48,2023.688 +49,1982.952 +50,1993.641 diff --git a/tests/results/2.5.1-8T.csv b/tests/results/2.5.1-8T.csv new file mode 100644 index 0000000..16fff42 --- /dev/null +++ b/tests/results/2.5.1-8T.csv @@ -0,0 +1,51 @@ +0,4007.679 +1,3963.986 +2,4222.243 +3,3640.707 +4,4388.553 +5,3636.047 +6,3644.611 +7,3547.39 +8,3412.162 +9,3632.367 +10,3536.655 +11,3820.019 +12,3677.177 +13,3366.323 +14,3353.031 +15,3392.423 +16,3330.368 +17,3363.272 +18,4027.34 +19,3467.982 +20,3607.754 +21,3767.614 +22,3340.544 +23,4086.612 +24,3784.164 +25,3496.518 +26,3543.808 +27,3453.934 +28,3546.188 +29,3458.804 +30,3728.609 +31,3697.624 +32,3698.191 +33,3673.973 +34,3690.046 +35,3663.799 +36,3540.004 +37,3857.604 +38,3426.215 +39,3704.176 +40,3796.133 +41,3604.623 +42,3650.508 +43,3501.861 +44,3685.992 +45,3623.404 +46,3728.601 +47,3844.994 +48,3820.046 +49,3680.976 +50,3797.432 diff --git a/tests/results/Benchmark-chart-with-load.png b/tests/results/Benchmark-chart-with-load.png new file mode 100644 index 0000000..54d48fb Binary files /dev/null and b/tests/results/Benchmark-chart-with-load.png differ diff --git a/tests/results/Benchmark-chart.png b/tests/results/Benchmark-chart.png index fce9993..39d9400 100644 Binary files a/tests/results/Benchmark-chart.png and b/tests/results/Benchmark-chart.png differ diff --git a/tests/results/test-firegex.csv b/tests/results/test-firegex.csv new file mode 100644 index 0000000..905d71e --- /dev/null +++ b/tests/results/test-firegex.csv @@ -0,0 +1,52 @@ +;2.3.3-1T;2.3.3-8T;2.4.0-1T;2.4.0-8T;2.5.1-1T;2.5.1-8T;;With Load;2.5.1-1T;2.5.1-8T +0;4090,616;3789,988;4216,05;4203,31;3245,763;4007,679;;0;710,619;1790,382 +1;2211,62;2069,487;4239,598;4283,392;3283,646;3963,986;;1;887,877;1933,881 +2;1165,45;1484,554;2418,527;2383,415;3741,157;4222,243;;2;981,431;1941,564 +3;849,39;956,972;2227,8;2419,701;3691,206;3640,707;;3;1081,412;1926,518 +4;828,635;1052,873;2045,351;2038,823;3365,134;4388,553;;4;1038,514;1945,295 +5;741,537;739,658;2066,161;2038;3691,457;3636,047;;5;1029,805;1734,462 +6;632,721;534,722;2214,416;2160,869;3354,807;3644,611;;6;928,317;2009,994 +7;624,772;638,524;2052,845;2192,641;3526,728;3547,39;;7;1130,938;2007,538 +8;529,234;573,833;2195,199;2216,766;3252,62;3412,162;;8;1165,42;2004,825 +9;469,688;531,658;2186,867;2762,56;3551,086;3632,367;;9;925,632;1848,551 +10;336,33;476,167;2147,534;2160,398;3561,506;3536,655;;10;949,483;1836,558 +11;427,783;443,746;2186,652;2147,886;3525,577;3820,019;;11;1021,973;1977,19 +12;400,662;406,027;2178,036;2146,47;2776,064;3677,177;;12;903,878;1987,207 +13;335,086;385,739;2182,151;2158,101;3541,86;3366,323;;13;1001,53;2007,422 +14;342,042;341,563;2185,324;2154,025;3501,34;3353,031;;14;895,351;1994,914 +15;307,283;318,699;1812,911;1997,694;3692,092;3392,423;;15;1026,722;1982,997 +16;239,694;303,722;2144,689;2028,288;3637,166;3330,368;;16;634,727;1955,828 +17;295,163;284,924;2163,525;2005,373;3617,031;3363,272;;17;744,758;1705,883 +18;285,787;284,336;2073,89;2153,945;3700,092;4027,34;;18;978,59;1983,501 +19;254,402;267,32;2071,682;2190,799;3176,831;3467,982;;19;962,375;1951,311 +20;250,553;202,74;2153,502;2169,302;3368,038;3607,754;;20;997,471;1921,772 +21;227,146;243,849;2144,04;2139,842;3716,577;3767,614;;21;929,785;1956,908 +22;238,747;226,082;2118,517;2155,307;3452,917;3340,544;;22;1200,83;1948,865 +23;234,718;214,348;2141,19;2152,223;3617,604;4086,612;;23;1257,741;1929,387 +24;210,484;216,8;2167,103;2124,155;3651,796;3784,164;;24;772,729;1814,539 +25;210,697;188,98;2168,631;2103,135;3552,053;3496,518;;25;683,913;2084,284 +26;205,943;158,68;2165,555;2148,053;3843,18;3543,808;;26;1188,17;1830,901 +27;202,568;166,556;2158,424;2163,366;3720,406;3453,934;;27;919,961;1946,713 +28;194,341;148,287;2188,376;2122,339;3431,1;3546,188;;28;922,225;1958,238 +29;189,916;149,681;2165,311;2064,701;3578,973;3458,804;;29;1066,286;1906,573 +30;154,228;177,043;2168,158;2134,748;3561,994;3728,609;;30;979,399;1895,341 +31;168,922;175,321;2108,045;1632,533;3524,566;3697,624;;31;978,917;1986,09 +32;173,623;165,312;2121,414;2082,309;3567,537;3698,191;;32;988,415;1943,785 +33;125,431;166,943;2022,533;1878,795;3626,767;3673,973;;33;1061,523;1879,917 +34;162,154;159,026;1888,759;2009,28;3498,361;3690,046;;34;942,85;1946,029 +35;149,865;156,759;2022,837;1987,424;3621,396;3663,799;;35;1045,949;1858,958 +36;150,088;150,216;2015,042;1748,364;3297,839;3540,004;;36;883,941;2009,44 +37;146,085;144,932;1920,401;1725,66;3541,207;3857,604;;37;958,41;1876,749 +38;137,182;146,088;2005,037;1967,877;3560,364;3426,215;;38;989,523;1967,254 +39;138,686;135,897;2028,856;1854,637;3589,746;3704,176;;39;1001,121;1968,595 +40;136,302;136,99;2010,43;1903,963;3686,673;3796,133;;40;1080,079;1846,438 +41;132,707;128,557;1522,342;1987,138;3463,811;3604,623;;41;1151,938;1955,897 +42;100,928;100,307;1525,635;1532,547;3428,408;3650,508;;42;1221,644;1986,446 +43;126,414;103,249;1912,05;1569,27;3753,139;3501,861;;43;991,855;1965,143 +44;125,271;123,49;1920,256;1535,941;3368,89;3685,992;;44;1088,344;1963,016 +45;117,839;120,39;1753,645;1941,715;3324,876;3623,404;;45;973,641;1890,88 +46;89,494;118,055;1476,977;2014,504;3614,895;3728,601;;46;952,35;1998,801 +47;116,939;115;1888,645;2005,794;3245,942;3844,994;;47;1089,644;1682,048 +48;112,517;112,593;1949,103;2022,972;3257,925;3820,046;;48;939,615;2023,688 +49;111,369;109,55;1684,633;1740,836;3200,585;3680,976;;49;1258,419;1982,952 +50;108,568;109,512;1493,935;1726,444;3321,55;3797,432;;50;949,414;1993,641 \ No newline at end of file diff --git a/tests/results/test-firegex.xlsx b/tests/results/test-firegex.xlsx deleted file mode 100644 index e9db96d..0000000 Binary files a/tests/results/test-firegex.xlsx and /dev/null differ