Fixes and improvements in thread managment

Dockerfile

@@ -19,9 +19,8 @@ ADD ./backend/requirements.txt /execute/requirements.txt
 RUN pip3 install --no-cache-dir -r /execute/requirements.txt --no-warn-script-location
 
 COPY ./backend/binsrc /execute/binsrc
-ARG GCC_PARAMS
 RUN g++ binsrc/nfqueue.cpp -o modules/cppqueue -O3 -march=native -lnetfilter_queue -pthread -lpcre2-8 -ltins -lmnl -lnfnetlink
-RUN g++ binsrc/proxy.cpp -o modules/proxy -O3 -march=native $GCC_PARAMS -pthread -lboost_system -lboost_thread -lpcre2-8
+RUN g++ binsrc/proxy.cpp -o modules/proxy -O3 -march=native -pthread -lboost_system -lboost_thread -lpcre2-8
 
 
 COPY ./backend/ /execute/

backend/app.py

@@ -141,6 +141,7 @@ if __name__ == '__main__':
         host="0.0.0.0",
         port=FIREGEX_PORT,
         reload=DEBUG,
-        access_log=DEBUG,
+        access_log=True,
-        workers=1
+        workers=1 # Multiple workers will cause a crash due to the creation
+                  # of multiple processes with separated memory
     )

backend/binsrc/nfqueue.cpp

@@ -45,9 +45,11 @@ bool filter_callback(const uint8_t *data, uint32_t len){
 int main(int argc, char *argv[])
 {
 	int n_of_threads = 1;
-	if (argc >= 2) n_of_threads = atoi(argv[1]);
+   	char * n_threads_str = getenv("NTHREADS");
+   	if (n_threads_str != NULL) n_of_threads = ::atoi(n_threads_str);
 	if(n_of_threads <= 0) n_of_threads = 1;
 	if (n_of_threads % 2 != 0 ) n_of_threads++;
+	cerr << "[info] [main] Using " << n_of_threads << " threads" << endl;
 	regex_config.reset(new regex_rules());
 	NFQueueSequence<filter_callback<true>> input_queues(n_of_threads/2);
 	input_queues.start();
@@ -55,6 +57,8 @@ int main(int argc, char *argv[])
 	output_queues.start();
 
 	cout << "QUEUES INPUT " << input_queues.init() << " " << input_queues.end() << " OUTPUT " << output_queues.init() << " " << output_queues.end() << endl;
+	cerr << "[info] [main] Input queues: " << input_queues.init() << ":" << input_queues.end() << " threads assigned: " << n_of_threads/2 << endl;
+	cerr << "[info] [main] Output queues: " << output_queues.init() << ":" << output_queues.end() << " threads assigned: " << n_of_threads/2 << endl;
 
 	config_updater();
 }

backend/binsrc/proxy.cpp

@@ -451,6 +451,10 @@ int main(int argc, char* argv[])
    const string local_host      = argv[1];
    const string forward_host    = argv[3];
    
+   int threads = 1;
+   char * n_threads_str = getenv("NTHREADS");
+   if (n_threads_str != NULL) threads = ::atoi(n_threads_str);
+
    boost::asio::io_context ios;
 
    boost::asio::streambuf buf;
@@ -470,19 +474,16 @@ int main(int argc, char* argv[])
                                            forward_host, forward_port);
 
       acceptor.accept_connections();
-      #ifdef MULTI_THREAD
+      
-      boost::thread_group tg;
+      if (threads > 1){
-      #ifdef THREAD_NUM
+         boost::thread_group tg;
-      for (unsigned i = 0; i < THREAD_NUM; ++i)
+         for (unsigned i = 0; i < threads; ++i)
-      #else
+            tg.create_thread(boost::bind(&boost::asio::io_context::run, &ios));
-      for (unsigned i = 0; i < thread::hardware_concurrency(); ++i)
-      #endif
-         tg.create_thread(boost::bind(&boost::asio::io_context::run, &ios));
 
       tg.join_all();
-      #else
+      }else{
-      ios.run();
+         ios.run();
-      #endif
+      }
    }
    catch(exception& e)
    {

backend/modules/nfregex/firegex.py

@@ -5,9 +5,6 @@ from modules.nfregex.models import Service, Regex
 import re, os, asyncio
 import traceback
 
-QUEUE_BASE_NUM = 1000
-
-
 class RegexFilter:
     def __init__(
         self, regex,
@@ -61,14 +58,12 @@ class FiregexInterceptor:
         self.regex_filters: Set[RegexFilter]
         self.update_config_lock:asyncio.Lock
         self.process:asyncio.subprocess.Process
-        self.n_queues:int
         self.update_task: asyncio.Task
     
     @classmethod
-    async def start(cls, filter: FiregexFilter, n_queues:int = int(os.getenv("NTHREADS","1"))):
+    async def start(cls, filter: FiregexFilter):
         self = cls()
         self.filter = filter
-        self.n_queues = n_queues
         self.filter_map_lock = asyncio.Lock()
         self.update_config_lock = asyncio.Lock()
         input_range, output_range = await self._start_binary()
@@ -81,7 +76,7 @@ class FiregexInterceptor:
     async def _start_binary(self):
         proxy_binary_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../cppqueue")
         self.process = await asyncio.create_subprocess_exec(
-            proxy_binary_path, str(self.n_queues),
+            proxy_binary_path,
             stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE
         )
         line_fut = self.process.stdout.readuntil()

backend/utils/firegextables.py

@@ -80,7 +80,7 @@ class FiregexTables:
             "expr": [
                     {'match': {'left': {'payload': {'protocol': ip_family(ip_int), 'field': 'saddr'}}, 'op': '==', 'right': {"prefix": {"addr": ip_addr, "len": ip_addr_cidr}}}},
                     {'match': {"left": { "payload": {"protocol": str(proto), "field": "sport"}}, "op": "==", "right": int(port)}},
-                    {"queue": {"num": str(init) if init == end else f"{init}-{end}", "flags": ["bypass"]}}
+                    {"queue": {"num": str(init) if init == end else {"range":[init, end] }, "flags": ["bypass"]}}
                 ]
         }}})
 
@@ -97,17 +97,17 @@ class FiregexTables:
             "expr": [
                     {'match': {'left': {'payload': {'protocol': ip_family(ip_int), 'field': 'daddr'}}, 'op': '==', 'right': {"prefix": {"addr": ip_addr, "len": ip_addr_cidr}}}},
                     {'match': {"left": { "payload": {"protocol": str(proto), "field": "dport"}}, "op": "==", "right": int(port)}},
-                    {"queue": {"num": str(init) if init == end else f"{init}-{end}", "flags": ["bypass"]}}
+                    {"queue": {"num": str(init) if init == end else {"range":[init, end] }, "flags": ["bypass"]}}
                 ]
         }}})
 
     def get(self) -> List[FiregexFilter]:
         res = []
         for filter in [ele["rule"] for ele in self.list() if "rule" in ele and ele["rule"]["table"] == self.table_name]:
-            queue_str = str(filter["expr"][2]["queue"]["num"]).split("-")
+            queue_str = filter["expr"][2]["queue"]["num"]
             queue = None
-            if len(queue_str) == 1: queue = int(queue_str[0]), int(queue_str[0])
+            if isinstance(queue_str,dict): queue = int(queue_str["range"][0]), int(queue_str["range"][1])
-            else: queue = int(queue_str[0]), int(queue_str[1])
+            else: queue = int(queue_str), int(queue_str)
             ip_int = None
             if isinstance(filter["expr"][0]["match"]["right"],str):
                 ip_int = str(ip_parse(filter["expr"][0]["match"]["right"]))

frontend/build/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "/static/css/main.96ef8f18.css",
-    "main.js": "/static/js/main.8840c3f9.js",
+    "main.js": "/static/js/main.8da55c0a.js",
     "index.html": "/index.html",
     "main.96ef8f18.css.map": "/static/css/main.96ef8f18.css.map",
-    "main.8840c3f9.js.map": "/static/js/main.8840c3f9.js.map"
+    "main.8da55c0a.js.map": "/static/js/main.8da55c0a.js.map"
   },
   "entrypoints": [
     "static/css/main.96ef8f18.css",
-    "static/js/main.8840c3f9.js"
+    "static/js/main.8da55c0a.js"
   ]
 }

frontend/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="manifest" href="/site.webmanifest"><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#FFFFFFFF"/><meta name="description" content="Firegex by Pwnzer0tt1"/><title>Firegex</title><script defer="defer" src="/static/js/main.8840c3f9.js"></script><link href="/static/css/main.96ef8f18.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="manifest" href="/site.webmanifest"><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#FFFFFFFF"/><meta name="description" content="Firegex by Pwnzer0tt1"/><title>Firegex</title><script defer="defer" src="/static/js/main.8da55c0a.js"></script><link href="/static/css/main.96ef8f18.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

frontend/src/components/MainLayout.tsx

@@ -19,6 +19,7 @@ function MainLayout({ children }:{ children:any }) {
     }
   },[location.pathname])
 
+
   return <>  
       
   <AppShell

frontend/src/js/utils.tsx

@@ -71,7 +71,7 @@ export function getapiobject(){
 
 export function HomeRedirector(){
     const section = sessionStorage.getItem("home_section")
-    const path = section?`/${section}`:`/nfqueue`
+    const path = section?`/${section}`:`/nfregex`
     return <Navigate to={path} />
 }
 

frontend/src/pages/NFRegex.tsx/index.tsx

@@ -21,7 +21,6 @@ function NFRegex({ children }: { children: any }) {
     const [tooltipAddOpened, setTooltipAddOpened] = useState(false);
     
     const [generalStats, setGeneralStats] = useState<GeneralStats>({closed:0, regexes:0, services:0});
-
     const updateInfo = async () => {
         
         await Promise.all([

start.py

@@ -36,8 +36,6 @@ if args.threads < 1:
 
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 
-gcc_params = f"-D MULTI_THREAD -D THREAD_NUM={args.threads}" if args.threads > 1 else ""
-
 with open("docker-compose.yml","wt") as compose:
 
     if "linux" in sys.platform and not 'microsoft-standard' in platform.uname().release: #Check if not is a wsl also
@@ -47,10 +45,7 @@ version: '3.9'
 services:
     firewall:
         restart: unless-stopped
-        build: 
+        build: .
-            context: .
-            args:
-                - GCC_PARAMS={gcc_params}
         network_mode: "host"
         environment:
             - PORT={args.port}
@@ -71,10 +66,7 @@ version: '3.9'
 services:
     firewall:
         restart: unless-stopped
-        build: 
+        build: .
-            context: .
-            args:
-                - GCC_PARAMS={gcc_params}
         ports:
             - {args.port}:{args.port}
         environment: