Fixes and improvements in thread managment

2022-08-02 19:45:28 +00:00
parent 81b5840375
commit af59ea0ef4
15 changed files with 40 additions and 48 deletions
--- a/3
+++ b/3
@@ -19,9 +19,8 @@ ADD ./backend/requirements.txt /execute/requirements.txt
 RUN pip3 install --no-cache-dir -r /execute/requirements.txt --no-warn-script-location

 COPY ./backend/binsrc /execute/binsrc
-ARG GCC_PARAMS
 RUN g++ binsrc/nfqueue.cpp -o modules/cppqueue -O3 -march=native -lnetfilter_queue -pthread -lpcre2-8 -ltins -lmnl -lnfnetlink
-RUN g++ binsrc/proxy.cpp -o modules/proxy -O3 -march=native $GCC_PARAMS -pthread -lboost_system -lboost_thread -lpcre2-8
+RUN g++ binsrc/proxy.cpp -o modules/proxy -O3 -march=native -pthread -lboost_system -lboost_thread -lpcre2-8


 COPY ./backend/ /execute/
--- a/backend/app.py
+++ b/backend/app.py
@@ -141,6 +141,7 @@ if __name__ == '__main__':
        host="0.0.0.0",
        port=FIREGEX_PORT,
        reload=DEBUG,
-        access_log=DEBUG,
-        workers=1
+        access_log=True,
+        workers=1 # Multiple workers will cause a crash due to the creation
+                  # of multiple processes with separated memory
    )
--- a/backend/binsrc/nfqueue.cpp
+++ b/backend/binsrc/nfqueue.cpp
@@ -45,9 +45,11 @@ bool filter_callback(const uint8_t *data, uint32_t len){
 int main(int argc, char *argv[])
 {
 	int n_of_threads = 1;
-	if (argc >= 2) n_of_threads = atoi(argv[1]);
+   	char * n_threads_str = getenv("NTHREADS");
+   	if (n_threads_str != NULL) n_of_threads = ::atoi(n_threads_str);
 	if(n_of_threads <= 0) n_of_threads = 1;
 	if (n_of_threads % 2 != 0 ) n_of_threads++;
+	cerr << "[info] [main] Using " << n_of_threads << " threads" << endl;
 	regex_config.reset(new regex_rules());
 	NFQueueSequence<filter_callback<true>> input_queues(n_of_threads/2);
 	input_queues.start();
@@ -55,6 +57,8 @@ int main(int argc, char *argv[])
 	output_queues.start();

 	cout << "QUEUES INPUT " << input_queues.init() << " " << input_queues.end() << " OUTPUT " << output_queues.init() << " " << output_queues.end() << endl;
+	cerr << "[info] [main] Input queues: " << input_queues.init() << ":" << input_queues.end() << " threads assigned: " << n_of_threads/2 << endl;
+	cerr << "[info] [main] Output queues: " << output_queues.init() << ":" << output_queues.end() << " threads assigned: " << n_of_threads/2 << endl;

 	config_updater();
 }
--- a/backend/binsrc/proxy.cpp
+++ b/backend/binsrc/proxy.cpp
@@ -451,6 +451,10 @@ int main(int argc, char* argv[])
   const string local_host      = argv[1];
   const string forward_host    = argv[3];
   
+   int threads = 1;
+   char * n_threads_str = getenv("NTHREADS");
+   if (n_threads_str != NULL) threads = ::atoi(n_threads_str);
+
   boost::asio::io_context ios;

   boost::asio::streambuf buf;
@@ -470,19 +474,16 @@ int main(int argc, char* argv[])
                                           forward_host, forward_port);

      acceptor.accept_connections();
-      #ifdef MULTI_THREAD
+      
+      if (threads > 1){
         boost::thread_group tg;
-      #ifdef THREAD_NUM
-      for (unsigned i = 0; i < THREAD_NUM; ++i)
-      #else
-      for (unsigned i = 0; i < thread::hardware_concurrency(); ++i)
-      #endif
+         for (unsigned i = 0; i < threads; ++i)
            tg.create_thread(boost::bind(&boost::asio::io_context::run, &ios));

      tg.join_all();
-      #else
+      }else{
         ios.run();
-      #endif
+      }
   }
   catch(exception& e)
   {
--- a/backend/modules/nfregex/firegex.py
+++ b/backend/modules/nfregex/firegex.py
@@ -5,9 +5,6 @@ from modules.nfregex.models import Service, Regex
 import re, os, asyncio
 import traceback

-QUEUE_BASE_NUM = 1000
-
-
 class RegexFilter:
    def __init__(
        self, regex,
@@ -61,14 +58,12 @@ class FiregexInterceptor:
        self.regex_filters: Set[RegexFilter]
        self.update_config_lock:asyncio.Lock
        self.process:asyncio.subprocess.Process
-        self.n_queues:int
        self.update_task: asyncio.Task
    
    @classmethod
-    async def start(cls, filter: FiregexFilter, n_queues:int = int(os.getenv("NTHREADS","1"))):
+    async def start(cls, filter: FiregexFilter):
        self = cls()
        self.filter = filter
-        self.n_queues = n_queues
        self.filter_map_lock = asyncio.Lock()
        self.update_config_lock = asyncio.Lock()
        input_range, output_range = await self._start_binary()
@@ -81,7 +76,7 @@ class FiregexInterceptor:
    async def _start_binary(self):
        proxy_binary_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../cppqueue")
        self.process = await asyncio.create_subprocess_exec(
-            proxy_binary_path, str(self.n_queues),
+            proxy_binary_path,
            stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE
        )
        line_fut = self.process.stdout.readuntil()
--- a/backend/utils/firegextables.py
+++ b/backend/utils/firegextables.py
@@ -80,7 +80,7 @@ class FiregexTables:
            "expr": [
                    {'match': {'left': {'payload': {'protocol': ip_family(ip_int), 'field': 'saddr'}}, 'op': '==', 'right': {"prefix": {"addr": ip_addr, "len": ip_addr_cidr}}}},
                    {'match': {"left": { "payload": {"protocol": str(proto), "field": "sport"}}, "op": "==", "right": int(port)}},
-                    {"queue": {"num": str(init) if init == end else f"{init}-{end}", "flags": ["bypass"]}}
+                    {"queue": {"num": str(init) if init == end else {"range":[init, end] }, "flags": ["bypass"]}}
                ]
        }}})

@@ -97,17 +97,17 @@ class FiregexTables:
            "expr": [
                    {'match': {'left': {'payload': {'protocol': ip_family(ip_int), 'field': 'daddr'}}, 'op': '==', 'right': {"prefix": {"addr": ip_addr, "len": ip_addr_cidr}}}},
                    {'match': {"left": { "payload": {"protocol": str(proto), "field": "dport"}}, "op": "==", "right": int(port)}},
-                    {"queue": {"num": str(init) if init == end else f"{init}-{end}", "flags": ["bypass"]}}
+                    {"queue": {"num": str(init) if init == end else {"range":[init, end] }, "flags": ["bypass"]}}
                ]
        }}})

    def get(self) -> List[FiregexFilter]:
        res = []
        for filter in [ele["rule"] for ele in self.list() if "rule" in ele and ele["rule"]["table"] == self.table_name]:
-            queue_str = str(filter["expr"][2]["queue"]["num"]).split("-")
+            queue_str = filter["expr"][2]["queue"]["num"]
            queue = None
-            if len(queue_str) == 1: queue = int(queue_str[0]), int(queue_str[0])
-            else: queue = int(queue_str[0]), int(queue_str[1])
+            if isinstance(queue_str,dict): queue = int(queue_str["range"][0]), int(queue_str["range"][1])
+            else: queue = int(queue_str), int(queue_str)
            ip_int = None
            if isinstance(filter["expr"][0]["match"]["right"],str):
                ip_int = str(ip_parse(filter["expr"][0]["match"]["right"]))
--- a/frontend/build/asset-manifest.json
+++ b/frontend/build/asset-manifest.json
@@ -1,13 +1,13 @@
 {
  "files": {
    "main.css": "/static/css/main.96ef8f18.css",
-    "main.js": "/static/js/main.8840c3f9.js",
+    "main.js": "/static/js/main.8da55c0a.js",
    "index.html": "/index.html",
    "main.96ef8f18.css.map": "/static/css/main.96ef8f18.css.map",
-    "main.8840c3f9.js.map": "/static/js/main.8840c3f9.js.map"
+    "main.8da55c0a.js.map": "/static/js/main.8da55c0a.js.map"
  },
  "entrypoints": [
    "static/css/main.96ef8f18.css",
-    "static/js/main.8840c3f9.js"
+    "static/js/main.8da55c0a.js"
  ]
 }
--- a/frontend/build/index.html
+++ b/frontend/build/index.html
@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="manifest" href="/site.webmanifest"><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#FFFFFFFF"/><meta name="description" content="Firegex by Pwnzer0tt1"/><title>Firegex</title><script defer="defer" src="/static/js/main.8840c3f9.js"></script><link href="/static/css/main.96ef8f18.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="manifest" href="/site.webmanifest"><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#FFFFFFFF"/><meta name="description" content="Firegex by Pwnzer0tt1"/><title>Firegex</title><script defer="defer" src="/static/js/main.8da55c0a.js"></script><link href="/static/css/main.96ef8f18.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
--- a/frontend/build/static/js/main.8da55c0a.js
+++ b/frontend/build/static/js/main.8da55c0a.js
--- a/frontend/build/static/js/main.8da55c0a.js.LICENSE.txt
+++ b/frontend/build/static/js/main.8da55c0a.js.LICENSE.txt
--- a/frontend/build/static/js/main.8da55c0a.js.map
+++ b/frontend/build/static/js/main.8da55c0a.js.map
--- a/frontend/src/components/MainLayout.tsx
+++ b/frontend/src/components/MainLayout.tsx
@@ -19,6 +19,7 @@ function MainLayout({ children }:{ children:any }) {
    }
  },[location.pathname])

+
  return <>  
      
  <AppShell
--- a/frontend/src/js/utils.tsx
+++ b/frontend/src/js/utils.tsx
@@ -71,7 +71,7 @@ export function getapiobject(){

 export function HomeRedirector(){
    const section = sessionStorage.getItem("home_section")
-    const path = section?`/${section}`:`/nfqueue`
+    const path = section?`/${section}`:`/nfregex`
    return <Navigate to={path} />
 }

--- a/frontend/src/pages/NFRegex.tsx/index.tsx
+++ b/frontend/src/pages/NFRegex.tsx/index.tsx
@@ -21,7 +21,6 @@ function NFRegex({ children }: { children: any }) {
    const [tooltipAddOpened, setTooltipAddOpened] = useState(false);
    
    const [generalStats, setGeneralStats] = useState<GeneralStats>({closed:0, regexes:0, services:0});
-
    const updateInfo = async () => {
        
        await Promise.all([
--- a/start.py
+++ b/start.py
@@ -36,8 +36,6 @@ if args.threads < 1:

 os.chdir(os.path.dirname(os.path.realpath(__file__)))

-gcc_params = f"-D MULTI_THREAD -D THREAD_NUM={args.threads}" if args.threads > 1 else ""
-
 with open("docker-compose.yml","wt") as compose:

    if "linux" in sys.platform and not 'microsoft-standard' in platform.uname().release: #Check if not is a wsl also
@@ -47,10 +45,7 @@ version: '3.9'
 services:
    firewall:
        restart: unless-stopped
-        build: 
-            context: .
-            args:
-                - GCC_PARAMS={gcc_params}
+        build: .
        network_mode: "host"
        environment:
            - PORT={args.port}
@@ -71,10 +66,7 @@ version: '3.9'
 services:
    firewall:
        restart: unless-stopped
-        build: 
-            context: .
-            args:
-                - GCC_PARAMS={gcc_params}
+        build: .
        ports:
            - {args.port}:{args.port}
        environment: