Moduled Firegex, Merging pt1 (not finished and not working yet)
This commit is contained in:
DomySh
0
backend/modules/nfregex/__init__.py
Normal file
0
backend/modules/nfregex/__init__.py
Normal file
151
backend/modules/nfregex/firegex.py
Normal file
151
backend/modules/nfregex/firegex.py
Normal file
@@ -0,0 +1,151 @@
|
||||
from typing import Dict, List, Set
|
||||
from utils.firegextables import FiregexFilter, FiregexTables
|
||||
from utils import ip_parse, ip_family, run_func
|
||||
from modules.nfregex.models import Service, Regex
|
||||
import re, os, asyncio
|
||||
import traceback
|
||||
|
||||
QUEUE_BASE_NUM = 1000
|
||||
|
||||
|
||||
class RegexFilter:
|
||||
def __init__(
|
||||
self, regex,
|
||||
is_case_sensitive=True,
|
||||
is_blacklist=True,
|
||||
input_mode=False,
|
||||
output_mode=False,
|
||||
blocked_packets=0,
|
||||
id=None,
|
||||
update_func = None
|
||||
):
|
||||
self.regex = regex
|
||||
self.is_case_sensitive = is_case_sensitive
|
||||
self.is_blacklist = is_blacklist
|
||||
if input_mode == output_mode: input_mode = output_mode = True # (False, False) == (True, True)
|
||||
self.input_mode = input_mode
|
||||
self.output_mode = output_mode
|
||||
self.blocked = blocked_packets
|
||||
self.id = id
|
||||
self.update_func = update_func
|
||||
self.compiled_regex = self.compile()
|
||||
|
||||
@classmethod
|
||||
def from_regex(cls, regex:Regex, update_func = None):
|
||||
return cls(
|
||||
id=regex.id, regex=regex.regex, is_case_sensitive=regex.is_case_sensitive,
|
||||
is_blacklist=regex.is_blacklist, blocked_packets=regex.blocked_packets,
|
||||
input_mode = regex.mode in ["C","B"], output_mode=regex.mode in ["S","B"],
|
||||
update_func = update_func
|
||||
)
|
||||
def compile(self):
|
||||
if isinstance(self.regex, str): self.regex = self.regex.encode()
|
||||
if not isinstance(self.regex, bytes): raise Exception("Invalid Regex Paramether")
|
||||
re.compile(self.regex) # raise re.error if it's invalid!
|
||||
case_sensitive = "1" if self.is_case_sensitive else "0"
|
||||
if self.input_mode:
|
||||
yield case_sensitive + "C" + self.regex.hex() if self.is_blacklist else case_sensitive + "c"+ self.regex.hex()
|
||||
if self.output_mode:
|
||||
yield case_sensitive + "S" + self.regex.hex() if self.is_blacklist else case_sensitive + "s"+ self.regex.hex()
|
||||
|
||||
async def update(self):
|
||||
if self.update_func:
|
||||
await run_func(self.update_func, self)
|
||||
|
||||
class FiregexInterceptor:
|
||||
|
||||
def __init__(self):
|
||||
self.filter:FiregexFilter
|
||||
self.filter_map_lock:asyncio.Lock
|
||||
self.filter_map: Dict[str, RegexFilter]
|
||||
self.regex_filters: Set[RegexFilter]
|
||||
self.update_config_lock:asyncio.Lock
|
||||
self.process:asyncio.subprocess.Process
|
||||
self.n_queues:int
|
||||
self.update_task: asyncio.Task
|
||||
|
||||
@classmethod
|
||||
async def start(cls, filter: FiregexFilter, n_queues:int = int(os.getenv("NTHREADS","1"))):
|
||||
self = cls()
|
||||
self.filter = filter
|
||||
self.n_queues = n_queues
|
||||
self.filter_map_lock = asyncio.Lock()
|
||||
self.update_config_lock = asyncio.Lock()
|
||||
input_range, output_range = await self._start_binary()
|
||||
self.update_task = asyncio.create_task(self.update_blocked())
|
||||
if not filter in FiregexTables().get():
|
||||
FiregexTables().add_input(queue_range=input_range, proto=self.filter.proto, port=self.filter.port, ip_int=self.filter.ip_int)
|
||||
FiregexTables().add_output(queue_range=output_range, proto=self.filter.proto, port=self.filter.port, ip_int=self.filter.ip_int)
|
||||
return self
|
||||
|
||||
async def _start_binary(self):
|
||||
proxy_binary_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../cppqueue")
|
||||
self.process = await asyncio.create_subprocess_exec(
|
||||
proxy_binary_path, str(self.n_queues),
|
||||
stdout=asyncio.subprocess.PIPE, stdin=asyncio.subprocess.PIPE
|
||||
)
|
||||
line_fut = self.process.stdout.readuntil()
|
||||
try:
|
||||
line_fut = await asyncio.wait_for(line_fut, timeout=3)
|
||||
except asyncio.TimeoutError:
|
||||
self.process.kill()
|
||||
raise Exception("Invalid binary output")
|
||||
line = line_fut.decode()
|
||||
if line.startswith("QUEUES "):
|
||||
params = line.split()
|
||||
return (int(params[2]), int(params[3])), (int(params[5]), int(params[6]))
|
||||
else:
|
||||
self.process.kill()
|
||||
raise Exception("Invalid binary output")
|
||||
|
||||
async def update_blocked(self):
|
||||
try:
|
||||
while True:
|
||||
line = (await self.process.stdout.readuntil()).decode()
|
||||
if line.startswith("BLOCKED"):
|
||||
regex_id = line.split()[1]
|
||||
async with self.filter_map_lock:
|
||||
if regex_id in self.filter_map:
|
||||
self.filter_map[regex_id].blocked+=1
|
||||
await self.filter_map[regex_id].update()
|
||||
except asyncio.CancelledError: pass
|
||||
except asyncio.IncompleteReadError: pass
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
async def stop(self):
|
||||
self.update_task.cancel()
|
||||
if self.process and self.process.returncode is None:
|
||||
self.process.kill()
|
||||
|
||||
async def _update_config(self, filters_codes):
|
||||
async with self.update_config_lock:
|
||||
self.process.stdin.write((" ".join(filters_codes)+"\n").encode())
|
||||
await self.process.stdin.drain()
|
||||
|
||||
async def reload(self, filters:List[RegexFilter]):
|
||||
async with self.filter_map_lock:
|
||||
self.filter_map = self.compile_filters(filters)
|
||||
filters_codes = self.get_filter_codes()
|
||||
await self._update_config(filters_codes)
|
||||
|
||||
def get_filter_codes(self):
|
||||
filters_codes = list(self.filter_map.keys())
|
||||
filters_codes.sort(key=lambda a: self.filter_map[a].blocked, reverse=True)
|
||||
return filters_codes
|
||||
|
||||
def compile_filters(self, filters:List[RegexFilter]):
|
||||
res = {}
|
||||
for filter_obj in filters:
|
||||
try:
|
||||
raw_filters = filter_obj.compile()
|
||||
for filter in raw_filters:
|
||||
res[filter] = filter_obj
|
||||
except Exception: pass
|
||||
return res
|
||||
|
||||
def delete_by_srv(srv:Service):
|
||||
nft = FiregexTables()
|
||||
for filter in nft.get():
|
||||
if filter.port == srv.port and filter.proto == srv.proto and ip_parse(filter.ip_int) == ip_parse(srv.ip_int):
|
||||
nft.cmd({"delete":{"rule": {"handle": filter.id, "table": nft.table_name, "chain": filter.target, "family": "inet"}}})
|
||||
114
backend/modules/nfregex/firewall.py
Normal file
114
backend/modules/nfregex/firewall.py
Normal file
@@ -0,0 +1,114 @@
|
||||
import asyncio
|
||||
from typing import Dict
|
||||
from modules.nfregex.firegex import FiregexFilter, FiregexInterceptor, FiregexTables, RegexFilter, delete_by_srv
|
||||
from modules.nfregex.models import Regex, Service
|
||||
from utils.sqlite import SQLite
|
||||
|
||||
class STATUS:
|
||||
STOP = "stop"
|
||||
ACTIVE = "active"
|
||||
|
||||
class FirewallManager:
|
||||
def __init__(self, db:SQLite):
|
||||
self.db = db
|
||||
self.proxy_table: Dict[str, ServiceManager] = {}
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def close(self):
|
||||
for key in list(self.proxy_table.keys()):
|
||||
await self.remove(key)
|
||||
|
||||
async def remove(self,srv_id):
|
||||
async with self.lock:
|
||||
if srv_id in self.proxy_table:
|
||||
await self.proxy_table[srv_id].next(STATUS.STOP)
|
||||
del self.proxy_table[srv_id]
|
||||
|
||||
async def init(self):
|
||||
FiregexTables().init()
|
||||
await self.reload()
|
||||
|
||||
async def reload(self):
|
||||
async with self.lock:
|
||||
for srv in self.db.query('SELECT * FROM services;'):
|
||||
srv = Service.from_dict(srv)
|
||||
if srv.id in self.proxy_table:
|
||||
continue
|
||||
self.proxy_table[srv.id] = ServiceManager(srv, self.db)
|
||||
await self.proxy_table[srv.id].next(srv.status)
|
||||
|
||||
def get(self,srv_id):
|
||||
if srv_id in self.proxy_table:
|
||||
return self.proxy_table[srv_id]
|
||||
else:
|
||||
raise ServiceNotFoundException()
|
||||
|
||||
class ServiceNotFoundException(Exception): pass
|
||||
|
||||
class ServiceManager:
|
||||
def __init__(self, srv: Service, db):
|
||||
self.srv = srv
|
||||
self.db = db
|
||||
self.status = STATUS.STOP
|
||||
self.filters: Dict[int, FiregexFilter] = {}
|
||||
self.lock = asyncio.Lock()
|
||||
self.interceptor = None
|
||||
|
||||
async def _update_filters_from_db(self):
|
||||
regexes = [
|
||||
Regex.from_dict(ele) for ele in
|
||||
self.db.query("SELECT * FROM regexes WHERE service_id = ? AND active=1;", self.srv.id)
|
||||
]
|
||||
#Filter check
|
||||
old_filters = set(self.filters.keys())
|
||||
new_filters = set([f.id for f in regexes])
|
||||
#remove old filters
|
||||
for f in old_filters:
|
||||
if not f in new_filters:
|
||||
del self.filters[f]
|
||||
#add new filters
|
||||
for f in new_filters:
|
||||
if not f in old_filters:
|
||||
filter = [ele for ele in regexes if ele.id == f][0]
|
||||
self.filters[f] = RegexFilter.from_regex(filter, self._stats_updater)
|
||||
if self.interceptor: await self.interceptor.reload(self.filters.values())
|
||||
|
||||
def __update_status_db(self, status):
|
||||
self.db.query("UPDATE services SET status = ? WHERE service_id = ?;", status, self.srv.id)
|
||||
|
||||
async def next(self,to):
|
||||
async with self.lock:
|
||||
if (self.status, to) == (STATUS.ACTIVE, STATUS.STOP):
|
||||
await self.stop()
|
||||
self._set_status(to)
|
||||
# PAUSE -> ACTIVE
|
||||
elif (self.status, to) == (STATUS.STOP, STATUS.ACTIVE):
|
||||
await self.restart()
|
||||
|
||||
def _stats_updater(self,filter:RegexFilter):
|
||||
self.db.query("UPDATE regexes SET blocked_packets = ? WHERE regex_id = ?;", filter.blocked, filter.id)
|
||||
|
||||
def _set_status(self,status):
|
||||
self.status = status
|
||||
self.__update_status_db(status)
|
||||
|
||||
async def start(self):
|
||||
if not self.interceptor:
|
||||
delete_by_srv(self.srv)
|
||||
self.interceptor = await FiregexInterceptor.start(FiregexFilter(self.srv.proto,self.srv.port, self.srv.ip_int))
|
||||
await self._update_filters_from_db()
|
||||
self._set_status(STATUS.ACTIVE)
|
||||
|
||||
async def stop(self):
|
||||
delete_by_srv(self.srv)
|
||||
if self.interceptor:
|
||||
await self.interceptor.stop()
|
||||
self.interceptor = None
|
||||
|
||||
async def restart(self):
|
||||
await self.stop()
|
||||
await self.start()
|
||||
|
||||
async def update_filters(self):
|
||||
async with self.lock:
|
||||
await self._update_filters_from_db()
|
||||
30
backend/modules/nfregex/models.py
Normal file
30
backend/modules/nfregex/models.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import base64
|
||||
|
||||
class Service:
|
||||
def __init__(self, id: str, status: str, port: int, name: str, proto: str, ip_int: str):
|
||||
self.id = id
|
||||
self.status = status
|
||||
self.port = port
|
||||
self.name = name
|
||||
self.proto = proto
|
||||
self.ip_int = ip_int
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, var: dict):
|
||||
return cls(id=var["service_id"], status=var["status"], port=var["port"], name=var["name"], proto=var["proto"], ip_int=var["ip_int"])
|
||||
|
||||
|
||||
class Regex:
|
||||
def __init__(self, id: int, regex: bytes, mode: str, service_id: str, is_blacklist: bool, blocked_packets: int, is_case_sensitive: bool, active: bool):
|
||||
self.regex = regex
|
||||
self.mode = mode
|
||||
self.service_id = service_id
|
||||
self.is_blacklist = is_blacklist
|
||||
self.blocked_packets = blocked_packets
|
||||
self.id = id
|
||||
self.is_case_sensitive = is_case_sensitive
|
||||
self.active = active
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, var: dict):
|
||||
return cls(id=var["regex_id"], regex=base64.b64decode(var["regex"]), mode=var["mode"], service_id=var["service_id"], is_blacklist=var["is_blacklist"], blocked_packets=var["blocked_packets"], is_case_sensitive=var["is_case_sensitive"], active=var["active"])
|
||||
Reference in New Issue
Block a user