diff --git a/CompuRacer_Core/main.py b/CompuRacer_Core/main.py index a4c10be..f21e307 100644 --- a/CompuRacer_Core/main.py +++ b/CompuRacer_Core/main.py @@ -11,12 +11,16 @@ import os, sys import time from multiprocessing import Queue +import warnings + +warnings.filterwarnings("ignore", category=UserWarning, message="resource_tracker: There appear to be") + # Check Python version >= 3.7 if not sys.version_info >= (3, 7): - sys.stderr.write("CompuRacer requires Python version >= 3.7\n\t" - "On Linux use: sudo apt-get install python3.7\n\t" - " python3.7 main.py\n") + sys.stderr.write("CompuRacer requires Python version >= 3.9\n\t" + "On Linux use: sudo apt-get install python3.9\n\t" + " python3.9 main.py\n") exit(1) # check whether external libs (from requirements.txt) are installed @@ -27,7 +31,7 @@ from bs4 import BeautifulSoup except ModuleNotFoundError: sys.stderr.write("Could not find external dependencies, please run:\n\t" - "python3.7 -m pip install -r requirements.txt\n") + "python3 -m pip install -r requirements.txt\n") exit(1) # check whether the system has a display @@ -53,7 +57,7 @@ __author__ = "R.J. van Emous @ Computest, B. van Wijk @ Computest" __license__ = "MIT License" __version__ = "v1.0.0 2023" -__email__ = "rvanemous@computest.nl, bvanwijk@computest.nl" +__email__ = "rvanemous@computest.nl, bartvwijkzk@outlook.com" __status__ = "Production v1" # --- Checking for arguments --- # diff --git a/CompuRacer_Core/requirements.txt b/CompuRacer_Core/requirements.txt index 0ab09de..d906395 100644 --- a/CompuRacer_Core/requirements.txt +++ b/CompuRacer_Core/requirements.txt @@ -1,31 +1,34 @@ #---- For rest_server.py ----# -Flask==1.0.2 # BSD -lxml==4.2.5 # BSD +Flask +lxml #---- For compu_racer_core.py ----# -tabulate==0.8.2 # MIT -tqdm==4.31.1 # MIT +tabulate +tqdm #---- For geo_dns_resolver.py ----# -cachetools==3.0.0 # MIT -IPy==1.00 # BSD -ratelimiter==1.2.0.post0 # Apache 1.1 +cachetools +IPy +ratelimiter #---- For batch_sender_async.py ----# -aiodns==1.1.1 # MIT -urllib3==1.24.2 # MIT -uvloop==0.12.2 # MIT -aiohttp==3.4.4 # Apache 2.0 -aiohttp_socks==0.2.2 # Apache 2.0 -async_timeout==3.0.1 # Apache 2.0 +aiodns +urllib3 +uvloop +aiohttp==3.7.4 +aiohttp_socks +async_timeout==3.0.1 #---- For batch.py ----# -beautifulsoup4==4.7.1 # MIT +beautifulsoup4 #---- Right dependencies ----# certifi==2019.11.28 -requests==2.22.0 -jinja==2.10 -markupsafe==1.0 -setuptools==39.1.0 -itsdangerous==1.1.0 +requests==2.25.1 +jinja2 +markupsafe +setuptools +itsdangerous + +#---- GUI dependencies ----# +PyQt5 \ No newline at end of file diff --git a/CompuRacer_Core/src/command_processor.py b/CompuRacer_Core/src/command_processor.py index abf391b..e08b0b5 100644 --- a/CompuRacer_Core/src/command_processor.py +++ b/CompuRacer_Core/src/command_processor.py @@ -16,12 +16,12 @@ import src.utils as utils -from src.maingui import MainGUI +from src.connectgui import ConnectGUI -from PyQt5.QtCore import QThread, QObject, pyqtSignal +from PyQt5.QtCore import QThread, pyqtSignal from PyQt5.QtWidgets import QApplication -__version__ = "v1" +__version__ = "v1.1.0" class GuiThread(QThread): start_gui_signal = pyqtSignal() @@ -140,7 +140,7 @@ def gui_interpreter(self, state): self.print_formatted("Starting GUI " + __version__ + "..", utils.QType.INFORMATION) app = QApplication([]) - MainGUI.show_requests_gui(self.racer, app, state, self) + ConnectGUI.show_requests_gui(self.racer, app, state, self) def command_interpreter(self): self.welcome_function(self.welcome_function_class) diff --git a/CompuRacer_Core/src/compu_racer_core.py b/CompuRacer_Core/src/compu_racer_core.py index cfe0174..7ce0f13 100644 --- a/CompuRacer_Core/src/compu_racer_core.py +++ b/CompuRacer_Core/src/compu_racer_core.py @@ -100,7 +100,7 @@ class CompuRacer: immediate_batch_name = "Imm" progress_bar_width = 100 - def __init__(self, port, proxy, queue, use_only_cli): + def __init__(self, port, proxy, queue, cli_check): """ Creates a new CompuRacer instance :param queue: the queue to be used when we want to display a filepicker dialog to the user @@ -110,7 +110,7 @@ def __init__(self, port, proxy, queue, use_only_cli): # if the queue is None, we cannot and will not show dialogs self.dialog_queue = queue - self.use_only_cli = use_only_cli + self.cli_check = cli_check # add shutdown hooks signal.signal(signal.SIGINT, self.force_shutdown) @@ -201,7 +201,7 @@ def set_unchanged(self): if self.command_processor.is_changed(): self.command_processor.set_changed(False) - def start(self, use_only_cli): + def start(self, cli_check): """ Starts the CompuRacer """ @@ -224,7 +224,7 @@ def start(self, use_only_cli): self.print_formatted("Starting command processor..", utils.QType.INFORMATION) time.sleep(0.25) utils.clear_output() - self.command_processor.start(use_only_cli, self, self.state) + self.command_processor.start(cli_check, self, self.state) def comm_general_save(self, do_print=True): """ @@ -895,19 +895,19 @@ def comm_requests_remove(self, request_id_first=None, request_id_last=None, ask_ self.print_formatted(f"Removal of all requests cancelled.", utils.QType.INFORMATION) return elif request_id_last is not None: - # remove a range of requests if not ask_confirmation or self.command_processor.accept_yes_no( f"Are you sure you want to remove requests with id between and including {request_id_first} and {request_id_last}?", utils.QType.WARNING): + # remove a range of requests for i, request_id in enumerate(copy.deepcopy(list(self.state['requests'].keys()))): if request_id_first <= request_id <= request_id_last: if self.rem_request(self, request_id, False) == -1: failed_requests.append(request_id) else: success_requests.append(request_id) - else: - self.print_formatted(f"Removal of range of requests cancelled.", utils.QType.INFORMATION) - return + else: + self.print_formatted(f"Removal of range of requests cancelled.", utils.QType.INFORMATION) + return else: # remove one request if self.rem_request(self, request_id_first, True) == -1: @@ -1062,7 +1062,6 @@ def add_request(self, a_request, used_from_interface=False, print_information=Tr return self.comm_batches_create_new(self, self.immediate_batch_name, False, not used_from_interface, allow_redirects, sync_last_byte, send_timeout) - immediate_batch = self.state['batches'][self.immediate_batch_name] try: immediate_batch.add(req_id, 0, par, seq, False) @@ -1094,8 +1093,7 @@ def colorprint_comp_results(self, results): self.print_formatted_multi( utils.tabbed_string(string, 1), - utils.QType.NONE, - {re.compile(r"^\t*-"): utils.QType.RED, re.compile(r"^\t*\+"): utils.QType.GREEN}) + utils.QType.NONE) # Note: only to be used internally with a valid request-id! @staticmethod @@ -1107,7 +1105,7 @@ def request_used_in(self, request_id): return used_in @staticmethod # do not add requests to this list in any other way - def rem_request(self, request_id, ask_confirmation=False): + def rem_request(self, request_id, ask_confirmation=True): with self.requests_list_lock: if request_id not in self.state['requests']: self.print_formatted(f"Cannot remove request:\n\t" @@ -1121,42 +1119,23 @@ def rem_request(self, request_id, ask_confirmation=False): f"The request with id '{request_id}' is (also) used by the immediate batch!", utils.QType.ERROR) return -1 - if not ask_confirmation: + if self.cli_check: self.print_formatted(f"The request with id '{request_id}' is used by batches: " f"{used_in}. It must be removed individually.", utils.QType.ERROR) return -1 - # remove request from the batches - if not self.command_processor.accept_yes_no(f"The request with id '{request_id}' is used by batches: " - f"{used_in}, continue?\n\tIt will be removed from these batches and their results are cleared!!", - utils.QType.WARNING): - return -1 - # remove request from the batches for batch_name in used_in: self.state['batches'][batch_name].remove(request_id) ask_confirmation = False - - if not ask_confirmation or self.command_processor.accept_yes_no( - f"Are you sure you want to remove the request with id '{request_id}'?", - utils.QType.WARNING): - self.__change_state('requests', sub_search=request_id, do_delete=True) - self.print_formatted(f"Request with id '{request_id}' is removed", utils.QType.INFORMATION) - else: - self.print_formatted(f"Removal of request cancelled.", utils.QType.INFORMATION) + self.__change_state('requests', sub_search=request_id, do_delete=True) + self.print_formatted(f"Request with id '{request_id}' is removed", utils.QType.INFORMATION) # --------------------------------------------------------------------------------------------------- # # ------------------------------------- Batch command functions ------------------------------------- # # --------------------------------------------------------------------------------------------------- # @staticmethod def get_batch_result_formatting(): - return {re.compile(r".*?\t\s{10}[12]\d\d\s\s.*?"): utils.QType.GREEN, - re.compile(r".*?\t\s{10}[3]\d\d\s\s.*?"): utils.QType.YELLOW, - re.compile(r".*?\t\s{10}[4]\d\d\s\s.*?"): utils.QType.RED, - re.compile(r".*?\t\s{10}[5]\d\d\s\s.*?"): utils.QType.BLUE, - re.compile(r"'status_code': [12].."): utils.QType.GREEN, - re.compile(r"'status_code': 3.."): utils.QType.YELLOW, - re.compile(r"'status_code': 4.."): utils.QType.RED, - re.compile(r"'status_code': 5.."): utils.QType.BLUE} + return None @staticmethod def comm_batches_send(self, index=None, print_results=True, immediate_allowed=False): @@ -1203,8 +1182,7 @@ def comm_batches_info(self): col_names = Batch.get_mini_summary_header() output = tabulate(contents, col_names, showindex="always", tablefmt="simple") + "\n" if self.state['current_batch']: - self.print_formatted_multi(output, utils.QType.NONE, - {f" {re.escape(self.state['current_batch'])} ": utils.QType.BLUE}) + self.print_formatted_multi(output, utils.QType.NONE) else: self.print_formatted(output, utils.QType.NONE) @@ -1912,21 +1890,23 @@ def comm_curr_remove(self, request_id=None, wait_time=None): self.print_formatted(f"Cannot remove a request from current batch: The current batch is empty!", utils.QType.ERROR) return -1 - if request_id is None: - # remove all items from the batch - question = "Are you sure you want to remove all requests from the current batch?" - elif wait_time is None: - # remove all items with a certain ID from the batch - question = f"Are you sure you want to remove all requests with id '{request_id}' from the current batch?" - else: - # remove a specific item with a certain ID and wait_time from the batch - question = f"Are you sure you want to remove the request with id '{request_id}' and wait_time '{wait_time}' from the current batch?" - if self.command_processor.accept_yes_no(question, utils.QType.WARNING): - num_removed = curr_batch.remove(request_id, wait_time) - self.print_formatted(f"All matching requests are removed from the current batch.\nNumber: {num_removed}", - utils.QType.INFORMATION) - else: - self.print_formatted(f"Removal of current batch requests cancelled.", utils.QType.INFORMATION) + if self.cli_check: + if request_id is None: + # remove all items from the batch + question = "Are you sure you want to remove all requests from the current batch?" + elif wait_time is None: + # remove all items with a certain ID from the batch + question = f"Are you sure you want to remove all requests with id '{request_id}' from the current batch?" + else: + # remove a specific item with a certain ID and wait_time from the batch + question = f"Are you sure you want to remove the request with id '{request_id}' and wait_time '{wait_time}' from the current batch?" + if self.command_processor.accept_yes_no(question, utils.QType.WARNING): + num_removed = curr_batch.remove(request_id, wait_time) + self.print_formatted(f"All matching requests are removed from the current batch.\nNumber: {num_removed}", + utils.QType.INFORMATION) + num_removed = curr_batch.remove(request_id, wait_time) + self.print_formatted(f"All matching requests are removed from the current batch.\nNumber: {num_removed}", + utils.QType.INFORMATION) # ------------------------------------------------------------------------------------------------- # # ------------------------------------- Main helper functions ------------------------------------- # diff --git a/CompuRacer_Core/src/maingui.py b/CompuRacer_Core/src/connectgui.py similarity index 59% rename from CompuRacer_Core/src/maingui.py rename to CompuRacer_Core/src/connectgui.py index d825be2..579b629 100644 --- a/CompuRacer_Core/src/maingui.py +++ b/CompuRacer_Core/src/connectgui.py @@ -1,16 +1,16 @@ import os import sys -from src.gui import RequestsGUI +from src.gui import MainGUI -class MainGUI: +class ConnectGUI: def __init__(self, racer): super().__init__() self.racer = racer def show_requests_gui(racer, app, state, cmdprocessor): - requests_gui = RequestsGUI(racer, state, cmdprocessor) + main_gui = MainGUI(racer, state, cmdprocessor) - requests_gui.show() + main_gui.show() sys.exit(app.exec_()) diff --git a/CompuRacer_Core/src/gui.py b/CompuRacer_Core/src/gui.py index 13de1a2..2a86169 100644 --- a/CompuRacer_Core/src/gui.py +++ b/CompuRacer_Core/src/gui.py @@ -1,13 +1,17 @@ import json import os import sys +from typing import List, Any from PyQt5.QtCore import Qt, QTimer -from PyQt5.QtWidgets import QPushButton, QSystemTrayIcon, QMenu, QAction, QMainWindow, QVBoxLayout, QLabel, QTableWidget, QTableWidgetItem, QTabWidget, QWidget, QMessageBox, QLineEdit, QHBoxLayout, QApplication -from PyQt5.QtGui import QIcon +from PyQt5.QtGui import QStandardItem, QStandardItemModel +from PyQt5.QtWidgets import QPushButton, QMainWindow, QVBoxLayout, QLabel, QTableWidget, QTableWidgetItem, QTabWidget, \ + QWidget, QLineEdit, QHBoxLayout, QApplication, QHeaderView, QTableView +from src.batch import Batch -def load_json_batches_names(directory) -> [str]: + +def load_json_batches(directory) -> List[Any]: file_names = [] for filename in os.listdir(directory): if filename.endswith(".json"): @@ -18,20 +22,27 @@ def load_json_batches_names(directory) -> [str]: return file_names -class RequestsGUI(QMainWindow): - def __init__(self, racer, state, cmdprocessor): +class MainGUI(QMainWindow): + def __init__(self, racer, state, command_processor) -> None: super().__init__() + self.request_window = None + self.general_window = None self.batch_window = None self.current_batch = None self.data_requests = None self.table_widget = None + self.table_widget_requests = None + self.table_widget_batches = None + self.file_names = None + self.directory = None - self.command_processor = cmdprocessor + self.command_processor = command_processor self.racer = racer self.state = state self.batch_buttons = [] + self.request_buttons = [] self.load_json_requests() @@ -41,6 +52,9 @@ def init_ui(self) -> None: self.showFullScreen() self.setWindowTitle("CompuRacer GUI") + self.directory = "state/batches" + self.file_names = load_json_batches(self.directory) + tabs = QTabWidget() general_tab = QWidget() logs_tab = QWidget() @@ -51,6 +65,7 @@ def init_ui(self) -> None: vbox_general = QVBoxLayout() vbox_logs = QVBoxLayout() + # --- Create and load in GUI --- # self.create_request_widget(vbox_general, general_tab) self.create_batch_widget(vbox_general, general_tab) self.create_logs_widget(vbox_logs, logs_tab) @@ -62,42 +77,38 @@ def init_ui(self) -> None: def create_request_widget(self, vbox, requests_tab) -> None: vbox.addWidget(QLabel("Requests Information")) - self.table_widget = QTableWidget(len(self.data_requests["requests"]), 6) - self.table_widget.setColumnWidth(0, 30) - self.table_widget.setColumnWidth(1, 400) - self.table_widget.setColumnWidth(3, 200) - self.table_widget.setColumnWidth(4, 100) - self.table_widget.setHorizontalHeaderLabels(["ID", "URL", "Method", "Timestamp", "Host", "Add To Batch"]) - vbox.addWidget(self.table_widget) - - self.load_requests(vbox) + # --- Creating Table --- # + self.table_widget_requests = QTableWidget() + self.table_widget_requests.setColumnCount(8) + self.table_widget_requests.setColumnWidth(0, 20) + self.table_widget_requests.setColumnWidth(1, 500) + self.table_widget_requests.setColumnWidth(3, 200) + self.table_widget_requests.setHorizontalHeaderLabels(["ID", "URL", "Method", "Timestamp", "Host", "Add To Batch", "Open", "Remove"]) + vbox.addWidget(self.table_widget_requests) requests_tab.setLayout(vbox) + self.table_widget_requests.show() + + self.load_requests() return None def create_batch_widget(self, vbox, batches_tab) -> None: vbox.addWidget(QLabel("Batches Information")) - directory = "state/batches" - file_names = load_json_batches_names(directory) - - self.table_widget = QTableWidget(len(file_names), 6) - self.table_widget.setColumnWidth(0, 400) - self.table_widget.setHorizontalHeaderLabels(["Name", "Allow Redirects", "Sync Last Byte", "Send Timeout", "Set Current Batch", "Open Batch"]) - vbox.addWidget(self.table_widget) - - # clear the table widget before loading new batches - self.table_widget.clearContents() - self.table_widget.setRowCount(0) - - current_batch = self.data_requests["current_batch"] + # --- Creating table --- # + self.table_widget_batches = QTableWidget() + self.table_widget_batches.setColumnCount(6) + self.table_widget_batches.setColumnWidth(0, 400) + self.table_widget_batches.setHorizontalHeaderLabels(["Name", "Allow Redirects", "Sync Last Byte", "Send Timeout", "Set Current Batch", "Open Batch"]) + vbox.addWidget(self.table_widget_batches) # --- Add new batch --- # add_batch_field = QLineEdit() add_batch_field_button = QPushButton("Add Batch", self) add_batch_field_button.clicked.connect(lambda _, input_field=add_batch_field: self.create_new_batch(input_field)) + # --- Create add batch button and field --- # hbox = QHBoxLayout() hbox.addWidget(add_batch_field) hbox.addWidget(add_batch_field_button) @@ -105,19 +116,27 @@ def create_batch_widget(self, vbox, batches_tab) -> None: # --- Add other important buttons --- # quit_button = QPushButton("Quit", self) - quit_button.clicked.connect(QApplication.quit) + quit_button.clicked.connect(self.shut_down) vbox.addWidget(quit_button) - self.load_batches(file_names, directory, vbox, current_batch) - - self.update_json_timer = QTimer() - self.update_json_timer.timeout.connect(self.reload_json) - self.update_json_timer.start(5000) + self.load_batches() batches_tab.setLayout(vbox) return None + def shut_down(self) -> None: + QApplication.quit() + + return None + + def update_json(self) -> None: + self.save_data() + self.load_requests() + self.load_batches() + + return None + def create_logs_widget(self, vbox, logs_tab) -> None: vbox.addWidget(QLabel("Logs")) @@ -127,7 +146,9 @@ def create_logs_widget(self, vbox, logs_tab) -> None: self.table_widget.setHorizontalHeaderLabels(["Commands"]) vbox.addWidget(self.table_widget) - vbox.addWidget(QPushButton("Save", self, clicked=self.save_data)) + save_button = QPushButton("Save") + save_button.clicked.connect(self.save_data) + vbox.addWidget(save_button) self.load_logs() @@ -135,36 +156,80 @@ def create_logs_widget(self, vbox, logs_tab) -> None: return None - def load_requests(self, vbox) -> None: - for idx, request in enumerate(self.data_requests["requests"]): - # --- Insert row number {forloopnumber} --- # - row = self.table_widget.rowCount() - self.table_widget.insertRow(row) + def create_requests_button_widget(self, request, row) -> None: + add_request_button = QPushButton("Add", self) + window_button = QPushButton("Open", self) + remove_button = QPushButton("Remove", self) - # --- Create Button --- # - add_request_button = QPushButton("Add", self) - add_request_button.clicked.connect(lambda _, request_id=str(request): self.add_request_to_batch(request_id)) + add_request_button.clicked.connect(lambda _, request_id=str(request): self.add_request_to_batch(request_id)) + window_button.clicked.connect(lambda _, request_id=request: self.new_request_window(request_id)) + remove_button.clicked.connect(lambda _, request_id=str(request): self.remove_request(request_id)) - # --- Insert data into row --- # - self.table_widget.setItem(row, 0, QTableWidgetItem(str(request))) - self.table_widget.setItem(row, 1, QTableWidgetItem(str(self.data_requests["requests"][request]["url"]))) - self.table_widget.setItem(row, 2, QTableWidgetItem(str(self.data_requests["requests"][request]["method"]))) - self.table_widget.setItem(row, 3, QTableWidgetItem(str(self.data_requests["requests"][request]["timestamp"]))) - headers = self.data_requests["requests"][request].get("headers", {}) - host = headers.get("Host", "") - self.table_widget.setItem(row, 4, QTableWidgetItem(str(host))) - self.table_widget.setCellWidget(row, 5, add_request_button) + self.request_buttons.append((add_request_button, window_button, remove_button)) - self.remove_empty_rows() + self.table_widget_requests.setCellWidget(row, 5, add_request_button) + self.table_widget_requests.setCellWidget(row, 6, window_button) + self.table_widget_requests.setCellWidget(row, 7, remove_button) return None - def load_batches(self, file_names, directory, vbox, current_batch) -> callable([]): + def load_requests(self) -> None: + self.load_json_requests() + + rows_to_delete = [] + + for row in range(self.table_widget_requests.rowCount()): + request_id = self.table_widget_requests.item(row, 0).text() + if request_id not in self.data_requests["requests"]: + rows_to_delete.append(row) + + for row in reversed(rows_to_delete): + self.table_widget_requests.removeRow(row) + + for request_id, request_data in self.data_requests["requests"].items(): + existing_row = None + for row in range(self.table_widget_requests.rowCount()): + if self.table_widget_requests.item(row, 0).text() == request_id: + existing_row = row + break + + if existing_row is not None: + self.table_widget_requests.setItem(existing_row, 1, QTableWidgetItem(str(request_data["url"]))) + self.table_widget_requests.setItem(existing_row, 2, QTableWidgetItem(str(request_data["method"]))) + self.table_widget_requests.setItem(existing_row, 3, QTableWidgetItem(str(request_data["timestamp"]))) + headers = request_data.get("headers", {}) + host = headers.get("Host", "") + self.table_widget_requests.setItem(existing_row, 4, QTableWidgetItem(str(host))) + else: + row = self.table_widget_requests.rowCount() + self.table_widget_requests.insertRow(row) + self.table_widget_requests.setItem(row, 0, QTableWidgetItem(str(request_id))) + self.table_widget_requests.setItem(row, 1, QTableWidgetItem(str(request_data["url"]))) + self.table_widget_requests.setItem(row, 2, QTableWidgetItem(str(request_data["method"]))) + self.table_widget_requests.setItem(row, 3, QTableWidgetItem(str(request_data["timestamp"]))) + headers = request_data.get("headers", {}) + host = headers.get("Host", "") + self.table_widget_requests.setItem(row, 4, QTableWidgetItem(str(host))) + + self.create_requests_button_widget(request_id, row) + + return None + + def load_batches(self) -> callable([]): + self.directory = "state/batches" + self.file_names = load_json_batches(self.directory) + self.batch_buttons.clear() + current_batch = self.data_requests["current_batch"] + + # remove existing rows + for row in reversed(range(self.table_widget_batches.rowCount())): + self.table_widget_batches.removeRow(row) + def load_table(): - for idx, name in enumerate(file_names): - # --- Create commandbuttons --- # + for idx, name in enumerate(self.file_names): + # --- Create command-buttons --- # current_button = QPushButton("Set Current", self) window_button = QPushButton("Open", self) @@ -174,28 +239,27 @@ def load_table(): self.batch_buttons.append((current_button, window_button)) # --- Insert row number {forloopnumber} --- # - row = self.table_widget.rowCount() - self.table_widget.insertRow(row) - self.table_widget.setItem(row, 0, QTableWidgetItem(str(name))) - self.table_widget.setCellWidget(row, 4, current_button) - self.table_widget.setCellWidget(row, 5, window_button) + row = self.table_widget_batches.rowCount() + self.table_widget_batches.insertRow(row) + self.table_widget_batches.setItem(row, 0, QTableWidgetItem(str(name))) + self.table_widget_batches.setCellWidget(row, 4, current_button) + self.table_widget_batches.setCellWidget(row, 5, window_button) self.check_current_batch(name, row, current_button, window_button, current_batch) - data = self.get_json_data(directory, name) + data = self.get_json_data(name) for col, col_name in enumerate(["Allow Redirects", "Sync Last Byte", "Send Timeout"]): value = data.get(col_name.lower().replace(" ", "_")) - self.table_widget.setItem(row, col + 1, QTableWidgetItem(str(value))) + self.table_widget_batches.setItem(row, col + 1, QTableWidgetItem(str(value))) if name == current_batch: - item = self.table_widget.item(row, col + 1) + item = self.table_widget_batches.item(row, col + 1) if item is not None: item.setBackground(Qt.gray) - self.remove_empty_rows() - load_table() + return load_table def load_logs(self) -> None: @@ -204,175 +268,274 @@ def load_logs(self) -> None: self.table_widget.insertRow(row) self.table_widget.setItem(row, 0, QTableWidgetItem(str(command))) - self.remove_empty_rows() return None - def add_request_to_batch(self, request_id): - self.showNotification("RequestID " + request_id + " has been added to active Batch!") - + def add_request_to_batch(self, request_id) -> None: self.racer.comm_curr_add(self.racer, request_id) - def get_json_data(self, directory, name): - with open(os.path.join(directory, name + ".json"), "r") as file: + self.update_json() + + return None + + def get_json_data(self, name) -> dict: + with open(os.path.join(self.directory, name + ".json"), "r") as file: data = json.load(file) return data - def check_current_batch(self, name, row, current_button, window_button, current_batch): + def check_current_batch(self, name, row, button1, button2, current_batch) -> None: if name == current_batch: - for col in range(self.table_widget.columnCount()): - item = self.table_widget.item(row, col) + for col in range(self.table_widget_batches.columnCount()): + item = self.table_widget_batches.item(row, col) if item is not None: item.setBackground(Qt.gray) - current_button.setEnabled(False) - window_button.setEnabled(True) + button1.setEnabled(False) + button2.setEnabled(True) else: - window_button.setEnabled(False) + button2.setEnabled(False) if name == "Imm": - current_button.setEnabled(False) - window_button.setEnabled(False) - - def remove_empty_rows(self) -> None: - for row in range(self.table_widget.rowCount() - 1, -1, -1): - empty = True - for col in range(self.table_widget.columnCount()): - item = self.table_widget.item(row, col) - if item is not None and not item.text().strip() == "": - empty = False - break - if empty: - self.table_widget.removeRow(row) + button1.setEnabled(False) + button2.setEnabled(False) return None def load_json_requests(self) -> None: with open('state/state.json', 'r') as f: self.data_requests = json.load(f) + return None def save_data(self) -> None: - self.racer.comm_general_save(True) - return None + self.racer.comm_general_save() - def reload_json(self): - if self.isActiveWindow(): - self.save_data() - self.update_json_timer.stop() - self.hide() - self.general_window = RequestsGUI(self.racer, self.state, self.command_processor) # Create a new window - self.general_window.show() - self.deleteLater() + return None def set_current_batch(self, batch_name) -> None: self.racer.set_curr_batch_by_name(self.racer, batch_name) self.current_batch = batch_name - self.showNotification("Set current batch to " + batch_name) + + self.update_json() + + return None + + def remove_request(self, request_id) -> None: + self.racer.comm_requests_remove(self.racer, request_id, None, False) + + self.update_json() + return None - def new_batch_window(self, batch_name): + def new_batch_window(self, batch_name) -> None: self.save_data() - self.update_json_timer.stop() self.batch_window = BatchWindow(batch_name, self.racer, self.state, self.command_processor) self.batch_window.show() self.hide() - def create_new_batch(self, batch_name): - batch_name = batch_name.text() - - self.racer.comm_batches_create_new(self.racer, batch_name) + return None - self.showNotification("Added New Batch " + batch_name + ". Add a request to your batch so you can open your batch") + def new_request_window(self, request_id) -> None: + self.update_json() + self.request_window = RequestWindow(request_id, self.racer, self.state, self.command_processor) + self.request_window.show() + self.hide() - def showNotification(self, notiText): - messageBox = QMessageBox() - messageBox.setIcon(QMessageBox.Information) - messageBox.setText(notiText) + return None - messageBox.setGeometry(0, 0, 500, 50) + def create_new_batch(self, batch_name) -> None: + batch_name = batch_name.text() + self.racer.comm_batches_create_new(self.racer, batch_name) - timer = QTimer() - timer.setSingleShot(True) - timer.timeout.connect(messageBox.close) - timer.start(5000) + self.update_json() - messageBox.exec() + return None class BatchWindow(QMainWindow): - def __init__(self, batch_name, racer, state, command_processor): + def __init__(self, batch_name, racer, state, command_processor) -> None: super().__init__() + self.table_widget_propperty = None self.showFullScreen() + self.setWindowTitle("Batch: " + batch_name) self.general_window = None - self.table_widget = None - self.batch_requests = [] + self.table_widget = QTableWidget() self.racer = racer self.batch_name = batch_name self.state = state self.command_processor = command_processor - self.setWindowTitle("Batch: " + batch_name) + self.batch_requests = [] self.init_ui() - def init_ui(self): + def init_ui(self) -> None: vbox = QVBoxLayout() batch_tab = QWidget() batch_tab.setLayout(vbox) - batch_tab.layout().addWidget(self.table_widget) tabs = QTabWidget() - tabs.addTab(batch_tab, "Batch") vbox.addStretch() - vbox.addWidget(QPushButton("Send Batch", self, clicked=self.send_batch), alignment=Qt.AlignBottom) - vbox.addWidget(QPushButton("Go Back", self, clicked=self.go_back), alignment=Qt.AlignBottom) - vbox.addWidget(QPushButton("Quit", self, clicked=QApplication.quit)) + self.add_batch_propperty_widget(vbox) + self.add_button_widget(vbox) self.setCentralWidget(tabs) + self.add_request_table(vbox) - self.create_requests_widget(vbox) vbox.insertWidget(0, self.table_widget) - self.update_json_timer = QTimer() - self.update_json_timer.timeout.connect(lambda: self.reload_json()) - self.update_json_timer.start(10000) + tabs.addTab(batch_tab, "Batch") + + return None + + def load_json(self, filepath) -> List[Any]: + with open(filepath, 'r') as file: + data = json.load(file) + + return data - def send_batch(self): + def update_json(self) -> None: self.save_data() - self.update_json_timer.stop() - self.racer.gui_send_batches() + self.load_requests() + self.load_propperties() - def create_requests_widget(self, vbox): - self.table_widget = QTableWidget() - vbox.addWidget(QLabel("")) - self.table_widget.setColumnCount(4) - self.table_widget.setHorizontalHeaderLabels(["ID", "URL", "Method", "Host"]) + return None - self.add_request_table() + def add_batch_propperty_widget(self, vbox) -> None: + self.table_widget_propperty = QTableWidget() + self.table_widget_propperty.setColumnCount(4) + self.table_widget_propperty.setHorizontalHeaderLabels(["Allow Redirects", "Sync Last Byte", "Send Timeout", "Batch Sent"]) + vbox.addWidget(self.table_widget_propperty) - def add_request_table(self) -> None: - items = self.load_json("state/batches/" + self.batch_name + ".json")["items"] - requests = self.load_json("state/state.json")["requests"] + self.add_propperty_widget(vbox) + self.load_propperties() + + return None + + def add_propperty_widget(self, vbox) -> None: + change_allow_redirects_button = QPushButton("Change Allow Redirects", self) + change_allow_redirects_button.clicked.connect(lambda _: self.change_allow_redirects()) + + change_sync_last_byte_button = QPushButton("Change Sync Last Byte", self) + change_sync_last_byte_button.clicked.connect(lambda _: self.change_sync_last_byte()) + + change_send_timeout_field = QLineEdit() + change_send_timeout_button = QPushButton("Change Send Timeout", self) + change_send_timeout_button.clicked.connect(lambda _, input_field=change_send_timeout_field: self.change_send_timeout(input_field)) - self.table_widget = QTableWidget(len(items), 4, self) - self.table_widget.setHorizontalHeaderLabels(["ID", "Method", "URL", "Host"]) + hbox = QHBoxLayout() + hbox.addWidget(change_allow_redirects_button) + hbox.addWidget(change_sync_last_byte_button) + + hbox.addWidget(change_send_timeout_field) + hbox.addWidget(change_send_timeout_button) + + vbox.addLayout(hbox) + + return None + + def load_propperties(self) -> None: + properties = self.load_json("state/batches/" + self.batch_name + ".json") + allow_redirects = properties["allow_redirects"] + sync_last_byte = properties["sync_last_byte"] + send_timeout = properties["send_timeout"] + sent_batch = properties["results"] + + self.table_widget_propperty.setRowCount(1) + + self.table_widget_propperty.setItem(0, 0, QTableWidgetItem(str(allow_redirects))) + self.table_widget_propperty.setItem(0, 1, QTableWidgetItem(str(sync_last_byte))) + self.table_widget_propperty.setItem(0, 2, QTableWidgetItem(str(send_timeout))) + if len(sent_batch) == 0: + self.table_widget_propperty.setItem(0, 3, QTableWidgetItem("False")) + else: + self.table_widget_propperty.setItem(0, 3, QTableWidgetItem("True")) + + return None + + def add_button_widget(self, vbox) -> None: + send_batch_button = QPushButton("Send Batch") + go_back_button = QPushButton("Go Back") + quit_button = QPushButton("Quit") + + send_batch_button.clicked.connect(self.send_batch) + go_back_button.clicked.connect(self.go_back) + quit_button.clicked.connect(QApplication.quit) + + vbox.addWidget(send_batch_button, alignment=Qt.AlignBottom) + vbox.addWidget(go_back_button, alignment=Qt.AlignBottom) + vbox.addWidget(quit_button, alignment=Qt.AlignBottom) + + return None + + def add_request_table(self, vbox) -> None: + self.table_widget.setColumnCount(8) + self.table_widget.setHorizontalHeaderLabels(["ID", "Method", "URL", "Host", "Delay Time", "Num Parallel", "Num Sequential", "Remove"]) self.table_widget.setColumnWidth(0, 50) self.table_widget.setColumnWidth(1, 50) - self.table_widget.setColumnWidth(2, 300) + self.table_widget.setColumnWidth(2, 600) self.table_widget.setColumnWidth(3, 100) + self.table_widget.setColumnWidth(4, 100) + + vbox.addWidget(self.table_widget) + + self.load_requests() + + return None + + def change_allow_redirects(self) -> None: + allow_redirects = self.load_json("state/batches/" + self.batch_name + ".json")["allow_redirects"] + + if allow_redirects is True: + self.racer.comm_curr_change_redirects(self.racer, False) + else: + self.racer.comm_curr_change_redirects(self.racer, True) + + self.update_json() + + return None - self.table_widget.verticalHeader().hide() + def change_sync_last_byte(self): + sync_last_byte = self.load_json("state/batches/" + self.batch_name + ".json")["sync_last_byte"] + + if sync_last_byte is True: + self.racer.comm_curr_change_sync(self.racer, False) + else: + self.racer.comm_curr_change_sync(self.racer, True) + + self.update_json() + + return None + + def change_send_timeout(self, value): + if value.text().isdigit(): + send_timeout = int(value.text()) + + self.racer.comm_curr_change_timeout(self.racer, send_timeout) + self.update_json() + + return None + + def load_requests(self) -> None: + items = self.load_json("state/batches/" + self.batch_name + ".json")["items"] + requests = self.load_json("state/state.json")["requests"] + remove_button = QPushButton("Remove", self) + + self.table_widget.setRowCount(len(items)) for i, item in enumerate(items): request_id = item["key"][0] + delay_time = item["key"][1] + num_parallel = item["value"][0] + num_sequential = item["value"][1] + request = requests[request_id] method = request["method"] @@ -383,14 +546,23 @@ def add_request_table(self) -> None: self.table_widget.setItem(i, 1, QTableWidgetItem(method)) self.table_widget.setItem(i, 2, QTableWidgetItem(url)) self.table_widget.setItem(i, 3, QTableWidgetItem(host)) + self.table_widget.setItem(i, 4, QTableWidgetItem(str(delay_time))) + self.table_widget.setItem(i, 5, QTableWidgetItem(str(num_parallel))) + self.table_widget.setItem(i, 6, QTableWidgetItem(str(num_sequential))) + self.table_widget.setCellWidget(i, 7, remove_button) - def load_json(self, filepath): - with open(filepath, 'r') as file: - data = json.load(file) - return data + remove_button.clicked.connect(lambda _, request_id=str(request_id): self.remove_request(request_id)) + + def send_batch(self) -> None: + self.save_data() + self.racer.comm_batches_send(self.racer) + + self.update_json() + + return None def go_back(self) -> None: - self.general_window = RequestsGUI(self.racer, self.state, self.command_processor) + self.general_window = MainGUI(self.racer, self.state, self.command_processor) self.general_window.show() self.hide() @@ -398,27 +570,123 @@ def go_back(self) -> None: def save_data(self) -> None: self.racer.comm_general_save(True) + return None - def reload_json(self): - if self.isActiveWindow(): - self.save_data() - self.update_json_timer.stop() - self.hide() - self.general_window = BatchWindow(self.batch_name, self.racer, self.state, self.command_processor) # Create a new window - self.general_window.show() - self.deleteLater() + def remove_request(self, request_id) -> None: + self.racer.comm_requests_remove(self.racer, request_id, None, False) - def showNotification(self, notiText): - messageBox = QMessageBox() - messageBox.setIcon(QMessageBox.Information) - messageBox.setText(notiText) + self.update_json() - messageBox.setGeometry(0, 0, 500, 50) + return None + + +class RequestWindow(QMainWindow): + def __init__(self, request_id, racer, state, command_processor): + super().__init__() + + self.request_id = request_id + self.racer = racer + self.state = state + self.command_processor = command_processor + + self.general_window = None + + self.table_widget = QWidget() + + self.init_ui() + + def init_ui(self): + vbox = QVBoxLayout() + + request_tab = QWidget() + request_tab.setLayout(vbox) + request_tab.layout().addWidget(self.table_widget) + + tabs = QTabWidget() + tabs.addTab(request_tab, "Request") + + vbox.addStretch() + + self.add_button_widget(vbox) + self.setCentralWidget(tabs) + self.load_request() + + vbox.insertWidget(0, self.table_widget) + + def load_request(self) -> None: + requests_data = self.load_json("state/state.json")["requests"] + request_data = requests_data.get(str(self.request_id)) + + if not request_data: + return + + # --- Ready the data --- # + body = request_data.get("body", "") + headers = request_data.get("headers", {}) + method = request_data.get("method", "") + timestamp = request_data.get("timestamp", "") + url = request_data.get("url", "") + request_id = request_data.get("id", "") + + # --- Create model and add headers --- # + model = QStandardItemModel() + model.setHorizontalHeaderLabels(["Field", "Value"]) + + # --- Insert data into rows --- # + model.appendRow([QStandardItem("Request ID"), QStandardItem(str(request_id))]) + model.appendRow([QStandardItem("URL"), QStandardItem(url)]) + model.appendRow([QStandardItem("Method"), QStandardItem(method)]) + model.appendRow([QStandardItem("Timestamp"), QStandardItem(str(timestamp))]) + model.appendRow([QStandardItem("Body"), QStandardItem(body)]) + for key, value in headers.items(): + model.appendRow([QStandardItem(key), QStandardItem(value)]) + + table_view = QTableView() + table_view.setModel(model) + + table_view.horizontalHeader().setStretchLastSection(True) + table_view.verticalHeader().setVisible(False) + table_view.setShowGrid(True) + table_view.setEditTriggers(QTableView.NoEditTriggers) + + # Set grid color to background color + table_view.setStyleSheet( + "QTableView::item {border-bottom: 1px solid black;} QTableView {background-color: white;}") + table_view.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents) + table_view.setColumnWidth(0, 300) + + self.table_widget = table_view + self.layout().addWidget(self.table_widget) + + return None + + def add_button_widget(self, vbox) -> None: + quit_button = QPushButton("Quit") + go_back_button = QPushButton("Go Back") + + quit_button.clicked.connect(QApplication.quit) + go_back_button.clicked.connect(self.go_back) + + vbox.addWidget(quit_button, alignment=Qt.AlignBottom) + vbox.addWidget(go_back_button, alignment=Qt.AlignBottom) + + return None + + def load_json(self, filepath): + with open(filepath, 'r') as file: + data = json.load(file) + + return data + + def go_back(self) -> None: + self.general_window = MainGUI(self.racer, self.state, self.command_processor) + self.general_window.show() + self.deleteLater() + + return None - timer = QTimer() - timer.setSingleShot(True) - timer.timeout.connect(messageBox.close) - timer.start(5000) + def save_data(self) -> None: + self.racer.comm_general_save(True) - messageBox.exec() \ No newline at end of file + return None diff --git a/CompuRacer_Extensions/Burp/compu_racer_extension_burp.py b/CompuRacer_Extensions/Burp/compu_racer_extension_burp.py index 802f69b..15897fc 100644 --- a/CompuRacer_Extensions/Burp/compu_racer_extension_burp.py +++ b/CompuRacer_Extensions/Burp/compu_racer_extension_burp.py @@ -27,6 +27,9 @@ import time import json import traceback +import sys + +print(sys.version, sys.executable) import requests diff --git a/README.md b/README.md index 7ffd3f5..178fc06 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The CompuRacer toolset for detection and exploitation of race conditions in web The toolset can be split in three separate parts: Core application in `CompuRacerCore`, Burp extension in `CompuRacerExtensionBurp` and browser extensions (Chrome & Firefox) in `CompuRacerExtensionChrome` and `CompuRacerExtensionFirefox`. The `TestWebAppVouchers` folder contains a Flask test web app for voucher redemption that contains race conditions. ## Recommended software versions -The toolset is only compatible with Python 3.7. It has been tested using Burp Suite Professional v1.7.37 & v2.1.03 (the Community Edition is also compatible), Firefox v. 69, Chrome v. 76 and Vagrant 2.1.5. It is tested on a MacBook Pro (2018) running macOS Mojave. Every individual tool is expected to be compatible with both Linux and Windows, but this is not fully tested. +The toolset is compatible with python <=3.9. It has been tested using Burp Suite Professional v1.7.37 & v2.1.03 (the Community Edition is also compatible), Firefox v. 69, Chrome v. 76 and Vagrant 2.1.5. It is tested on a MacBook Pro (2018) running macOS Mojave. Every individual tool is expected to be compatible with both Linux and Windows, but this is not fully tested. ## Installation #### Clone the repository @@ -13,10 +13,10 @@ The toolset is only compatible with Python 3.7. It has been tested using Burp Su * Go to the [`CompuRacer_Core/`](CompuRacer_Core/) folder. * Run: `$ pip3 install -r requirements.txt` #### Install CompuRacer Burp Suite extension -* First, download the Jython standalone JAR file at https://www.jython.org/download and install the Requests library dependancy using: `$ pip3 install requests`. -* In the Burp Suite, go to: Extender > Options > Python Environment and select the downloaded JAR file. -* Then, point to the folder where the Requests library is installed. On a mac, this is probably: `/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages` or `~/Library/Python/3.7/lib/python/site-packages`. -* Next, go to: Extender > Extensions > Add and select `Python` as the extension type. +* First, download the Jython standalone JAR file at https://www.jython.org/download +* In the Burp Suite, go to: Extensions > Extension settings > Python Environment and select the downloaded JAR file. +* Then, point to the folder where the Requests library is installed. On a mac, this is probably: `/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages` or `~/Library/Python/3.9/lib/python/site-packages`. +* Next, go to: Extensions > Add and select `Python` as the extension type. * Regarding the extension file, go to the [`CompuRacer_Extensions/Burp/`](CompuRacer_Extensions/Burp/) folder and select: `compu_racer_extension_burp.py`. * Click 'next' and after loading the extension, close the window. #### Install CompuRacer Firefox extension (optional) @@ -38,7 +38,7 @@ The Firefox, Chrome, Burp Suite extensions and test web app do not need any conf ## Running The Firefox, Chrome, Burp Suite extensions and test web app are already started after the install. The Computest Core can be started by running the following command within the `CompuRacer_Core` folder:
-`$ python3.7 main.py [-h] [--port [PORT]] [--proxy [PROXY]]` +`$ python3 main.py [-h] [--cli] [--port [PORT]] [--proxy [PROXY]]` ## How to use An elaborate manual on how to use the toolset can be found in [`CompuRacer_Manual.pdf`](CompuRacer_Manual.pdf).