import json import re import time from datetime import datetime from pathlib import Path from selenium.common.exceptions import NoAlertPresentException, TimeoutException, WebDriverException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait TM_ID_PATTERN = re.compile(r"([CS]\d{4})", re.IGNORECASE) SCREEN_FOLDER_NAMES = { "single_bid": "Single", "block_bid": "Block", "single_and_block_bid": "Single", } BUYSELL_SIDE = "BUYSELL" BUYSELL_COMPONENT_SIDES = ("Buy", "Sell") def get_folder_upload_entries(config): product = config["product"] bidding_config = config["exchange_bidding"] upload_root = Path( bidding_config.get("folder_upload_root") or bidding_config.get("upload_input_root", "") ) base_paths = _resolve_folder_upload_base_paths(upload_root, product, bidding_config) entries = [] for base_path, side_name in base_paths: for folder_path in sorted(path for path in base_path.iterdir() if path.is_dir()): if folder_path.name.startswith("."): continue entries.append( { "folder_path": str(folder_path.resolve()), "folder_name": folder_path.name, "side": side_name, "tm_id": _derive_tm_id_from_folder_name(folder_path.name), } ) if not entries: searched_paths = [str(base_path) for base_path, _side_name in base_paths] raise FileNotFoundError(f"No TM folders were found in {searched_paths}") return entries def upload_folder(driver, folder_path): folder = Path(folder_path) if not folder.exists() or not folder.is_dir(): raise FileNotFoundError(f"Upload folder not found: {folder}") wait = WebDriverWait(driver, 30) folder_input = wait.until( lambda _driver: _folder_file_input(_driver) or False, "No folder upload input was found. Expected an input[type='file'][webkitdirectory]." ) driver.execute_script( """ arguments[0].style.display = 'block'; arguments[0].style.visibility = 'visible'; arguments[0].style.opacity = 1; arguments[0].style.height = '1px'; arguments[0].style.width = '1px'; arguments[0].removeAttribute('disabled'); arguments[0].value = ''; """, folder_input ) folder_input.send_keys(str(folder.resolve())) _accept_upload_confirmation_if_present(driver) wait.until(lambda _driver: _folder_file_rows(_driver) or False) def submit_all_and_wait_for_progress( driver, timeout_seconds=300, poll_seconds=2, keep_alive_interval_seconds=30, progress_log_interval_seconds=60, progress_log_label="folder_upload", continue_when_last_file_complete=False, ): wait = WebDriverWait(driver, 30) submit_all_button = wait.until( EC.element_to_be_clickable( ( By.XPATH, "//table[@id='tblFileList']//button[normalize-space()='Submit All']" "|//button[normalize-space()='Submit All' or contains(normalize-space(),'Submit All')]" ) ) ) driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", submit_all_button) _click(driver, submit_all_button) submit_all_clicked_at = datetime.now() submit_all_monotonic = time.monotonic() progress_rows = _wait_for_all_progress_100( driver, timeout_seconds, poll_seconds, keep_alive_interval_seconds, progress_log_interval_seconds, progress_log_label, continue_when_last_file_complete, ) all_progress_100_at = datetime.now() incomplete_rows = [ row for row in progress_rows if row.get("progress", 0) < 100 ] return { "submit_all_clicked_at": submit_all_clicked_at.isoformat(timespec="seconds"), "all_progress_100_at": all_progress_100_at.isoformat(timespec="seconds"), "completion_condition": _completion_condition(progress_rows), "duration_seconds": round(time.monotonic() - submit_all_monotonic, 3), "file_count": len(progress_rows), "incomplete_count": len(incomplete_rows), "incomplete_files": incomplete_rows, "files": progress_rows, } def fetch_folder_upload_statuses(driver, timeout_seconds=30): results = [] rows = _folder_file_rows(driver) for index, row in enumerate(rows, start=1): file_name = _row_cell_text(row, 1) remarks = _row_cell_text(row, 2) progress_text = _row_cell_text(row, 4) progress = _row_progress_value(row, progress_text) if progress < 100: results.append( { "index": index, "file_name": file_name, "status": "upload_incomplete", "raw_status": remarks, "remarks": remarks, "progress": progress, "progress_text": progress_text, "fetch_attempted": False, } ) print( f"Skipping fetch for incomplete upload row: " f"file={file_name}, progress={progress}%, remarks={remarks}" ) continue previous_status = _current_status_text(driver) fetch_button = _row_fetch_button(row) if fetch_button is None: results.append( { "index": index, "file_name": file_name, "status": "missing_fetch_button", "raw_status": "", "remarks": remarks, "progress": progress, "progress_text": progress_text, "fetch_attempted": False, } ) continue driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", fetch_button) _click(driver, fetch_button) try: status_text = _wait_for_refreshed_status(driver, previous_status, timeout_seconds) except TimeoutException: status_text = _current_status_text(driver) results.append( { "index": index, "file_name": file_name, "status": "fetch_timeout", "raw_status": status_text, "previous_status": previous_status, "remarks": remarks, "progress": progress, "progress_text": progress_text, "fetch_attempted": True, } ) print( f"Fetch status timeout for uploaded row: " f"file={file_name}, progress={progress}%, " f"previous_status={previous_status}, current_status={status_text}" ) continue lowered_status = status_text.lower() if "reject" in lowered_status: status = "rejected" elif "open" in lowered_status or "accept" in lowered_status or "submitted" in lowered_status: status = "open" else: status = "unknown" results.append( { "index": index, "file_name": file_name, "status": status, "raw_status": status_text, "remarks": remarks, "progress": progress, "progress_text": progress_text, "fetch_attempted": True, } ) return results def save_folder_upload_log(test_context, market, entry, timing, fetch_results): folder = Path(test_context.get("log_folder", test_context["folder"])) folder.mkdir(parents=True, exist_ok=True) side_name = entry.get("side", "") log_name_parts = ["folder_upload", market] if side_name: log_name_parts.append(side_name) log_name_parts.append(entry["tm_id"]) path = folder / f"{'_'.join(log_name_parts)}.json" rejected_files = [item for item in fetch_results if item["status"] == "rejected"] unresolved_files = [ item for item in fetch_results if item["status"] in {"upload_incomplete", "fetch_timeout", "missing_fetch_button", "unknown"} ] payload = { "market": market, "side": side_name, "tm_id": entry["tm_id"], "folder_name": entry["folder_name"], "folder_path": entry["folder_path"], "timing": timing, "fetch_results": fetch_results, "rejected_files": rejected_files, "unresolved_files": unresolved_files, } path.write_text(json.dumps(payload, indent=2), encoding="utf-8") return str(path) def _resolve_folder_upload_base_paths(upload_root, product, bidding_config): if not upload_root: raise ValueError("Set exchange_bidding.folder_upload_root or upload_input_root in config/runtime.json.") base_path = upload_root.resolve() if not base_path.exists(): raise FileNotFoundError(f"Folder upload root does not exist: {base_path}") market_path = base_path / product["trading_market"] if not market_path.exists(): available_markets = [ path.name for path in base_path.iterdir() if path.is_dir() and not path.name.startswith(".") ] raise FileNotFoundError( f"Folder upload market path does not exist: {market_path}. " f"Available market folders under {base_path}: {available_markets}. " "Set active_selection.trading_market to an available market, " "or create folders for every market when using ALL." ) base_path = market_path screen_folder = SCREEN_FOLDER_NAMES.get(product.get("bidding_screen", "single_bid")) if screen_folder: screen_path = base_path / screen_folder if not screen_path.exists(): raise FileNotFoundError(f"Folder upload screen path does not exist: {screen_path}") base_path = screen_path sides = bidding_config.get("folder_upload_sides") or [bidding_config.get("folder_upload_side", "Buy")] if isinstance(sides, str): sides = [sides] expanded_sides = _expand_folder_upload_sides(base_path, sides) existing_side_paths = [ (base_path / side, side) for side in expanded_sides if (base_path / side).exists() ] if existing_side_paths: return existing_side_paths raise FileNotFoundError( f"Folder upload side path does not exist under {base_path}. " f"Configured folder_upload_sides: {sides}. Expanded sides: {expanded_sides}" ) def _expand_folder_upload_sides(base_path, sides): expanded_sides = [] for side in sides: side_name = str(side).strip() if not side_name: continue if side_name.upper() != BUYSELL_SIDE: expanded_sides.append(side_name) continue if (base_path / BUYSELL_SIDE).exists(): expanded_sides.append(BUYSELL_SIDE) continue expanded_sides.extend(BUYSELL_COMPONENT_SIDES) return list(dict.fromkeys(expanded_sides)) def _derive_tm_id_from_folder_name(folder_name): match = TM_ID_PATTERN.search(folder_name) if not match: raise ValueError(f"Could not derive TM ID from folder name '{folder_name}'.") return match.group(1).upper() def _folder_file_input(driver): inputs = driver.find_elements( By.XPATH, "//input[@type='file' and (@webkitdirectory or @id='fileInput' or @name='fileInput')]" ) if inputs: return inputs[0] return None def _accept_upload_confirmation_if_present(driver): try: alert = driver.switch_to.alert alert.accept() except NoAlertPresentException: pass def _folder_file_rows(driver): rows = driver.find_elements(By.XPATH, "//table[@id='tblFileList']//tbody/tr[td]") return [row for row in rows if row.is_displayed()] def _wait_for_all_progress_100( driver, timeout_seconds, poll_seconds, keep_alive_interval_seconds, progress_log_interval_seconds, progress_log_label, continue_when_last_file_complete, ): end_time = time.monotonic() + timeout_seconds next_keep_alive = time.monotonic() + keep_alive_interval_seconds next_progress_log = time.monotonic() + progress_log_interval_seconds last_rows = [] while time.monotonic() < end_time: try: rows = _read_progress_rows(driver) except WebDriverException as exc: raise RuntimeError( "Browser session disconnected while waiting for folder upload progress. " "This usually means Chrome was closed, crashed, or Windows/Chrome " "suspended the renderer during the long upload. Re-run with the browser " "windows kept open and the PC sleep/lock settings disabled for the upload window." ) from exc if rows: last_rows = rows if all(row["progress"] >= 100 for row in rows): _print_progress_summary(progress_log_label, rows) return rows if continue_when_last_file_complete and _last_file_is_complete(rows): _print_progress_summary(progress_log_label, rows) _print_incomplete_rows(progress_log_label, rows) return rows if keep_alive_interval_seconds and time.monotonic() >= next_keep_alive: _keep_session_active(driver) next_keep_alive = time.monotonic() + keep_alive_interval_seconds if progress_log_interval_seconds and time.monotonic() >= next_progress_log: _print_progress_summary(progress_log_label, last_rows) next_progress_log = time.monotonic() + progress_log_interval_seconds time.sleep(poll_seconds) raise TimeoutException( "Folder upload progress did not reach 100% for all files. " f"Last progress rows: {last_rows}" ) def _print_progress_summary(label, rows): if not rows: print(f"Folder upload progress [{label}]: waiting for progress rows...") return completed = sum(1 for row in rows if row["progress"] >= 100) total = len(rows) max_progress = max(row["progress"] for row in rows) min_progress = min(row["progress"] for row in rows) print( f"Folder upload progress [{label}]: " f"{completed}/{total} files at 100%, " f"min={min_progress}%, max={max_progress}%" ) def _print_incomplete_rows(label, rows): incomplete_rows = [ row for row in rows if row.get("progress", 0) < 100 ] if not incomplete_rows: return print( f"Folder upload progress [{label}]: continuing because last file is 100%. " "Incomplete/error rows: " + "; ".join( f"{row.get('file_name', '')} progress={row.get('progress', 0)}% " f"remarks={row.get('remarks', '')}" for row in incomplete_rows ) ) def _last_file_is_complete(rows): return bool(rows) and rows[-1].get("progress", 0) >= 100 def _completion_condition(rows): if all(row.get("progress", 0) >= 100 for row in rows): return "all_files_100" if _last_file_is_complete(rows): return "last_file_100_with_incomplete_rows" return "incomplete" def _read_progress_rows(driver): rows = [] for row in _folder_file_rows(driver): file_name = _row_cell_text(row, 1) remarks = _row_cell_text(row, 2) progress_text = _row_cell_text(row, 4) progress = _row_progress_value(row, progress_text) rows.append( { "file_name": file_name, "remarks": remarks, "progress": progress, "progress_text": progress_text, } ) return rows def _keep_session_active(driver): driver.execute_script( """ const scrollY = window.scrollY || document.documentElement.scrollTop || 0; window.scrollBy(0, 1); window.scrollTo(0, scrollY); document.dispatchEvent(new MouseEvent('mousemove', { bubbles: true, clientX: 1, clientY: 1 })); document.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, key: 'Shift' })); """ ) def _row_progress_value(row, fallback_text): progress_bars = row.find_elements(By.XPATH, ".//*[contains(@class,'progress-bar')]") for progress_bar in progress_bars: aria_value = progress_bar.get_attribute("aria-valuenow") progress = _parse_progress(aria_value) if progress > 0: return progress style = progress_bar.get_attribute("style") or "" progress = _parse_progress(style) if progress > 0: return progress text = progress_bar.text.strip() progress = _parse_progress(text) if progress > 0: return progress return _parse_progress(fallback_text) def _row_cell_text(row, zero_based_index): cells = row.find_elements(By.XPATH, "./td") if len(cells) <= zero_based_index: return "" return cells[zero_based_index].text.strip() def _parse_progress(progress_text): progress_text = str(progress_text or "") match = re.search(r"(\d+)\s*%", progress_text) if match: return int(match.group(1)) match = re.search(r"\b(\d{1,3})\b", progress_text) if match: return int(match.group(1)) if "100" in progress_text: return 100 return 0 def _row_fetch_button(row): buttons = row.find_elements(By.XPATH, ".//*[self::button or self::a][normalize-space()='Fetch']") for button in buttons: if button.is_displayed() and button.is_enabled(): return button return None def _current_status_text(driver): by_id = driver.find_elements(By.ID, "lblStatus") for candidate in by_id: text = candidate.text.strip() if text: return text candidates = driver.find_elements( By.XPATH, "//*[contains(normalize-space(),'Status :') or contains(normalize-space(),'Status:')]" ) for candidate in candidates: text = candidate.text.strip() if text: return text return "" def _wait_for_refreshed_status(driver, previous_status, timeout_seconds): wait = WebDriverWait(driver, timeout_seconds) return wait.until( lambda _driver: _refreshed_status_text(_driver, previous_status) or False ) def _refreshed_status_text(driver, previous_status): current_status = _current_status_text(driver).strip() if not current_status: return "" normalized_current = " ".join(current_status.split()).lower() normalized_previous = " ".join((previous_status or "").split()).lower() terminal_tokens = ("reject", "submitted", "open", "accept") if any(token in normalized_current for token in terminal_tokens): if normalized_current != normalized_previous: return current_status if "order id" in normalized_current: return current_status return "" def _click(driver, element): try: element.click() except Exception: driver.execute_script("arguments[0].click();", element)