Initial commit - Selenium automation framework
This commit is contained in:
@@ -0,0 +1,622 @@
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from selenium.common.exceptions import NoAlertPresentException, TimeoutException, WebDriverException
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
|
||||
TM_ID_PATTERN = re.compile(r"([CS]\d{4})", re.IGNORECASE)
|
||||
SCREEN_FOLDER_NAMES = {
|
||||
"single_bid": "Single",
|
||||
"block_bid": "Block",
|
||||
"single_and_block_bid": "Single",
|
||||
}
|
||||
BUYSELL_SIDE = "BUYSELL"
|
||||
BUYSELL_COMPONENT_SIDES = ("Buy", "Sell")
|
||||
|
||||
|
||||
def get_folder_upload_entries(config):
|
||||
|
||||
product = config["product"]
|
||||
bidding_config = config["exchange_bidding"]
|
||||
upload_root = Path(
|
||||
bidding_config.get("folder_upload_root")
|
||||
or bidding_config.get("upload_input_root", "")
|
||||
)
|
||||
|
||||
base_paths = _resolve_folder_upload_base_paths(upload_root, product, bidding_config)
|
||||
entries = []
|
||||
|
||||
for base_path, side_name in base_paths:
|
||||
for folder_path in sorted(path for path in base_path.iterdir() if path.is_dir()):
|
||||
if folder_path.name.startswith("."):
|
||||
continue
|
||||
|
||||
entries.append(
|
||||
{
|
||||
"folder_path": str(folder_path.resolve()),
|
||||
"folder_name": folder_path.name,
|
||||
"side": side_name,
|
||||
"tm_id": _derive_tm_id_from_folder_name(folder_path.name),
|
||||
}
|
||||
)
|
||||
|
||||
if not entries:
|
||||
searched_paths = [str(base_path) for base_path, _side_name in base_paths]
|
||||
raise FileNotFoundError(f"No TM folders were found in {searched_paths}")
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def upload_folder(driver, folder_path):
|
||||
|
||||
folder = Path(folder_path)
|
||||
if not folder.exists() or not folder.is_dir():
|
||||
raise FileNotFoundError(f"Upload folder not found: {folder}")
|
||||
|
||||
wait = WebDriverWait(driver, 30)
|
||||
folder_input = wait.until(
|
||||
lambda _driver: _folder_file_input(_driver) or False,
|
||||
"No folder upload input was found. Expected an input[type='file'][webkitdirectory]."
|
||||
)
|
||||
driver.execute_script(
|
||||
"""
|
||||
arguments[0].style.display = 'block';
|
||||
arguments[0].style.visibility = 'visible';
|
||||
arguments[0].style.opacity = 1;
|
||||
arguments[0].style.height = '1px';
|
||||
arguments[0].style.width = '1px';
|
||||
arguments[0].removeAttribute('disabled');
|
||||
arguments[0].value = '';
|
||||
""",
|
||||
folder_input
|
||||
)
|
||||
folder_input.send_keys(str(folder.resolve()))
|
||||
|
||||
_accept_upload_confirmation_if_present(driver)
|
||||
wait.until(lambda _driver: _folder_file_rows(_driver) or False)
|
||||
|
||||
|
||||
def submit_all_and_wait_for_progress(
|
||||
driver,
|
||||
timeout_seconds=300,
|
||||
poll_seconds=2,
|
||||
keep_alive_interval_seconds=30,
|
||||
progress_log_interval_seconds=60,
|
||||
progress_log_label="folder_upload",
|
||||
continue_when_last_file_complete=False,
|
||||
):
|
||||
|
||||
wait = WebDriverWait(driver, 30)
|
||||
submit_all_button = wait.until(
|
||||
EC.element_to_be_clickable(
|
||||
(
|
||||
By.XPATH,
|
||||
"//table[@id='tblFileList']//button[normalize-space()='Submit All']"
|
||||
"|//button[normalize-space()='Submit All' or contains(normalize-space(),'Submit All')]"
|
||||
)
|
||||
)
|
||||
)
|
||||
driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", submit_all_button)
|
||||
_click(driver, submit_all_button)
|
||||
|
||||
submit_all_clicked_at = datetime.now()
|
||||
submit_all_monotonic = time.monotonic()
|
||||
progress_rows = _wait_for_all_progress_100(
|
||||
driver,
|
||||
timeout_seconds,
|
||||
poll_seconds,
|
||||
keep_alive_interval_seconds,
|
||||
progress_log_interval_seconds,
|
||||
progress_log_label,
|
||||
continue_when_last_file_complete,
|
||||
)
|
||||
all_progress_100_at = datetime.now()
|
||||
incomplete_rows = [
|
||||
row for row in progress_rows
|
||||
if row.get("progress", 0) < 100
|
||||
]
|
||||
|
||||
return {
|
||||
"submit_all_clicked_at": submit_all_clicked_at.isoformat(timespec="seconds"),
|
||||
"all_progress_100_at": all_progress_100_at.isoformat(timespec="seconds"),
|
||||
"completion_condition": _completion_condition(progress_rows),
|
||||
"duration_seconds": round(time.monotonic() - submit_all_monotonic, 3),
|
||||
"file_count": len(progress_rows),
|
||||
"incomplete_count": len(incomplete_rows),
|
||||
"incomplete_files": incomplete_rows,
|
||||
"files": progress_rows,
|
||||
}
|
||||
|
||||
|
||||
def fetch_folder_upload_statuses(driver, timeout_seconds=30):
|
||||
|
||||
results = []
|
||||
rows = _folder_file_rows(driver)
|
||||
|
||||
for index, row in enumerate(rows, start=1):
|
||||
file_name = _row_cell_text(row, 1)
|
||||
remarks = _row_cell_text(row, 2)
|
||||
progress_text = _row_cell_text(row, 4)
|
||||
progress = _row_progress_value(row, progress_text)
|
||||
|
||||
if progress < 100:
|
||||
results.append(
|
||||
{
|
||||
"index": index,
|
||||
"file_name": file_name,
|
||||
"status": "upload_incomplete",
|
||||
"raw_status": remarks,
|
||||
"remarks": remarks,
|
||||
"progress": progress,
|
||||
"progress_text": progress_text,
|
||||
"fetch_attempted": False,
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"Skipping fetch for incomplete upload row: "
|
||||
f"file={file_name}, progress={progress}%, remarks={remarks}"
|
||||
)
|
||||
continue
|
||||
|
||||
previous_status = _current_status_text(driver)
|
||||
fetch_button = _row_fetch_button(row)
|
||||
|
||||
if fetch_button is None:
|
||||
results.append(
|
||||
{
|
||||
"index": index,
|
||||
"file_name": file_name,
|
||||
"status": "missing_fetch_button",
|
||||
"raw_status": "",
|
||||
"remarks": remarks,
|
||||
"progress": progress,
|
||||
"progress_text": progress_text,
|
||||
"fetch_attempted": False,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", fetch_button)
|
||||
_click(driver, fetch_button)
|
||||
|
||||
try:
|
||||
status_text = _wait_for_refreshed_status(driver, previous_status, timeout_seconds)
|
||||
except TimeoutException:
|
||||
status_text = _current_status_text(driver)
|
||||
results.append(
|
||||
{
|
||||
"index": index,
|
||||
"file_name": file_name,
|
||||
"status": "fetch_timeout",
|
||||
"raw_status": status_text,
|
||||
"previous_status": previous_status,
|
||||
"remarks": remarks,
|
||||
"progress": progress,
|
||||
"progress_text": progress_text,
|
||||
"fetch_attempted": True,
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"Fetch status timeout for uploaded row: "
|
||||
f"file={file_name}, progress={progress}%, "
|
||||
f"previous_status={previous_status}, current_status={status_text}"
|
||||
)
|
||||
continue
|
||||
|
||||
lowered_status = status_text.lower()
|
||||
if "reject" in lowered_status:
|
||||
status = "rejected"
|
||||
elif "open" in lowered_status or "accept" in lowered_status or "submitted" in lowered_status:
|
||||
status = "open"
|
||||
else:
|
||||
status = "unknown"
|
||||
|
||||
results.append(
|
||||
{
|
||||
"index": index,
|
||||
"file_name": file_name,
|
||||
"status": status,
|
||||
"raw_status": status_text,
|
||||
"remarks": remarks,
|
||||
"progress": progress,
|
||||
"progress_text": progress_text,
|
||||
"fetch_attempted": True,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def save_folder_upload_log(test_context, market, entry, timing, fetch_results):
|
||||
|
||||
folder = Path(test_context.get("log_folder", test_context["folder"]))
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
side_name = entry.get("side", "")
|
||||
log_name_parts = ["folder_upload", market]
|
||||
if side_name:
|
||||
log_name_parts.append(side_name)
|
||||
log_name_parts.append(entry["tm_id"])
|
||||
path = folder / f"{'_'.join(log_name_parts)}.json"
|
||||
rejected_files = [item for item in fetch_results if item["status"] == "rejected"]
|
||||
unresolved_files = [
|
||||
item for item in fetch_results
|
||||
if item["status"] in {"upload_incomplete", "fetch_timeout", "missing_fetch_button", "unknown"}
|
||||
]
|
||||
payload = {
|
||||
"market": market,
|
||||
"side": side_name,
|
||||
"tm_id": entry["tm_id"],
|
||||
"folder_name": entry["folder_name"],
|
||||
"folder_path": entry["folder_path"],
|
||||
"timing": timing,
|
||||
"fetch_results": fetch_results,
|
||||
"rejected_files": rejected_files,
|
||||
"unresolved_files": unresolved_files,
|
||||
}
|
||||
|
||||
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
|
||||
def _resolve_folder_upload_base_paths(upload_root, product, bidding_config):
|
||||
|
||||
if not upload_root:
|
||||
raise ValueError("Set exchange_bidding.folder_upload_root or upload_input_root in config/runtime.json.")
|
||||
|
||||
base_path = upload_root.resolve()
|
||||
if not base_path.exists():
|
||||
raise FileNotFoundError(f"Folder upload root does not exist: {base_path}")
|
||||
|
||||
market_path = base_path / product["trading_market"]
|
||||
if not market_path.exists():
|
||||
available_markets = [
|
||||
path.name for path in base_path.iterdir()
|
||||
if path.is_dir() and not path.name.startswith(".")
|
||||
]
|
||||
raise FileNotFoundError(
|
||||
f"Folder upload market path does not exist: {market_path}. "
|
||||
f"Available market folders under {base_path}: {available_markets}. "
|
||||
"Set active_selection.trading_market to an available market, "
|
||||
"or create folders for every market when using ALL."
|
||||
)
|
||||
|
||||
base_path = market_path
|
||||
|
||||
screen_folder = SCREEN_FOLDER_NAMES.get(product.get("bidding_screen", "single_bid"))
|
||||
if screen_folder:
|
||||
screen_path = base_path / screen_folder
|
||||
if not screen_path.exists():
|
||||
raise FileNotFoundError(f"Folder upload screen path does not exist: {screen_path}")
|
||||
base_path = screen_path
|
||||
|
||||
sides = bidding_config.get("folder_upload_sides") or [bidding_config.get("folder_upload_side", "Buy")]
|
||||
if isinstance(sides, str):
|
||||
sides = [sides]
|
||||
|
||||
expanded_sides = _expand_folder_upload_sides(base_path, sides)
|
||||
existing_side_paths = [
|
||||
(base_path / side, side)
|
||||
for side in expanded_sides
|
||||
if (base_path / side).exists()
|
||||
]
|
||||
if existing_side_paths:
|
||||
return existing_side_paths
|
||||
|
||||
raise FileNotFoundError(
|
||||
f"Folder upload side path does not exist under {base_path}. "
|
||||
f"Configured folder_upload_sides: {sides}. Expanded sides: {expanded_sides}"
|
||||
)
|
||||
|
||||
|
||||
def _expand_folder_upload_sides(base_path, sides):
|
||||
|
||||
expanded_sides = []
|
||||
for side in sides:
|
||||
side_name = str(side).strip()
|
||||
if not side_name:
|
||||
continue
|
||||
|
||||
if side_name.upper() != BUYSELL_SIDE:
|
||||
expanded_sides.append(side_name)
|
||||
continue
|
||||
|
||||
if (base_path / BUYSELL_SIDE).exists():
|
||||
expanded_sides.append(BUYSELL_SIDE)
|
||||
continue
|
||||
|
||||
expanded_sides.extend(BUYSELL_COMPONENT_SIDES)
|
||||
|
||||
return list(dict.fromkeys(expanded_sides))
|
||||
|
||||
|
||||
def _derive_tm_id_from_folder_name(folder_name):
|
||||
|
||||
match = TM_ID_PATTERN.search(folder_name)
|
||||
if not match:
|
||||
raise ValueError(f"Could not derive TM ID from folder name '{folder_name}'.")
|
||||
return match.group(1).upper()
|
||||
|
||||
|
||||
def _folder_file_input(driver):
|
||||
|
||||
inputs = driver.find_elements(
|
||||
By.XPATH,
|
||||
"//input[@type='file' and (@webkitdirectory or @id='fileInput' or @name='fileInput')]"
|
||||
)
|
||||
if inputs:
|
||||
return inputs[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _accept_upload_confirmation_if_present(driver):
|
||||
|
||||
try:
|
||||
alert = driver.switch_to.alert
|
||||
alert.accept()
|
||||
except NoAlertPresentException:
|
||||
pass
|
||||
|
||||
|
||||
def _folder_file_rows(driver):
|
||||
|
||||
rows = driver.find_elements(By.XPATH, "//table[@id='tblFileList']//tbody/tr[td]")
|
||||
return [row for row in rows if row.is_displayed()]
|
||||
|
||||
|
||||
def _wait_for_all_progress_100(
|
||||
driver,
|
||||
timeout_seconds,
|
||||
poll_seconds,
|
||||
keep_alive_interval_seconds,
|
||||
progress_log_interval_seconds,
|
||||
progress_log_label,
|
||||
continue_when_last_file_complete,
|
||||
):
|
||||
|
||||
end_time = time.monotonic() + timeout_seconds
|
||||
next_keep_alive = time.monotonic() + keep_alive_interval_seconds
|
||||
next_progress_log = time.monotonic() + progress_log_interval_seconds
|
||||
last_rows = []
|
||||
|
||||
while time.monotonic() < end_time:
|
||||
try:
|
||||
rows = _read_progress_rows(driver)
|
||||
except WebDriverException as exc:
|
||||
raise RuntimeError(
|
||||
"Browser session disconnected while waiting for folder upload progress. "
|
||||
"This usually means Chrome was closed, crashed, or Windows/Chrome "
|
||||
"suspended the renderer during the long upload. Re-run with the browser "
|
||||
"windows kept open and the PC sleep/lock settings disabled for the upload window."
|
||||
) from exc
|
||||
|
||||
if rows:
|
||||
last_rows = rows
|
||||
if all(row["progress"] >= 100 for row in rows):
|
||||
_print_progress_summary(progress_log_label, rows)
|
||||
return rows
|
||||
if continue_when_last_file_complete and _last_file_is_complete(rows):
|
||||
_print_progress_summary(progress_log_label, rows)
|
||||
_print_incomplete_rows(progress_log_label, rows)
|
||||
return rows
|
||||
|
||||
if keep_alive_interval_seconds and time.monotonic() >= next_keep_alive:
|
||||
_keep_session_active(driver)
|
||||
next_keep_alive = time.monotonic() + keep_alive_interval_seconds
|
||||
|
||||
if progress_log_interval_seconds and time.monotonic() >= next_progress_log:
|
||||
_print_progress_summary(progress_log_label, last_rows)
|
||||
next_progress_log = time.monotonic() + progress_log_interval_seconds
|
||||
|
||||
time.sleep(poll_seconds)
|
||||
|
||||
raise TimeoutException(
|
||||
"Folder upload progress did not reach 100% for all files. "
|
||||
f"Last progress rows: {last_rows}"
|
||||
)
|
||||
|
||||
|
||||
def _print_progress_summary(label, rows):
|
||||
|
||||
if not rows:
|
||||
print(f"Folder upload progress [{label}]: waiting for progress rows...")
|
||||
return
|
||||
|
||||
completed = sum(1 for row in rows if row["progress"] >= 100)
|
||||
total = len(rows)
|
||||
max_progress = max(row["progress"] for row in rows)
|
||||
min_progress = min(row["progress"] for row in rows)
|
||||
print(
|
||||
f"Folder upload progress [{label}]: "
|
||||
f"{completed}/{total} files at 100%, "
|
||||
f"min={min_progress}%, max={max_progress}%"
|
||||
)
|
||||
|
||||
|
||||
def _print_incomplete_rows(label, rows):
|
||||
|
||||
incomplete_rows = [
|
||||
row for row in rows
|
||||
if row.get("progress", 0) < 100
|
||||
]
|
||||
if not incomplete_rows:
|
||||
return
|
||||
|
||||
print(
|
||||
f"Folder upload progress [{label}]: continuing because last file is 100%. "
|
||||
"Incomplete/error rows: "
|
||||
+ "; ".join(
|
||||
f"{row.get('file_name', '')} progress={row.get('progress', 0)}% "
|
||||
f"remarks={row.get('remarks', '')}"
|
||||
for row in incomplete_rows
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _last_file_is_complete(rows):
|
||||
|
||||
return bool(rows) and rows[-1].get("progress", 0) >= 100
|
||||
|
||||
|
||||
def _completion_condition(rows):
|
||||
|
||||
if all(row.get("progress", 0) >= 100 for row in rows):
|
||||
return "all_files_100"
|
||||
if _last_file_is_complete(rows):
|
||||
return "last_file_100_with_incomplete_rows"
|
||||
return "incomplete"
|
||||
|
||||
|
||||
def _read_progress_rows(driver):
|
||||
|
||||
rows = []
|
||||
for row in _folder_file_rows(driver):
|
||||
file_name = _row_cell_text(row, 1)
|
||||
remarks = _row_cell_text(row, 2)
|
||||
progress_text = _row_cell_text(row, 4)
|
||||
progress = _row_progress_value(row, progress_text)
|
||||
rows.append(
|
||||
{
|
||||
"file_name": file_name,
|
||||
"remarks": remarks,
|
||||
"progress": progress,
|
||||
"progress_text": progress_text,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _keep_session_active(driver):
|
||||
|
||||
driver.execute_script(
|
||||
"""
|
||||
const scrollY = window.scrollY || document.documentElement.scrollTop || 0;
|
||||
window.scrollBy(0, 1);
|
||||
window.scrollTo(0, scrollY);
|
||||
document.dispatchEvent(new MouseEvent('mousemove', {
|
||||
bubbles: true,
|
||||
clientX: 1,
|
||||
clientY: 1
|
||||
}));
|
||||
document.dispatchEvent(new KeyboardEvent('keydown', {
|
||||
bubbles: true,
|
||||
key: 'Shift'
|
||||
}));
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _row_progress_value(row, fallback_text):
|
||||
|
||||
progress_bars = row.find_elements(By.XPATH, ".//*[contains(@class,'progress-bar')]")
|
||||
for progress_bar in progress_bars:
|
||||
aria_value = progress_bar.get_attribute("aria-valuenow")
|
||||
progress = _parse_progress(aria_value)
|
||||
if progress > 0:
|
||||
return progress
|
||||
|
||||
style = progress_bar.get_attribute("style") or ""
|
||||
progress = _parse_progress(style)
|
||||
if progress > 0:
|
||||
return progress
|
||||
|
||||
text = progress_bar.text.strip()
|
||||
progress = _parse_progress(text)
|
||||
if progress > 0:
|
||||
return progress
|
||||
|
||||
return _parse_progress(fallback_text)
|
||||
|
||||
|
||||
def _row_cell_text(row, zero_based_index):
|
||||
|
||||
cells = row.find_elements(By.XPATH, "./td")
|
||||
if len(cells) <= zero_based_index:
|
||||
return ""
|
||||
return cells[zero_based_index].text.strip()
|
||||
|
||||
|
||||
def _parse_progress(progress_text):
|
||||
|
||||
progress_text = str(progress_text or "")
|
||||
match = re.search(r"(\d+)\s*%", progress_text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
match = re.search(r"\b(\d{1,3})\b", progress_text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
if "100" in progress_text:
|
||||
return 100
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _row_fetch_button(row):
|
||||
|
||||
buttons = row.find_elements(By.XPATH, ".//*[self::button or self::a][normalize-space()='Fetch']")
|
||||
for button in buttons:
|
||||
if button.is_displayed() and button.is_enabled():
|
||||
return button
|
||||
return None
|
||||
|
||||
|
||||
def _current_status_text(driver):
|
||||
|
||||
by_id = driver.find_elements(By.ID, "lblStatus")
|
||||
for candidate in by_id:
|
||||
text = candidate.text.strip()
|
||||
if text:
|
||||
return text
|
||||
|
||||
candidates = driver.find_elements(
|
||||
By.XPATH,
|
||||
"//*[contains(normalize-space(),'Status :') or contains(normalize-space(),'Status:')]"
|
||||
)
|
||||
for candidate in candidates:
|
||||
text = candidate.text.strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _wait_for_refreshed_status(driver, previous_status, timeout_seconds):
|
||||
|
||||
wait = WebDriverWait(driver, timeout_seconds)
|
||||
return wait.until(
|
||||
lambda _driver: _refreshed_status_text(_driver, previous_status) or False
|
||||
)
|
||||
|
||||
|
||||
def _refreshed_status_text(driver, previous_status):
|
||||
|
||||
current_status = _current_status_text(driver).strip()
|
||||
if not current_status:
|
||||
return ""
|
||||
|
||||
normalized_current = " ".join(current_status.split()).lower()
|
||||
normalized_previous = " ".join((previous_status or "").split()).lower()
|
||||
terminal_tokens = ("reject", "submitted", "open", "accept")
|
||||
|
||||
if any(token in normalized_current for token in terminal_tokens):
|
||||
if normalized_current != normalized_previous:
|
||||
return current_status
|
||||
if "order id" in normalized_current:
|
||||
return current_status
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _click(driver, element):
|
||||
|
||||
try:
|
||||
element.click()
|
||||
except Exception:
|
||||
driver.execute_script("arguments[0].click();", element)
|
||||
Reference in New Issue
Block a user