623 lines
19 KiB
Python
623 lines
19 KiB
Python
import json
|
|
import re
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from selenium.common.exceptions import NoAlertPresentException, TimeoutException, WebDriverException
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
|
|
|
|
TM_ID_PATTERN = re.compile(r"([CS]\d{4})", re.IGNORECASE)
|
|
SCREEN_FOLDER_NAMES = {
|
|
"single_bid": "Single",
|
|
"block_bid": "Block",
|
|
"single_and_block_bid": "Single",
|
|
}
|
|
BUYSELL_SIDE = "BUYSELL"
|
|
BUYSELL_COMPONENT_SIDES = ("Buy", "Sell")
|
|
|
|
|
|
def get_folder_upload_entries(config):
|
|
|
|
product = config["product"]
|
|
bidding_config = config["exchange_bidding"]
|
|
upload_root = Path(
|
|
bidding_config.get("folder_upload_root")
|
|
or bidding_config.get("upload_input_root", "")
|
|
)
|
|
|
|
base_paths = _resolve_folder_upload_base_paths(upload_root, product, bidding_config)
|
|
entries = []
|
|
|
|
for base_path, side_name in base_paths:
|
|
for folder_path in sorted(path for path in base_path.iterdir() if path.is_dir()):
|
|
if folder_path.name.startswith("."):
|
|
continue
|
|
|
|
entries.append(
|
|
{
|
|
"folder_path": str(folder_path.resolve()),
|
|
"folder_name": folder_path.name,
|
|
"side": side_name,
|
|
"tm_id": _derive_tm_id_from_folder_name(folder_path.name),
|
|
}
|
|
)
|
|
|
|
if not entries:
|
|
searched_paths = [str(base_path) for base_path, _side_name in base_paths]
|
|
raise FileNotFoundError(f"No TM folders were found in {searched_paths}")
|
|
|
|
return entries
|
|
|
|
|
|
def upload_folder(driver, folder_path):
|
|
|
|
folder = Path(folder_path)
|
|
if not folder.exists() or not folder.is_dir():
|
|
raise FileNotFoundError(f"Upload folder not found: {folder}")
|
|
|
|
wait = WebDriverWait(driver, 30)
|
|
folder_input = wait.until(
|
|
lambda _driver: _folder_file_input(_driver) or False,
|
|
"No folder upload input was found. Expected an input[type='file'][webkitdirectory]."
|
|
)
|
|
driver.execute_script(
|
|
"""
|
|
arguments[0].style.display = 'block';
|
|
arguments[0].style.visibility = 'visible';
|
|
arguments[0].style.opacity = 1;
|
|
arguments[0].style.height = '1px';
|
|
arguments[0].style.width = '1px';
|
|
arguments[0].removeAttribute('disabled');
|
|
arguments[0].value = '';
|
|
""",
|
|
folder_input
|
|
)
|
|
folder_input.send_keys(str(folder.resolve()))
|
|
|
|
_accept_upload_confirmation_if_present(driver)
|
|
wait.until(lambda _driver: _folder_file_rows(_driver) or False)
|
|
|
|
|
|
def submit_all_and_wait_for_progress(
|
|
driver,
|
|
timeout_seconds=300,
|
|
poll_seconds=2,
|
|
keep_alive_interval_seconds=30,
|
|
progress_log_interval_seconds=60,
|
|
progress_log_label="folder_upload",
|
|
continue_when_last_file_complete=False,
|
|
):
|
|
|
|
wait = WebDriverWait(driver, 30)
|
|
submit_all_button = wait.until(
|
|
EC.element_to_be_clickable(
|
|
(
|
|
By.XPATH,
|
|
"//table[@id='tblFileList']//button[normalize-space()='Submit All']"
|
|
"|//button[normalize-space()='Submit All' or contains(normalize-space(),'Submit All')]"
|
|
)
|
|
)
|
|
)
|
|
driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", submit_all_button)
|
|
_click(driver, submit_all_button)
|
|
|
|
submit_all_clicked_at = datetime.now()
|
|
submit_all_monotonic = time.monotonic()
|
|
progress_rows = _wait_for_all_progress_100(
|
|
driver,
|
|
timeout_seconds,
|
|
poll_seconds,
|
|
keep_alive_interval_seconds,
|
|
progress_log_interval_seconds,
|
|
progress_log_label,
|
|
continue_when_last_file_complete,
|
|
)
|
|
all_progress_100_at = datetime.now()
|
|
incomplete_rows = [
|
|
row for row in progress_rows
|
|
if row.get("progress", 0) < 100
|
|
]
|
|
|
|
return {
|
|
"submit_all_clicked_at": submit_all_clicked_at.isoformat(timespec="seconds"),
|
|
"all_progress_100_at": all_progress_100_at.isoformat(timespec="seconds"),
|
|
"completion_condition": _completion_condition(progress_rows),
|
|
"duration_seconds": round(time.monotonic() - submit_all_monotonic, 3),
|
|
"file_count": len(progress_rows),
|
|
"incomplete_count": len(incomplete_rows),
|
|
"incomplete_files": incomplete_rows,
|
|
"files": progress_rows,
|
|
}
|
|
|
|
|
|
def fetch_folder_upload_statuses(driver, timeout_seconds=30):
|
|
|
|
results = []
|
|
rows = _folder_file_rows(driver)
|
|
|
|
for index, row in enumerate(rows, start=1):
|
|
file_name = _row_cell_text(row, 1)
|
|
remarks = _row_cell_text(row, 2)
|
|
progress_text = _row_cell_text(row, 4)
|
|
progress = _row_progress_value(row, progress_text)
|
|
|
|
if progress < 100:
|
|
results.append(
|
|
{
|
|
"index": index,
|
|
"file_name": file_name,
|
|
"status": "upload_incomplete",
|
|
"raw_status": remarks,
|
|
"remarks": remarks,
|
|
"progress": progress,
|
|
"progress_text": progress_text,
|
|
"fetch_attempted": False,
|
|
}
|
|
)
|
|
print(
|
|
f"Skipping fetch for incomplete upload row: "
|
|
f"file={file_name}, progress={progress}%, remarks={remarks}"
|
|
)
|
|
continue
|
|
|
|
previous_status = _current_status_text(driver)
|
|
fetch_button = _row_fetch_button(row)
|
|
|
|
if fetch_button is None:
|
|
results.append(
|
|
{
|
|
"index": index,
|
|
"file_name": file_name,
|
|
"status": "missing_fetch_button",
|
|
"raw_status": "",
|
|
"remarks": remarks,
|
|
"progress": progress,
|
|
"progress_text": progress_text,
|
|
"fetch_attempted": False,
|
|
}
|
|
)
|
|
continue
|
|
|
|
driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", fetch_button)
|
|
_click(driver, fetch_button)
|
|
|
|
try:
|
|
status_text = _wait_for_refreshed_status(driver, previous_status, timeout_seconds)
|
|
except TimeoutException:
|
|
status_text = _current_status_text(driver)
|
|
results.append(
|
|
{
|
|
"index": index,
|
|
"file_name": file_name,
|
|
"status": "fetch_timeout",
|
|
"raw_status": status_text,
|
|
"previous_status": previous_status,
|
|
"remarks": remarks,
|
|
"progress": progress,
|
|
"progress_text": progress_text,
|
|
"fetch_attempted": True,
|
|
}
|
|
)
|
|
print(
|
|
f"Fetch status timeout for uploaded row: "
|
|
f"file={file_name}, progress={progress}%, "
|
|
f"previous_status={previous_status}, current_status={status_text}"
|
|
)
|
|
continue
|
|
|
|
lowered_status = status_text.lower()
|
|
if "reject" in lowered_status:
|
|
status = "rejected"
|
|
elif "open" in lowered_status or "accept" in lowered_status or "submitted" in lowered_status:
|
|
status = "open"
|
|
else:
|
|
status = "unknown"
|
|
|
|
results.append(
|
|
{
|
|
"index": index,
|
|
"file_name": file_name,
|
|
"status": status,
|
|
"raw_status": status_text,
|
|
"remarks": remarks,
|
|
"progress": progress,
|
|
"progress_text": progress_text,
|
|
"fetch_attempted": True,
|
|
}
|
|
)
|
|
|
|
return results
|
|
|
|
|
|
def save_folder_upload_log(test_context, market, entry, timing, fetch_results):
|
|
|
|
folder = Path(test_context.get("log_folder", test_context["folder"]))
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
side_name = entry.get("side", "")
|
|
log_name_parts = ["folder_upload", market]
|
|
if side_name:
|
|
log_name_parts.append(side_name)
|
|
log_name_parts.append(entry["tm_id"])
|
|
path = folder / f"{'_'.join(log_name_parts)}.json"
|
|
rejected_files = [item for item in fetch_results if item["status"] == "rejected"]
|
|
unresolved_files = [
|
|
item for item in fetch_results
|
|
if item["status"] in {"upload_incomplete", "fetch_timeout", "missing_fetch_button", "unknown"}
|
|
]
|
|
payload = {
|
|
"market": market,
|
|
"side": side_name,
|
|
"tm_id": entry["tm_id"],
|
|
"folder_name": entry["folder_name"],
|
|
"folder_path": entry["folder_path"],
|
|
"timing": timing,
|
|
"fetch_results": fetch_results,
|
|
"rejected_files": rejected_files,
|
|
"unresolved_files": unresolved_files,
|
|
}
|
|
|
|
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
return str(path)
|
|
|
|
|
|
def _resolve_folder_upload_base_paths(upload_root, product, bidding_config):
|
|
|
|
if not upload_root:
|
|
raise ValueError("Set exchange_bidding.folder_upload_root or upload_input_root in config/runtime.json.")
|
|
|
|
base_path = upload_root.resolve()
|
|
if not base_path.exists():
|
|
raise FileNotFoundError(f"Folder upload root does not exist: {base_path}")
|
|
|
|
market_path = base_path / product["trading_market"]
|
|
if not market_path.exists():
|
|
available_markets = [
|
|
path.name for path in base_path.iterdir()
|
|
if path.is_dir() and not path.name.startswith(".")
|
|
]
|
|
raise FileNotFoundError(
|
|
f"Folder upload market path does not exist: {market_path}. "
|
|
f"Available market folders under {base_path}: {available_markets}. "
|
|
"Set active_selection.trading_market to an available market, "
|
|
"or create folders for every market when using ALL."
|
|
)
|
|
|
|
base_path = market_path
|
|
|
|
screen_folder = SCREEN_FOLDER_NAMES.get(product.get("bidding_screen", "single_bid"))
|
|
if screen_folder:
|
|
screen_path = base_path / screen_folder
|
|
if not screen_path.exists():
|
|
raise FileNotFoundError(f"Folder upload screen path does not exist: {screen_path}")
|
|
base_path = screen_path
|
|
|
|
sides = bidding_config.get("folder_upload_sides") or [bidding_config.get("folder_upload_side", "Buy")]
|
|
if isinstance(sides, str):
|
|
sides = [sides]
|
|
|
|
expanded_sides = _expand_folder_upload_sides(base_path, sides)
|
|
existing_side_paths = [
|
|
(base_path / side, side)
|
|
for side in expanded_sides
|
|
if (base_path / side).exists()
|
|
]
|
|
if existing_side_paths:
|
|
return existing_side_paths
|
|
|
|
raise FileNotFoundError(
|
|
f"Folder upload side path does not exist under {base_path}. "
|
|
f"Configured folder_upload_sides: {sides}. Expanded sides: {expanded_sides}"
|
|
)
|
|
|
|
|
|
def _expand_folder_upload_sides(base_path, sides):
|
|
|
|
expanded_sides = []
|
|
for side in sides:
|
|
side_name = str(side).strip()
|
|
if not side_name:
|
|
continue
|
|
|
|
if side_name.upper() != BUYSELL_SIDE:
|
|
expanded_sides.append(side_name)
|
|
continue
|
|
|
|
if (base_path / BUYSELL_SIDE).exists():
|
|
expanded_sides.append(BUYSELL_SIDE)
|
|
continue
|
|
|
|
expanded_sides.extend(BUYSELL_COMPONENT_SIDES)
|
|
|
|
return list(dict.fromkeys(expanded_sides))
|
|
|
|
|
|
def _derive_tm_id_from_folder_name(folder_name):
|
|
|
|
match = TM_ID_PATTERN.search(folder_name)
|
|
if not match:
|
|
raise ValueError(f"Could not derive TM ID from folder name '{folder_name}'.")
|
|
return match.group(1).upper()
|
|
|
|
|
|
def _folder_file_input(driver):
|
|
|
|
inputs = driver.find_elements(
|
|
By.XPATH,
|
|
"//input[@type='file' and (@webkitdirectory or @id='fileInput' or @name='fileInput')]"
|
|
)
|
|
if inputs:
|
|
return inputs[0]
|
|
|
|
return None
|
|
|
|
|
|
def _accept_upload_confirmation_if_present(driver):
|
|
|
|
try:
|
|
alert = driver.switch_to.alert
|
|
alert.accept()
|
|
except NoAlertPresentException:
|
|
pass
|
|
|
|
|
|
def _folder_file_rows(driver):
|
|
|
|
rows = driver.find_elements(By.XPATH, "//table[@id='tblFileList']//tbody/tr[td]")
|
|
return [row for row in rows if row.is_displayed()]
|
|
|
|
|
|
def _wait_for_all_progress_100(
|
|
driver,
|
|
timeout_seconds,
|
|
poll_seconds,
|
|
keep_alive_interval_seconds,
|
|
progress_log_interval_seconds,
|
|
progress_log_label,
|
|
continue_when_last_file_complete,
|
|
):
|
|
|
|
end_time = time.monotonic() + timeout_seconds
|
|
next_keep_alive = time.monotonic() + keep_alive_interval_seconds
|
|
next_progress_log = time.monotonic() + progress_log_interval_seconds
|
|
last_rows = []
|
|
|
|
while time.monotonic() < end_time:
|
|
try:
|
|
rows = _read_progress_rows(driver)
|
|
except WebDriverException as exc:
|
|
raise RuntimeError(
|
|
"Browser session disconnected while waiting for folder upload progress. "
|
|
"This usually means Chrome was closed, crashed, or Windows/Chrome "
|
|
"suspended the renderer during the long upload. Re-run with the browser "
|
|
"windows kept open and the PC sleep/lock settings disabled for the upload window."
|
|
) from exc
|
|
|
|
if rows:
|
|
last_rows = rows
|
|
if all(row["progress"] >= 100 for row in rows):
|
|
_print_progress_summary(progress_log_label, rows)
|
|
return rows
|
|
if continue_when_last_file_complete and _last_file_is_complete(rows):
|
|
_print_progress_summary(progress_log_label, rows)
|
|
_print_incomplete_rows(progress_log_label, rows)
|
|
return rows
|
|
|
|
if keep_alive_interval_seconds and time.monotonic() >= next_keep_alive:
|
|
_keep_session_active(driver)
|
|
next_keep_alive = time.monotonic() + keep_alive_interval_seconds
|
|
|
|
if progress_log_interval_seconds and time.monotonic() >= next_progress_log:
|
|
_print_progress_summary(progress_log_label, last_rows)
|
|
next_progress_log = time.monotonic() + progress_log_interval_seconds
|
|
|
|
time.sleep(poll_seconds)
|
|
|
|
raise TimeoutException(
|
|
"Folder upload progress did not reach 100% for all files. "
|
|
f"Last progress rows: {last_rows}"
|
|
)
|
|
|
|
|
|
def _print_progress_summary(label, rows):
|
|
|
|
if not rows:
|
|
print(f"Folder upload progress [{label}]: waiting for progress rows...")
|
|
return
|
|
|
|
completed = sum(1 for row in rows if row["progress"] >= 100)
|
|
total = len(rows)
|
|
max_progress = max(row["progress"] for row in rows)
|
|
min_progress = min(row["progress"] for row in rows)
|
|
print(
|
|
f"Folder upload progress [{label}]: "
|
|
f"{completed}/{total} files at 100%, "
|
|
f"min={min_progress}%, max={max_progress}%"
|
|
)
|
|
|
|
|
|
def _print_incomplete_rows(label, rows):
|
|
|
|
incomplete_rows = [
|
|
row for row in rows
|
|
if row.get("progress", 0) < 100
|
|
]
|
|
if not incomplete_rows:
|
|
return
|
|
|
|
print(
|
|
f"Folder upload progress [{label}]: continuing because last file is 100%. "
|
|
"Incomplete/error rows: "
|
|
+ "; ".join(
|
|
f"{row.get('file_name', '')} progress={row.get('progress', 0)}% "
|
|
f"remarks={row.get('remarks', '')}"
|
|
for row in incomplete_rows
|
|
)
|
|
)
|
|
|
|
|
|
def _last_file_is_complete(rows):
|
|
|
|
return bool(rows) and rows[-1].get("progress", 0) >= 100
|
|
|
|
|
|
def _completion_condition(rows):
|
|
|
|
if all(row.get("progress", 0) >= 100 for row in rows):
|
|
return "all_files_100"
|
|
if _last_file_is_complete(rows):
|
|
return "last_file_100_with_incomplete_rows"
|
|
return "incomplete"
|
|
|
|
|
|
def _read_progress_rows(driver):
|
|
|
|
rows = []
|
|
for row in _folder_file_rows(driver):
|
|
file_name = _row_cell_text(row, 1)
|
|
remarks = _row_cell_text(row, 2)
|
|
progress_text = _row_cell_text(row, 4)
|
|
progress = _row_progress_value(row, progress_text)
|
|
rows.append(
|
|
{
|
|
"file_name": file_name,
|
|
"remarks": remarks,
|
|
"progress": progress,
|
|
"progress_text": progress_text,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def _keep_session_active(driver):
|
|
|
|
driver.execute_script(
|
|
"""
|
|
const scrollY = window.scrollY || document.documentElement.scrollTop || 0;
|
|
window.scrollBy(0, 1);
|
|
window.scrollTo(0, scrollY);
|
|
document.dispatchEvent(new MouseEvent('mousemove', {
|
|
bubbles: true,
|
|
clientX: 1,
|
|
clientY: 1
|
|
}));
|
|
document.dispatchEvent(new KeyboardEvent('keydown', {
|
|
bubbles: true,
|
|
key: 'Shift'
|
|
}));
|
|
"""
|
|
)
|
|
|
|
|
|
def _row_progress_value(row, fallback_text):
|
|
|
|
progress_bars = row.find_elements(By.XPATH, ".//*[contains(@class,'progress-bar')]")
|
|
for progress_bar in progress_bars:
|
|
aria_value = progress_bar.get_attribute("aria-valuenow")
|
|
progress = _parse_progress(aria_value)
|
|
if progress > 0:
|
|
return progress
|
|
|
|
style = progress_bar.get_attribute("style") or ""
|
|
progress = _parse_progress(style)
|
|
if progress > 0:
|
|
return progress
|
|
|
|
text = progress_bar.text.strip()
|
|
progress = _parse_progress(text)
|
|
if progress > 0:
|
|
return progress
|
|
|
|
return _parse_progress(fallback_text)
|
|
|
|
|
|
def _row_cell_text(row, zero_based_index):
|
|
|
|
cells = row.find_elements(By.XPATH, "./td")
|
|
if len(cells) <= zero_based_index:
|
|
return ""
|
|
return cells[zero_based_index].text.strip()
|
|
|
|
|
|
def _parse_progress(progress_text):
|
|
|
|
progress_text = str(progress_text or "")
|
|
match = re.search(r"(\d+)\s*%", progress_text)
|
|
if match:
|
|
return int(match.group(1))
|
|
|
|
match = re.search(r"\b(\d{1,3})\b", progress_text)
|
|
if match:
|
|
return int(match.group(1))
|
|
|
|
if "100" in progress_text:
|
|
return 100
|
|
|
|
return 0
|
|
|
|
|
|
def _row_fetch_button(row):
|
|
|
|
buttons = row.find_elements(By.XPATH, ".//*[self::button or self::a][normalize-space()='Fetch']")
|
|
for button in buttons:
|
|
if button.is_displayed() and button.is_enabled():
|
|
return button
|
|
return None
|
|
|
|
|
|
def _current_status_text(driver):
|
|
|
|
by_id = driver.find_elements(By.ID, "lblStatus")
|
|
for candidate in by_id:
|
|
text = candidate.text.strip()
|
|
if text:
|
|
return text
|
|
|
|
candidates = driver.find_elements(
|
|
By.XPATH,
|
|
"//*[contains(normalize-space(),'Status :') or contains(normalize-space(),'Status:')]"
|
|
)
|
|
for candidate in candidates:
|
|
text = candidate.text.strip()
|
|
if text:
|
|
return text
|
|
return ""
|
|
|
|
|
|
def _wait_for_refreshed_status(driver, previous_status, timeout_seconds):
|
|
|
|
wait = WebDriverWait(driver, timeout_seconds)
|
|
return wait.until(
|
|
lambda _driver: _refreshed_status_text(_driver, previous_status) or False
|
|
)
|
|
|
|
|
|
def _refreshed_status_text(driver, previous_status):
|
|
|
|
current_status = _current_status_text(driver).strip()
|
|
if not current_status:
|
|
return ""
|
|
|
|
normalized_current = " ".join(current_status.split()).lower()
|
|
normalized_previous = " ".join((previous_status or "").split()).lower()
|
|
terminal_tokens = ("reject", "submitted", "open", "accept")
|
|
|
|
if any(token in normalized_current for token in terminal_tokens):
|
|
if normalized_current != normalized_previous:
|
|
return current_status
|
|
if "order id" in normalized_current:
|
|
return current_status
|
|
|
|
return ""
|
|
|
|
|
|
def _click(driver, element):
|
|
|
|
try:
|
|
element.click()
|
|
except Exception:
|
|
driver.execute_script("arguments[0].click();", element)
|