S.
# ============================================================================== # FILE: C:\Scripts\MegaQueueEditor.py # DESCRIPTION: TURBO HYBRID PRE-PROCESSOR (Pre-Flight & ETA) # ============================================================================== import tkinter as tk from tkinter import ttk, messagebox, scrolledtext import json import os import re import urllib.request import urllib.error import threading import subprocess import time from datetime import datetime LANG_MAP = { "de": "German", "en": "English", "fr": "French", "ru-old": "Russian (strictly use pre-1918 historical orthography)", "ru": "Russian (use modern post-1918 orthography)", "es": "Spanish", "pt": "Portuguese", "it": "Italian", "nl": "Dutch", "sv": "Swedish", "da": "Danish", "hu": "Hungarian", "cs": "Czech", "pl": "Polish", "ja": "Japanese", "zh": "Chinese", "und": "Unknown/Mixed" } CORE_TEXT = """You are an expert archival OCR engine for the "Sigmund Freud Edition", processing historical printed texts. Output ONLY clean HTML fragments. No JSON wrappers, no markdown code blocks, no explanations. Your response must start directly with the first open HTML bracket (<). Absolutely no conversational text or structural analysis is permitted. ## STRICT ALIGNMENT & GEOMETRY RULES - ZERO TOLERANCE ALIGNMENT: Every single
,
,
, or other structural element MUST strictly be forced to align="left". Absolutely NO align="center", class="central", or right-alignment is permitted. - RUNNING HEADERS (SKIP): Identify running text headings at the very top of the page, but DO NOT output them. Skip them entirely to provide a clean text stream. (Note: This does not apply to page numbers). - NO INDENTATION: DO NOT USE ANY spaces to replicate visual indents. Ignore all original physical indentations and start every paragraph and line flush left. - HORIZONTAL ARTIFACTS & DASHES: Never print long, repetitive sequences of hyphens, underscores, or dashes to reproduce visual dividing lines, print boundaries, or scanning artifacts. If a literal typographic divider line is present, use a single
tag. If it is a minor layout artifact or tracking dash sequence, ignore it entirely. - DOT LEADERS (FÜLLPUNKTE): In tables of contents or lists, NEVER transcribe long sequences of periods (dots) connecting text to page numbers. Omit the dots entirely and replace them with a single space. - PRINTER'S MARKS (BOGENSIGNATUREN): Do NOT transcribe catchwords (Kustoden), signature marks (Bogennummern like '3*'), printer's marks, or periodical imprints located at the very bottom margin. Skip them entirely. - TYPOGRAPHY & UNICODE: Output correct typographic characters. Do not use generic ASCII substitutes. * Quotation marks: Use typographic double and single quotes appropriate for the language context (Unicode U+201E, U+201C, etc.). * Apostrophe: Use typographic apostrophe (Unicode U+2019). * Dashes (Gedankenstriche): STANDARDIZE all dashes used as sentence separators to the Halbgeviertstrich / En-dash (Unicode U+2013). NEVER use the longer Em-dash (U+2014) or a simple hyphen (-). * Percent Signs: Old Fraktur percent signs often resemble '0/0' with a horizontal bar. Transcribe them simply as the standard % symbol. * Ellipsis: Use horizontal ellipsis (Unicode U+2026), never three periods. * Special Symbols: Use correct Unicode symbols for fractions (1/4, 1/2, 3/4) and the 'gestorben' dagger cross (Unicode U+2020). * Overlines (Gemination): If a letter has a printed overline, wrap the letter exactly like this: m. - ILLEGIBLE TEXT: Do not hallucinate or guess. If a character or word is completely unreadable due to smudges or print errors, replace each unreadable character with an inverted question mark (Unicode U+00BF). - NO LAYOUT TABLES: NEVER use , , or
tags to force elements side-by-side. Output elements sequentially. - Do NOT use a global container or wrapper div element. - LINE BREAKS: Map exact visual line breaks using exactly one
tag after each visible line. - Always use what is visible in the original. Use exact OCR readings, capturing deletions () and superscripts () exactly where they appear.""" PROMPT_MODULES = { "Seitenzahlen (Drupal)": "### OVERRIDE: PAGE NUMBERS\n- Extract the page number and place it at the very top of your output (flush left), even if it is printed on the right side of the physical page. Wrap it exactly as:...", "Kopfzeilen ausgeben (Running Headers)": "### OVERRIDE: RUNNING HEADERS\n- Extract the running header (if present) and place it sequentially on the next line immediately below the page number, wrapped exactly as:...", "Gesperrt (Smallcaps)": "### OVERRIDE: GESPERRT (SPERRDRUCK / LETTER-SPACING)\n- HISTORICAL RULE: In German Fraktur texts, emphasized words and author names (e.g., R a d e s t o c k, V o l k e l t, G r i e s i n g e r) are ALWAYS emphasized using wide letter-spacing (Sperrdruck).\n- VISUAL PATTERN: Look for words with unnatural blank spaces between EVERY single letter.\n- STRIP THE SPACES: You MUST remove the spaces and wrap the word like this: Radestock. NEVER output R a d e s t o c k.\n- CRITICAL: Do NOT tag these letter-spaced names asor Italics! Use the smallcaps span.", "Kursiv (Antiqua/Italics)": "### OVERRIDE: KURSIV (LATIN LETTERS / ANTIQUA)\n- Identify text printed in standard Latin/Antiqua font (different typeface than the surrounding Fraktur). This was historically used for foreign words or Latin terms.\n- ONLY use ... for actual font style changes (Latin/Antiqua letters).\n- FATAL ERROR PREVENTION: Do NOT use for German author names just because they are emphasized. A spaced-out name in Fraktur is SPERRDRUCK (use smallcaps), absolutely NOT Kursiv.", "Zitate Inline (Drupal)": "### OVERRIDE: INLINE QUOTES\n- Identify short quotations or highlighted phrases within the regular text flow that are enclosed in quotation marks (whether German, French, or English style).\n- Wrap the quoted text AND its quotation marks exactly like this: \"Quote goes here\"\n- CRITICAL: This is only for short, inline text. Do NOT use this for large, indented blockquotes (which use).", "Fußnoten (Drupal)": "### OVERRIDE: FOOTNOTES\n- REFERENCE IN TEXT: Identify footnote indicators (e.g., numbers or asterisks) within the main text flow. You MUST force a superscript tag inside the span. Wrap them EXACTLY like this: 1).\n- FOOTNOTE BODY: Identify the footnote text at the bottom of the page. If a single footnote contains multiple paragraphs, EVERY paragraph of that footnote must be wrapped in.\n- CRITICAL DRUPAL FORMAT: You MUST wrap the footnote number here with the same tag as in the main text. Do NOT put any space between this number tag and the footnote text. Wrap the ENTIRE entry in.", "Endnoten (Drupal)": "### OVERRIDE: ENDNOTES\n- REFERENCE IN TEXT: Identify endnote indicators within the main text flow (if present). You MUST force a superscript tag inside the span. Wrap them EXACTLY like this: 1).\n- ENDNOTE BODY: Identify terminal chapter annotations, reference lists, or end-of-text notes.\n- CRITICAL DRUPAL FORMAT: You MUST wrap the endnote number here with the same tag as in the main text. Do NOT put any space between this number tag and the endnote text. Wrap the ENTIRE entry in.", "Blockquotes & Poetry": "### OVERRIDE: BLOCKQUOTES & POETRY\n- Identify extended prose quotations, indented blocks, poetry, stanzas, or verse extracts that are visually set off and indented from the main body text.\n- Wrap each block or stanza cleanly EXACTLY like this:...\n- Map the internal lines using precisetags at the end of each line.\n- CRITICAL: Do NOT use standard paragraph HTML spaces for indents within a blockquote; thetag automatically handles the block indentation.", "Marginalia": "### OVERRIDE: MARGINALIA & TEXT FLOW\n- Identify notes or headings physically located in the wide outer margins, clearly separated from the central main text column.\n- CRITICAL ANTI-WEAVING RULE: Read the main text body as one continuous semantic stream, REGARDLESS of the language. NEVER interleave or weave marginal words into the middle of a main text sentence just because they sit on the same horizontal line in the scan.\n- Extract the marginalia completely from the main flow and output it as a standalone paragraph:...\n- Place thisblock sequentially IMMEDIATELY BEFORE theblock of the main text it refers to.", "Fortlaufende Absätze (ff)": "### OVERRIDE: CONTINUATIONS (SEITENÜBERGÄNGE)\n- Check the very first word of the main body text immediately below the header area.\n- If it starts with a lowercase letter, it is a paragraph continued from the previous page.\n- CRITICAL: You MUST open this specific paragraph exactly like this:\n- Ignore any visual indents for this continued paragraph.", "Silbentrennung (Hyphens)": "### OVERRIDE: HYPHENATION vs. SUSPENSION HYPHENS\n- END-OF-LINE HYPHENS: If a word is split at the far right margin of the page, wrap the printed end-of-line hyphen like this: -\n- NEVER insert atag for hyphens in the middle of a continuous line.\n- SUSPENSION HYPHENS (Ergänzungsstriche): Hyphens followed by a space in the middle of a sentence (e.g., 'Zwangs- und Wahnideen') are NOT line breaks! Output them exactly as printed on the same line. Do NOT wrap them, do NOT add atag.\n- CRITICAL ANTI-OMISSION RULE: Never stop reading a line just because you encounter a hyphen! You must transcribe the entire line up to the right margin." } DEFAULT_MODULE_STATES = { "Seitenzahlen (Drupal)": True, "Kopfzeilen ausgeben (Running Headers)": False, "Gesperrt (Smallcaps)": True, "Kursiv (Antiqua/Italics)": True, "Zitate Inline (Drupal)": False, "Fußnoten (Drupal)": True, "Endnoten (Drupal)": False, "Blockquotes & Poetry": True, "Marginalia": True, "Fortlaufende Absätze (ff)": True, "Silbentrennung (Hyphens)": True } class MegaWorkshopV1(tk.Tk): def __init__(self): super().__init__() self.title("MEGA Queue Editor - Asynchronous Ticket Pipelining") self.geometry("1480x920") self.minsize(1200, 700) self.username = os.environ.get("USERNAME", "DefaultUser") self.base_dir = f"C:/OCR/{self.username}" os.makedirs(self.base_dir, exist_ok=True) self.tickets_dir = f"{self.base_dir}/tickets" os.makedirs(self.tickets_dir, exist_ok=True) self.status_file = f"{self.base_dir}/status.txt" self.omitted_file = f"{self.base_dir}/omittedpages.txt" self.entries = [] self.module_vars = {} self.main_frame = ttk.Frame(self, padding=15) self.main_frame.pack(fill="both", expand=True) self.setup_task_queue() self.setup_workshop_panel() self.add_row() def setup_task_queue(self): task_header = ttk.LabelFrame(self.main_frame, text="Task Queue (Tragen Sie einfach die Node-ID oder URL ein)", padding=8) task_header.pack(fill="both", expand=True, pady=(0, 10)) self.canvas = tk.Canvas(task_header, highlightthickness=0) self.scrollbar = ttk.Scrollbar(task_header, orient="vertical", command=self.canvas.yview) self.scrollable_inner = ttk.Frame(self.canvas) self.scroll_window = self.canvas.create_window((0, 0), window=self.scrollable_inner, anchor="nw") self.canvas.configure(yscrollcommand=self.scrollbar.set) self.canvas.pack(side="left", fill="both", expand=True) self.scrollbar.pack(side="right", fill="y") self.scrollable_inner.bind("", lambda e: self.canvas.configure(scrollregion=self.canvas.bbox("all"))) self.canvas.bind("", lambda e: self.canvas.itemconfig(self.scroll_window, width=e.width)) def setup_workshop_panel(self): ws_frame = ttk.LabelFrame(self.main_frame, text="Prompt Workshop & Execution", padding=10) ws_frame.pack(fill="x", side="bottom") left_f = ttk.Frame(ws_frame) left_f.pack(side="left", fill="y", padx=(0, 15)) mod_f = ttk.LabelFrame(left_f, text="Prompt Modules", padding=8) mod_f.pack(fill="y", pady=(0, 10)) for label in PROMPT_MODULES.keys(): var = tk.BooleanVar(value=DEFAULT_MODULE_STATES.get(label, True)) self.module_vars[label] = var ttk.Checkbutton(mod_f, text=label, variable=var, command=self.refresh_prompt).pack(anchor="w") edit_f = ttk.Frame(ws_frame) edit_f.pack(side="left", fill="both", expand=True) pre_f = ttk.Frame(edit_f) pre_f.pack(fill="x", pady=(0, 5)) self.tokens_var = tk.StringVar(value="8192") ttk.Entry(pre_f, textvariable=self.tokens_var, width=10).pack(side="right") ttk.Label(pre_f, text="Max Tokens:").pack(side="right", padx=5) self.prompt_text = scrolledtext.ScrolledText(edit_f, height=24, font=("Consolas", 10)) self.prompt_text.pack(fill="x", pady=5) btn_f = ttk.Frame(edit_f) btn_f.pack(fill="x") ttk.Button(btn_f, text="+ New Row", command=self.add_row).pack(side="left") ttk.Button(btn_f, text="Paste from Clipboard", command=self.paste_from_clipboard).pack(side="left", padx=8) ttk.Button(btn_f, text="Clear Queue", command=self.clear_queue).pack(side="left") start_btn = tk.Button(btn_f, text="🚀 START MEGA PIPELINE", bg="#A3BE8C", fg="black", font=("Segoe UI", 10, "bold"), command=self.start_mega_pipeline) start_btn.pack(side="right", padx=10) self.refresh_prompt() def refresh_prompt(self): core = CORE_TEXT if "Kopfzeilen ausgeben (Running Headers)" in self.module_vars and self.module_vars["Kopfzeilen ausgeben (Running Headers)"].get(): core = core.replace("- RUNNING HEADERS (SKIP): Identify running text headings at the very top of the page, but DO NOT output them. Skip them entirely to provide a clean text stream. (Note: This does not apply to page numbers).\n", "") prompt = core + "\n\n" active_modules = [PROMPT_MODULES[lbl] for lbl, var in self.module_vars.items() if var.get()] if active_modules: prompt += "## CRITICAL OVERRIDES (STRICT HIERARCHY: Footnotes/Endnotes supersedes Blockquotes supersedes Marginalia)\n" prompt += "\n\n".join(active_modules) self.prompt_text.delete("1.0", tk.END) self.prompt_text.insert("1.0", prompt.strip()) def add_row(self, url="", pages="", lang="de", model="gemini-2.5-pro"): row = ttk.Frame(self.scrollable_inner) row.pack(fill="x", pady=3) u_v = tk.StringVar(value=url) p_v = tk.StringVar(value=pages) l_v = tk.StringVar(value=lang) m_v = tk.StringVar(value=model) ttk.Entry(row, textvariable=u_v).pack(side="left", fill="x", expand=True, padx=3) ttk.Entry(row, textvariable=p_v, width=12).pack(side="left", padx=3) ttk.Combobox(row, textvariable=l_v, values=list(LANG_MAP.keys()), width=8).pack(side="left", padx=3) ttk.Combobox(row, textvariable=m_v, values=["gemini-2.5-flash", "gemini-2.5-pro"], width=25).pack(side="left", padx=3) ttk.Button(row, text="X", width=3, command=lambda r=row: self.remove_row(r)).pack(side="left", padx=3) self.entries.append({"f": row, "u": u_v, "p": p_v, "l": l_v, "m": m_v}) def paste_from_clipboard(self): try: items = [u.strip() for u in self.clipboard_get().split('\n') if u.strip()] for item in items: if item.startswith('http') or item.isdigit(): self.add_row(url=item) except: pass def clear_queue(self): if messagebox.askyesno("Clear Queue", "Delete all entries?"): for e in self.entries[:]: e["f"].destroy() self.entries.clear() self.add_row() def remove_row(self, row_frame): row_frame.destroy() self.entries = [e for e in self.entries if e["f"] != row_frame] def clear_old_tickets(self): for f in os.listdir(self.tickets_dir): if f.endswith(".ticket"): try: os.remove(os.path.join(self.tickets_dir, f)) except: pass def write_ticket(self, node_id): open(f"{self.tickets_dir}/{node_id}.ticket", 'w').close() def write_done_ticket(self): open(f"{self.tickets_dir}/DONE.ticket", 'w').close() def start_mega_pipeline(self): if messagebox.askyesno("Mega Pipeline", "Lokale Vorverarbeitung starten? Sie können das UI.Vision Makro starten, sobald das erste Bild heruntergeladen wird!"): self.log_win = tk.Toplevel(self) self.log_win.title("Mega Pipeline Execution Log") self.log_win.geometry("800x600") self.log_text = scrolledtext.ScrolledText(self.log_win, bg="#2E3440", fg="#D8DEE9", font=("Consolas", 10)) self.log_text.pack(fill="both", expand=True, padx=10, pady=10) threading.Thread(target=self.run_pipeline_thread, daemon=True).start() def log(self, message): timestamp = datetime.now().strftime("%H:%M:%S") self.log_text.insert(tk.END, f"[{timestamp}] {message}\n") self.log_text.see(tk.END) def write_status(self, q_idx, q_len, g_idx, g_tot, l_idx, l_tot): try: with open(self.status_file, "w", encoding="utf-8-sig") as f: f.write(f"{q_idx},{q_len},{g_idx},{g_tot},{l_idx},{l_tot}") except: pass def log_omission(self, node_id, reason): try: timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") with open(self.omitted_file, "a", encoding="utf-8") as f: f.write(f"[{timestamp}] Node/SubNode: {node_id} | Reason: {reason}\n") except: pass def run_pipeline_thread(self): self.log("🚀 PRE-FLIGHT CHECK: Zähle alle Seiten zusammen...") self.clear_old_tickets() self.refresh_prompt() base_prompt = self.prompt_text.get("1.0", tk.END).strip() valid_entries = [e for e in self.entries if e["u"].get().strip()] total_queue_len = len(valid_entries) # ── PRE-FLIGHT LOOP (Sammelt alle Daten für ETA) ── queue_tasks = [] global_total_pages = 0 for q_idx, entry in enumerate(valid_entries): raw_input = entry["u"].get().strip() pages_input = entry["p"].get().strip() lang_code = entry["l"].get().strip() model = entry["m"].get().strip() max_tokens = self.tokens_var.get() match = re.search(r'(\d+)', raw_input) if not match: continue main_nid = match.group(1) try: # Nutzt die Core API um den Filter-Bock auszutricksen! url_manif = f"https://freudedition.net/node/{main_nid}?_format=json" req = urllib.request.Request(url_manif, headers={'User-Agent': 'Mozilla/5.0'}) resp = urllib.request.urlopen(req, timeout=45) data = json.loads(resp.read()) seiten = data.get('field_seiten', []) if isinstance(data, dict) else [] if not seiten: continue sub_nids = [s['target_id'] for s in seiten if 'target_id' in s] if not pages_input or pages_input.lower() == 'all' or pages_input == '*': target_sub_nids = sub_nids else: parts = re.findall(r'\d+', pages_input) start = max(0, int(parts[0]) - 1 if parts else 0) end = min(len(sub_nids) - 1, int(parts[1]) - 1 if len(parts) > 1 else start) target_sub_nids = sub_nids[start:end+1] global_total_pages += len(target_sub_nids) queue_tasks.append({ "q_idx": q_idx, "main_nid": main_nid, "sub_nids": target_sub_nids, "lang_code": lang_code, "model": model, "max_tokens": max_tokens }) except Exception as e: self.log(f"❌ API-Fehler bei Akte {main_nid} im Pre-Flight: {str(e)}") self.log(f"✅ PRE-FLIGHT BEENDET: {total_queue_len} Akten mit insgesamt {global_total_pages} Seiten.") # ── MAIN PROCESSING LOOP ── global_p_idx = 0 for task in queue_tasks: q_idx, main_nid, target_sub_nids = task["q_idx"], task["main_nid"], task["sub_nids"] lang_code, model, max_tokens = task["lang_code"], task["model"], task["max_tokens"] total_pages = len(target_sub_nids) self.log(f"--- BEARBEITE AKTE: {main_nid} ---") for p_idx, sub_nid in enumerate(target_sub_nids): self.write_status(q_idx, total_queue_len, global_p_idx, global_total_pages, p_idx, total_pages) self.log(f" -> Verarbeite Seite {p_idx+1}/{total_pages} (Gesamt: {global_p_idx+1}/{global_total_pages}) | Node {sub_nid}") try: url_node = f"https://freudedition.net/node/{sub_nid}?_format=json" req = urllib.request.Request(url_node, headers={'User-Agent': 'Mozilla/5.0'}) resp = urllib.request.urlopen(req, timeout=45) node_data = json.loads(resp.read()) faksimile = node_data.get('field_faksimile', []) if not faksimile or not faksimile[0].get('url'): continue img_url = faksimile[0].get('url') except Exception as e: self.log_omission(sub_nid, "Node API Fetch Failed") continue lang_name = LANG_MAP.get(lang_code, "Unknown") page_prompt = base_prompt if lang_code != "und": page_prompt = f"CRITICAL CONTEXT: The primary language of the following text is {lang_name}.\n\n" + page_prompt with open(f"{self.base_dir}/prompt_{sub_nid}.txt", "w", encoding="utf-8") as f: f.write(page_prompt) dl_cmd = ['powershell.exe', '-ExecutionPolicy', 'Bypass', '-File', 'C:/Scripts/megadownload_image.ps1', '-url', img_url, '-index', str(sub_nid), '-lang', lang_code] if subprocess.run(dl_cmd, capture_output=True).returncode != 0: continue gem_cmd = ['powershell.exe', '-ExecutionPolicy', 'Bypass', '-File', 'C:/Scripts/megacall_gemini.ps1', '-pIdx', str(sub_nid), '-p_lang', lang_code, '-p_model', model, '-p_temp', '0.0', '-p_topP', '0.05', '-p_tokens', str(max_tokens)] gem_result = subprocess.run(gem_cmd, capture_output=True) html_path = f"{self.base_dir}/html_{sub_nid}.txt" if gem_result.returncode == 0 and os.path.exists(html_path) and os.path.getsize(html_path) > 5: self.log(" ✅ HTML generiert. Ticket für UI.Vision erstellt!") self.write_ticket(sub_nid) else: self.log_omission(sub_nid, f"Gemini Failed (Code {gem_result.returncode})") global_p_idx += 1 self.log("\n✅ Alle API- und Gemini-Aufrufe beendet.") self.log("🏁 DONE Ticket für UI.Vision erstellt. Feierabend für Python!") self.write_status(total_queue_len, total_queue_len, global_p_idx, global_total_pages, 0, 0) self.write_done_ticket() if __name__ == "__main__": app = MegaWorkshopV1() app.mainloop()S.
You are here
Drömtydning I.1900-001/1927.sv