Add continuity dashboard and scene authoring workflow

2026-08-27 17:31:01 -05:00
parent 4499d4bd8f
commit 50b11264a9
247 changed files with 24024 additions and 9464 deletions
--- a/prototype/app.py
+++ b/prototype/app.py
@@ -89,16 +89,25 @@ class ScrivenerReader:
        self.root_items = self._load_binder()
        self.by_uuid = {item.uuid: item for item in self.walk()}

-    def bindings(self) -> dict[str, str]:
+    def binding_metadata(self) -> dict:
        if not self.bindings_path.exists():
            return {}
        try:
            data = json.loads(self.bindings_path.read_text(encoding="utf-8"))
-            bindings = data.get("scene_to_brief", {})
-            return {str(scene_uuid): str(brief_uuid) for scene_uuid, brief_uuid in bindings.items()} if isinstance(bindings, dict) else {}
+            return data if isinstance(data, dict) else {}
        except (json.JSONDecodeError, OSError):
            return {}

+    def bindings(self) -> dict[str, str]:
+        bindings = self.binding_metadata().get("scene_to_brief", {})
+        return {str(scene_uuid): str(brief_uuid) for scene_uuid, brief_uuid in bindings.items()} if isinstance(bindings, dict) else {}
+
+    def thread_predecessors(self) -> dict[str, str | None]:
+        predecessors = self.binding_metadata().get("scene_predecessors", {})
+        if not isinstance(predecessors, dict):
+            return {}
+        return {str(scene_uuid): (str(predecessor) if predecessor is not None else None) for scene_uuid, predecessor in predecessors.items()}
+
    def _load_binder(self) -> list[Item]:
        root = ET.parse(self.project_file).getroot()
        binder = root.find("Binder")
@@ -168,6 +177,39 @@ class ScrivenerReader:
                    )
        return scenes

+    def manuscript_chapters(self) -> list[dict]:
+        manuscript = self.top_level("Manuscript")
+        if manuscript is None:
+            return []
+        return [{"uuid": item.uuid, "title": item.title} for item in manuscript.children if item.item_type == "Folder"]
+
+    @staticmethod
+    def lexicon_entries(text: str) -> list[dict]:
+        """Read author-maintained ``Term — definition`` paragraphs from Master Lexicon."""
+        entries: list[dict] = []
+        entry_lines: list[str] = []
+
+        def save_entry() -> None:
+            if not entry_lines:
+                return
+            entry = "\n".join(entry_lines).strip()
+            title, _, definition = entry.partition(" — ")
+            if title.strip() and definition.strip():
+                entries.append({"title": title.strip(), "text": entry})
+
+        for raw_line in text.splitlines():
+            line = raw_line.strip()
+            if " — " in line:
+                save_entry()
+                entry_lines = [line]
+            elif not line:
+                save_entry()
+                entry_lines = []
+            elif entry_lines:
+                entry_lines.append(line)
+        save_entry()
+        return entries
+
    def summary(self) -> dict:
        authoring = self.top_level("Authoring System")
        setup = self.child(authoring, "Book Setup")
@@ -175,6 +217,7 @@ class ScrivenerReader:
        briefs_folder = self.child(authoring, "Scene Briefs")
        summaries_folder = self.child(continuity, "Scene Summaries")
        archive = self.child(authoring, "Drafting Archive")
+        accepted_folder = self.child(archive, "Accepted for Editing")
        snapshots_folder = self.child(archive, "Package Snapshots")
        style = self.child(setup, "STYLE")
        instructions = self.child(setup, "Frontier-Model Instructions")
@@ -185,8 +228,14 @@ class ScrivenerReader:
        brief_by_title = {brief["title"]: brief for brief in briefs}
        brief_by_uuid = {brief["uuid"]: brief for brief in briefs}
        bindings = self.bindings()
+        predecessors = self.thread_predecessors()
        summary_by_title = {record["title"]: record for record in self.records(summaries_folder, self)}
        character_by_title = {character["title"]: character["uuid"] for character in self.records(self.top_level("Characters"), self)}
+        working_by_title: dict[str, dict] = {}
+        for record in self.records(accepted_folder, self):
+            match = re.match(r"^(.*?) — working copy — ", record["title"])
+            if match:
+                working_by_title[match.group(1)] = record
        package_context: dict[str, dict] = {}
        for snapshot in self.records(snapshots_folder, self):
            scene_title = re.search(r"^Scene:\s*(.+)$", snapshot["text"], re.MULTILINE)
@@ -200,7 +249,24 @@ class ScrivenerReader:
            else:
                character_block = self._package_section(text, "CHARACTERS")
                character_uuids = [character_by_title[line.strip()] for line in re.findall(r"^(.+?)\n", character_block) if line.strip() in character_by_title]
-            package_context[title] = {"character_uuids": character_uuids, "notes": self._package_section(text, "NOTES")}
+            notes = self._package_section(text, "NOTES")
+            place_uuids: list[str] = []
+            for place in self.records(self.top_level("Places"), self):
+                place_block = f"{place['title']} — {place['text'].strip()}"
+                if place_block and place_block in notes:
+                    place_uuids.append(place["uuid"])
+                    notes = notes.replace(place_block, "").strip()
+            package_context[title] = {
+                "character_uuids": character_uuids,
+                "place_uuids": place_uuids,
+                "lexicon": self._package_section(text, "LEXICON"),
+                "story": self._package_section(text, "STORY"),
+                "recall": self._package_section(text, "RECALL"),
+                "previous": self._package_section(text, "PREVIOUS"),
+                "notes": notes,
+                "directive": self._package_section(text, "DIRECTIVE"),
+                "exact_package": text.split("\n\n", 1)[1] if "\n\n" in text else text,
+            }

        paired = []
        bound_brief_uuids: set[str] = set()
@@ -219,7 +285,10 @@ class ScrivenerReader:
                    "brief_uuid": (brief or {}).get("uuid"),
                    "binding_state": binding_state,
                    "summary": summary_by_title.get(scene["title"], {}).get("text", ""),
-                    "package_context": package_context.get(scene["title"], {"character_uuids": [], "notes": ""}),
+                    "working_copy": working_by_title.get(scene["title"]),
+                    "package_context": package_context.get(scene["title"], {"character_uuids": [], "place_uuids": [], "lexicon": "", "story": "", "recall": "", "previous": "", "notes": "", "directive": "", "exact_package": ""}),
+                    "predecessor_uuid": predecessors.get(scene["uuid"]),
+                    "thread_state": "new-thread" if scene["uuid"] in predecessors and predecessors[scene["uuid"]] is None else "continues" if predecessors.get(scene["uuid"]) else "unassigned",
                }
            )

@@ -229,8 +298,10 @@ class ScrivenerReader:
            "style": self.content(style),
            "frontier_instructions": self.content(instructions),
            "master_lexicon": self.content(lexicon),
+            "lexicon_entries": self.lexicon_entries(self.content(lexicon)),
            "world_notes": self.content(world_notes),
            "scenes": paired,
+            "chapters": self.manuscript_chapters(),
            "characters": self.records(self.top_level("Characters"), self),
            "places": self.records(self.top_level("Places"), self),
            "unmatched_briefs": [brief for brief in briefs if brief["uuid"] not in bound_brief_uuids],
@@ -259,16 +330,40 @@ class ScrivenerWriter:
        if (self.project / "Files" / "user.lock").exists():
            raise ValueError("Scrivener is open (Files/user.lock exists). Close the project before writing.")

-    def _write_bindings(self, bindings: dict[str, str]) -> None:
-        payload = {
+    def _write_bindings(self, bindings: dict[str, str], predecessors: dict[str, str | None] | None = None) -> None:
+        payload = self.reader.binding_metadata()
+        payload.update({
            "version": 1,
            "updated_at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
            "scene_to_brief": bindings,
-        }
+        })
+        if predecessors is not None:
+            payload["scene_predecessors"] = predecessors
        temporary = self.reader.bindings_path.with_suffix(".tmp")
        temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
        os.replace(temporary, self.reader.bindings_path)

+    def assign_thread_predecessor(self, scene_uuid: str, predecessor_uuid: str | None) -> None:
+        """Store the author's explicit narrative-thread relationship for a scene."""
+        self._assert_unlocked()
+        scenes = {scene["uuid"]: scene for scene in self.reader.summary()["scenes"]}
+        if scene_uuid not in scenes:
+            raise ValueError("The selected Manuscript scene no longer exists.")
+        if predecessor_uuid is not None:
+            if predecessor_uuid not in scenes or not scenes[predecessor_uuid]["text"].strip():
+                raise ValueError("A narrative predecessor must be a canonical Manuscript scene.")
+            if predecessor_uuid == scene_uuid:
+                raise ValueError("A scene cannot continue from itself.")
+        predecessors = self.reader.thread_predecessors()
+        cursor = predecessor_uuid
+        while cursor is not None:
+            if cursor == scene_uuid:
+                raise ValueError("That narrative-thread link would create a cycle.")
+            cursor = predecessors.get(cursor)
+        predecessors[scene_uuid] = predecessor_uuid
+        self._write_bindings(self.reader.bindings(), predecessors)
+        self.reader = ScrivenerReader(self.project)
+
    def reconcile_scene_bindings(self) -> dict:
        """Persist currently resolved scene/BRIEF pairs without relying on their titles."""
        self._assert_unlocked()
@@ -310,6 +405,18 @@ class ScrivenerWriter:
        assert parent is not None
        return parent

+    @staticmethod
+    def _element_by_uuid(parent: ET.Element, item_uuid: str) -> ET.Element | None:
+        for item in parent.findall("BinderItem"):
+            if item.attrib.get("UUID") == item_uuid:
+                return item
+            children = item.find("Children")
+            if children is not None:
+                found = ScrivenerWriter._element_by_uuid(children, item_uuid)
+                if found is not None:
+                    return found
+        return None
+
    def _backup_binder(self) -> None:
        backup_dir = self.project / "Files" / "authoring-backups"
        backup_dir.mkdir(parents=True, exist_ok=True)
@@ -390,6 +497,35 @@ class ScrivenerWriter:
        self.reader = ScrivenerReader(self.project)
        return {"created": True, "brief_uuid": brief_uuid}

+    def create_planned_scene(self, chapter_uuid: str, title: str) -> dict:
+        """Create a blank Manuscript scene and its blank, stably bound BRIEF."""
+        self._assert_unlocked()
+        manuscript = self.reader.top_level("Manuscript")
+        chapter = self.reader.by_uuid.get(chapter_uuid)
+        if manuscript is None or chapter is None or chapter_uuid not in {item.uuid for item in manuscript.children}:
+            raise ValueError("Choose a current Manuscript chapter.")
+        title = title.strip()
+        existing_titles = {scene["title"] for scene in self.reader.summary()["scenes"]}
+        if not title:
+            numbers = [int(match.group(1)) for scene_title in existing_titles if (match := re.fullmatch(r"Scene\s+(\d+)", scene_title, re.IGNORECASE))]
+            title = f"Scene {max(numbers, default=0) + 1}"
+        if title in existing_titles:
+            raise ValueError(f"A Manuscript scene named {title!r} already exists.")
+        self._backup_binder()
+        tree, binder = self._tree_and_binder()
+        chapter_element = self._element_by_uuid(binder, chapter_uuid)
+        if chapter_element is None:
+            raise ValueError("The selected chapter no longer exists.")
+        briefs_folder = self._folder(binder, ["Authoring System", "Scene Briefs"])
+        scene_uuid = self._add_text(chapter_element, title, "")
+        brief_uuid = self._add_text(briefs_folder, title, "")
+        self._save_tree(tree)
+        bindings = self.reader.bindings()
+        bindings[scene_uuid] = brief_uuid
+        self._write_bindings(bindings)
+        self.reader = ScrivenerReader(self.project)
+        return {"scene_uuid": scene_uuid, "brief_uuid": brief_uuid, "title": title}
+
    def accept_for_editing(self, scene_uuid: str, draft_text: str, package: str, model: str, character_uuids: list[str]) -> dict:
        self._assert_unlocked()
        if not draft_text.strip() or not package.strip():
@@ -422,7 +558,7 @@ class ScrivenerWriter:
        path.write_bytes(content)
        self._update_checksum(working_uuid, content)

-    def accept_into_book(self, scene_uuid: str, text: str) -> None:
+    def accept_into_book(self, scene_uuid: str, text: str, predecessor_uuid: str | None) -> None:
        self._assert_unlocked()
        item = self.reader.by_uuid.get(scene_uuid)
        if item is None or item.item_type != "Text":
@@ -435,6 +571,7 @@ class ScrivenerWriter:
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_bytes(content)
        self._update_checksum(scene_uuid, content)
+        self.assign_thread_predecessor(scene_uuid, predecessor_uuid)

    def save_scene_summary(self, scene_uuid: str, summary: str) -> str:
        """Create or update the one author-approved dense summary for a scene."""
@@ -644,16 +781,19 @@ def propose_continuity_updates(project: ScrivenerReader, payload: dict) -> dict:
        raise ValueError("A canonical scene is required to propose continuity updates.")
    data = project.summary()
    context = {"characters": data["characters"], "master_lexicon": data["master_lexicon"], "world_notes": data["world_notes"]}
-    system = "You propose factual continuity updates for a fiction-authoring system. Return JSON only, with exactly these keys: character_updates (array of objects with character_uuid and text), new_characters (array of objects with name and text), lexicon_additions (string), world_note_additions (string). Suggest only information newly established or materially changed by the canonical scene and absent from the supplied records. Character text must be a concise factual update, without a heading. Use a supplied UUID only when updating an existing character. Put a newly introduced named person in new_characters, never in lexicon_additions; name is the character's display name and text is an initial concise factual Character record. Do not create a new character for a passing unnamed person. Use empty arrays or empty strings when there is no justified addition."
+    system = "You propose factual continuity updates for a fiction-authoring system. Return one strict JSON object and nothing else: no Markdown, explanation, or code fences. It must contain exactly these keys: character_updates (array of objects with character_uuid and text), new_characters (array of objects with name and text), lexicon_additions (string), world_note_additions (string). Suggest only information newly established or materially changed by the canonical scene and absent from the supplied records. Keep every proposed text field to at most two concise sentences and return no more than eight total proposed records across all arrays. Character text must be a concise factual update, without a heading. Use a supplied UUID only when updating an existing character. Put a newly introduced named person in new_characters, never in lexicon_additions; name is the character's display name and text is an initial concise factual Character record. Do not create a new character for a passing unnamed person. Use empty arrays or empty strings when there is no justified addition."
    user = "Existing continuity records:\n" + json.dumps(context, ensure_ascii=False) + f"\n\nCanonical scene ({scene.title}):\n{text}"
-    body = json.dumps({"model": OPENROUTER_MODEL, "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}], "max_tokens": 1200}).encode()
+    body = json.dumps({"model": OPENROUTER_MODEL, "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}], "max_tokens": 2500, "temperature": 0, "response_format": {"type": "json_object"}}).encode()
    request = urllib.request.Request("https://openrouter.ai/api/v1/chat/completions", data=body, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, method="POST")
    try:
        with urllib.request.urlopen(request, timeout=300) as response:
            result = json.loads(response.read().decode())
    except urllib.error.HTTPError as error:
        raise ValueError(f"OpenRouter returned HTTP {error.code}: {error.read().decode(errors='replace')}") from error
-    raw = result.get("choices", [{}])[0].get("message", {}).get("content")
+    choice = result.get("choices", [{}])[0]
+    if choice.get("finish_reason") == "length":
+        raise ValueError("OpenRouter stopped the continuity proposal at its output limit; nothing was saved. Please retry after restarting with the updated proposal budget.")
+    raw = choice.get("message", {}).get("content")
    if not isinstance(raw, str):
        raise ValueError("OpenRouter response did not contain continuity proposals.")
    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw.strip(), flags=re.IGNORECASE)
@@ -664,10 +804,18 @@ def propose_continuity_updates(project: ScrivenerReader, payload: dict) -> dict:
    last_brace = raw.rfind("}")
    if first_brace >= 0 and last_brace > first_brace:
        raw = raw[first_brace:last_brace + 1]
+    # Some otherwise valid model responses use Markdown-style escapes for
+    # underscores or double-escape quotes inside a JSON string.  Neither is
+    # valid JSON; normalize only these two unambiguous formatting mistakes.
+    raw = re.sub(r"\\+_", "_", raw)
+    raw = re.sub(r'(?:\\){2,}(?=")', r"\\", raw)
    try:
        proposals = json.loads(raw)
    except json.JSONDecodeError as error:
-        raise ValueError("OpenRouter returned proposals in an unexpected format; please try again.") from error
+        start = max(0, error.pos - 120)
+        end = min(len(raw), error.pos + 180)
+        context_excerpt = re.sub(r"\s+", " ", raw[start:end]).strip()
+        raise ValueError(f"OpenRouter returned proposals in an unexpected format near character {error.pos}: {context_excerpt!r}") from error
    if not isinstance(proposals, dict):
        raise ValueError("OpenRouter returned proposals in an unexpected format; please try again.")
    valid_uuids = {character["uuid"] for character in data["characters"]}
@@ -721,7 +869,7 @@ class AppHandler(SimpleHTTPRequestHandler):

    def do_POST(self) -> None:
        path = urlparse(self.path).path
-        if path not in {"/api/package-preview", "/api/draft", "/api/accept-for-editing", "/api/save-working-copy", "/api/accept-into-book", "/api/propose-summary", "/api/save-summary", "/api/propose-continuity-updates", "/api/save-continuity-updates", "/api/reconcile-bindings", "/api/save-brief"}:
+        if path not in {"/api/package-preview", "/api/draft", "/api/accept-for-editing", "/api/save-working-copy", "/api/accept-into-book", "/api/propose-summary", "/api/save-summary", "/api/propose-continuity-updates", "/api/save-continuity-updates", "/api/reconcile-bindings", "/api/save-brief", "/api/save-thread-link", "/api/create-scene"}:
            self.send_error(HTTPStatus.NOT_FOUND)
            return
        try:
@@ -745,8 +893,8 @@ class AppHandler(SimpleHTTPRequestHandler):
                if not payload.get("final_confirmation"):
                    raise ValueError("Confirm final acceptance before writing canonical Manuscript text.")
                writer = ScrivenerWriter(self.reader.project)
-                writer.accept_into_book(payload.get("scene_uuid", ""), payload.get("text", ""))
-                AppHandler.reader = ScrivenerReader(self.reader.project)
+                writer.accept_into_book(payload.get("scene_uuid", ""), payload.get("text", ""), payload.get("predecessor_uuid"))
+                AppHandler.reader = writer.reader
                self.send_json({"accepted": True})
            elif path == "/api/propose-summary":
                self.send_json(propose_summary(self.reader, payload))
@@ -776,6 +924,16 @@ class AppHandler(SimpleHTTPRequestHandler):
                result = writer.save_brief(payload.get("scene_uuid", ""), payload.get("text", ""))
                AppHandler.reader = writer.reader
                self.send_json(result)
+            elif path == "/api/save-thread-link":
+                writer = ScrivenerWriter(self.reader.project)
+                writer.assign_thread_predecessor(payload.get("scene_uuid", ""), payload.get("predecessor_uuid"))
+                AppHandler.reader = writer.reader
+                self.send_json({"saved": True})
+            elif path == "/api/create-scene":
+                writer = ScrivenerWriter(self.reader.project)
+                result = writer.create_planned_scene(payload.get("chapter_uuid", ""), payload.get("title", ""))
+                AppHandler.reader = writer.reader
+                self.send_json(result)
            else:
                self.send_json({"package": build_package(self.reader, payload), "warnings": preflight(self.reader, payload)})
        except (ValueError, json.JSONDecodeError) as error: