From 5643f201fe20bac6db581ffb0bf562dfad7bcbf7 Mon Sep 17 00:00:00 2001 From: noonghunna Date: Tue, 30 Jun 2026 04:12:03 +0500 Subject: [PATCH] Director: make the LLM the intent driver, demote keyword detection to a bare fallback (#524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "dig history of pakistan? … ok do it" chat built a film literally titled "ok do it": the LLM controller returned brief='' (it read "dig history of pakistan?" as a research request, not a film), so the brittle keyword floor took over and grabbed the confirm phrase as the brief. We'd been patching the floor's keyword lists to cover for the LLM — treating the symptom. Fix it where it belongs — in the LLM's prompt — and stop leaning on keyword intent detection: - CONTROLLER PROMPT (build_controller_system): brief extraction now INFERS the subject from indirect phrasing — a topic asked to be researched/dug/searched ("dig history of pakistan" -> "the history of pakistan"), a bare topic, or a changed subject. A live A/B: this one prompt change flips the failing transcript from brief='' to brief='the history of pakistan' with no false positives (greetings/capability-questions still -> ''). Its reply guidance is also truthful about web research (no more "I can search the web!" fantasy). - TRUST THE LLM: the pipe no longer ORs the LLM brief with the keyword floor. When the controller answers, its brief/intent/reply are authoritative; the keyword floor is consulted ONLY when the controller is unreachable (and even then never guesses a brief from a confirm). - CONFIRM stays a tiny CLOSED vocabulary (is_confirm now catches compound bare confirms like "ok do it" / "yes go ahead") — affirmation is a closed set where keywords are reliable; the irreversible render is still gated by a real go-word (safety latch). Brief (open-ended) is the LLM's job; confirm (closed) keeps a generic latch. - _classify runs at temperature 0 so the brief is stable across turns. Live end-to-end (controller -> merge -> decide_action), 5/5 correct: the failing transcript builds "the history of pakistan", mid-chat revision swaps the brief, questions/greetings chat. 147 offline unittests green; pipe rebuilt + py_compile-clean. Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm Co-authored-by: noonghunna <10742901+noonghunna@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- services/studio/build_studio_pipe.py | 19 +++++-- services/studio/director_intent.py | 37 +++++++++--- .../production/tests/test_director_intent.py | 21 +++++++ services/studio/studio_pipe.py | 56 ++++++++++++++----- 4 files changed, 107 insertions(+), 26 deletions(-) diff --git a/services/studio/build_studio_pipe.py b/services/studio/build_studio_pipe.py index 2ea7d12c..0bf402e7 100644 --- a/services/studio/build_studio_pipe.py +++ b/services/studio/build_studio_pipe.py @@ -495,7 +495,7 @@ class Pipe: body = json.dumps({"model": self.valves.chat_model, "messages": [{"role": "system", "content": build_controller_system()}, {"role": "user", "content": u}], - "max_tokens": 320, "temperature": 0.2, + "max_tokens": 320, "temperature": 0.0, # deterministic intent across turns "chat_template_kwargs": {"enable_thinking": False}}).encode() try: req = urllib.request.Request(self.valves.chat_url + "/chat/completions", data=body, @@ -872,13 +872,22 @@ class Pipe: (", ~%d shots" % _pshots) if _pshots else "") _decision = await loop.run_in_executor(None, self._classify, users, _prelim_ctx) if _decision: - brief = _decision["brief"] or brief_kw - for _k, _v in _decision["stack_patch"].items(): - ov.setdefault(_k, _v) # keyword floor wins; the LLM only fills a lane it left blank - confirmed = (_decision["confirm"] or confirm_kw) and has_confirm_word(last) + # The LLM is the DRIVER — trust its read of the conversation (brief, intent, reply). + # The keyword floor is NOT consulted for the brief; it only supplies the explicit + # stack toggles the user typed ("use ltx", "no music", "research") the LLM might miss. + # Brief extraction is the LLM's job — that's what the controller prompt is for. + brief = _decision["brief"] intent = _decision["intent"] llm_reply = _decision["reply"] + for _k, _v in _decision["stack_patch"].items(): + ov.setdefault(_k, _v) # explicit keyword toggles win; the LLM fills gaps + # render is irreversible → fire on a BARE confirm ("go", "ok do it") or an + # LLM-confirmed compound ("go with ltx"), ALWAYS gated by a real go-word (safety latch). + confirmed = confirm_kw or (_decision["confirm"] and has_confirm_word(last)) else: + # LLM unreachable → minimal keyword fallback (the 4B planner is down too, so this + # mostly keeps the lane honest until the director is back — it never guesses a brief + # from a confirm phrase). brief = brief_kw confirmed = confirm_kw intent = ("confirm" if confirm_kw diff --git a/services/studio/director_intent.py b/services/studio/director_intent.py index cad52475..33e35a54 100644 --- a/services/studio/director_intent.py +++ b/services/studio/director_intent.py @@ -36,12 +36,25 @@ _GEN_NOUN = (r"(?:films?|movies?|shorts?|videos?|clips?|documentar\w*|docu\w*|tr _GEN_RE = re.compile(_GEN_VERB + r"\b.{0,40}?\b" + _GEN_NOUN, re.I) +# Tokens that, on their own, only ever mean "yes, proceed" — so a SHORT turn made up entirely of +# them is a confirm even if the exact phrase isn't in CONFIRM (e.g. "ok do it", "yes go ahead"). +# This stops a compound confirm from leaking into brief detection (the "ok do it" became the film +# brief bug, 2026-06-29). +_CONFIRM_TOKENS = {"go", "yes", "y", "ya", "yeah", "yep", "yup", "sure", "ok", "okay", "k", + "do", "it", "that", "start", "render", "build", "proceed", "confirm", + "please", "now", "lets", "let's", "ahead", "on", "make", "run", "begin"} + + def _norm(t): return (t or "").strip().lower().strip(" .!?") def is_confirm(t): - return _norm(t) in CONFIRM + norm = _norm(t) + if norm in CONFIRM: + return True + toks = [w.strip(".,!?") for w in norm.split() if w] + return bool(toks) and len(toks) <= 4 and all(w in _CONFIRM_TOKENS for w in toks) def is_greeting(t): @@ -89,7 +102,10 @@ def is_brief_candidate(t, pure_override=False): Otherwise it's a brief only if it isn't a greeting / confirm / pure stack-tweak / question. `pure_override` is supplied by the caller (it needs valve/seconds context to compute). """ - if is_generation_request(t): + # A creation ask OR a named factual subject ("dig history of pakistan?", "the history of jazz") + # is a brief even when question-shaped — the old gate dropped "dig history of pakistan?" as a + # question and the confirm phrase after it became the brief (2026-06-29). + if is_generation_request(t) or looks_documentary(t): return True return not (is_confirm(t) or is_greeting(t) or pure_override or is_question(t)) @@ -152,18 +168,23 @@ def build_controller_system(): ' "reply": ""\n' "}\n\n" "Rules:\n" - "- brief: the film the user wants, using their LATEST description. If they CHANGED the subject " - "(\"actually make it a bookstore promo\"), use the NEW one. Empty if they haven't described a film yet.\n" + "- brief: the SUBJECT the user wants a film about — INFER it even from indirect phrasing: a topic " + "they asked you to research / dig / search (\"dig history of pakistan\" -> \"the history of pakistan\"), " + "a bare topic (\"history of jazz\"), or a changed subject (\"actually make it a bookstore promo\" -> use " + "the NEW one). If they named ANY subject to film or research, THAT subject is the brief. Leave it empty " + "ONLY if they named no topic at all (pure greetings, or questions about what you can do).\n" "- stack_patch: include ONLY settings the user explicitly asked for; OMIT keys they didn't mention. " "video_lane ∈ {wan, ltx, sulphur, 10eros}; keyframe_lane ∈ {chroma, zimage, krea, hidream}; " "continuity ∈ {storyboard, hero, chain, none}; music/narration are booleans; seconds is the requested length.\n" - "- confirm: true ONLY if the user is telling you to START rendering now (\"go\", \"yes do it\", " - "\"go with ltx\", \"render it\"). A change request like \"make it 30 seconds\" is NOT a confirm.\n" + "- confirm: true ONLY if the user is telling you to START rendering now (\"go\", \"ok do it\", " + "\"yes do it\", \"go with ltx\", \"render it\"). A change request like \"make it 30 seconds\" is NOT a confirm.\n" "- intent: brief = first/main film description; revise = changing the film; stack = changing a " "setting/model; question = asking something; confirm = start now; smalltalk = greeting/chit-chat; " "cancel = stop/never mind.\n" - "- reply: what to say back, warm and concise. For a question, ANSWER it. Never claim rendering " - "has started (only the word \"go\" starts it).\n" + "- reply: what to say back, warm and concise. For a question, ANSWER it truthfully. You CAN research " + "real facts on the web (SearXNG) to ground a DOCUMENTARY you're making — so \"can you search the web?\" " + "is YES, for grounding a documentary — but you canNOT browse arbitrary pages or fetch live info to chat " + "about. Never claim rendering or searching has already started (only \"go\" starts a render).\n" "Output ONLY the JSON object." ) diff --git a/services/studio/production/tests/test_director_intent.py b/services/studio/production/tests/test_director_intent.py index cb380ce2..13769e62 100644 --- a/services/studio/production/tests/test_director_intent.py +++ b/services/studio/production/tests/test_director_intent.py @@ -57,6 +57,14 @@ class TestClassifiers(unittest.TestCase): for t in ["a lone detective in the rain", "make a noir short"]: self.assertFalse(di.is_question(t), t) + def test_compound_confirms(self): + # "ok do it" etc. must read as a confirm so they don't leak into brief detection + for t in ["ok do it", "yes go ahead", "do it now", "ok go", "yes please", "go on", "sure lets go"]: + self.assertTrue(di.is_confirm(t), t) + # NOT bare confirms: a compound confirm+stack, or any real content + for t in ["go with ltx", "make a noir film", "do some research", "a noir short"]: + self.assertFalse(di.is_confirm(t), t) + class TestBriefCandidate(unittest.TestCase): def test_generation_request_beats_question_shape(self): @@ -95,6 +103,19 @@ class TestPickBrief(unittest.TestCase): def test_all_smalltalk_yields_no_brief(self): self.assertEqual(di.pick_brief(["hi", "hello", "what can you do?"]), "") + def test_documentary_subject_is_a_brief_even_as_a_question(self): + self.assertTrue(di.is_brief_candidate("dig history of pakistan?")) + self.assertTrue(di.is_brief_candidate("the history of jazz")) + + def test_the_ok_do_it_transcript_captures_the_real_brief(self): + # Regression (2026-06-29): "dig history of pakistan?" was dropped as a question and the + # confirm phrase "ok do it" became the film brief. Now the subject is the brief, and + # "ok do it" is a confirm. + turns = ["hi", "can you search the web?", "can you research?", + "dig history of pakistan?", "ok do it"] + self.assertEqual(di.pick_brief(turns), "dig history of pakistan?") + self.assertTrue(di.is_confirm("ok do it")) + class TestConfirmWord(unittest.TestCase): """has_confirm_word corroborates the LLM confirm so a render never fires on a hallucination.""" diff --git a/services/studio/studio_pipe.py b/services/studio/studio_pipe.py index 0cd01e01..db7b44c9 100644 --- a/services/studio/studio_pipe.py +++ b/services/studio/studio_pipe.py @@ -79,12 +79,25 @@ _GEN_NOUN = (r"(?:films?|movies?|shorts?|videos?|clips?|documentar\w*|docu\w*|tr _GEN_RE = re.compile(_GEN_VERB + r"\b.{0,40}?\b" + _GEN_NOUN, re.I) +# Tokens that, on their own, only ever mean "yes, proceed" — so a SHORT turn made up entirely of +# them is a confirm even if the exact phrase isn't in CONFIRM (e.g. "ok do it", "yes go ahead"). +# This stops a compound confirm from leaking into brief detection (the "ok do it" became the film +# brief bug, 2026-06-29). +_CONFIRM_TOKENS = {"go", "yes", "y", "ya", "yeah", "yep", "yup", "sure", "ok", "okay", "k", + "do", "it", "that", "start", "render", "build", "proceed", "confirm", + "please", "now", "lets", "let's", "ahead", "on", "make", "run", "begin"} + + def _norm(t): return (t or "").strip().lower().strip(" .!?") def is_confirm(t): - return _norm(t) in CONFIRM + norm = _norm(t) + if norm in CONFIRM: + return True + toks = [w.strip(".,!?") for w in norm.split() if w] + return bool(toks) and len(toks) <= 4 and all(w in _CONFIRM_TOKENS for w in toks) def is_greeting(t): @@ -132,7 +145,10 @@ def is_brief_candidate(t, pure_override=False): Otherwise it's a brief only if it isn't a greeting / confirm / pure stack-tweak / question. `pure_override` is supplied by the caller (it needs valve/seconds context to compute). """ - if is_generation_request(t): + # A creation ask OR a named factual subject ("dig history of pakistan?", "the history of jazz") + # is a brief even when question-shaped — the old gate dropped "dig history of pakistan?" as a + # question and the confirm phrase after it became the brief (2026-06-29). + if is_generation_request(t) or looks_documentary(t): return True return not (is_confirm(t) or is_greeting(t) or pure_override or is_question(t)) @@ -195,18 +211,23 @@ def build_controller_system(): ' "reply": ""\n' "}\n\n" "Rules:\n" - "- brief: the film the user wants, using their LATEST description. If they CHANGED the subject " - "(\"actually make it a bookstore promo\"), use the NEW one. Empty if they haven't described a film yet.\n" + "- brief: the SUBJECT the user wants a film about — INFER it even from indirect phrasing: a topic " + "they asked you to research / dig / search (\"dig history of pakistan\" -> \"the history of pakistan\"), " + "a bare topic (\"history of jazz\"), or a changed subject (\"actually make it a bookstore promo\" -> use " + "the NEW one). If they named ANY subject to film or research, THAT subject is the brief. Leave it empty " + "ONLY if they named no topic at all (pure greetings, or questions about what you can do).\n" "- stack_patch: include ONLY settings the user explicitly asked for; OMIT keys they didn't mention. " "video_lane ∈ {wan, ltx, sulphur, 10eros}; keyframe_lane ∈ {chroma, zimage, krea, hidream}; " "continuity ∈ {storyboard, hero, chain, none}; music/narration are booleans; seconds is the requested length.\n" - "- confirm: true ONLY if the user is telling you to START rendering now (\"go\", \"yes do it\", " - "\"go with ltx\", \"render it\"). A change request like \"make it 30 seconds\" is NOT a confirm.\n" + "- confirm: true ONLY if the user is telling you to START rendering now (\"go\", \"ok do it\", " + "\"yes do it\", \"go with ltx\", \"render it\"). A change request like \"make it 30 seconds\" is NOT a confirm.\n" "- intent: brief = first/main film description; revise = changing the film; stack = changing a " "setting/model; question = asking something; confirm = start now; smalltalk = greeting/chit-chat; " "cancel = stop/never mind.\n" - "- reply: what to say back, warm and concise. For a question, ANSWER it. Never claim rendering " - "has started (only the word \"go\" starts it).\n" + "- reply: what to say back, warm and concise. For a question, ANSWER it truthfully. You CAN research " + "real facts on the web (SearXNG) to ground a DOCUMENTARY you're making — so \"can you search the web?\" " + "is YES, for grounding a documentary — but you canNOT browse arbitrary pages or fetch live info to chat " + "about. Never claim rendering or searching has already started (only \"go\" starts a render).\n" "Output ONLY the JSON object." ) @@ -665,7 +686,7 @@ class Pipe: body = json.dumps({"model": self.valves.chat_model, "messages": [{"role": "system", "content": build_controller_system()}, {"role": "user", "content": u}], - "max_tokens": 320, "temperature": 0.2, + "max_tokens": 320, "temperature": 0.0, # deterministic intent across turns "chat_template_kwargs": {"enable_thinking": False}}).encode() try: req = urllib.request.Request(self.valves.chat_url + "/chat/completions", data=body, @@ -1042,13 +1063,22 @@ class Pipe: (", ~%d shots" % _pshots) if _pshots else "") _decision = await loop.run_in_executor(None, self._classify, users, _prelim_ctx) if _decision: - brief = _decision["brief"] or brief_kw - for _k, _v in _decision["stack_patch"].items(): - ov.setdefault(_k, _v) # keyword floor wins; the LLM only fills a lane it left blank - confirmed = (_decision["confirm"] or confirm_kw) and has_confirm_word(last) + # The LLM is the DRIVER — trust its read of the conversation (brief, intent, reply). + # The keyword floor is NOT consulted for the brief; it only supplies the explicit + # stack toggles the user typed ("use ltx", "no music", "research") the LLM might miss. + # Brief extraction is the LLM's job — that's what the controller prompt is for. + brief = _decision["brief"] intent = _decision["intent"] llm_reply = _decision["reply"] + for _k, _v in _decision["stack_patch"].items(): + ov.setdefault(_k, _v) # explicit keyword toggles win; the LLM fills gaps + # render is irreversible → fire on a BARE confirm ("go", "ok do it") or an + # LLM-confirmed compound ("go with ltx"), ALWAYS gated by a real go-word (safety latch). + confirmed = confirm_kw or (_decision["confirm"] and has_confirm_word(last)) else: + # LLM unreachable → minimal keyword fallback (the 4B planner is down too, so this + # mostly keeps the lane honest until the director is back — it never guesses a brief + # from a confirm phrase). brief = brief_kw confirmed = confirm_kw intent = ("confirm" if confirm_kw