From 049864efd49c518bbf280cc1168c5e893574be3b Mon Sep 17 00:00:00 2001 From: Bl0ck <36800583+Bl0ck154@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:27:53 +0200 Subject: [PATCH] feat(flow): optional trusted UI path for Omni 1.2 text video --- agent/services/flow_client.py | 13 +- agent/services/omni_12_ui.py | 249 ++++++++++++++++++++++++++ agent/services/omni_model_versions.py | 32 ++++ docs/OMNI_12.md | 62 +++++++ tests/unit/test_omni_12_ui.py | 100 +++++++++++ 5 files changed, 455 insertions(+), 1 deletion(-) create mode 100644 agent/services/omni_12_ui.py create mode 100644 agent/services/omni_model_versions.py create mode 100644 docs/OMNI_12.md create mode 100644 tests/unit/test_omni_12_ui.py diff --git a/agent/services/flow_client.py b/agent/services/flow_client.py index 4cadab44..036fa744 100644 --- a/agent/services/flow_client.py +++ b/agent/services/flow_client.py @@ -20,6 +20,7 @@ import asyncio import json import logging +import os import time import uuid from typing import Optional @@ -626,7 +627,17 @@ async def batch_rpc(self, rpcid: str, freq: str, await asyncio.sleep(delay) self._generation_last_submit_at = time.monotonic() - result = await self._send("batch_rpc", params, timeout=timeout) + if rpcid == fb.RPC_GEN_VIDEO_TEXT and os.environ.get("FLOW_OMNI12_CDP_URL"): + from agent.services.omni_12_ui import is_omni_12_ui_envelope, run_omni_12_ui_generation + else: + is_omni_12_ui_envelope = lambda _freq: False + if is_omni_12_ui_envelope(freq): + result = await run_omni_12_ui_generation( + freq, project_id=self._batch_active_project or FLOW_PROJECT_ID or None, + timeout=timeout, + ) + else: + result = await self._send("batch_rpc", params, timeout=timeout) blob = f"{result.get('error', '')} {result.get('data', '')}" is_hijack = "extension_hijack" in blob.lower() is_unusual = ( diff --git a/agent/services/omni_12_ui.py b/agent/services/omni_12_ui.py new file mode 100644 index 00000000..7adaa464 --- /dev/null +++ b/agent/services/omni_12_ui.py @@ -0,0 +1,249 @@ +"""Opt-in Omni 1.2 text-to-video through Flow's own signed-in composer. + +Flow 1.2 has a 3–15s slider and versioned model selection. The +macaron_cookie keys are documented by the separate live-captured FlowKit PR #85. +A dedicated Chrome CDP target is explicitly configured by the operator. +No account data, cookies, or CAPTCHA tokens are read or copied by this module. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +from typing import Any +from urllib.parse import urlparse + +from agent.services import flow_batch as fb +from agent.services.omni_model_versions import ( + estimate_omni_12_credits, matches_model_label, +) + +logger = logging.getLogger(__name__) + + +def is_omni_12_ui_envelope(freq: str) -> bool: + """Match the model slot, never arbitrary user prompt text.""" + try: + inner = json.loads(json.loads(freq)[0][0][1]) + model = str(inner[0][0][1]) + except (ValueError, KeyError, IndexError, TypeError): + return False + return bool(re.fullmatch(r"macaron_cookie_t2v_\d{1,2}s(?:_360p)?", model)) + + +def parse_ui_request(freq: str) -> tuple[str, str, int, str, int]: + """Decode the intent from a FlowKit envelope without submitting it.""" + wrapper = json.loads(freq) + inner = json.loads(wrapper[0][0][1]) + request = inner[0][0] + model = str(request[1]) + match = re.fullmatch(r"macaron_cookie_t2v_(\d{1,2})s(_360p)?", model) + if not match: + raise ValueError("Not an Omni 1.2 UI request") + seconds = int(match.group(1)) + resolution = "360p" if match.group(2) else "720p" + aspect = int(request[2]) + prompt = str(request[0][2][0][0][0]) + project_id = str(inner[1][5]) + estimate_omni_12_credits(seconds, resolution) # validates duration and resolution + if aspect not in (fb.VIDEO_ASPECT_PORTRAIT, fb.VIDEO_ASPECT_LANDSCAPE): + raise ValueError("Invalid Omni 1.2 aspect") + return prompt, project_id, seconds, resolution, aspect + + +def read_credit_quote(body: str) -> int: + """Parse a visible live quote; unknown UI locale fails without spending.""" + text = body[-1500:] + patterns = ( + r"кредитів:\s*(\d+)", + r"(\d+)\s+credits", + r"credits?:\s*(\d+)", + r"(\d+)\s+Credits", + ) + for pattern in patterns: + found = re.search(pattern, text, flags=re.IGNORECASE) + if found: + return int(found.group(1)) + raise RuntimeError("Flow's live credit quote was not visible; refusing generation") + + +async def _click_radio(page: Any, label: str, *, icon: bool = False) -> None: + radios = page.locator('button[role="radio"]:visible') + for i in range(await radios.count()): + item = radios.nth(i) + if icon: + val = await item.locator("mat-icon").first.inner_text() if await item.locator("mat-icon").count() else "" + ok = val.strip() == label + else: + ok = label in (await item.inner_text()) + if ok: + if await item.get_attribute("aria-checked") != "true": + await item.click() + await page.wait_for_timeout(160) + return + raise RuntimeError(f"Flow UI option not found: {label}") + + +async def _pick_model(page: Any) -> None: + triggers = page.locator("button").filter( + has=page.locator("mat-icon", has_text="arrow_drop_down") + ) + for i in range(await triggers.count()): + button = triggers.nth(i) + if not await button.is_visible(): + continue + text = (await button.inner_text()).strip() + if not any(word in text for word in ("Omni", "Veo", "Nano Banana")): + continue + if matches_model_label(text, "Omni 1.2 Flash"): + return + await button.click() + items = page.locator('[role="menuitem"]:visible') + for j in range(await items.count()): + item = items.nth(j) + if matches_model_label(await item.inner_text(), "Omni 1.2 Flash"): + await item.click() + await page.wait_for_timeout(220) + return + raise RuntimeError("Omni 1.2 Flash is not offered by the signed-in Flow account") + raise RuntimeError("Flow UI model menu was not found") + + +async def _set_duration(page: Any, seconds: int) -> None: + slider = page.locator('input[type="range"][min="0"][max="12"]:visible').first + await slider.wait_for(state="visible", timeout=7000) + bounds = await slider.bounding_box() + if not bounds: + raise RuntimeError("Omni 1.2 duration slider is not attached") + await slider.click(position={"x": min(7, bounds["width"]/2), "y": bounds["height"]/2}) + for _ in range(seconds - 3): + await slider.press("ArrowRight") + if int(await slider.input_value()) != seconds - 3: + raise RuntimeError("Flow did not select the requested Omni 1.2 duration") + + +async def _configure(page: Any, *, seconds: int, resolution: str, aspect: int) -> None: + settings = page.locator("button.settings-trigger-button").first + if not await settings.is_visible(): + if await page.locator("flow-creative-agent-prompt-box").is_visible(): + raise RuntimeError("This account uses the new agent-first composer; support for that Flow UI remains pending") + raise RuntimeError("Classic Flow settings trigger is unavailable") + await settings.click() + await _click_radio(page, "videocam", icon=True) + await _click_radio(page, "chrome_extension", icon=True) + await _click_radio(page, "crop_9_16" if aspect == fb.VIDEO_ASPECT_PORTRAIT else "crop_16_9", icon=True) + await _pick_model(page) + await _click_radio(page, resolution) + await _set_duration(page, seconds) + await _click_radio(page, "x1") + await page.wait_for_timeout(320) + quoted = read_credit_quote(await page.locator("body").inner_text()) + expected = estimate_omni_12_credits(seconds, resolution) + cap = int(os.environ.get("FLOW_OMNI12_MAX_CREDITS", "20")) + if cap < 0: + raise ValueError("FLOW_OMNI12_MAX_CREDITS must be non-negative") + if quoted > cap: + raise RuntimeError(f"Flow quotes {quoted} credits; exceeds configured max {cap}") + if quoted > expected: + raise RuntimeError( + f"Flow quotes {quoted} credits, above the known {expected}-credit limit; refusing generation" + ) + await settings.click() + await page.wait_for_timeout(180) + + +async def _capture_generation_response(route: Any, future: asyncio.Future, timeout: float) -> None: + """Proxy the UI-generated POST only once; capture body before navigation.""" + try: + response = await route.fetch(timeout=int(timeout * 1000)) + body = await response.text() + await route.fulfill(response=response) + if not future.done(): + future.set_result((int(response.status), body)) + except Exception as exc: + if not future.done(): + future.set_exception(exc) + try: + await route.abort() + except Exception: + pass + + +async def run_omni_12_ui_generation( + freq: str, *, project_id: str | None, timeout: float = 120 +) -> dict: + """Submit at most once via Flow's own UI, never by replaying a model id.""" + cdp = os.environ.get("FLOW_OMNI12_CDP_URL", "").strip() + if not cdp or urlparse(cdp).hostname not in ("127.0.0.1", "localhost"): + return {"status": 422, "error": "FLOW_OMNI12_CDP_URL must be an explicit local Chrome CDP address"} + try: + prompt, pid, seconds, resolution, aspect = parse_ui_request(freq) + if project_id and pid != project_id: + raise ValueError("Requested project and UI generation project differ") + configured_cap = int(os.environ.get("FLOW_OMNI12_MAX_CREDITS", "20")) + if configured_cap < 0: + raise ValueError("FLOW_OMNI12_MAX_CREDITS cannot be negative") + if estimate_omni_12_credits(seconds, resolution) > configured_cap: + return {"status": 422, "error": f"Omni 1.2 generation exceeds max {configured_cap} credits"} + except (ValueError, KeyError, IndexError, TypeError) as exc: + return {"status": 422, "error": str(exc)} + + try: + from playwright.async_api import async_playwright + except ImportError: + return {"status": 503, "error": "Omni 1.2 UI backend requires optional playwright (pip install playwright)"} + + async with async_playwright() as playwright: + browser = await playwright.chromium.connect_over_cdp(cdp) + contexts = browser.contexts + if not contexts: + return {"status": 503, "error": "The configured Chrome has no signed-in Flow profile"} + # Always use an isolated tab in the existing browser profile; never + # close the user's persistent Chrome or replace their active Flow tab. + page = await contexts[0].new_page() + try: + await page.goto( + f"https://flow.google.com/project/{pid}", + wait_until="domcontentloaded", + timeout=30000, + ) + await page.wait_for_timeout(1100) + await _configure(page, seconds=seconds, resolution=resolution, aspect=aspect) + editor = page.locator('[contenteditable="true"]:visible').first + if not await editor.count(): + editor = page.locator("textarea:visible").first + await editor.wait_for(state="visible", timeout=9000) + await editor.fill(prompt) + await page.wait_for_timeout(250) + if (await editor.inner_text()).strip() != prompt.strip() and await editor.evaluate("e=>e.isContentEditable"): + raise RuntimeError("Flow composer did not accept the prompt") + + future = asyncio.get_running_loop().create_future() + async def on_route(route: Any) -> None: + await _capture_generation_response(route, future, timeout) + + def matcher(url: Any) -> bool: + text = str(url) + return "batchexecute" in text and f"rpcids={fb.RPC_GEN_VIDEO_TEXT}" in text + + await page.route(matcher, on_route) + try: + generate = page.locator("button.generate-icon-button:visible").first + await generate.wait_for(state="visible", timeout=7000) + if await generate.is_disabled(): + raise RuntimeError("Flow Generate button is disabled") + await generate.click() # ONE submit; no automatic retry + status, body = await asyncio.wait_for(future, timeout=timeout) + finally: + await page.unroute(matcher, on_route) + + if "PUBLIC_ERROR_UNUSUAL_ACTIVITY" in body: + return {"status": 429, "error": "PUBLIC_ERROR_UNUSUAL_ACTIVITY"} + return {"status": status, "data": body} + except Exception as exc: + logger.warning("Omni 1.2 UI submit failed (no automatic retry): %s", exc) + return {"status": 502, "error": f"OMNI_12_UI_SUBMIT_FAILED: {type(exc).__name__}: {exc}"} + finally: + await page.close() diff --git a/agent/services/omni_model_versions.py b/agent/services/omni_model_versions.py new file mode 100644 index 00000000..28b78204 --- /dev/null +++ b/agent/services/omni_model_versions.py @@ -0,0 +1,32 @@ +"""User-facing Omni model labels and Flow credit preflight. + +Ported from the alias and duration work in GTAI-1/flow-mcp PR #30. +This module handles UI-facing labels and published credit preflight. +""" +from __future__ import annotations + +import re + +OMNI_11_DURATIONS = (4, 6, 8, 10) +OMNI_12_DURATIONS = tuple(range(3, 16)) + + +def is_omni_12_label(value: str) -> bool: + tokens = re.findall(r"[a-z]+|[0-9]+(?:\.[0-9]+)*", str(value).lower()) + return all(term in tokens for term in ("omni", "1.2", "flash")) + + +def matches_model_label(shown: str, requested: str) -> bool: + if is_omni_12_label(requested): + return is_omni_12_label(shown) + return str(requested).strip().casefold() in str(shown).casefold() + + +def estimate_omni_12_credits(duration: int, resolution: str) -> int: + """Flow Nov-2026 posted per-second pricing for one clip.""" + if type(duration) is not int or duration not in OMNI_12_DURATIONS: + raise ValueError("Omni 1.2 supports integer seconds from 3 to 15") + if resolution not in ("360p", "720p"): + raise ValueError("Omni 1.2 resolution must be 360p or 720p") + unit = 1 if resolution == "360p" else 3 + return unit * min(duration, 10) + 2 * unit * max(0, duration - 10) diff --git a/docs/OMNI_12.md b/docs/OMNI_12.md new file mode 100644 index 00000000..26d6a6b4 --- /dev/null +++ b/docs/OMNI_12.md @@ -0,0 +1,62 @@ +# Optional Omni 1.2 trusted-UI transport (complements PR #85) + +This adds an opt-in, signed-in Chrome UI path for Omni Flash 1.2 +text-to-video requests. It is complementary to the captured +macaron_cookie wire-key integration in [PR #85](https://github.com/crisng95/flowkit/pull/85). +Once PR #85 is merged, clients continue to use the same +/api/flow/generate-video-omni-text API with omni_version = "1.2". + +## Why offer a browser UI path? + +The Flow frontend owns the model menu, duration slider and reCAPTCHA +evaluation. A direct extension batch request can be rejected by Google's +account risk controls even when the normal signed-in UI works. +This optional backend navigates the *existing dedicated Flow Chrome session* +and clicks Generate once. It captures the response before Flow navigates. + +The model aliases and durations are adapted from +[GTAI-1/flow-mcp #29](https://github.com/GTAI-1/flow-mcp/pull/29) and +[#30](https://github.com/GTAI-1/flow-mcp/pull/30). +The optional transport is for the classic Flow composer. Agent-first Flow +support from #29 is not included and fails explicitly without charging. + +## Setup + +1. Install the optional Python Playwright package. This uses an existing + Chrome browser, so it does not download Playwright Chromium. +2. Connect a **dedicated** local Chrome with a signed-in Flow profile and CDP + on a free localhost port. Do not reuse another app's Chrome. +3. Configure FLOW_OMNI12_CDP_URL=http://127.0.0.1:9224 (change the port as + appropriate) and FLOW_OMNI12_MAX_CREDITS=20 (default). + +If FLOW_OMNI12_CDP_URL is unset, **nothing changes**: PR #85 continues to +submit through the existing extension batch RPC. The FlowKit extension must +remain connected for account and project bookkeeping. + +The optional UI path recognizes Omni 1.2 text-to-video model keys +macaron_cookie_t2v_s[_360p] for 3–15 integer seconds. Other Omni +modes continue through the regular extension transport. + +The UI checks the 360p/720p resolution, 3–15-second slider and x1 variants, +then reads Flow's *live* credit quote before clicking Generate. A quote +over FLOW_OMNI12_MAX_CREDITS or over the official 1.2 rate is refused, as +is an unrecognized quote locale. Official posted rates are 1 credit/sec +at 360p or 3 credits/sec at 720p through 10s; additional seconds 11–15 +are twice as expensive. + +There are **no automatic retries** of ambiguous requests. This code does +not bypass CAPTCHA, quotas or account risk controls. If Flow flags unusual +activity, stop submitting until the account is permitted to generate again. + +### Validation + +- Unit tests cover model matching, price tiers, duration parsing and + exactly-once response forwarding. +- The classic Flow model picker, 3s duration slider and 3-credit 360p + quote were checked against a signed-in session without submitting. +- A real Omni 1.2 generation was attempted on 2026-11-07, but Flow's + unusual-activity warning prevented successful end-to-end validation. + Do not mark this experimental transport as live-verified. + +Google references: https://support.google.com/flow/answer/16352836 +and https://support.google.com/flow/answer/16526234 diff --git a/tests/unit/test_omni_12_ui.py b/tests/unit/test_omni_12_ui.py new file mode 100644 index 00000000..697ac39b --- /dev/null +++ b/tests/unit/test_omni_12_ui.py @@ -0,0 +1,100 @@ +"""Offline tests for the opt-in browser UI Omni 1.2 bridge.""" +import asyncio +from types import SimpleNamespace + +import pytest + +from agent.services import flow_batch as fb +from agent.services.omni_12_ui import ( + parse_ui_request, read_credit_quote, _capture_generation_response, + run_omni_12_ui_generation, is_omni_12_ui_envelope, +) +from agent.services.omni_model_versions import ( + OMNI_12_DURATIONS, estimate_omni_12_credits, + is_omni_12_label, matches_model_label, +) + + +def test_aliases_match_12_not_11(): + assert matches_model_label("Omni 1.2 Flash", "Omni Flash 1.2") + assert matches_model_label("Gemini Omni Flash 1.2", "Omni 1.2 Flash") + assert not matches_model_label("Omni 1.1 Flash", "Omni Flash 1.2") + assert not is_omni_12_label("Omni Flash 1.1") + + +@pytest.mark.parametrize("seconds",range(3,16)) +def test_ui_marker_parses_all_durations(seconds): + assert seconds in OMNI_12_DURATIONS + freq=fb.text_video_request( + "paper boat", "f7c87337-60dd-4d1b-b109-40158e34a9ab", + aspect="VIDEO_ASPECT_RATIO_LANDSCAPE", + model=f"macaron_cookie_t2v_{seconds}s", + resolution="360p", + ) + prompt,pid,duration,resolution,aspect=parse_ui_request(freq) + assert prompt=="paper boat" + assert pid=="f7c87337-60dd-4d1b-b109-40158e34a9ab" + assert duration==seconds + assert resolution=="360p" + assert aspect==fb.VIDEO_ASPECT_LANDSCAPE + + +@pytest.mark.parametrize("duration,cost360,cost720",[(3,3,9),(10,10,30),(11,12,36),(15,20,60)]) +def test_costs_include_higher_rate_for_11_to_15(duration,cost360,cost720): + assert estimate_omni_12_credits(duration,"360p")==cost360 + assert estimate_omni_12_credits(duration,"720p")==cost720 + + +def test_unknown_quote_fails_closed(): + assert read_credit_quote("Для створення буде використано стільки кредитів: 3")==3 + assert read_credit_quote("This will use 30 credits")==30 + with pytest.raises(RuntimeError,match="live credit quote"): + read_credit_quote("Choose your video model") + + +@pytest.mark.asyncio +async def test_missing_explicit_ui_config_refuses_request(monkeypatch): + monkeypatch.delenv("FLOW_OMNI12_CDP_URL",raising=False) + freq=fb.text_video_request("test","project",model="macaron_cookie_t2v_3s",resolution="360p") + response=await run_omni_12_ui_generation(freq,project_id="project") + assert response["status"]==422 + + +@pytest.mark.asyncio +async def test_low_credit_cap_rejects_15s_without_opening_browser(monkeypatch): + monkeypatch.setenv("FLOW_OMNI12_CDP_URL","http://127.0.0.1:9224") + monkeypatch.setenv("FLOW_OMNI12_MAX_CREDITS","20") + freq=fb.text_video_request("test","project",model="macaron_cookie_t2v_15s") + response=await run_omni_12_ui_generation(freq,project_id="project") + assert response["status"]==422 + assert "max 20 credits" in response["error"] + + +def test_model_check_does_not_match_user_prompt_containing_key(): + freq=fb.text_video_request( + "Please explain the model name macaron_cookie_t2v_3s", + "project",model="abra_t2v_4s") + assert not is_omni_12_ui_envelope(freq) + freq12=fb.text_video_request("test","project",model="macaron_cookie_t2v_3s") + assert is_omni_12_ui_envelope(freq12) + + +@pytest.mark.asyncio +async def test_body_captured_once_before_ui_navigation(): + calls=[] + class Response: + status=200 + async def text(self): + calls.append("read_body") + return "rpc-body" + class Route: + async def fetch(self,**kwargs): + calls.append("forward_once") + return Response() + async def fulfill(self,**kwargs): + calls.append("return_to_flow") + async def abort(self):calls.append("abort") + f=asyncio.get_running_loop().create_future() + await _capture_generation_response(Route(),f,120) + assert await f==(200,"rpc-body") + assert calls==["forward_once","read_body","return_to_flow"]