feat: add image locate step type with template matching, grid click, and config panel

- Backend: image storage endpoints for task reference images
- Backend: click() supports grid=(col,row) and splits=(h,v) for region clicks
- Backend: _exists() catches TimeoutError for image-based checks
- Backend: ws_run_task copies referenced images to temp dir for execution
- Backend: parse/generate # @imglocate JSON metadata in .py steps
- Frontend: new 'imglocate' slash command for image-based positioning
- Frontend: screenshot canvas drag-to-crop for template image capture
- Frontend: image step card with preview, action badge, and grid params
- Frontend: config dialog with grid overlay, cell click selection, sliders
- Frontend: code generation for click/wait_show/wait_hide image actions
- CSS: styles for image step cards and config panel

Author: caishilong-collab

ha4t/api.py

@@ -83,8 +83,32 @@ def perform_click(x, y, duration):
                 device.driver(**args[0], **kwargs).long_click(duration=duration)
     elif kwargs.get("image"):
         path = os.path.join(_CF.CURRENT_PATH, kwargs["image"])
-        pos = match_loop(screenshot_func=device.driver.screenshot, template=path, timeout=kwargs.get("timeout", 10),
-                         threshold=kwargs.get("threshold", 0.8))
+        grid = kwargs.pop("grid", None)
+        splits = kwargs.pop("splits", None)
+        if grid and splits:
+            # Grid-based click: match template, then click center of specified grid cell
+            import cv2
+            from ha4t.aircv.cv import Template
+            source_image = device.driver.screenshot()
+            source_image = np.array(source_image.resize((_CF.SCREEN_WIDTH, _CF.SCREEN_HEIGHT)))
+            template = Template(path, threshold=kwargs.get("threshold", 0.8), rgb=kwargs.get("rgb", False),
+                                scale_max=kwargs.get("scale_max", 800), scale_step=kwargs.get("scale_step", 0.005))
+            result = template._cv_match(source_image)
+            if not result:
+                raise TimeoutError(f"图片匹配失败: {path}")
+            rect = result["rectangle"]  # [(x1,y1), (x2,y2), (x3,y3), (x4,y4)]
+            x1, y1 = rect[0]
+            x3, y3 = rect[2]
+            w = x3 - x1
+            h = y3 - y1
+            cell_w = w / splits[0]
+            cell_h = h / splits[1]
+            target_x = x1 + cell_w * (grid[0] + 0.5)
+            target_y = y1 + cell_h * (grid[1] + 0.5)
+            pos = (int(target_x), int(target_y))
+        else:
+            pos = match_loop(screenshot_func=device.driver.screenshot, template=path, timeout=kwargs.get("timeout", 10),
+                             threshold=kwargs.get("threshold", 0.8))
         perform_click(*pos, duration)
     else:
         timeout = kwargs.pop("timeout", 3)
@@ -143,11 +167,11 @@ def _exists(*args, **kwargs) -> bool:
     else:
         if kwargs.get("image"):
             path = os.path.join(_CF.CURRENT_PATH, kwargs["image"])
-            pos = match_loop(screenshot_func=device.driver.screenshot, template=path, timeout=kwargs.get("timeout", 10),
-                             threshold=kwargs.get("threshold", 0.8))
-            if pos:
-                return True
-            else:
+            try:
+                pos = match_loop(screenshot_func=device.driver.screenshot, template=path, timeout=kwargs.get("timeout", 10),
+                                 threshold=kwargs.get("threshold", 0.8))
+                return True if pos else False
+            except Exception:
                 return False
         else:
             return device.driver(**kwargs).exists

ha4t/editor/routers/api.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import base64
 import json
 import os
 import re
@@ -22,6 +23,7 @@
 from ha4t.editor._models import (    ApiResponse, XPathLiteRequest,
     TaskFile, TaskSaveRequest, TaskRunRequest, TaskStepResult, TaskRunResponse
 )
+from pydantic import BaseModel
 from ha4t.editor.parser.xpath_lite import XPathLiteGenerator
 
 
@@ -134,24 +136,35 @@ def press_key(platform: str, serial: str, key: str):
 # ── Task API (.py + # --step--) ──────────────────────────
 
 
+import json as _json
+
 def _parse_py_steps(content: str) -> list:
-    """Parse # --step-- markers, return list of {code}"""
+    """Parse # --step-- markers, return list of {code, meta}"""
     lines = content.split('\n')
     steps = []
     buf = []
+    meta = None
     in_step = False
     for line in lines:
         stripped = line.strip()
         if stripped == _STEP_MARKER:
             if in_step and buf:
-                steps.append('\n'.join(buf).strip())
+                steps.append({'code': '\n'.join(buf).strip(), 'meta': meta})
                 buf = []
+                meta = None
             in_step = True
             continue
-        if in_step and stripped and not stripped.startswith('#'):
-            buf.append(line)
+        if in_step:
+            if stripped.startswith('# @imglocate'):
+                try:
+                    meta = _json.loads(stripped[len('# @imglocate'):].strip())
+                except Exception:
+                    pass
+                continue
+            if stripped and not stripped.startswith('#'):
+                buf.append(line)
     if in_step and buf:
-        steps.append('\n'.join(buf).strip())
+        steps.append({'code': '\n'.join(buf).strip(), 'meta': meta})
     return steps
 
 
@@ -168,7 +181,7 @@ def _extract_meta(content: str) -> dict:
     return {'name': name, 'desc': desc, 'platform': platform}
 
 
-def _generate_py(name, desc, platform, step_codes, extra_lines=None):
+def _generate_py(name, desc, platform, steps, extra_lines=None):
     lines = ['# name: ' + name]
     if desc:
         lines.append('# desc: ' + desc)
@@ -181,9 +194,12 @@ def _generate_py(name, desc, platform, step_codes, extra_lines=None):
     lines.append('')
     if extra_lines:
         lines.extend(extra_lines)
-    for code in step_codes:
+    for step in steps:
         lines.append(_STEP_MARKER)
-        lines.append(code)
+        meta = step.get('meta')
+        if meta and meta.get('_type') == 'imglocate':
+            lines.append('# @imglocate ' + _json.dumps(meta, ensure_ascii=False, separators=(',', ':')))
+        lines.append(step['code'])
     lines.append('')
     return '\n'.join(lines)
 
@@ -221,6 +237,42 @@ def save_task(filename: str, req: TaskSaveRequest):
     return ApiResponse.doSuccess({"filename": filename, "saved": True})
 
 
+@router.get("/tasks/{filename:path}/images", response_model=ApiResponse)
+def list_task_images(filename: str):
+    stem = Path(filename).stem
+    img_dir = TASKS_DIR / f"{stem}_imgs"
+    if not img_dir.exists():
+        return ApiResponse.doSuccess([])
+    files = [p.name for p in sorted(img_dir.glob("*.png"))]
+    return ApiResponse.doSuccess(files)
+
+
+@router.get("/tasks/{filename:path}/images/{imgname}", response_model=ApiResponse)
+def get_task_image(filename: str, imgname: str):
+    stem = Path(filename).stem
+    img_path = TASKS_DIR / f"{stem}_imgs" / imgname
+    if not img_path.exists():
+        return ApiResponse.doError(f"Image not found: {imgname}")
+    data = base64.b64encode(img_path.read_bytes()).decode("utf-8")
+    return ApiResponse.doSuccess({"filename": imgname, "data": data})
+
+
+class ImageUploadRequest(BaseModel):
+    data: str
+
+@router.post("/tasks/{filename:path}/images/{imgname}", response_model=ApiResponse)
+def save_task_image(filename: str, imgname: str, req: ImageUploadRequest):
+    stem = Path(filename).stem
+    img_dir = TASKS_DIR / f"{stem}_imgs"
+    img_dir.mkdir(parents=True, exist_ok=True)
+    img_path = img_dir / imgname
+    data = req.data or ""
+    if data.startswith("data:image"):
+        data = data.split(",", 1)[1]
+    img_path.write_bytes(base64.b64decode(data))
+    return ApiResponse.doSuccess({"filename": imgname, "saved": True})
+
+
 @router.websocket("/ws/run")
 async def ws_run_task(ws: WebSocket):
     await ws.accept()
@@ -242,6 +294,16 @@ async def ws_run_task(ws: WebSocket):
     tmppy = tmpdir / (filename or "untitled.py")
     tmppy.write_text(content, encoding="utf-8")
 
+    # Copy referenced images to temp dir so the script can find them
+    stem = Path(filename).stem
+    img_dir = TASKS_DIR / f"{stem}_imgs"
+    if img_dir.exists():
+        for img in img_dir.glob("*.png"):
+            try:
+                shutil.copy(img, tmpdir / img.name)
+            except Exception:
+                pass
+
     step_codes = _parse_py_steps(content)
     total = len(step_codes)
 

ha4t/editor/static/css/style.css

@@ -417,4 +417,116 @@ code {
 .log-info { color: #888; }
 .log-ok { color: #67c23a; }
 .log-fail { color: #f56c6c; }
-.log-warn { color: #e6a23c; }
+.log-warn { color: #e6a23c; }
+
+.capture-mode #hierarchyCanvas {
+    cursor: crosshair !important;
+}
+
+.capture-indicator {
+    position: absolute;
+    top: 35px;
+    left: 50%;
+    transform: translateX(-50%);
+    background: rgba(54, 121, 227, 0.9);
+    color: #fff;
+    padding: 4px 12px;
+    border-radius: 4px;
+    font-size: 12px;
+    z-index: 20;
+    pointer-events: none;
+}
+
+/* ── Image Step Card ── */
+.step-img {
+    padding: 6px 12px;
+    min-height: auto;
+}
+.img-step-card {
+    display: flex;
+    gap: 10px;
+    width: 100%;
+    height: 200px;
+}
+.img-step-preview {
+    width: 160px;
+    height: 100%;
+    flex-shrink: 0;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background: #1a2530;
+    border-radius: 4px;
+    overflow: hidden;
+    border: 1px solid #2a3040;
+}
+.img-step-preview img {
+    max-width: 100%;
+    max-height: 100%;
+    object-fit: contain;
+}
+.img-step-info {
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    gap: 4px;
+    overflow: hidden;
+}
+.img-step-title {
+    display: flex;
+    align-items: center;
+    gap: 4px;
+    font-size: 12px;
+}
+.img-step-badge {
+    background: #3679E3;
+    color: #fff;
+    padding: 1px 6px;
+    border-radius: 3px;
+    font-size: 11px;
+}
+.img-step-params {
+    display: flex;
+    flex-direction: column;
+    gap: 2px;
+    color: #8492a6;
+    font-size: 11px;
+}
+.img-step-btns {
+    display: flex;
+    gap: 2px;
+    margin-top: auto;
+}
+
+/* ── Image Config Panel ── */
+.img-config-body {
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+}
+.img-config-preview-wrap {
+    position: relative;
+    width: 100%;
+    height: 220px;
+    background: #1a2530;
+    border-radius: 4px;
+    overflow: hidden;
+    border: 1px solid #2a3040;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+.img-config-preview {
+    max-width: 100%;
+    max-height: 100%;
+    object-fit: contain;
+}
+.img-config-grid {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    pointer-events: auto;
+    cursor: crosshair;
+}