From 554f6ce9c5ad4d9744d4783222406f6ef2efb658 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 8 Sep 2026 17:19:55 +0200 Subject: [PATCH 1/2] Migrate legacy notebooks in shared readers --- aidialog/_modidx.py | 4 + aidialog/ipynb.py | 115 ++++++++++---- nbs/02_ipynb.ipynb | 373 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 407 insertions(+), 85 deletions(-) diff --git a/aidialog/_modidx.py b/aidialog/_modidx.py index c86cd09..27e5c91 100644 --- a/aidialog/_modidx.py +++ b/aidialog/_modidx.py @@ -226,12 +226,16 @@ 'aidialog.ipynb.Message.to_cell': ('ipynb.html#message.to_cell', 'aidialog/ipynb.py'), 'aidialog.ipynb._clean_out_meta': ('ipynb.html#_clean_out_meta', 'aidialog/ipynb.py'), 'aidialog.ipynb._output_from_cell': ('ipynb.html#_output_from_cell', 'aidialog/ipynb.py'), + 'aidialog.ipynb._v1_to_v2': ('ipynb.html#_v1_to_v2', 'aidialog/ipynb.py'), + 'aidialog.ipynb._v2_to_v3': ('ipynb.html#_v2_to_v3', 'aidialog/ipynb.py'), 'aidialog.ipynb.att2dict': ('ipynb.html#att2dict', 'aidialog/ipynb.py'), 'aidialog.ipynb.conv_old_prompts': ('ipynb.html#conv_old_prompts', 'aidialog/ipynb.py'), 'aidialog.ipynb.dict2att': ('ipynb.html#dict2att', 'aidialog/ipynb.py'), 'aidialog.ipynb.get_ipynb': ('ipynb.html#get_ipynb', 'aidialog/ipynb.py'), 'aidialog.ipynb.home_atts': ('ipynb.html#home_atts', 'aidialog/ipynb.py'), 'aidialog.ipynb.ipynb_cells': ('ipynb.html#ipynb_cells', 'aidialog/ipynb.py'), + 'aidialog.ipynb.migrate_nb': ('ipynb.html#migrate_nb', 'aidialog/ipynb.py'), + 'aidialog.ipynb.nb_ver': ('ipynb.html#nb_ver', 'aidialog/ipynb.py'), 'aidialog.ipynb.read_ipynb': ('ipynb.html#read_ipynb', 'aidialog/ipynb.py'), 'aidialog.ipynb.reads_ipynb': ('ipynb.html#reads_ipynb', 'aidialog/ipynb.py'), 'aidialog.ipynb.safe_mtime': ('ipynb.html#safe_mtime', 'aidialog/ipynb.py'), diff --git a/aidialog/ipynb.py b/aidialog/ipynb.py index cb12c12..6cf70e6 100644 --- a/aidialog/ipynb.py +++ b/aidialog/ipynb.py @@ -5,8 +5,8 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_ipynb.ipynb. # %% auto #0 -__all__ = ['att2dict', 'home_atts', 'unhome_atts', 'get_ipynb', 'safe_mtime', 'write_ipynb', 'ipynb_cells', 'dict2att', - 'reads_ipynb', 'read_ipynb', 'conv_old_prompts'] +__all__ = ['CUR_VER', 'att2dict', 'home_atts', 'unhome_atts', 'conv_old_prompts', 'nb_ver', 'migrate_nb', 'get_ipynb', + 'safe_mtime', 'write_ipynb', 'ipynb_cells', 'dict2att', 'reads_ipynb', 'read_ipynb'] # %% ../nbs/02_ipynb.ipynb #d3d0463b from fastcore.utils import * @@ -16,6 +16,7 @@ from contextlib import suppress from base64 import b64encode,b64decode from .dialog import * +from .msg_parts import conv_tools # %% ../nbs/02_ipynb.ipynb #aaaeb85a def att2dict(att): return {att.content_type: b64encode(att.data).decode('ascii') if isinstance(att.data, bytes) else att.data} @@ -59,7 +60,7 @@ def cell_meta(self:Message): _prompt_magic = '%%prompt' @patch -def to_cell(self:Message, version=2): +def to_cell(self:Message): "Convert message to a notebook cell" meta = self.cell_meta() src = self.source @@ -67,21 +68,81 @@ def to_cell(self:Message, version=2): meta['solveit_ai'] = True src = f'{_prompt_magic}\n{src}' outkw = {} - if self.msg_type in (scode,sprompt) and self.output: - outputs = self.output - if version==1 and self.msg_type==scode: outputs = json.loads(outputs) - outkw['outputs'] = [_clean_out_meta(o) for o in outputs] + if self.msg_type in (scode,sprompt) and self.output: outkw['outputs'] = [_clean_out_meta(o) for o in self.output] atts = {att.id: att2dict(att) for att in (self.attachments or [])} if atts: outkw['attachments'] = atts cell = mk_cell(src, self.cell_type, id=self.id, metadata=meta, **outkw) if repairs := repair_cell(cell): print('NB repair:', '; '.join(repairs)) return cell +# %% ../nbs/02_ipynb.ipynb #ded8e1b4 +_reply_sep = "\n\n##### 🤖Reply🤖\n\n" + +def conv_old_prompts(nb): + "Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids" + changed = [] + for c in nb['cells']: + if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue + if isinstance(c['source'], list): c['source'] = ''.join(c['source']) + content,*reply = c['source'].split(_reply_sep) + c['cell_type'] = 'code' + c['source'] = f'{_prompt_magic}\n{content}' + c['outputs'] = prompt_output(reply[0]) if reply else [] + c['execution_count'] = None + changed.append(c['id']) + if changed: home_atts(nb) + return changed + +# %% ../nbs/02_ipynb.ipynb #migrations +CUR_VER = 3 + +def nb_ver(nb): + "Stored notebook version; unversioned files start at v1" + meta = nb.get('metadata', {}) + return meta.get('solveit', {}).get('ver', meta.get('solveit_ver', 1)) + +# %% ../nbs/02_ipynb.ipynb #migratev1 +def _v1_to_v2(nb): + meta = nb.setdefault('metadata', {}) + sv = meta.get('solveit', {}) + modes = {1:'standard', 2:'learning', 3:'concise'} + if isinstance(sv.get('mode'), int): sv['mode'] = modes[sv['mode']] + if isinstance(meta.get('solveit_dialog_mode'), int): meta['solveit_dialog_mode'] = modes[meta['solveit_dialog_mode']] + +# %% ../nbs/02_ipynb.ipynb #migratev2 +def _v2_to_v3(nb): + meta = nb.setdefault('metadata', {}) + sv = meta.setdefault('solveit', {}) + if 'solveit_dialog_mode' in meta: sv.setdefault('mode', meta.pop('solveit_dialog_mode')) + meta.pop('solveit_ver', None) + conv_old_prompts(nb) + for c in nb['cells']: + if not c['metadata'].get('solveit_ai'): continue + for o in c.get('outputs', []): + md = nested_idx(o, 'metadata', 'is_ai_res') and nested_idx(o, 'data', 'text/markdown') + if not md: continue + if isinstance(md, list): md = ''.join(md) + if '-usage-details' in md: o['data']['text/markdown'] = conv_tools(md) + +# %% ../nbs/02_ipynb.ipynb #migratenb +_migrations = {1:_v1_to_v2, 2:_v2_to_v3} + +def migrate_nb(nb): + "Upgrade a notebook in place. Return whether its version changed" + ver = nb_ver(nb) + if not 1 <= ver <= CUR_VER: raise ValueError(f'Unsupported notebook version: {ver}') + for v in range(ver, CUR_VER): + _migrations[v](nb) + nb.setdefault('metadata', {}).setdefault('solveit', {})['ver'] = v+1 + return ver < CUR_VER + # %% ../nbs/02_ipynb.ipynb #10a73cfc -def get_ipynb(dlg:Dialog, version=2, msgs=None): +def get_ipynb(dlg:Dialog, msgs=None): "Notebook object for `dlg`; `msgs` defaults to all its messages" - cells = [m.to_cell(version=version) for m in (dlg.messages if msgs is None else msgs)] - nb = new_nb(cells=cells, meta=dict(dlg.meta)) + cells = [m.to_cell() for m in (dlg.messages if msgs is None else msgs)] + meta = dict(dlg.meta) + meta['solveit'] = dict(meta.get('solveit', {}), ver=CUR_VER) + nb = new_nb(cells=cells, meta=meta) home_atts(nb) if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs)) return nb @@ -94,18 +155,19 @@ def safe_mtime(p): with suppress(FileNotFoundError): return p.stat(follow_symlinks=False).st_mtime # %% ../nbs/02_ipynb.ipynb #ea5fa1dd -def write_ipynb(dlg:Dialog, fname=None, version=2, msgs=None, **kwargs): +def write_ipynb(dlg:Dialog, fname=None, msgs=None, **kwargs): "Write `dlg` as a notebook, or return the JSON string if `fname` is None; `kwargs` (e.g. `uid`/`gid`) pass to `atomic_save`" - res = nb2str(get_ipynb(dlg, version=version, msgs=msgs)) + res = nb2str(get_ipynb(dlg, msgs=msgs)) if not fname: return res fname = Path(fname).expanduser() with atomic_save(fname, mode='w', encoding='utf-8', **kwargs) as f: f.write(res) dlg.mtime_ = safe_mtime(fname) + dlg.migrated_ = False # %% ../nbs/02_ipynb.ipynb #5527e596 @patch -def write(self:Dialog, base_path, version=2, msgs=None, **kwargs): - write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', version=version, msgs=msgs, **kwargs) +def write(self:Dialog, base_path, msgs=None, **kwargs): + write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', msgs=msgs, **kwargs) # %% ../nbs/02_ipynb.ipynb #0e16e93a def ipynb_cells(path, nm, prefix=None, suffix=None): @@ -114,6 +176,8 @@ def ipynb_cells(path, nm, prefix=None, suffix=None): try: nb = read_nb(tmpl) except json.JSONDecodeError: return [] if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs)) + migrate_nb(nb) + unhome_atts(nb) return listify(prefix) + nb.cells + listify(suffix) # %% ../nbs/02_ipynb.ipynb #ba0bb61e @@ -153,9 +217,12 @@ def reads_ipynb(txt, cls=Dialog, name='dialog', verbose=False): "Read a dialog from notebook JSON string `txt`, constructing via `cls`" nb = json.loads(txt) if (repairs := repair_nb(nb)) and verbose: print('NB repair:', '; '.join(repairs)) + migrated = migrate_nb(nb) unhome_atts(nb) nb = dict2nb(nb) - return cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells) + res = cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells) + res.migrated_ = migrated + return res # %% ../nbs/02_ipynb.ipynb #7e8912e8 def read_ipynb(fname, cls=Dialog, name=None, verbose=False): @@ -176,21 +243,3 @@ def save(self:Dialog, fname=None): fname = fname or self.path_ if not fname: raise ValueError('no fname passed, and no `path_` stamped by read_ipynb') write_ipynb(self, fname) - -# %% ../nbs/02_ipynb.ipynb #ded8e1b4 -_reply_sep = "\n\n##### 🤖Reply🤖\n\n" - -def conv_old_prompts(nb): - "Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids" - changed = [] - for c in nb['cells']: - if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue - if isinstance(c['source'], list): c['source'] = ''.join(c['source']) - content,*reply = c['source'].split(_reply_sep) - c['cell_type'] = 'code' - c['source'] = f'{_prompt_magic}\n{content}' - c['outputs'] = prompt_output(reply[0]) if reply else [] - c['execution_count'] = None - changed.append(c['id']) - if changed: home_atts(nb) - return changed diff --git a/nbs/02_ipynb.ipynb b/nbs/02_ipynb.ipynb index 8bbe306..de68671 100644 --- a/nbs/02_ipynb.ipynb +++ b/nbs/02_ipynb.ipynb @@ -51,7 +51,8 @@ "import json\n", "from contextlib import suppress\n", "from base64 import b64encode,b64decode\n", - "from aidialog.dialog import *" + "from aidialog.dialog import *\n", + "from aidialog.msg_parts import conv_tools" ] }, { @@ -67,6 +68,16 @@ "from fastcore.test import *" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f982b4c", + "metadata": {}, + "outputs": [], + "source": [ + "from aidialog.msg_parts import ToolUse, ToolResult, fmt2hist" + ] + }, { "cell_type": "code", "execution_count": null, @@ -243,7 +254,7 @@ "_prompt_magic = '%%prompt'\n", "\n", "@patch\n", - "def to_cell(self:Message, version=2):\n", + "def to_cell(self:Message):\n", " \"Convert message to a notebook cell\"\n", " meta = self.cell_meta()\n", " src = self.source\n", @@ -251,10 +262,7 @@ " meta['solveit_ai'] = True\n", " src = f'{_prompt_magic}\\n{src}'\n", " outkw = {}\n", - " if self.msg_type in (scode,sprompt) and self.output:\n", - " outputs = self.output\n", - " if version==1 and self.msg_type==scode: outputs = json.loads(outputs)\n", - " outkw['outputs'] = [_clean_out_meta(o) for o in outputs]\n", + " if self.msg_type in (scode,sprompt) and self.output: outkw['outputs'] = [_clean_out_meta(o) for o in self.output]\n", " atts = {att.id: att2dict(att) for att in (self.attachments or [])}\n", " if atts: outkw['attachments'] = atts\n", " cell = mk_cell(src, self.cell_type, id=self.id, metadata=meta, **outkw)\n", @@ -358,6 +366,139 @@ "pr_cell\n" ] }, + { + "cell_type": "markdown", + "id": "53ee134e", + "metadata": {}, + "source": [ + "## Converting old prompts" + ] + }, + { + "cell_type": "markdown", + "id": "dcd7920b", + "metadata": {}, + "source": [ + "`conv_old_prompts` rewrites a notebook's markdown-form prompt cells to the code-cell form, in place, and returns the ids it changed — so a host can convert a file the first time it opens it, and write back only when something changed. Converted cells no longer match the predicate, so it is idempotent by construction. A retype makes any prompt attachments illegal where they sit, so it finishes with `home_atts`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ded8e1b4", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_reply_sep = \"\\n\\n##### 🤖Reply🤖\\n\\n\"\n", + "\n", + "def conv_old_prompts(nb):\n", + " \"Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids\"\n", + " changed = []\n", + " for c in nb['cells']:\n", + " if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue\n", + " if isinstance(c['source'], list): c['source'] = ''.join(c['source'])\n", + " content,*reply = c['source'].split(_reply_sep)\n", + " c['cell_type'] = 'code'\n", + " c['source'] = f'{_prompt_magic}\\n{content}'\n", + " c['outputs'] = prompt_output(reply[0]) if reply else []\n", + " c['execution_count'] = None\n", + " changed.append(c['id'])\n", + " if changed: home_atts(nb)\n", + " return changed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migrations", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "CUR_VER = 3\n", + "\n", + "def nb_ver(nb):\n", + " \"Stored notebook version; unversioned files start at v1\"\n", + " meta = nb.get('metadata', {})\n", + " return meta.get('solveit', {}).get('ver', meta.get('solveit_ver', 1))" + ] + }, + { + "cell_type": "markdown", + "id": "b8ff7e95", + "metadata": {}, + "source": [ + "`nb_ver` reads the version without scanning cell contents. Writers stamp `CUR_VER` in `metadata.solveit.ver`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migratev1", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _v1_to_v2(nb):\n", + " meta = nb.setdefault('metadata', {})\n", + " sv = meta.get('solveit', {})\n", + " modes = {1:'standard', 2:'learning', 3:'concise'}\n", + " if isinstance(sv.get('mode'), int): sv['mode'] = modes[sv['mode']]\n", + " if isinstance(meta.get('solveit_dialog_mode'), int): meta['solveit_dialog_mode'] = modes[meta['solveit_dialog_mode']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migratev2", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _v2_to_v3(nb):\n", + " meta = nb.setdefault('metadata', {})\n", + " sv = meta.setdefault('solveit', {})\n", + " if 'solveit_dialog_mode' in meta: sv.setdefault('mode', meta.pop('solveit_dialog_mode'))\n", + " meta.pop('solveit_ver', None)\n", + " conv_old_prompts(nb)\n", + " for c in nb['cells']:\n", + " if not c['metadata'].get('solveit_ai'): continue\n", + " for o in c.get('outputs', []):\n", + " md = nested_idx(o, 'metadata', 'is_ai_res') and nested_idx(o, 'data', 'text/markdown')\n", + " if not md: continue\n", + " if isinstance(md, list): md = ''.join(md)\n", + " if '-usage-details' in md: o['data']['text/markdown'] = conv_tools(md)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migratenb", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "_migrations = {1:_v1_to_v2, 2:_v2_to_v3}\n", + "\n", + "def migrate_nb(nb):\n", + " \"Upgrade a notebook in place. Return whether its version changed\"\n", + " ver = nb_ver(nb)\n", + " if not 1 <= ver <= CUR_VER: raise ValueError(f'Unsupported notebook version: {ver}')\n", + " for v in range(ver, CUR_VER):\n", + " _migrations[v](nb)\n", + " nb.setdefault('metadata', {}).setdefault('solveit', {})['ver'] = v+1\n", + " return ver < CUR_VER" + ] + }, + { + "cell_type": "markdown", + "id": "bb0af64e", + "metadata": {}, + "source": [ + "`migrate_nb` applies each version step in order. Version 1 stores mode metadata in the old form. Version 2 permits markdown prompts and legacy tool envelopes. Version 3 stores code prompts and fenced tool records. The notebook stays in memory until a caller saves it. A newer, unsupported version raises an error." + ] + }, { "cell_type": "code", "execution_count": null, @@ -366,10 +507,12 @@ "outputs": [], "source": [ "#| export\n", - "def get_ipynb(dlg:Dialog, version=2, msgs=None):\n", + "def get_ipynb(dlg:Dialog, msgs=None):\n", " \"Notebook object for `dlg`; `msgs` defaults to all its messages\"\n", - " cells = [m.to_cell(version=version) for m in (dlg.messages if msgs is None else msgs)]\n", - " nb = new_nb(cells=cells, meta=dict(dlg.meta))\n", + " cells = [m.to_cell() for m in (dlg.messages if msgs is None else msgs)]\n", + " meta = dict(dlg.meta)\n", + " meta['solveit'] = dict(meta.get('solveit', {}), ver=CUR_VER)\n", + " nb = new_nb(cells=cells, meta=meta)\n", " home_atts(nb)\n", " if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))\n", " return nb\n" @@ -397,13 +540,14 @@ "outputs": [], "source": [ "#| export\n", - "def write_ipynb(dlg:Dialog, fname=None, version=2, msgs=None, **kwargs):\n", + "def write_ipynb(dlg:Dialog, fname=None, msgs=None, **kwargs):\n", " \"Write `dlg` as a notebook, or return the JSON string if `fname` is None; `kwargs` (e.g. `uid`/`gid`) pass to `atomic_save`\"\n", - " res = nb2str(get_ipynb(dlg, version=version, msgs=msgs))\n", + " res = nb2str(get_ipynb(dlg, msgs=msgs))\n", " if not fname: return res\n", " fname = Path(fname).expanduser()\n", " with atomic_save(fname, mode='w', encoding='utf-8', **kwargs) as f: f.write(res)\n", - " dlg.mtime_ = safe_mtime(fname)" + " dlg.mtime_ = safe_mtime(fname)\n", + " dlg.migrated_ = False" ] }, { @@ -413,7 +557,7 @@ "metadata": {}, "outputs": [], "source": [ - "write_ipynb(dlg, fname=tstdir/'dlg.ipynb', version=2)" + "write_ipynb(dlg, fname=tstdir/'dlg.ipynb')" ] }, { @@ -425,8 +569,8 @@ "source": [ "#| export\n", "@patch\n", - "def write(self:Dialog, base_path, version=2, msgs=None, **kwargs):\n", - " write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', version=version, msgs=msgs, **kwargs)" + "def write(self:Dialog, base_path, msgs=None, **kwargs):\n", + " write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', msgs=msgs, **kwargs)" ] }, { @@ -453,9 +597,19 @@ " try: nb = read_nb(tmpl)\n", " except json.JSONDecodeError: return []\n", " if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))\n", + " migrate_nb(nb)\n", + " unhome_atts(nb)\n", " return listify(prefix) + nb.cells + listify(suffix)" ] }, + { + "cell_type": "markdown", + "id": "7759cc58", + "metadata": {}, + "source": [ + "`ipynb_cells` upgrades file contents before adding the caller's prefix and suffix cells. CRAFT and template loaders use this reader. It restores attachments to cells and does not write the file." + ] + }, { "cell_type": "code", "execution_count": null, @@ -597,9 +751,20 @@ " \"Read a dialog from notebook JSON string `txt`, constructing via `cls`\"\n", " nb = json.loads(txt)\n", " if (repairs := repair_nb(nb)) and verbose: print('NB repair:', '; '.join(repairs))\n", + " migrated = migrate_nb(nb)\n", " unhome_atts(nb)\n", " nb = dict2nb(nb)\n", - " return cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)" + " res = cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)\n", + " res.migrated_ = migrated\n", + " return res" + ] + }, + { + "cell_type": "markdown", + "id": "a29a996a", + "metadata": {}, + "source": [ + "`reads_ipynb` marks an upgraded dialog with `migrated_`. This attribute is not notebook metadata. A successful file save clears it." ] }, { @@ -863,44 +1028,10 @@ }, { "cell_type": "markdown", - "id": "53ee134e", + "id": "a23cbaa7", "metadata": {}, "source": [ - "## Converting old prompts" - ] - }, - { - "cell_type": "markdown", - "id": "dcd7920b", - "metadata": {}, - "source": [ - "`conv_old_prompts` rewrites a notebook's markdown-form prompt cells to the code-cell form, in place, and returns the ids it changed — so a host can convert a file the first time it opens it, and write back only when something changed. Converted cells no longer match the predicate, so it is idempotent by construction. A retype makes any prompt attachments illegal where they sit, so it finishes with `home_atts`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ded8e1b4", - "metadata": {}, - "outputs": [], - "source": [ - "#| export\n", - "_reply_sep = \"\\n\\n##### 🤖Reply🤖\\n\\n\"\n", - "\n", - "def conv_old_prompts(nb):\n", - " \"Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids\"\n", - " changed = []\n", - " for c in nb['cells']:\n", - " if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue\n", - " if isinstance(c['source'], list): c['source'] = ''.join(c['source'])\n", - " content,*reply = c['source'].split(_reply_sep)\n", - " c['cell_type'] = 'code'\n", - " c['source'] = f'{_prompt_magic}\\n{content}'\n", - " c['outputs'] = prompt_output(reply[0]) if reply else []\n", - " c['execution_count'] = None\n", - " changed.append(c['id'])\n", - " if changed: home_atts(nb)\n", - " return changed" + "### Legacy prompt conversion" ] }, { @@ -934,6 +1065,144 @@ "back.ai_res" ] }, + { + "cell_type": "markdown", + "id": "d7acdf0c", + "metadata": {}, + "source": [ + "## Notebook migrations" + ] + }, + { + "cell_type": "markdown", + "id": "5eafab6c", + "metadata": {}, + "source": [ + "Old CRAFT notebooks must produce structured tool calls when read. Reading upgrades the in-memory copy. The source file stays unchanged until it is saved." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migfixture", + "metadata": {}, + "outputs": [], + "source": [ + "legacy_reply = \"\"\"
\n", + "```json\n", + "{\"id\":\"call1\",\"call\":{\"function\":\"python\",\"arguments\":{\"code\":\"1+1\"}},\"result\":\"2\"}\n", + "```\n", + "
\"\"\"\n", + "legacy_nb = new_nb([mk_cell('Calculate.' + _reply_sep + legacy_reply, 'markdown',\n", + " id='legacy1', metadata=dict(solveit_ai=True), attachments={'att1': {'image/png': 'aGk='}})],\n", + " meta=dict(solveit=dict(ver=2, mode='learning')))\n", + "legacy_text = nb2str(legacy_nb)\n", + "(tstdir/'CRAFT.ipynb').write_text(legacy_text);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migread", + "metadata": {}, + "outputs": [], + "source": [ + "upgraded = read_ipynb(tstdir/'CRAFT.ipynb')\n", + "test_eq(upgraded.messages[0].content, 'Calculate.')\n", + "assert upgraded.migrated_\n", + "test_eq(upgraded.meta['solveit']['ver'], 3)\n", + "test_eq(upgraded.messages[0].attachments[0].data, b'hi')\n", + "parts = [p for m in fmt2hist(upgraded.messages[0].ai_res) for p in m.content if isinstance(p, (ToolUse, ToolResult))]\n", + "test_eq([type(p) for p in parts], [ToolUse, ToolResult])\n", + "test_eq((parts[0].name, parts[0].arguments, parts[1].text), ('python', {'code':'1+1'}, '2'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migsave", + "metadata": {}, + "outputs": [], + "source": [ + "craft = Dialog().from_cells(ipynb_cells(tstdir, 'CRAFT'))\n", + "test_eq(craft.messages[0].ai_res, upgraded.messages[0].ai_res)\n", + "test_eq(craft.messages[0].attachments[0].data, b'hi')\n", + "test_eq((tstdir/'CRAFT.ipynb').read_text(), legacy_text)\n", + "upgraded.save()\n", + "assert not upgraded.migrated_\n", + "assert not read_ipynb(tstdir/'CRAFT.ipynb').migrated_\n", + "nbformat.validate(nbformat.read(tstdir/'CRAFT.ipynb', as_version=4))" + ] + }, + { + "cell_type": "markdown", + "id": "mixedintro", + "metadata": {}, + "source": [ + "Version 2 files can already contain code prompts with old tool envelopes. Migration changes the AI reply and leaves quoted examples in notes unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migmixed", + "metadata": {}, + "outputs": [], + "source": [ + "mixed_nb = new_nb([Message('Calculate.', msg_type=sprompt, output=legacy_reply).to_cell(),\n", + " mk_cell(legacy_reply, 'markdown')], meta=dict(solveit=dict(ver=2)))\n", + "mixed_nb.cells[0].outputs[0]['data']['text/markdown'] = legacy_reply.splitlines(keepends=True)\n", + "mixed = reads_ipynb(nb2str(mixed_nb))\n", + "test_eq(mixed.messages[0].ai_res, upgraded.messages[0].ai_res)\n", + "test_eq(mixed.messages[1].content, legacy_reply)" + ] + }, + { + "cell_type": "markdown", + "id": "0f4da874", + "metadata": {}, + "source": [ + "Older files used top-level Solveit metadata. The migration preserves the mode and unrelated metadata across both version steps. Files without a version follow the same chain." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migversions", + "metadata": {}, + "outputs": [], + "source": [ + "for version in (1, 2):\n", + " oldmeta = dict(solveit_ver=version, solveit_dialog_mode=2 if version==1 else 'learning', custom='keep')\n", + " oldfile = new_nb(meta=oldmeta)\n", + " assert migrate_nb(oldfile)\n", + " test_eq(oldfile.metadata, dict(solveit=dict(ver=3, mode='learning'), custom='keep'))\n", + " assert not migrate_nb(oldfile)\n", + "unversioned = new_nb()\n", + "assert migrate_nb(unversioned)\n", + "test_eq(nb_ver(unversioned), CUR_VER)" + ] + }, + { + "cell_type": "markdown", + "id": "f8050898", + "metadata": {}, + "source": [ + "A file from a newer writer is rejected before any migration runs. Saving it through an older reader could otherwise discard information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "migfuture", + "metadata": {}, + "outputs": [], + "source": [ + "future = new_nb(meta=dict(solveit=dict(ver=CUR_VER+1)))\n", + "with expect_fail(ValueError, 'Unsupported notebook version'): migrate_nb(future)\n", + "test_eq(nb_ver(future), CUR_VER+1)" + ] + }, { "cell_type": "markdown", "id": "5bad95b9", From 412228930e03c3005e84a4b64bfe9ba5a758f560 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 9 Sep 2026 13:18:37 +0200 Subject: [PATCH 2/2] Normalize notebook text before migrations --- aidialog/ipynb.py | 6 +++--- nbs/02_ipynb.ipynb | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/aidialog/ipynb.py b/aidialog/ipynb.py index 6cf70e6..f4200ff 100644 --- a/aidialog/ipynb.py +++ b/aidialog/ipynb.py @@ -121,14 +121,13 @@ def _v2_to_v3(nb): for o in c.get('outputs', []): md = nested_idx(o, 'metadata', 'is_ai_res') and nested_idx(o, 'data', 'text/markdown') if not md: continue - if isinstance(md, list): md = ''.join(md) if '-usage-details' in md: o['data']['text/markdown'] = conv_tools(md) # %% ../nbs/02_ipynb.ipynb #migratenb _migrations = {1:_v1_to_v2, 2:_v2_to_v3} def migrate_nb(nb): - "Upgrade a notebook in place. Return whether its version changed" + "Upgrade a normalized notebook in place. Return whether its version changed" ver = nb_ver(nb) if not 1 <= ver <= CUR_VER: raise ValueError(f'Unsupported notebook version: {ver}') for v in range(ver, CUR_VER): @@ -217,9 +216,10 @@ def reads_ipynb(txt, cls=Dialog, name='dialog', verbose=False): "Read a dialog from notebook JSON string `txt`, constructing via `cls`" nb = json.loads(txt) if (repairs := repair_nb(nb)) and verbose: print('NB repair:', '; '.join(repairs)) - migrated = migrate_nb(nb) unhome_atts(nb) nb = dict2nb(nb) + migrated = migrate_nb(nb) + unhome_atts(nb) res = cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells) res.migrated_ = migrated return res diff --git a/nbs/02_ipynb.ipynb b/nbs/02_ipynb.ipynb index de68671..36613ec 100644 --- a/nbs/02_ipynb.ipynb +++ b/nbs/02_ipynb.ipynb @@ -467,7 +467,6 @@ " for o in c.get('outputs', []):\n", " md = nested_idx(o, 'metadata', 'is_ai_res') and nested_idx(o, 'data', 'text/markdown')\n", " if not md: continue\n", - " if isinstance(md, list): md = ''.join(md)\n", " if '-usage-details' in md: o['data']['text/markdown'] = conv_tools(md)" ] }, @@ -482,7 +481,7 @@ "_migrations = {1:_v1_to_v2, 2:_v2_to_v3}\n", "\n", "def migrate_nb(nb):\n", - " \"Upgrade a notebook in place. Return whether its version changed\"\n", + " \"Upgrade a normalized notebook in place. Return whether its version changed\"\n", " ver = nb_ver(nb)\n", " if not 1 <= ver <= CUR_VER: raise ValueError(f'Unsupported notebook version: {ver}')\n", " for v in range(ver, CUR_VER):\n", @@ -496,7 +495,7 @@ "id": "bb0af64e", "metadata": {}, "source": [ - "`migrate_nb` applies each version step in order. Version 1 stores mode metadata in the old form. Version 2 permits markdown prompts and legacy tool envelopes. Version 3 stores code prompts and fenced tool records. The notebook stays in memory until a caller saves it. A newer, unsupported version raises an error." + "`migrate_nb` accepts a notebook normalized by `dict2nb` or `read_nb`, with text held as strings. It applies each version step in order. Version 1 stores mode metadata in the old form. Version 2 permits markdown prompts and legacy tool envelopes. Version 3 uses code prompts and structured tool markup. Migrations do not rename tools. `nb2str` restores lists of lines when writing." ] }, { @@ -751,9 +750,10 @@ " \"Read a dialog from notebook JSON string `txt`, constructing via `cls`\"\n", " nb = json.loads(txt)\n", " if (repairs := repair_nb(nb)) and verbose: print('NB repair:', '; '.join(repairs))\n", - " migrated = migrate_nb(nb)\n", " unhome_atts(nb)\n", " nb = dict2nb(nb)\n", + " migrated = migrate_nb(nb)\n", + " unhome_atts(nb)\n", " res = cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)\n", " res.migrated_ = migrated\n", " return res" @@ -1131,7 +1131,10 @@ "upgraded.save()\n", "assert not upgraded.migrated_\n", "assert not read_ipynb(tstdir/'CRAFT.ipynb').migrated_\n", - "nbformat.validate(nbformat.read(tstdir/'CRAFT.ipynb', as_version=4))" + "nbformat.validate(nbformat.read(tstdir/'CRAFT.ipynb', as_version=4))\n", + "saved_craft = json.loads((tstdir/'CRAFT.ipynb').read_text())['cells'][0]\n", + "assert isinstance(saved_craft['source'], list)\n", + "assert isinstance(saved_craft['outputs'][0]['data']['text/markdown'], list)" ] }, { @@ -1151,7 +1154,6 @@ "source": [ "mixed_nb = new_nb([Message('Calculate.', msg_type=sprompt, output=legacy_reply).to_cell(),\n", " mk_cell(legacy_reply, 'markdown')], meta=dict(solveit=dict(ver=2)))\n", - "mixed_nb.cells[0].outputs[0]['data']['text/markdown'] = legacy_reply.splitlines(keepends=True)\n", "mixed = reads_ipynb(nb2str(mixed_nb))\n", "test_eq(mixed.messages[0].ai_res, upgraded.messages[0].ai_res)\n", "test_eq(mixed.messages[1].content, legacy_reply)"