fix(trainer): detect OpenShift pip permission error and document workaround

Signed-off-by: priyank <priyank8445@gmail.com>

Author: priyank766

kubeflow_mcp/trainer/__init__.py

@@ -370,6 +370,7 @@
 
 PLATFORM:
 - If pre_flight() returns platform=openshift, ALWAYS pass emptyDir volumes for /.local, /.cache, /tmp — without these, jobs fail on read-only filesystem. Read trainer://guides/platform-fixes for copy-paste JSON
+- OpenShift packages restriction: if platform=openshift, do NOT use the packages parameter in run_custom_training() as the pre-script pip install step will fail with PermissionError on /.local. Instead, install packages inside your training script to /workspace/lib using subprocess and append to sys.path
 - Gated HuggingFace models require hf_token parameter
 - Training jobs consume GPU resources — be conservative with num_nodes
 - Use get_training_events() to debug stuck/failed jobs. Read trainer://guides/troubleshooting for error-to-fix tables""",

kubeflow_mcp/trainer/api/monitoring.py

@@ -58,6 +58,14 @@
         "FILE_NOT_FOUND",
         "Check dataset/model paths and volume mounts.",
     ),
+    (
+        re.compile(
+            r"PermissionError: \[Errno 13\] Permission denied: '/\.local|Permission denied: '/\.local",
+            re.IGNORECASE,
+        ),
+        "OPENSHIFT_PIP_ERROR",
+        "On OpenShift under a restricted SCC, pip install --user fails on read-only /.local. Do NOT use the 'packages' parameter in run_custom_training(). Instead, install packages inside your script to /workspace/lib using subprocess and append to sys.path. Read trainer://guides/platform-fixes for details.",
+    ),
     (
         re.compile(r"PermissionError|Access Denied", re.IGNORECASE),
         "PERMISSION_ERROR",

kubeflow_mcp/trainer/resources/platform-fixes.md

@@ -96,7 +96,24 @@ For `fine_tune()`, these volumes apply to ALL replicated jobs (node, dataset-ini
 "env": {"NCCL_P2P_DISABLE": "1", "NCCL_SHM_DISABLE": "1"}
 ```
 
-**4. If emptyDirs are not enough** — escalate to cluster admin:
+**4. Pip package installation permission denied in run_custom_training()** — when using `run_custom_training(packages=[...])` on OpenShift under a restricted SCC, the SDK's pre-script pip install step attempts to run `pip install --user` which writes to `/.local`. Since user-defined emptyDir volumes are not mounted on the training container during this step, the job will fail immediately with:
+`PermissionError: [Errno 13] Permission denied: '/.local'`
+
+*Workaround*: Do **NOT** use the `packages` parameter on OpenShift. Instead, write a workaround directly in your training script to install the required packages to `/workspace/lib` (which is a writable emptyDir) and append it to `sys.path`:
+
+```python
+import subprocess, sys, os
+lib_dir = '/workspace/lib'
+os.makedirs(lib_dir, exist_ok=True)
+subprocess.run([
+    sys.executable, '-m', 'pip', 'install',
+    '--target', lib_dir, '--quiet',
+    'transformers', 'peft', 'trl'
+], check=True)
+sys.path.insert(0, lib_dir)
+```
+
+**5. If emptyDirs are not enough** — escalate to cluster admin:
 
 ```bash
 oc adm policy add-scc-to-user anyuid -z <service-account> -n <namespace>

tests/unit/trainer/test_monitoring.py

@@ -0,0 +1,64 @@
+# Copyright The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for log monitoring and failure pattern extraction."""
+
+from kubeflow_mcp.trainer.api.monitoring import _extract_failure_hint
+
+
+def test_extract_failure_hint_openshift_pip_error():
+    # Test exact PermissionError trace
+    logs = (
+        "Installing collected packages: torch\n"
+        "PermissionError: [Errno 13] Permission denied: '/.local'\n"
+        "ERROR: Job failed"
+    )
+    hint = _extract_failure_hint(logs)
+    assert hint is not None
+    assert hint["category"] == "OPENSHIFT_PIP_ERROR"
+    assert "On OpenShift under a restricted SCC" in hint["suggestion"]
+    assert "Do NOT use the 'packages' parameter" in hint["suggestion"]
+
+    # Test generic permission denied on /.local
+    logs_generic = "Permission denied: '/.local/bin/pip'"
+    hint_generic = _extract_failure_hint(logs_generic)
+    assert hint_generic is not None
+    assert hint_generic["category"] == "OPENSHIFT_PIP_ERROR"
+
+
+def test_extract_failure_hint_generic_permission_error():
+    # Test a generic permission error does not trigger OpenShift pip error
+    logs = "PermissionError: [Errno 13] Permission denied: '/workspace/data.csv'"
+    hint = _extract_failure_hint(logs)
+    assert hint is not None
+    assert hint["category"] == "PERMISSION_ERROR"
+    assert "Check service account permissions" in hint["suggestion"]
+
+
+def test_extract_failure_hint_other_patterns():
+    # Test CUDA OOM
+    oom_logs = "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB"
+    oom_hint = _extract_failure_hint(oom_logs)
+    assert oom_hint is not None
+    assert oom_hint["category"] == "OOM"
+
+    # Test Missing Module
+    module_logs = "ModuleNotFoundError: No module named 'peft'"
+    module_hint = _extract_failure_hint(module_logs)
+    assert module_hint is not None
+    assert module_hint["category"] == "MISSING_MODULE"
+
+    # Test no match
+    clean_logs = "Training completed successfully. Epoch 5/5 finished."
+    assert _extract_failure_hint(clean_logs) is None