Skip to content

fix: ensure boot order is properly set before deployment reboot (#567) #620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
skatsaounis wants to merge 1 commit into
base: 3.8
Choose a base branch
from
+441 −25
Open

fix: ensure boot order is properly set before deployment reboot (#567) #620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 88 additions & 1 deletion src/maastemporalworker/workflow/deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
from maastemporalworker.workflow.power import (
POWER_ACTION_ACTIVITY_TIMEOUT,
POWER_CYCLE_ACTIVITY_NAME,
POWER_OFF_ACTIVITY_NAME,
POWER_ON_ACTIVITY_NAME,
POWER_QUERY_ACTIVITY_NAME,
SET_POWER_STATE_ACTIVITY_NAME,
Expand All @@ -61,6 +62,12 @@
DEFAULT_DEPLOY_ACTIVITY_TIMEOUT = timedelta(seconds=30)
DEFAULT_DEPLOY_RETRY_TIMEOUT = timedelta(seconds=60)

# _switch_to_local_boot()'s power-on is fire-and-forget, so poll afterwards
# to persist the real state. Some drivers (e.g. IBM Z HMC/DPM) can take
# minutes to settle.
CONFIRM_POWERED_ON_POLL_INTERVAL = timedelta(seconds=15)
CONFIRM_POWERED_ON_MAX_ATTEMPTS = 20 # ~5 minutes total

# Activities names
GET_BOOT_ORDER_ACTIVITY_NAME = "get-boot-order"
SET_NODE_STATUS_ACTIVITY_NAME = "set-node-status"
Expand Down Expand Up @@ -482,6 +489,86 @@ async def _set_boot_order(self, params: DeployParam) -> None:
else:
raise InvalidMachineStateException("no boot order found")

async def _power(self, params: DeployParam, activity_name: str) -> None:
"""Run a power activity (on/off) on the agent for this machine."""
await workflow.execute_activity(
activity_name,
params.power_params,
task_queue=params.power_params.task_queue,
start_to_close_timeout=POWER_ACTION_ACTIVITY_TIMEOUT,
retry_policy=RetryPolicy(maximum_attempts=3),
)

async def _confirm_powered_on(self, params: DeployParam) -> None:
"""Poll and persist the node's power state until it reads "on".

The preceding power-on is fire-and-forget, so query the state on a
bounded poll and persist changes as they settle, reusing the
query-then-persist pattern from _start_deployment(). A reading is
only acted on once seen on two consecutive polls, to ignore
transient flaky reports from the BMC/HMC.
"""
persisted_state = None
candidate_state = None
for _ in range(CONFIRM_POWERED_ON_MAX_ATTEMPTS):
result = await workflow.execute_activity(
POWER_QUERY_ACTIVITY_NAME,
PowerQueryParam(
system_id=params.power_params.system_id,
driver_type=params.power_params.driver_type,
driver_opts=params.power_params.driver_opts,
task_queue=params.power_params.task_queue,
is_dpu=params.power_params.is_dpu,
),
task_queue=params.power_params.task_queue,
start_to_close_timeout=POWER_ACTION_ACTIVITY_TIMEOUT,
retry_policy=RetryPolicy(maximum_attempts=3),
)
state = PowerState(result["state"])
if state != candidate_state:
candidate_state = state
elif state != persisted_state:
await workflow.execute_activity(
SET_POWER_STATE_ACTIVITY_NAME,
SetPowerStateParam(
system_id=params.power_params.system_id,
state=state,
),
task_queue="region",
start_to_close_timeout=DEFAULT_DEPLOY_ACTIVITY_TIMEOUT,
retry_policy=RetryPolicy(
maximum_interval=DEFAULT_DEPLOY_RETRY_TIMEOUT,
),
)
persisted_state = state
if persisted_state == PowerState.ON:
return
await asyncio.sleep(
CONFIRM_POWERED_ON_POLL_INTERVAL.total_seconds()
)

async def _switch_to_local_boot(self, params: DeployParam) -> None:
"""Point a boot-order-capable machine at its local disk to boot.

MAAS drives the power transition itself (power off -> confirm off ->
set boot order to disk -> power on) instead of relying on the
in-installer reboot.

This avoids a race specific to BMCs that MAAS controls the boot order
for (``can_set_boot_order``) and whose firmware cannot fall through
from a netboot config to the local disk -- notably IBM Z DPM. Once the
installer finishes, MAAS stops serving a netboot kernel; if the guest
reboots before the boot device is flipped to disk, the machine
network-IPLs into an empty config and hangs ("No value found for
kernel", IPL failed 110). Powering off and confirming the machine is
actually down before the flip closes that window (and also sidesteps
the transient HMC "busy" state seen while an operation is in flight).
"""
await self._power(params, POWER_OFF_ACTIVITY_NAME)
await self._set_boot_order(params, netboot=False)
await self._power(params, POWER_ON_ACTIVITY_NAME)
await self._confirm_powered_on(params)

@workflow_run_with_context
async def run(self, params: DeployParam) -> DeployResult:
await self._start_deployment(params)
Expand All @@ -490,7 +577,7 @@ async def run(self, params: DeployParam) -> DeployResult:
await workflow.wait_condition(lambda: self._has_netbooted)

if params.can_set_boot_order:
await self._set_boot_order(params)
await self._switch_to_local_boot(params)

await workflow.wait_condition(lambda: self._deployed_os_ready)

Expand Down
121 changes: 110 additions & 11 deletions src/provisioningserver/drivers/power/hmcz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""

import contextlib
import time

from twisted.internet.defer import inlineCallbacks

Expand All @@ -28,19 +29,84 @@
from provisioningserver.utils.twisted import asynchronous, threadDeferred

try:
from zhmcclient import Client, NotFound, Session, StatusTimeout
from zhmcclient import Client, HTTPError, NotFound, Session, StatusTimeout
except ImportError:
no_zhmcclient = True
else:
no_zhmcclient = False

maaslog = get_maas_logger("drivers.power.hmcz")


def _retry_on_busy(
func,
*args,
op_desc,
system_id,
max_attempts=12,
retry_delay=5,
**kwargs,
):
"""Run a zhmcclient call, retrying while the partition is 409,2 busy.

IBM Z rejects a write against a partition with HTTP 409 reason 2 while
another operation is in flight on that partition -- e.g. a fire-and-forget
start/stop still settling on the HMC, or a guest re-IPL after the install.
That busy window is transient and is not reflected in the partition status,
so retry the call for a bounded time before giving up. Any other error is
re-raised immediately.

These power/boot operations run inside the short-lived maas.power CLI under
a Temporal power activity whose start_to_close timeout is 5 minutes, so the
~60s budget here stays comfortably under that.
"""
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except HTTPError as exc:
if not (exc.http_status == 409 and exc.reason == 2):
raise
if attempt >= max_attempts:
maaslog.error(
"%s: %s still 409,2 busy after %d attempts; giving up. "
"HMC error: %s [%s %s]",
system_id,
op_desc,
attempt,
exc.message,
exc.request_method,
exc.request_uri,
)
raise
maaslog.warning(
"%s: %s got 409,2 busy (attempt %d/%d); retrying in %ds. "
"HMC error: %s [%s %s]",
system_id,
op_desc,
attempt,
max_attempts,
retry_delay,
exc.message,
exc.request_method,
exc.request_uri,
)
time.sleep(retry_delay)


VERIFY_SSL_YES = "y"
VERIFY_SSL_NO = "n"

VERIFY_SSL_CHOICES = [[VERIFY_SSL_NO, "No"], [VERIFY_SSL_YES, "Yes"]]

# set_boot_order shares the single 5 minute (300s) Temporal power activity
# budget with an upfront partition.wait_for_status() that can block up to 120s.
# Keep the busy-retry budget small enough that 120s + retries + the HMC session
# setup still finish well under 300s; otherwise Temporal aborts and retries the
# activity while this Twisted thread keeps writing to the HMC in the background.
# 8 attempts => 7 x 15s = 105s of retries, so ~225s worst case (120s + 105s).
SET_BOOT_ORDER_RETRY_MAX_ATTEMPTS = 8
SET_BOOT_ORDER_RETRY_DELAY = 15


class HMCZPowerDriver(PowerDriver):
name = "hmcz"
Expand Down Expand Up @@ -118,7 +184,12 @@ def power_on(self, system_id: str, context: dict):
# the stop action completes. This holds the thread in MAAS for ~30s.
# IBM is aware this isn't optimal for us so they are looking into
# modifying IBM Z to go into a stopped state.
partition.stop(wait_for_completion=True)
_retry_on_busy(
partition.stop,
wait_for_completion=True,
op_desc="power_on pre-start stop",
system_id=system_id,
)
elif status == "stopping":
# The HMC does not allow a machine to be powered on if its
# currently stopping. Wait 120s for it which should be more
Expand All @@ -134,7 +205,12 @@ def power_on(self, system_id: str, context: dict):
f"{partition.get_property('status')} state!"
)

partition.start(wait_for_completion=False)
_retry_on_busy(
partition.start,
wait_for_completion=False,
op_desc="power_on start",
system_id=system_id,
)

@asynchronous
@threadDeferred
Expand All @@ -156,7 +232,12 @@ def power_off(self, system_id: str, context: dict):
"Partition is stuck in a "
f"{partition.get_property('status')} state!"
)
partition.stop(wait_for_completion=False)
_retry_on_busy(
partition.stop,
wait_for_completion=False,
op_desc="power_off stop",
system_id=system_id,
)

@asynchronous
@threadDeferred
Expand Down Expand Up @@ -202,21 +283,34 @@ def set_boot_order(self, system_id: str, context: dict, order: list):

if status in {"starting", "stopping"}:
# The HMC does not allow a machine's boot order to be reconfigured
# while in a transitional state. Wait for it to complete. If this
# times out allow it to be raised so the region can log it.
partition.wait_for_status(["stopped", "active"], 120)
# while in a transitional (starting/stopping) state. Wait for it to
# settle. Accept any non-transitional state -- including "paused",
# which IBM Z can land in (e.g. after `shutdown -h now` or a quick
# release/redeploy) and which never resolves to stopped/active on
# its own. Waiting only for ["stopped", "active"] would then hang
# the full 120s and raise StatusTimeout. If it times out anyway,
# allow it to be raised so the region can log it.
partition.wait_for_status(
["stopped", "active", "degraded", "paused", "terminated"],
120,
)

# You can only specify one boot device on IBM Z
boot_device = order[0]
if boot_device.get("mac_address"):
nic = partition.nics.find(
**{"mac-address": boot_device["mac_address"]}
)
partition.update_properties(
_retry_on_busy(
partition.update_properties,
{
"boot-device": "network-adapter",
"boot-network-device": nic.uri,
}
},
op_desc="set_boot_order network-adapter",
system_id=system_id,
max_attempts=SET_BOOT_ORDER_RETRY_MAX_ATTEMPTS,
retry_delay=SET_BOOT_ORDER_RETRY_DELAY,
)
else:
for storage_group in partition.list_attached_storage_groups():
Expand All @@ -229,11 +323,16 @@ def set_boot_order(self, system_id: str, context: dict, order: list):
vol.properties.get("uuid", "").strip(),
vol.properties.get("serial-number", "").strip(),
]:
partition.update_properties(
_retry_on_busy(
partition.update_properties,
{
"boot-device": "storage-volume",
"boot-storage-volume": vol.uri,
}
},
op_desc="set_boot_order storage-volume",
system_id=system_id,
max_attempts=SET_BOOT_ORDER_RETRY_MAX_ATTEMPTS,
retry_delay=SET_BOOT_ORDER_RETRY_DELAY,
)
return

Expand Down
Loading
Loading