Skip to content

Commit dc8eb1d

Browse files
committed
improve topology error reporting
1 parent bb40b54 commit dc8eb1d

2 files changed

Lines changed: 148 additions & 2 deletions

File tree

ds4_distributed.c

Lines changed: 110 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,108 @@ static bool dist_route_search_workers(
21012101
return false;
21022102
}
21032103

2104+
static void dist_format_range_end(
2105+
uint32_t layer_end,
2106+
bool has_output,
2107+
char *buf,
2108+
size_t buflen) {
2109+
if (!buf || buflen == 0) return;
2110+
if (has_output) snprintf(buf, buflen, "output");
2111+
else snprintf(buf, buflen, "%u", layer_end);
2112+
}
2113+
2114+
static bool dist_route_diag_local_overlap(
2115+
const ds4_dist_coordinator_state *state,
2116+
const ds4_dist_worker_entry *w,
2117+
char *err,
2118+
size_t errlen) {
2119+
if (!state || !w) return false;
2120+
if (w->layer_end < state->local_start || w->layer_start > state->local_end) return false;
2121+
2122+
uint32_t overlap_start = w->layer_start > state->local_start ? w->layer_start : state->local_start;
2123+
uint32_t overlap_end = w->layer_end < state->local_end ? w->layer_end : state->local_end;
2124+
char worker_end[32];
2125+
char local_end[32];
2126+
dist_format_range_end(w->layer_end, w->has_output != 0, worker_end, sizeof(worker_end));
2127+
dist_format_range_end(state->local_end, state->local_has_output, local_end, sizeof(local_end));
2128+
if (errlen) {
2129+
snprintf(err,
2130+
errlen,
2131+
"distributed %s route invalid: worker %s:%u layers=%u:%s overlap coordinator local range %u:%s at layers %u:%u",
2132+
dist_topology_name(state->topology),
2133+
w->peer_host,
2134+
w->listen_port,
2135+
w->layer_start,
2136+
worker_end,
2137+
state->local_start,
2138+
local_end,
2139+
overlap_start,
2140+
overlap_end);
2141+
}
2142+
return true;
2143+
}
2144+
2145+
static bool dist_route_diag_worker_overlap(
2146+
const ds4_dist_coordinator_state *state,
2147+
const ds4_dist_worker_entry *prev,
2148+
const ds4_dist_worker_entry *cur,
2149+
char *err,
2150+
size_t errlen) {
2151+
if (!state || !prev || !cur) return false;
2152+
if (cur->layer_start > prev->layer_end) return false;
2153+
2154+
uint32_t overlap_start = cur->layer_start;
2155+
uint32_t overlap_end = prev->layer_end < cur->layer_end ? prev->layer_end : cur->layer_end;
2156+
char prev_end[32];
2157+
char cur_end[32];
2158+
dist_format_range_end(prev->layer_end, prev->has_output != 0, prev_end, sizeof(prev_end));
2159+
dist_format_range_end(cur->layer_end, cur->has_output != 0, cur_end, sizeof(cur_end));
2160+
if (errlen) {
2161+
snprintf(err,
2162+
errlen,
2163+
"distributed %s route invalid: worker %s:%u layers=%u:%s overlap worker %s:%u layers=%u:%s at layers %u:%u",
2164+
dist_topology_name(state->topology),
2165+
prev->peer_host,
2166+
prev->listen_port,
2167+
prev->layer_start,
2168+
prev_end,
2169+
cur->peer_host,
2170+
cur->listen_port,
2171+
cur->layer_start,
2172+
cur_end,
2173+
overlap_start,
2174+
overlap_end);
2175+
}
2176+
return true;
2177+
}
2178+
2179+
static void dist_route_diagnose_failure(
2180+
const ds4_dist_coordinator_state *state,
2181+
ds4_dist_worker_entry **workers,
2182+
uint32_t n,
2183+
uint32_t required_start,
2184+
uint32_t required_end,
2185+
uint32_t missing_layer,
2186+
char *err,
2187+
size_t errlen) {
2188+
if (!state || !err || errlen == 0) return;
2189+
2190+
ds4_dist_worker_entry *prev = NULL;
2191+
for (uint32_t i = 0; i < n; i++) {
2192+
ds4_dist_worker_entry *w = workers[i];
2193+
if (w->layer_end < required_start || w->layer_start > required_end) continue;
2194+
if (dist_route_diag_local_overlap(state, w, err, errlen)) return;
2195+
if (prev && dist_route_diag_worker_overlap(state, prev, w, err, errlen)) return;
2196+
prev = w;
2197+
}
2198+
2199+
snprintf(err,
2200+
errlen,
2201+
"distributed %s route incomplete: missing layer %u",
2202+
dist_topology_name(state->topology),
2203+
missing_layer);
2204+
}
2205+
21042206
static void dist_coordinator_report_plan(ds4_dist_coordinator_state *state) {
21052207
if (!dist_coordinator_debug_enabled(state)) return;
21062208
pthread_mutex_lock(&state->mu);
@@ -2411,11 +2513,17 @@ static bool dist_coordinator_build_route_plan(
24112513
path,
24122514
&path_len,
24132515
&missing)) {
2414-
const char *topology_name = dist_topology_name(state->topology);
24152516
pthread_mutex_unlock(&state->mu);
2517+
dist_route_diagnose_failure(state,
2518+
workers,
2519+
n,
2520+
required_start,
2521+
required_end,
2522+
missing,
2523+
err,
2524+
errlen);
24162525
free(workers);
24172526
free(path);
2418-
if (errlen) snprintf(err, errlen, "distributed %s route incomplete: missing layer %u", topology_name, missing);
24192527
return false;
24202528
}
24212529

tests/ds4_test.c

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2329,6 +2329,44 @@ static void test_distributed_topology_logic_group(void) {
23292329
state.workers = test_dist_worker("127.0.0.1", 20012, 2, 1, 19, false, true, 1024);
23302330
memset(&route, 0, sizeof(route));
23312331
TEST_ASSERT(!ds4_dist_test_build_route_plan(&state, &route, err, sizeof(err)));
2332+
TEST_ASSERT(strstr(err, "missing layer 0") != NULL);
2333+
test_dist_free_workers(state.workers);
2334+
pthread_mutex_destroy(&state.mu);
2335+
2336+
memset(&state, 0, sizeof(state));
2337+
state.n_layers = 43;
2338+
state.local_start = 21;
2339+
state.local_end = 42;
2340+
state.topology = DS4_DIST_TOPOLOGY_REVERSE;
2341+
state.local_has_output = true;
2342+
state.local_can_output_head = true;
2343+
state.use_control_for_work = false;
2344+
pthread_mutex_init(&state.mu, NULL);
2345+
state.workers = test_dist_worker("127.0.0.1", 20013, 2, 0, 21, false, true, 1024);
2346+
memset(&route, 0, sizeof(route));
2347+
err[0] = '\0';
2348+
TEST_ASSERT(!ds4_dist_test_build_route_plan(&state, &route, err, sizeof(err)));
2349+
TEST_ASSERT(strstr(err, "overlap coordinator local range 21:output") != NULL);
2350+
TEST_ASSERT(strstr(err, "layers 21:21") != NULL);
2351+
test_dist_free_workers(state.workers);
2352+
pthread_mutex_destroy(&state.mu);
2353+
2354+
memset(&state, 0, sizeof(state));
2355+
state.n_layers = 43;
2356+
state.local_start = 21;
2357+
state.local_end = 42;
2358+
state.topology = DS4_DIST_TOPOLOGY_REVERSE;
2359+
state.local_has_output = true;
2360+
state.local_can_output_head = true;
2361+
state.use_control_for_work = false;
2362+
pthread_mutex_init(&state.mu, NULL);
2363+
state.workers = test_dist_worker("127.0.0.1", 20014, 2, 0, 10, false, true, 1024);
2364+
state.workers->next = test_dist_worker("127.0.0.1", 20015, 2, 10, 20, false, true, 1024);
2365+
memset(&route, 0, sizeof(route));
2366+
err[0] = '\0';
2367+
TEST_ASSERT(!ds4_dist_test_build_route_plan(&state, &route, err, sizeof(err)));
2368+
TEST_ASSERT(strstr(err, "overlap worker") != NULL);
2369+
TEST_ASSERT(strstr(err, "layers 10:10") != NULL);
23322370
test_dist_free_workers(state.workers);
23332371
pthread_mutex_destroy(&state.mu);
23342372
}

0 commit comments

Comments
 (0)