From 81e3e85cdbf98d3fc83a346dc24fb6bc5b67e0d1 Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Wed, 2 Jul 2025 12:57:21 +0300 Subject: [PATCH 01/23] Implement mining cache anomaly search --- apps/arweave/src/ar_mining_cache.erl | 81 +++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index 2d1ba72ec7..8f0f0dff24 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -146,8 +146,11 @@ release_for_session(SessionId, Size, Cache0) -> -spec drop_session(SessionId :: term(), Cache0 :: #ar_mining_cache{}) -> Cache1 :: #ar_mining_cache{}. drop_session(SessionId, Cache0) -> + ?LOG_DEBUG([{event, drop_session}, {session_id, SessionId}]), + {Session, Sessions} = maps:take(SessionId, Cache0#ar_mining_cache.mining_cache_sessions), + maybe_search_for_anomalies(SessionId, Session), Cache0#ar_mining_cache{ - mining_cache_sessions = maps:remove(SessionId, Cache0#ar_mining_cache.mining_cache_sessions), + mining_cache_sessions = Sessions, mining_cache_sessions_queue = queue:filter( fun(SessionId0) -> SessionId0 =/= SessionId end, Cache0#ar_mining_cache.mining_cache_sessions_queue @@ -286,6 +289,82 @@ with_mining_cache_session(SessionId, Fun, Cache0) -> {error, session_not_found} end. +%% Searches for anomalies in the mining cache session. +%% If the actual cache size is different from the expected cache size, +%% it will log a warning. +%% If the reserved cache size is different from 0, it will log a warning. +%% It will also search for invalid cache values, e.g. missing chunks, or failed +%% invariants. +%% +%% Perhaps it is a good idea to put this under a config flag, disabled by default. +maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ + mining_cache = MiningCache, + mining_cache_size_bytes = MiningCacheSize, + reserved_mining_cache_bytes = ReservedMiningCacheBytes +}) -> + ActualCacheSize = maybe_search_for_anomalies_cache_values(SessionId, MiningCache), + case ActualCacheSize =/= MiningCacheSize of + true -> ?LOG_WARNING([ + {event, mining_cache_anomaly}, {anomaly, cache_size_mismatch}, + {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); + false -> ok + end, + case ReservedMiningCacheBytes of + 0 -> ok; + _ -> ?LOG_WARNING([ + {event, mining_cache_anomaly}, {anomaly, reserved_size_mismatch}, + {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) + end. + +maybe_search_for_anomalies_cache_values(SessionId, MiningCache) -> + OuterAcc0 = {_Anomalies = #{}, _ActualSize = 0}, + {Anomalies, ActualSize} = maps:fold(fun(_Key, Value, {Anomalies0, ActualSize0}) -> + Anomalies1 = lists:foldl(fun(Check, Anomalies) -> Check(Value, Anomalies) end, Anomalies0, [ + fun maybe_search_for_anomalies_cache_values_chunk1_missing/2, + fun maybe_search_for_anomalies_cache_values_chunk2_missing/2, + fun maybe_search_for_anomalies_cache_values_h1_missing/2, + fun maybe_search_for_anomalies_cache_values_h2_missing/2, + fun maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present/2 + ]), + {Anomalies1, ActualSize0 + cached_value_size(Value)} + end, OuterAcc0, MiningCache), + case maps:size(Anomalies) > 0 of + true -> ?LOG_WARNING([ + {event, mining_cache_anomaly}, {anomaly, cached_values_anomalies}, + {anomalies, Anomalies}, {session_id, SessionId}]); + false -> ok + end, + ActualSize. + +maybe_search_for_anomalies_cache_values_chunk1_missing(#ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false}, Anomalies) -> + maps:update_with(chunk1_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_chunk1_missing(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_chunk2_missing(#ar_mining_cache_value{chunk2 = undefined, chunk2_missing = false}, Anomalies) -> + maps:update_with(chunk2_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_chunk2_missing(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_h1_missing(#ar_mining_cache_value{h1 = undefined, chunk1 = Chunk1}, Anomalies) +when undefined =/= Chunk1 -> + maps:update_with(h1_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_h1_missing(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_h2_missing(#ar_mining_cache_value{h2 = undefined, chunk2 = Chunk2}, Anomalies) +when undefined =/= Chunk2 -> + maps:update_with(h2_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_h2_missing(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(#ar_mining_cache_value{h1_passes_diff_checks = true}, Anomalies) -> + maps:update_with(h1_passes_diff_checks_present, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(_, Anomalies) -> + Anomalies. + + + %%%=================================================================== %%% Tests. %%%=================================================================== From 7527e2bbefa1c61933f99de0df4c8f6f9b6b4800 Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Wed, 2 Jul 2025 13:08:50 +0300 Subject: [PATCH 02/23] Add mining values type checks --- apps/arweave/src/ar_mining_cache.erl | 33 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index 8f0f0dff24..fdd1e27610 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -147,15 +147,18 @@ release_for_session(SessionId, Size, Cache0) -> Cache1 :: #ar_mining_cache{}. drop_session(SessionId, Cache0) -> ?LOG_DEBUG([{event, drop_session}, {session_id, SessionId}]), - {Session, Sessions} = maps:take(SessionId, Cache0#ar_mining_cache.mining_cache_sessions), - maybe_search_for_anomalies(SessionId, Session), - Cache0#ar_mining_cache{ - mining_cache_sessions = Sessions, - mining_cache_sessions_queue = queue:filter( - fun(SessionId0) -> SessionId0 =/= SessionId end, - Cache0#ar_mining_cache.mining_cache_sessions_queue - ) - }. + case maps:take(SessionId, Cache0#ar_mining_cache.mining_cache_sessions) of + {Session, Sessions} -> + maybe_search_for_anomalies(SessionId, Session), + Cache0#ar_mining_cache{ + mining_cache_sessions = Sessions, + mining_cache_sessions_queue = queue:filter( + fun(SessionId0) -> SessionId0 =/= SessionId end, + Cache0#ar_mining_cache.mining_cache_sessions_queue + ) + }; + _ -> Cache0 + end. %% @doc Checks if a session exists in the cache. -spec session_exists(SessionId :: term(), Cache0 :: #ar_mining_cache{}) -> @@ -314,9 +317,12 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ _ -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, reserved_size_mismatch}, {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) - end. + end; +maybe_search_for_anomalies(SessionId, _InvalidSession) -> + ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_session_type}, {session_id, SessionId}]), + ok. -maybe_search_for_anomalies_cache_values(SessionId, MiningCache) -> +maybe_search_for_anomalies_cache_values(SessionId, MiningCache) when is_map(MiningCache) -> OuterAcc0 = {_Anomalies = #{}, _ActualSize = 0}, {Anomalies, ActualSize} = maps:fold(fun(_Key, Value, {Anomalies0, ActualSize0}) -> Anomalies1 = lists:foldl(fun(Check, Anomalies) -> Check(Value, Anomalies) end, Anomalies0, [ @@ -334,7 +340,10 @@ maybe_search_for_anomalies_cache_values(SessionId, MiningCache) -> {anomalies, Anomalies}, {session_id, SessionId}]); false -> ok end, - ActualSize. + ActualSize; +maybe_search_for_anomalies_cache_values(SessionId, _InvalidCache) -> + ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_cache_type}, {session_id, SessionId}]), + 0. maybe_search_for_anomalies_cache_values_chunk1_missing(#ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false}, Anomalies) -> maps:update_with(chunk1_missing, fun(V) -> V + 1 end, 1, Anomalies); From 8f696773554f13b5b0df6729e6b26e6eb846bcfd Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Mon, 7 Jul 2025 16:30:08 +0300 Subject: [PATCH 03/23] Add more debugging information --- apps/arweave/src/ar_mining_cache.erl | 52 ++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index fdd1e27610..e701a41b99 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -306,18 +306,23 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ reserved_mining_cache_bytes = ReservedMiningCacheBytes }) -> ActualCacheSize = maybe_search_for_anomalies_cache_values(SessionId, MiningCache), - case ActualCacheSize =/= MiningCacheSize of - true -> ?LOG_WARNING([ + case {ActualCacheSize, MiningCacheSize} of + {0, 0} -> ok; + {EqualSize, EqualSize} -> ?LOG_WARNING([ + {event, mining_cache_anomaly}, {anomaly, cache_size_non_zero}, + {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); + {_, 0} -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, cache_size_mismatch}, {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); - false -> ok + _ -> ok end, case ReservedMiningCacheBytes of 0 -> ok; _ -> ?LOG_WARNING([ - {event, mining_cache_anomaly}, {anomaly, reserved_size_mismatch}, + {event, mining_cache_anomaly}, {anomaly, reserved_size_non_zero}, {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) end; + maybe_search_for_anomalies(SessionId, _InvalidSession) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_session_type}, {session_id, SessionId}]), ok. @@ -327,7 +332,9 @@ maybe_search_for_anomalies_cache_values(SessionId, MiningCache) when is_map(Mini {Anomalies, ActualSize} = maps:fold(fun(_Key, Value, {Anomalies0, ActualSize0}) -> Anomalies1 = lists:foldl(fun(Check, Anomalies) -> Check(Value, Anomalies) end, Anomalies0, [ fun maybe_search_for_anomalies_cache_values_chunk1_missing/2, + fun maybe_search_for_anomalies_cache_values_chunk1_stale/2, fun maybe_search_for_anomalies_cache_values_chunk2_missing/2, + fun maybe_search_for_anomalies_cache_values_chunk2_stale/2, fun maybe_search_for_anomalies_cache_values_h1_missing/2, fun maybe_search_for_anomalies_cache_values_h2_missing/2, fun maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present/2 @@ -345,30 +352,47 @@ maybe_search_for_anomalies_cache_values(SessionId, _InvalidCache) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_cache_type}, {session_id, SessionId}]), 0. -maybe_search_for_anomalies_cache_values_chunk1_missing(#ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false}, Anomalies) -> - maps:update_with(chunk1_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_chunk1_missing(#ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false} = Value, Anomalies) -> + maps:update_with(chunk1_missing, fun(V) -> V + 1 end, 1, + maps:update_with(chunk1_missing_sample, fun(V) -> V end, Value, Anomalies)); maybe_search_for_anomalies_cache_values_chunk1_missing(_, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_chunk2_missing(#ar_mining_cache_value{chunk2 = undefined, chunk2_missing = false}, Anomalies) -> - maps:update_with(chunk2_missing, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_chunk1_stale(#ar_mining_cache_value{chunk1 = Chunk1, chunk1_missing = true} = Value, Anomalies) when undefined =/= Chunk1 -> + maps:update_with(chunk1_stale, fun(V) -> V + 1 end, 1, + maps:update_with(chunk1_stale_sample, fun(V) -> V end, Value, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk1_stale(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_chunk2_missing(#ar_mining_cache_value{chunk2 = undefined, chunk2_missing = false} = Value, Anomalies) -> + maps:update_with(chunk2_missing, fun(V) -> V + 1 end, 1, + maps:update_with(chunk2_missing_sample, fun(V) -> V end, Value, Anomalies)); maybe_search_for_anomalies_cache_values_chunk2_missing(_, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h1_missing(#ar_mining_cache_value{h1 = undefined, chunk1 = Chunk1}, Anomalies) +maybe_search_for_anomalies_cache_values_chunk2_stale(#ar_mining_cache_value{chunk2 = Chunk2, chunk2_missing = true} = Value, Anomalies) when undefined =/= Chunk2 -> + maps:update_with(chunk2_stale, fun(V) -> V + 1 end, 1, + maps:update_with(chunk2_stale_sample, fun(V) -> V end, Value, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk2_stale(_, Anomalies) -> + Anomalies. + +maybe_search_for_anomalies_cache_values_h1_missing(#ar_mining_cache_value{h1 = undefined, chunk1 = Chunk1} = Value, Anomalies) when undefined =/= Chunk1 -> - maps:update_with(h1_missing, fun(V) -> V + 1 end, 1, Anomalies); + maps:update_with(h1_missing, fun(V) -> V + 1 end, 1, + maps:update_with(h1_missing_sample, fun(V) -> V end, Value, Anomalies)); maybe_search_for_anomalies_cache_values_h1_missing(_, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h2_missing(#ar_mining_cache_value{h2 = undefined, chunk2 = Chunk2}, Anomalies) +maybe_search_for_anomalies_cache_values_h2_missing(#ar_mining_cache_value{h2 = undefined, chunk2 = Chunk2} = Value, Anomalies) when undefined =/= Chunk2 -> - maps:update_with(h2_missing, fun(V) -> V + 1 end, 1, Anomalies); + maps:update_with(h2_missing, fun(V) -> V + 1 end, 1, + maps:update_with(h2_missing_sample, fun(V) -> V end, Value, Anomalies)); maybe_search_for_anomalies_cache_values_h2_missing(_, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(#ar_mining_cache_value{h1_passes_diff_checks = true}, Anomalies) -> - maps:update_with(h1_passes_diff_checks_present, fun(V) -> V + 1 end, 1, Anomalies); +maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(#ar_mining_cache_value{h1_passes_diff_checks = true} = Value, Anomalies) -> + maps:update_with(h1_passes_diff_checks_present, fun(V) -> V + 1 end, 1, + maps:update_with(h1_passes_diff_checks_present_sample, fun(V) -> V end, Value, Anomalies)); maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(_, Anomalies) -> Anomalies. From 5bcba79272d6f0564e2b3cb932e62886f0639b55 Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Tue, 8 Jul 2025 00:20:21 +0300 Subject: [PATCH 04/23] Add more logging and adjust anomaly detection --- apps/arweave/src/ar_mining_cache.erl | 73 +++++++++++++++++---------- apps/arweave/src/ar_mining_worker.erl | 7 +-- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index e701a41b99..b611658ec8 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -311,10 +311,9 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ {EqualSize, EqualSize} -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, cache_size_non_zero}, {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); - {_, 0} -> ?LOG_WARNING([ + {_, _} -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, cache_size_mismatch}, - {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); - _ -> ok + {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]) end, case ReservedMiningCacheBytes of 0 -> ok; @@ -322,15 +321,14 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ {event, mining_cache_anomaly}, {anomaly, reserved_size_non_zero}, {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) end; - maybe_search_for_anomalies(SessionId, _InvalidSession) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_session_type}, {session_id, SessionId}]), ok. maybe_search_for_anomalies_cache_values(SessionId, MiningCache) when is_map(MiningCache) -> OuterAcc0 = {_Anomalies = #{}, _ActualSize = 0}, - {Anomalies, ActualSize} = maps:fold(fun(_Key, Value, {Anomalies0, ActualSize0}) -> - Anomalies1 = lists:foldl(fun(Check, Anomalies) -> Check(Value, Anomalies) end, Anomalies0, [ + {Anomalies, ActualSize} = maps:fold(fun(Key, Value, {Anomalies0, ActualSize0}) -> + Anomalies1 = lists:foldl(fun(Check, Anomalies) -> Check({Key, Value}, Anomalies) end, Anomalies0, [ fun maybe_search_for_anomalies_cache_values_chunk1_missing/2, fun maybe_search_for_anomalies_cache_values_chunk1_stale/2, fun maybe_search_for_anomalies_cache_values_chunk2_missing/2, @@ -352,48 +350,69 @@ maybe_search_for_anomalies_cache_values(SessionId, _InvalidCache) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_cache_type}, {session_id, SessionId}]), 0. -maybe_search_for_anomalies_cache_values_chunk1_missing(#ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false} = Value, Anomalies) -> +maybe_search_for_anomalies_cache_values_chunk1_missing({ + Key, + #ar_mining_cache_value{chunk1 = undefined, chunk1_missing = false} = Value +}, Anomalies) -> maps:update_with(chunk1_missing, fun(V) -> V + 1 end, 1, - maps:update_with(chunk1_missing_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_chunk1_missing(_, Anomalies) -> + maps:update_with(chunk1_missing_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk1_missing({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_chunk1_stale(#ar_mining_cache_value{chunk1 = Chunk1, chunk1_missing = true} = Value, Anomalies) when undefined =/= Chunk1 -> +maybe_search_for_anomalies_cache_values_chunk1_stale({ + Key, + #ar_mining_cache_value{chunk1 = Chunk1, chunk1_missing = true} = Value +}, Anomalies) when undefined =/= Chunk1 -> maps:update_with(chunk1_stale, fun(V) -> V + 1 end, 1, - maps:update_with(chunk1_stale_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_chunk1_stale(_, Anomalies) -> + maps:update_with(chunk1_stale_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk1_stale({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_chunk2_missing(#ar_mining_cache_value{chunk2 = undefined, chunk2_missing = false} = Value, Anomalies) -> +maybe_search_for_anomalies_cache_values_chunk2_missing({ + Key, + #ar_mining_cache_value{chunk2 = undefined, chunk2_missing = false} = Value +}, Anomalies) -> maps:update_with(chunk2_missing, fun(V) -> V + 1 end, 1, - maps:update_with(chunk2_missing_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_chunk2_missing(_, Anomalies) -> + maps:update_with(chunk2_missing_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk2_missing({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_chunk2_stale(#ar_mining_cache_value{chunk2 = Chunk2, chunk2_missing = true} = Value, Anomalies) when undefined =/= Chunk2 -> +maybe_search_for_anomalies_cache_values_chunk2_stale({ + Key, + #ar_mining_cache_value{chunk2 = Chunk2, chunk2_missing = true} = Value +}, Anomalies) when undefined =/= Chunk2 -> maps:update_with(chunk2_stale, fun(V) -> V + 1 end, 1, - maps:update_with(chunk2_stale_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_chunk2_stale(_, Anomalies) -> + maps:update_with(chunk2_stale_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_chunk2_stale({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h1_missing(#ar_mining_cache_value{h1 = undefined, chunk1 = Chunk1} = Value, Anomalies) +maybe_search_for_anomalies_cache_values_h1_missing({ + Key, + #ar_mining_cache_value{h1 = undefined, chunk1 = Chunk1} = Value +}, Anomalies) when undefined =/= Chunk1 -> maps:update_with(h1_missing, fun(V) -> V + 1 end, 1, - maps:update_with(h1_missing_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_h1_missing(_, Anomalies) -> + maps:update_with(h1_missing_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_h1_missing({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h2_missing(#ar_mining_cache_value{h2 = undefined, chunk2 = Chunk2} = Value, Anomalies) +maybe_search_for_anomalies_cache_values_h2_missing({ + Key, + #ar_mining_cache_value{h2 = undefined, chunk2 = Chunk2} = Value +}, Anomalies) when undefined =/= Chunk2 -> maps:update_with(h2_missing, fun(V) -> V + 1 end, 1, - maps:update_with(h2_missing_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_h2_missing(_, Anomalies) -> + maps:update_with(h2_missing_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_h2_missing({_, _}, Anomalies) -> Anomalies. -maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(#ar_mining_cache_value{h1_passes_diff_checks = true} = Value, Anomalies) -> +maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present({ + Key, + #ar_mining_cache_value{h1_passes_diff_checks = true} = Value +}, Anomalies) -> maps:update_with(h1_passes_diff_checks_present, fun(V) -> V + 1 end, 1, - maps:update_with(h1_passes_diff_checks_present_sample, fun(V) -> V end, Value, Anomalies)); -maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present(_, Anomalies) -> + maps:update_with(h1_passes_diff_checks_present_sample, fun(V) -> V end, {Key, Value}, Anomalies)); +maybe_search_for_anomalies_cache_values_h1_passes_diff_checks_present({_, _}, Anomalies) -> Anomalies. diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index bc1a66d278..a2cea0e20d 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -943,12 +943,9 @@ mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% We've already read the chunk2 from disk, so we can just drop the cached value. %% The cache reservation for corresponding chunk2 was already consumed. {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; - (CachedValue) -> + (#ar_mining_cache_value{chunk2 = undefined} = CachedValue) -> %% Mark the chunk1 as missing. - %% If the corresponding chunk2 will be read from disk, it will be dropped immediately. - %% If we didn't read the chunk2 from disk, we didn't reserve the cache space for it; - %% in this case the cached value will hang in the cache until the session will be dropped, - %% but it will not contain any large binaries, so it will not consume any significant memory. + %% When the corresponding chunk2 will be read from disk, it will be dropped immediately. {ok, CachedValue#ar_mining_cache_value{chunk1_missing = true}, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} end ) of From 30f37277c88e7a092a6b187962d926936b3fa7cc Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Wed, 9 Jul 2025 13:25:01 +0300 Subject: [PATCH 05/23] Tune cache logic --- apps/arweave/src/ar_mining_worker.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index a2cea0e20d..764e7d2444 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -939,7 +939,7 @@ mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% We've already marked the chunk2 as missing, so there was no reservation for it. %% We can just drop the cached value. {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; - (#ar_mining_cache_value{chunk2 = Chunk2}) when undefined /= Chunk2 -> + (#ar_mining_cache_value{chunk2 = Chunk2}) when is_binary(Chunk2) -> %% We've already read the chunk2 from disk, so we can just drop the cached value. %% The cache reservation for corresponding chunk2 was already consumed. {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; @@ -973,12 +973,12 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% We've already marked the chunk1 as missing, so the reservation for it was released. %% We can just drop the cached value and release the reservation for a single subchunk. {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; - (#ar_mining_cache_value{chunk1 = Chunk1, h1 = undefined} = CachedValue) when undefined /= Chunk1 -> + (#ar_mining_cache_value{chunk1 = Chunk1, h1 = undefined} = CachedValue) when is_binary(Chunk1) -> %% We have the corresponding chunk1, but we didn't calculate H1 yet. %% Mark chunk2 as missing to drop the cached value after we calculate H1. %% Drop the reservation for a single subchunk. {ok, CachedValue#ar_mining_cache_value{chunk2_missing = true}, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; - (#ar_mining_cache_value{h1 = H1}) when undefined /= H1 -> + (#ar_mining_cache_value{h1 = H1}) when is_binary(H1) -> %% We've already calculated H1, so we can drop the cached value. %% Drop the reservation for a single subchunk. {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; From 0555af44fba97793008f60d82d9f9179a7863720 Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Wed, 9 Jul 2025 17:58:44 +0300 Subject: [PATCH 06/23] Fix chunk2 being stored when chunk1 is missing --- apps/arweave/src/ar_mining_worker.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 764e7d2444..ad66df0b41 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -736,6 +736,10 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> Candidate#mining_candidate.session_key, State#state.chunk_cache, fun + (#ar_mining_cache_value{chunk1_missing = true}) -> + %% We've already marked the chunk1 as missing, so there was no reservation for it. + %% Since there is no need to calculate H2, we can just drop the cached value. + {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; (#ar_mining_cache_value{h1_passes_diff_checks = true} = _CachedValue) -> %% H1 passes diff checks, so we skip H2 for this nonce. %% Drop the cached data, we don't need it anymore. From dffd496c12c7a97353020540ed46b3dacfbd7f60 Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Wed, 9 Jul 2025 18:29:22 +0300 Subject: [PATCH 07/23] Fix chunk2 not being marked as missing --- apps/arweave/src/ar_mining_cache.erl | 3 ++- apps/arweave/src/ar_mining_worker.erl | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index b611658ec8..baf815b221 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -320,7 +320,8 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ _ -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, reserved_size_non_zero}, {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) - end; + end, + ?LOG_INFO([{event, mining_cache_anomaly}, {anomaly, mining_cache_anomaly_search_completed}, {session_id, SessionId}]); maybe_search_for_anomalies(SessionId, _InvalidSession) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_session_type}, {session_id, SessionId}]), ok. diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index ad66df0b41..78127eccee 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -668,15 +668,20 @@ process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 ); - {_, true, _} -> + {_, true, Kind} -> %% Skip this chunk. %% Nonce falls in a chunk beyond the current chunk offset, (for example, because we %% read extra chunk in the beginning of recall range). Move ahead to the next %% chunk offset. - %% No need to remove anything from cache, as the nonce is still in the recall range. + %% No need to remove anything from cache, as the nonce is still in the recall range, + %% but we need to mark the chunk as missing if it's not already. + State1 = case Kind of + chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State); + chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) + end, process_chunks( WhichChunk, Candidate, RangeStart, Nonce, NoncesPerChunk, - NoncesPerRecallRange, ChunkOffsets, SubChunkSize, Count, State + NoncesPerRecallRange, ChunkOffsets, SubChunkSize, Count, State1 ); {false, false, _} -> %% Process all sub-chunks in Chunk, and then advance to the next chunk. From 6438ed5f1d87ca950214ecd9b46e3f2b372c0afd Mon Sep 17 00:00:00 2001 From: Denis Fakhrtdinov Date: Thu, 10 Jul 2025 12:57:30 +0300 Subject: [PATCH 08/23] Fix logging --- apps/arweave/src/ar_mining_cache.erl | 4 ++-- apps/arweave/src/ar_mining_worker.erl | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index baf815b221..524704567e 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -310,10 +310,10 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ {0, 0} -> ok; {EqualSize, EqualSize} -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, cache_size_non_zero}, - {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]); + {session_id, SessionId}, {actual_size, ActualCacheSize}, {reported_size, MiningCacheSize}]); {_, _} -> ?LOG_WARNING([ {event, mining_cache_anomaly}, {anomaly, cache_size_mismatch}, - {session_id, SessionId}, {actual_size, ActualCacheSize}, {expected_size, MiningCacheSize}]) + {session_id, SessionId}, {actual_size, ActualCacheSize}, {reported_size, MiningCacheSize}]) end, case ReservedMiningCacheBytes of 0 -> ok; diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 78127eccee..34333b2597 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -668,20 +668,14 @@ process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 ); - {_, true, Kind} -> + {_, true, _} -> %% Skip this chunk. %% Nonce falls in a chunk beyond the current chunk offset, (for example, because we %% read extra chunk in the beginning of recall range). Move ahead to the next %% chunk offset. - %% No need to remove anything from cache, as the nonce is still in the recall range, - %% but we need to mark the chunk as missing if it's not already. - State1 = case Kind of - chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State); - chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) - end, process_chunks( WhichChunk, Candidate, RangeStart, Nonce, NoncesPerChunk, - NoncesPerRecallRange, ChunkOffsets, SubChunkSize, Count, State1 + NoncesPerRecallRange, ChunkOffsets, SubChunkSize, Count, State ); {false, false, _} -> %% Process all sub-chunks in Chunk, and then advance to the next chunk. From 0a13504b39a7f32f9df7abb6eca72136708211a4 Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 15:17:46 +0300 Subject: [PATCH 09/23] Add tag tracing for cached values and mining candidates --- apps/arweave/include/ar_mining.hrl | 3 +- apps/arweave/include/ar_mining_cache.hrl | 3 +- apps/arweave/src/ar_mining_worker.erl | 145 ++++++++++++----------- 3 files changed, 82 insertions(+), 69 deletions(-) diff --git a/apps/arweave/include/ar_mining.hrl b/apps/arweave/include/ar_mining.hrl index 72f70596f9..93ad506748 100644 --- a/apps/arweave/include/ar_mining.hrl +++ b/apps/arweave/include/ar_mining.hrl @@ -31,7 +31,8 @@ step_number = not_set, %% serialized packing_difficulty = 0, %% serialized replica_format = 0, %% serialized - label = <<"not_set">> %% not atom, for prevent atom table pollution DoS + label = <<"not_set">>, %% not atom, for prevent atom table pollution DoS + tags = [] }). -record(mining_solution, { diff --git a/apps/arweave/include/ar_mining_cache.hrl b/apps/arweave/include/ar_mining_cache.hrl index 062da200f8..772b731249 100644 --- a/apps/arweave/include/ar_mining_cache.hrl +++ b/apps/arweave/include/ar_mining_cache.hrl @@ -8,7 +8,8 @@ chunk2_missing = false :: boolean(), h1 :: binary() | undefined, h1_passes_diff_checks = false :: boolean(), - h2 :: binary() | undefined + h2 :: binary() | undefined, + tags = [] :: [atom()] }). -record(ar_mining_cache_session, { diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 34333b2597..2f1cb2f228 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -106,11 +106,11 @@ chunks_read(Worker, WhichChunk, Candidate, RangeStart, ChunkOffsets) -> Candidate :: #mining_candidate{} ) -> ok. computed_hash(Worker, computed_h0, H0, undefined, Candidate) -> - add_task(Worker, computed_h0, Candidate#mining_candidate{ h0 = H0 }); + add_task(Worker, computed_h0, Candidate#mining_candidate{ h0 = H0, tags = [computed_h0 | Candidate#mining_candidate.tags] }); computed_hash(Worker, computed_h1, H1, Preimage, Candidate) -> - add_task(Worker, computed_h1, Candidate#mining_candidate{ h1 = H1, preimage = Preimage }); + add_task(Worker, computed_h1, Candidate#mining_candidate{ h1 = H1, preimage = Preimage, tags = [computed_h1 | Candidate#mining_candidate.tags] }); computed_hash(Worker, computed_h2, H2, Preimage, Candidate) -> - add_task(Worker, computed_h2, Candidate#mining_candidate{ h2 = H2, preimage = Preimage }). + add_task(Worker, computed_h2, Candidate#mining_candidate{ h2 = H2, preimage = Preimage, tags = [computed_h2 | Candidate#mining_candidate.tags] }). %% @doc Set the new mining difficulty. We do not recalculate it inside the mining %% server or worker because we want to completely detach the mining server from the block @@ -317,7 +317,7 @@ handle_task({compute_h0, Candidate, _ExtraArgs}, State) -> case try_to_reserve_cache_range_space(2, Candidate#mining_candidate.session_key, State1) of {true, State2} -> %% Cache space reserved, compute h0. - ar_mining_hash:compute_h0(self(), Candidate), + ar_mining_hash:compute_h0(self(), Candidate#mining_candidate{ tags = [compute_h0 | Candidate#mining_candidate.tags] }), State2#state{ latest_vdf_step_number = max(StepNumber, LatestVDFStepNumber) }; false -> %% We don't have enough cache space to read the recall ranges, so we'll try again later. @@ -336,7 +336,7 @@ handle_task({computed_h0, Candidate, _ExtraArgs}, State) -> } = Candidate, {RecallRange1Start, RecallRange2Start} = ar_block:get_recall_range(H0, Partition1, PartitionUpperBound), Partition2 = ar_node:get_partition_number(RecallRange2Start), - Candidate2 = generate_cache_ref(Candidate#mining_candidate{ partition_number2 = Partition2 }), + Candidate2 = generate_cache_ref(Candidate#mining_candidate{ partition_number2 = Partition2, tags = [computed_h0_add_partition2 | Candidate#mining_candidate.tags] }), %% Check if the recall ranges are readable to avoid reserving cache space for non-existent data. Range1Exists = ar_mining_io:is_recall_range_readable(Candidate2, RecallRange1Start), Range2Exists = ar_mining_io:is_recall_range_readable(Candidate2, RecallRange2Start), @@ -347,17 +347,17 @@ handle_task({computed_h0, Candidate, _ExtraArgs}, State) -> {true, true} -> %% Both recall ranges are readable, no release needed. %% Read the recall ranges; the result of the read will be reported by the `chunk1` and `chunk2` tasks. - ar_mining_io:read_recall_range(chunk1, self(), Candidate2, RecallRange1Start), - ar_mining_io:read_recall_range(chunk2, self(), Candidate2, RecallRange2Start), + ar_mining_io:read_recall_range(chunk1, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk1 | Candidate2#mining_candidate.tags] }, RecallRange1Start), + ar_mining_io:read_recall_range(chunk2, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk2 | Candidate2#mining_candidate.tags] }, RecallRange2Start), State; {true, false} -> %% Only the first recall range is readable, so we need to release the reserved space for the second %% recall range. State1 = release_cache_range_space(1, Candidate2#mining_candidate.session_key, State), %% Mark second recall range as missing, not to wait for it to arrive. - State2 = mark_second_recall_range_missing(Candidate2, State1), + State2 = mark_second_recall_range_missing(Candidate2, State1, range_not_readable), %% Read the recall range; the result of the read will be reported by the `chunk1` task. - ar_mining_io:read_recall_range(chunk1, self(), Candidate2, RecallRange1Start), + ar_mining_io:read_recall_range(chunk1, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk1_only | Candidate2#mining_candidate.tags] }, RecallRange1Start), State2; {false, _} -> %% We don't have the recall ranges, so we need to release the reserved space for both partitions. @@ -385,7 +385,7 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> H1PassesDiffChecks = h1_passes_diff_checks(H1, Candidate, State1), case H1PassesDiffChecks of false -> ok; - partial -> ar_mining_server:prepare_and_post_solution(Candidate); + partial -> ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h1_partial_solution | Candidate#mining_candidate.tags] }); true -> %% H1 solution found, report it. ?LOG_INFO([{event, found_h1_solution}, @@ -394,7 +394,7 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> {h1, ar_util:encode(H1)}, {p1, Candidate#mining_candidate.partition_number}, {difficulty, get_difficulty(State1, Candidate)}]), - ar_mining_server:prepare_and_post_solution(Candidate), + ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h1_solution | Candidate#mining_candidate.tags] }), ar_mining_stats:h1_solution() end, %% Check if we need to compute H2. @@ -416,18 +416,24 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> not_set -> get_difficulty(State1, Candidate); PartialDiffPair -> PartialDiffPair end, - ar_coordination:computed_h1(Candidate, DiffPair) + ar_coordination:computed_h1(Candidate#mining_candidate{ tags = [computed_h1_coordination | Candidate#mining_candidate.tags] }, DiffPair) end, %% Remove the cached value from the cache. {ok, drop}; (#ar_mining_cache_value{chunk2 = undefined} = CachedValue) -> %% chunk2 hasn't been read yet, so we cache H1 and wait for it. %% If H1 passes diff checks, we will skip H2 for this nonce. - {ok, CachedValue#ar_mining_cache_value{h1 = H1, h1_passes_diff_checks = H1PassesDiffChecks}}; + {ok, CachedValue#ar_mining_cache_value{ + h1 = H1, h1_passes_diff_checks = H1PassesDiffChecks, + tags = [computed_h1 | CachedValue#ar_mining_cache_value.tags] + }}; (#ar_mining_cache_value{chunk2 = Chunk2} = CachedValue) when not H1PassesDiffChecks -> %% chunk2 has already been read, so we can compute H2 now. - ar_mining_hash:compute_h2(self(), Candidate#mining_candidate{ chunk2 = Chunk2 }), - {ok, CachedValue#ar_mining_cache_value{h1 = H1}}; + ar_mining_hash:compute_h2(self(), Candidate#mining_candidate{ chunk2 = Chunk2, tags = [computed_h1_compute_h2 | Candidate#mining_candidate.tags] }), + {ok, CachedValue#ar_mining_cache_value{ + h1 = H1, + tags = [computed_h1 | CachedValue#ar_mining_cache_value.tags] + }}; (#ar_mining_cache_value{chunk2 = _Chunk2} = _CachedValue) when H1PassesDiffChecks -> %% H1 passes diff checks, so we skip H2 for this nonce. %% Might as well drop the cached data, we don't need it anymore. @@ -481,22 +487,11 @@ handle_task({computed_h2, Candidate, _ExtraArgs}, State) -> %% In case of solo mining, the `Check` will always be `true`. %% In case of pool mining, the `Check` will be `partial` or `true`. %% In either case, we prepare and post the solution. - case Candidate#mining_candidate.chunk1 of - not_set -> - ?LOG_ERROR([{event, received_solution_candidate_without_chunk1_in_solo_mining}, - {worker, State#state.name}, - {step, Candidate#mining_candidate.step_number}, - {nonce, Candidate#mining_candidate.nonce}, - {session_key, ar_nonce_limiter:encode_session_key(Candidate#mining_candidate.session_key)}, - {h2, ar_util:encode(H2)}]), - ok; - _ -> - ar_mining_server:prepare_and_post_solution(Candidate) - end; + ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h2_solution | Candidate#mining_candidate.tags] }); {Check, _} when partial == Check orelse true == Check -> %% This branch only handles the case where we're part of a coordinated mining set. %% In this case, we prepare the PoA2 and send it to the lead peer. - ar_coordination:computed_h2_for_peer(Candidate) + ar_coordination:computed_h2_for_peer(Candidate#mining_candidate{ tags = [computed_h2_coordination | Candidate#mining_candidate.tags] }) end, %% Remove the cached value from the cache. case ar_mining_cache:with_cached_value( @@ -526,17 +521,17 @@ handle_task({compute_h2_for_peer, Candidate, _ExtraArgs}, State) -> cm_lead_peer = Peer } = Candidate, {_, RecallRange2Start} = ar_block:get_recall_range(H0, Partition1, PartitionUpperBound), - Candidate2 = generate_cache_ref(Candidate), + Candidate2 = generate_cache_ref(Candidate#mining_candidate{ tags = [compute_h2_for_peer_generate_cache_ref | Candidate#mining_candidate.tags] }), %% Clear the list so we aren't copying it around all over the place. - Candidate3 = Candidate2#mining_candidate{ cm_h1_list = [] }, - Range2Exists = ar_mining_io:read_recall_range(chunk2, self(), Candidate3, RecallRange2Start), + Candidate3 = Candidate2#mining_candidate{ cm_h1_list = [], tags = [compute_h2_for_peer_clear_h1_list | Candidate2#mining_candidate.tags] }, + Range2Exists = ar_mining_io:read_recall_range(chunk2, self(), Candidate3#mining_candidate{ tags = [compute_h2_for_peer_read_chunk2 | Candidate3#mining_candidate.tags] }, RecallRange2Start), case Range2Exists of true -> ar_mining_stats:h1_received_from_peer(Peer, length(H1List)), %% Mark second recall range as missing, just like if we didn't have the recall range %% in a regular mining setup. %% This is done because we only have a part of H1 list that passes diff checks. - State1 = mark_second_recall_range_missing(Candidate3, State), + State1 = mark_second_recall_range_missing(Candidate3, State, compute_for_peer), %% After we marked the whole second recall range as missing, we can cache the H1 list. %% During this process, we also reset the chunk2_missing flag to false for the entries %% we have H1 for. @@ -622,8 +617,8 @@ process_chunks( %% No more ChunkOffsets means no more chunks have been read. Iterate through all the %% remaining nonces and remove the full chunks from the cache. State1 = case WhichChunk of - chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State); - chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) + chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, beyond_end_of_recall_range); + chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, beyond_end_of_recall_range) end, %% Drop the reservation for the current nonce group (from Nonce to Nonce + NoncesPerChunk - 1). State2 = case ar_mining_cache:release_for_session( @@ -653,7 +648,7 @@ process_chunks( %% Nonce falls in a chunk which wasn't read from disk (for example, because there are holes %% in the recall range), e.g. the nonce is in the middle of a non-existent chunk. %% Mark single chunk1 as missing or remove it if the corresponding chunk is already read or marked as missing. - State1 = mark_single_chunk1_missing_or_drop(Nonce, Candidate, State), + State1 = mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, missing_start_of_recall_range), process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 @@ -663,7 +658,7 @@ process_chunks( %% Nonce falls in a chunk which wasn't read from disk (for example, because there are holes %% in the recall range), e.g. the nonce is in the middle of a non-existent chunk. %% Mark single chunk2 as missing or remove it if the corresponding chunk is already read and H1 is calculated. - State1 = mark_single_chunk2_missing_or_drop(Nonce, Candidate, State), + State1 = mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, missing_start_of_recall_range), process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 @@ -690,7 +685,7 @@ process_all_sub_chunks(_WhichChunk, <<>>, _Candidate, _Nonce, State) -> State; process_all_sub_chunks(WhichChunk, Chunk, Candidate, Nonce, State) when Candidate#mining_candidate.packing_difficulty == 0 -> %% Spora 2.6 packing (aka difficulty 0). - Candidate2 = Candidate#mining_candidate{ nonce = Nonce }, + Candidate2 = Candidate#mining_candidate{ nonce = Nonce, tags = [process_all_sub_chunks_spora | Candidate#mining_candidate.tags] }, process_sub_chunk(WhichChunk, Candidate2, Chunk, State); process_all_sub_chunks( WhichChunk, @@ -698,7 +693,7 @@ process_all_sub_chunks( Candidate, Nonce, State ) -> %% Composite packing / replica packing (aka difficulty 1+). - Candidate2 = Candidate#mining_candidate{ nonce = Nonce }, + Candidate2 = Candidate#mining_candidate{ nonce = Nonce, tags = [process_all_sub_chunks_replica | Candidate#mining_candidate.tags] }, State1 = process_sub_chunk(WhichChunk, Candidate2, SubChunk, State), process_all_sub_chunks(WhichChunk, Rest, Candidate2, Nonce + 1, State1); process_all_sub_chunks(WhichChunk, Rest, _Candidate, Nonce, State) -> @@ -711,13 +706,13 @@ process_all_sub_chunks(WhichChunk, Rest, _Candidate, Nonce, State) -> process_sub_chunk(chunk1, Candidate, SubChunk, State) -> %% Compute h1. - ar_mining_hash:compute_h1(self(), Candidate#mining_candidate{ chunk1 = SubChunk }), + ar_mining_hash:compute_h1(self(), Candidate#mining_candidate{ chunk1 = SubChunk, tags = [process_sub_chunk_chunk1_compute_h1 | Candidate#mining_candidate.tags] }), %% Store the chunk1 in the cache. case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Candidate#mining_candidate.nonce), Candidate#mining_candidate.session_key, State#state.chunk_cache, - fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{chunk1 = SubChunk}} end + fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{chunk1 = SubChunk, tags = [process_chunk1 | CachedValue#ar_mining_cache_value.tags]}} end ) of {ok, ChunkCache2} -> State#state{ chunk_cache = ChunkCache2 }; {error, Reason} -> @@ -729,7 +724,7 @@ process_sub_chunk(chunk1, Candidate, SubChunk, State) -> State end; process_sub_chunk(chunk2, Candidate, SubChunk, State) -> - Candidate2 = Candidate#mining_candidate{ chunk2 = SubChunk }, + Candidate2 = Candidate#mining_candidate{ chunk2 = SubChunk, tags = [process_sub_chunk_chunk2 | Candidate#mining_candidate.tags] }, case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate2#mining_candidate.cache_ref, Candidate2#mining_candidate.nonce), Candidate#mining_candidate.session_key, @@ -747,11 +742,11 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; (#ar_mining_cache_value{h1 = undefined} = CachedValue) -> %% H1 is not yet calculated, cache the chunk2 for this nonce. - {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk}}; + {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2_wait_for_h1 | CachedValue#ar_mining_cache_value.tags]}}; (#ar_mining_cache_value{h1 = H1} = CachedValue) -> %% H1 is already calculated, compute H2 and cache the chunk2 for this nonce. - ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1 }), - {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk}} + ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), + {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2 | CachedValue#ar_mining_cache_value.tags]}} end ) of {ok, ChunkCache2} -> State#state{ chunk_cache = ChunkCache2 }; @@ -924,13 +919,13 @@ release_cache_range_space(Multiplier, SessionKey, #state{ %% @doc Mark the chunk1 as missing or drop the cache and reservation for this chunk. %% This function is called for one chunk1. -mark_single_chunk1_missing_or_drop(Nonce, Candidate, State) -> +mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, Reason) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, SubChunksPerChunk = ar_block:get_nonces_per_chunk(PackingDifficulty), - mark_single_chunk1_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State). + mark_single_chunk1_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State, Reason). -mark_single_chunk1_missing_or_drop(_Nonce, 0, _Candidate, State) -> State; -mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> +mark_single_chunk1_missing_or_drop(_Nonce, 0, _Candidate, State, _Reason) -> State; +mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) -> %% Mark the chunk1 as missing. %% The cache reservation for this chunk1 will be dropped in the final (first) clause of the function. case ar_mining_cache:with_cached_value( @@ -949,24 +944,27 @@ mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> (#ar_mining_cache_value{chunk2 = undefined} = CachedValue) -> %% Mark the chunk1 as missing. %% When the corresponding chunk2 will be read from disk, it will be dropped immediately. - {ok, CachedValue#ar_mining_cache_value{chunk1_missing = true}, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} + {ok, CachedValue#ar_mining_cache_value{ + chunk1_missing = true, + tags = [{chunk1_missing, Reason} | CachedValue#ar_mining_cache_value.tags] + }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} end ) of {ok, ChunkCache1} -> - mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); + mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); {error, Reason} -> ?LOG_ERROR([{event, mining_worker_failed_to_mark_chunk1_missing}, {reason, Reason}]), - mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State) + mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) end. %% @doc Mark the chunk2 as missing for a single chunk. -mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) -> +mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, Reason) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, SubChunksPerChunk = ar_block:get_nonces_per_chunk(PackingDifficulty), - mark_single_chunk2_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State). + mark_single_chunk2_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State, Reason). -mark_single_chunk2_missing_or_drop(_Nonce, 0, _Candidate, State) -> State; -mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> +mark_single_chunk2_missing_or_drop(_Nonce, 0, _Candidate, State, _Reason) -> State; +mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) -> case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Nonce), Candidate#mining_candidate.session_key, @@ -980,7 +978,10 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% We have the corresponding chunk1, but we didn't calculate H1 yet. %% Mark chunk2 as missing to drop the cached value after we calculate H1. %% Drop the reservation for a single subchunk. - {ok, CachedValue#ar_mining_cache_value{chunk2_missing = true}, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; + {ok, CachedValue#ar_mining_cache_value{ + chunk2_missing = true, + tags = [{chunk2_missing, Reason} | CachedValue#ar_mining_cache_value.tags] + }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; (#ar_mining_cache_value{h1 = H1}) when is_binary(H1) -> %% We've already calculated H1, so we can drop the cached value. %% Drop the reservation for a single subchunk. @@ -989,38 +990,44 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% The corresponding chunk1 is not missing but we didn't read it yet, so %% we just mark the chunk2 as missing and continue. %% Drop the reservation for a single subchunk. - {ok, CachedValue#ar_mining_cache_value{chunk2_missing = true}, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} + {ok, CachedValue#ar_mining_cache_value{ + chunk2_missing = true, + tags = [{chunk2_missing_waiting_for_h1, Reason} | CachedValue#ar_mining_cache_value.tags] + }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} end ) of {ok, ChunkCache1} -> - mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); + mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); {error, Reason} -> %% NB: this clause may cause a memory leak, because mining worker will wait for %% chunk2 to arrive. ?LOG_ERROR([{event, mining_worker_failed_to_mark_chunk2_missing}, {reason, Reason}]), - mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State) + mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) end. %% @doc Mark the chunk2 as missing for the whole recall range. -mark_second_recall_range_missing(Candidate, State) -> +mark_second_recall_range_missing(Candidate, State, Reason) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, - mark_second_recall_range_missing(0, ar_block:get_max_nonce(PackingDifficulty), Candidate, State). + mark_second_recall_range_missing(0, ar_block:get_max_nonce(PackingDifficulty), Candidate, State, Reason). -mark_second_recall_range_missing(_Nonce, 0, _Candidate, State) -> State; -mark_second_recall_range_missing(Nonce, NoncesLeft, Candidate, State) -> +mark_second_recall_range_missing(_Nonce, 0, _Candidate, State, _Reason) -> State; +mark_second_recall_range_missing(Nonce, NoncesLeft, Candidate, State, Reason) -> case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Nonce), Candidate#mining_candidate.session_key, State#state.chunk_cache, - fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{chunk2_missing = true}} end + fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{ + chunk2_missing = true, + tags = [{chunk2_missing_second_recall_range, Reason} | CachedValue#ar_mining_cache_value.tags] + }} end ) of {ok, ChunkCache1} -> - mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); + mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); {error, Reason} -> %% NB: this clause may cause a memory leak, because mining worker will wait for %% chunk2 to arrive. ?LOG_ERROR([{event, mining_worker_failed_to_add_chunk_to_cache}, {reason, Reason}]), - mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State) + mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) end. cache_h1_list(_Candidate, [], State) -> State; @@ -1033,7 +1040,11 @@ cache_h1_list(Candidate, [ {H1, Nonce} | H1List ], State) -> fun(CachedValue) -> %% Store the H1 received from peer, and set chunk2_missing to false, %% marking that we have a recall range for this H1 list. - {ok, CachedValue#ar_mining_cache_value{h1 = H1, chunk2_missing = false}} + {ok, CachedValue#ar_mining_cache_value{ + h1 = H1, + chunk2_missing = false, + tags = [h1_list_received | CachedValue#ar_mining_cache_value.tags] + }} end ) of {ok, ChunkCache1} -> From 650028c19bcdec703f29113b724724bc31988f7d Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 16:20:59 +0300 Subject: [PATCH 10/23] Fix tagging clause --- apps/arweave/src/ar_mining_worker.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 2f1cb2f228..60223115c9 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -425,14 +425,14 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> %% If H1 passes diff checks, we will skip H2 for this nonce. {ok, CachedValue#ar_mining_cache_value{ h1 = H1, h1_passes_diff_checks = H1PassesDiffChecks, - tags = [computed_h1 | CachedValue#ar_mining_cache_value.tags] + tags = [computed_h1_wait_for_chunk2 | CachedValue#ar_mining_cache_value.tags] }}; (#ar_mining_cache_value{chunk2 = Chunk2} = CachedValue) when not H1PassesDiffChecks -> %% chunk2 has already been read, so we can compute H2 now. ar_mining_hash:compute_h2(self(), Candidate#mining_candidate{ chunk2 = Chunk2, tags = [computed_h1_compute_h2 | Candidate#mining_candidate.tags] }), {ok, CachedValue#ar_mining_cache_value{ h1 = H1, - tags = [computed_h1 | CachedValue#ar_mining_cache_value.tags] + tags = [computed_h1_compute_h2 | CachedValue#ar_mining_cache_value.tags] }}; (#ar_mining_cache_value{chunk2 = _Chunk2} = _CachedValue) when H1PassesDiffChecks -> %% H1 passes diff checks, so we skip H2 for this nonce. From 5dd3bb2eb3d1a896b499592b2721da814c042c82 Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 17:24:16 +0300 Subject: [PATCH 11/23] Assign chunk1 along with h1 when computing h2 --- apps/arweave/src/ar_mining_worker.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 60223115c9..e988eed8e8 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -743,9 +743,9 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> (#ar_mining_cache_value{h1 = undefined} = CachedValue) -> %% H1 is not yet calculated, cache the chunk2 for this nonce. {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2_wait_for_h1 | CachedValue#ar_mining_cache_value.tags]}}; - (#ar_mining_cache_value{h1 = H1} = CachedValue) -> + (#ar_mining_cache_value{h1 = H1, chunk1 = Chunk1} = CachedValue) -> %% H1 is already calculated, compute H2 and cache the chunk2 for this nonce. - ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), + ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, chunk1 = Chunk1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2 | CachedValue#ar_mining_cache_value.tags]}} end ) of From 23b73dca538f2b264310da229201fbf3165758ec Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 17:51:47 +0300 Subject: [PATCH 12/23] Fix chunk2 not marked as missing --- apps/arweave/src/ar_mining_worker.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index e988eed8e8..74e7564ebf 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -745,7 +745,7 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2_wait_for_h1 | CachedValue#ar_mining_cache_value.tags]}}; (#ar_mining_cache_value{h1 = H1, chunk1 = Chunk1} = CachedValue) -> %% H1 is already calculated, compute H2 and cache the chunk2 for this nonce. - ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, chunk1 = Chunk1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), + ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2 | CachedValue#ar_mining_cache_value.tags]}} end ) of @@ -1008,7 +1008,7 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) %% @doc Mark the chunk2 as missing for the whole recall range. mark_second_recall_range_missing(Candidate, State, Reason) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, - mark_second_recall_range_missing(0, ar_block:get_max_nonce(PackingDifficulty), Candidate, State, Reason). + mark_second_recall_range_missing(0, ar_block:get_nonces_per_recall_range(PackingDifficulty), Candidate, State, Reason). mark_second_recall_range_missing(_Nonce, 0, _Candidate, State, _Reason) -> State; mark_second_recall_range_missing(Nonce, NoncesLeft, Candidate, State, Reason) -> From a2679fa7ff703f0e141b86c89e7564bb2f836e6e Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 18:07:47 +0300 Subject: [PATCH 13/23] Fix chunk1 being not_set when posting solution --- apps/arweave/src/ar_mining_worker.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 74e7564ebf..0ced7d7fba 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -745,7 +745,7 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2_wait_for_h1 | CachedValue#ar_mining_cache_value.tags]}}; (#ar_mining_cache_value{h1 = H1, chunk1 = Chunk1} = CachedValue) -> %% H1 is already calculated, compute H2 and cache the chunk2 for this nonce. - ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), + ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, chunk1 = Chunk1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2 | CachedValue#ar_mining_cache_value.tags]}} end ) of From c58937eeb4b2738ececfab35310945fa5ac3aaca Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 20:54:55 +0300 Subject: [PATCH 14/23] Revert tag tracer --- apps/arweave/include/ar_mining.hrl | 3 +- apps/arweave/include/ar_mining_cache.hrl | 3 +- apps/arweave/src/ar_mining_worker.erl | 154 ++++++++++------------- 3 files changed, 68 insertions(+), 92 deletions(-) diff --git a/apps/arweave/include/ar_mining.hrl b/apps/arweave/include/ar_mining.hrl index 93ad506748..72f70596f9 100644 --- a/apps/arweave/include/ar_mining.hrl +++ b/apps/arweave/include/ar_mining.hrl @@ -31,8 +31,7 @@ step_number = not_set, %% serialized packing_difficulty = 0, %% serialized replica_format = 0, %% serialized - label = <<"not_set">>, %% not atom, for prevent atom table pollution DoS - tags = [] + label = <<"not_set">> %% not atom, for prevent atom table pollution DoS }). -record(mining_solution, { diff --git a/apps/arweave/include/ar_mining_cache.hrl b/apps/arweave/include/ar_mining_cache.hrl index 772b731249..062da200f8 100644 --- a/apps/arweave/include/ar_mining_cache.hrl +++ b/apps/arweave/include/ar_mining_cache.hrl @@ -8,8 +8,7 @@ chunk2_missing = false :: boolean(), h1 :: binary() | undefined, h1_passes_diff_checks = false :: boolean(), - h2 :: binary() | undefined, - tags = [] :: [atom()] + h2 :: binary() | undefined }). -record(ar_mining_cache_session, { diff --git a/apps/arweave/src/ar_mining_worker.erl b/apps/arweave/src/ar_mining_worker.erl index 0ced7d7fba..1fcb12e9fd 100644 --- a/apps/arweave/src/ar_mining_worker.erl +++ b/apps/arweave/src/ar_mining_worker.erl @@ -106,11 +106,11 @@ chunks_read(Worker, WhichChunk, Candidate, RangeStart, ChunkOffsets) -> Candidate :: #mining_candidate{} ) -> ok. computed_hash(Worker, computed_h0, H0, undefined, Candidate) -> - add_task(Worker, computed_h0, Candidate#mining_candidate{ h0 = H0, tags = [computed_h0 | Candidate#mining_candidate.tags] }); + add_task(Worker, computed_h0, Candidate#mining_candidate{ h0 = H0 }); computed_hash(Worker, computed_h1, H1, Preimage, Candidate) -> - add_task(Worker, computed_h1, Candidate#mining_candidate{ h1 = H1, preimage = Preimage, tags = [computed_h1 | Candidate#mining_candidate.tags] }); + add_task(Worker, computed_h1, Candidate#mining_candidate{ h1 = H1, preimage = Preimage }); computed_hash(Worker, computed_h2, H2, Preimage, Candidate) -> - add_task(Worker, computed_h2, Candidate#mining_candidate{ h2 = H2, preimage = Preimage, tags = [computed_h2 | Candidate#mining_candidate.tags] }). + add_task(Worker, computed_h2, Candidate#mining_candidate{ h2 = H2, preimage = Preimage }). %% @doc Set the new mining difficulty. We do not recalculate it inside the mining %% server or worker because we want to completely detach the mining server from the block @@ -317,7 +317,7 @@ handle_task({compute_h0, Candidate, _ExtraArgs}, State) -> case try_to_reserve_cache_range_space(2, Candidate#mining_candidate.session_key, State1) of {true, State2} -> %% Cache space reserved, compute h0. - ar_mining_hash:compute_h0(self(), Candidate#mining_candidate{ tags = [compute_h0 | Candidate#mining_candidate.tags] }), + ar_mining_hash:compute_h0(self(), Candidate), State2#state{ latest_vdf_step_number = max(StepNumber, LatestVDFStepNumber) }; false -> %% We don't have enough cache space to read the recall ranges, so we'll try again later. @@ -336,7 +336,7 @@ handle_task({computed_h0, Candidate, _ExtraArgs}, State) -> } = Candidate, {RecallRange1Start, RecallRange2Start} = ar_block:get_recall_range(H0, Partition1, PartitionUpperBound), Partition2 = ar_node:get_partition_number(RecallRange2Start), - Candidate2 = generate_cache_ref(Candidate#mining_candidate{ partition_number2 = Partition2, tags = [computed_h0_add_partition2 | Candidate#mining_candidate.tags] }), + Candidate2 = generate_cache_ref(Candidate#mining_candidate{ partition_number2 = Partition2 }), %% Check if the recall ranges are readable to avoid reserving cache space for non-existent data. Range1Exists = ar_mining_io:is_recall_range_readable(Candidate2, RecallRange1Start), Range2Exists = ar_mining_io:is_recall_range_readable(Candidate2, RecallRange2Start), @@ -347,17 +347,17 @@ handle_task({computed_h0, Candidate, _ExtraArgs}, State) -> {true, true} -> %% Both recall ranges are readable, no release needed. %% Read the recall ranges; the result of the read will be reported by the `chunk1` and `chunk2` tasks. - ar_mining_io:read_recall_range(chunk1, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk1 | Candidate2#mining_candidate.tags] }, RecallRange1Start), - ar_mining_io:read_recall_range(chunk2, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk2 | Candidate2#mining_candidate.tags] }, RecallRange2Start), + ar_mining_io:read_recall_range(chunk1, self(), Candidate2, RecallRange1Start), + ar_mining_io:read_recall_range(chunk2, self(), Candidate2, RecallRange2Start), State; {true, false} -> %% Only the first recall range is readable, so we need to release the reserved space for the second %% recall range. State1 = release_cache_range_space(1, Candidate2#mining_candidate.session_key, State), %% Mark second recall range as missing, not to wait for it to arrive. - State2 = mark_second_recall_range_missing(Candidate2, State1, range_not_readable), + State2 = mark_second_recall_range_missing(Candidate2, State1), %% Read the recall range; the result of the read will be reported by the `chunk1` task. - ar_mining_io:read_recall_range(chunk1, self(), Candidate2#mining_candidate{ tags = [computed_h0_read_chunk1_only | Candidate2#mining_candidate.tags] }, RecallRange1Start), + ar_mining_io:read_recall_range(chunk1, self(), Candidate2, RecallRange1Start), State2; {false, _} -> %% We don't have the recall ranges, so we need to release the reserved space for both partitions. @@ -385,7 +385,7 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> H1PassesDiffChecks = h1_passes_diff_checks(H1, Candidate, State1), case H1PassesDiffChecks of false -> ok; - partial -> ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h1_partial_solution | Candidate#mining_candidate.tags] }); + partial -> ar_mining_server:prepare_and_post_solution(Candidate); true -> %% H1 solution found, report it. ?LOG_INFO([{event, found_h1_solution}, @@ -394,7 +394,7 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> {h1, ar_util:encode(H1)}, {p1, Candidate#mining_candidate.partition_number}, {difficulty, get_difficulty(State1, Candidate)}]), - ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h1_solution | Candidate#mining_candidate.tags] }), + ar_mining_server:prepare_and_post_solution(Candidate), ar_mining_stats:h1_solution() end, %% Check if we need to compute H2. @@ -416,24 +416,18 @@ handle_task({computed_h1, Candidate, _ExtraArgs}, State) -> not_set -> get_difficulty(State1, Candidate); PartialDiffPair -> PartialDiffPair end, - ar_coordination:computed_h1(Candidate#mining_candidate{ tags = [computed_h1_coordination | Candidate#mining_candidate.tags] }, DiffPair) + ar_coordination:computed_h1(Candidate, DiffPair) end, %% Remove the cached value from the cache. {ok, drop}; (#ar_mining_cache_value{chunk2 = undefined} = CachedValue) -> %% chunk2 hasn't been read yet, so we cache H1 and wait for it. %% If H1 passes diff checks, we will skip H2 for this nonce. - {ok, CachedValue#ar_mining_cache_value{ - h1 = H1, h1_passes_diff_checks = H1PassesDiffChecks, - tags = [computed_h1_wait_for_chunk2 | CachedValue#ar_mining_cache_value.tags] - }}; + {ok, CachedValue#ar_mining_cache_value{ h1 = H1, h1_passes_diff_checks = H1PassesDiffChecks }}; (#ar_mining_cache_value{chunk2 = Chunk2} = CachedValue) when not H1PassesDiffChecks -> %% chunk2 has already been read, so we can compute H2 now. - ar_mining_hash:compute_h2(self(), Candidate#mining_candidate{ chunk2 = Chunk2, tags = [computed_h1_compute_h2 | Candidate#mining_candidate.tags] }), - {ok, CachedValue#ar_mining_cache_value{ - h1 = H1, - tags = [computed_h1_compute_h2 | CachedValue#ar_mining_cache_value.tags] - }}; + ar_mining_hash:compute_h2(self(), Candidate#mining_candidate{ chunk2 = Chunk2 }), + {ok, CachedValue#ar_mining_cache_value{ h1 = H1 }}; (#ar_mining_cache_value{chunk2 = _Chunk2} = _CachedValue) when H1PassesDiffChecks -> %% H1 passes diff checks, so we skip H2 for this nonce. %% Might as well drop the cached data, we don't need it anymore. @@ -487,11 +481,11 @@ handle_task({computed_h2, Candidate, _ExtraArgs}, State) -> %% In case of solo mining, the `Check` will always be `true`. %% In case of pool mining, the `Check` will be `partial` or `true`. %% In either case, we prepare and post the solution. - ar_mining_server:prepare_and_post_solution(Candidate#mining_candidate{ tags = [computed_h2_solution | Candidate#mining_candidate.tags] }); + ar_mining_server:prepare_and_post_solution(Candidate); {Check, _} when partial == Check orelse true == Check -> %% This branch only handles the case where we're part of a coordinated mining set. %% In this case, we prepare the PoA2 and send it to the lead peer. - ar_coordination:computed_h2_for_peer(Candidate#mining_candidate{ tags = [computed_h2_coordination | Candidate#mining_candidate.tags] }) + ar_coordination:computed_h2_for_peer(Candidate) end, %% Remove the cached value from the cache. case ar_mining_cache:with_cached_value( @@ -521,17 +515,17 @@ handle_task({compute_h2_for_peer, Candidate, _ExtraArgs}, State) -> cm_lead_peer = Peer } = Candidate, {_, RecallRange2Start} = ar_block:get_recall_range(H0, Partition1, PartitionUpperBound), - Candidate2 = generate_cache_ref(Candidate#mining_candidate{ tags = [compute_h2_for_peer_generate_cache_ref | Candidate#mining_candidate.tags] }), + Candidate2 = generate_cache_ref(Candidate), %% Clear the list so we aren't copying it around all over the place. - Candidate3 = Candidate2#mining_candidate{ cm_h1_list = [], tags = [compute_h2_for_peer_clear_h1_list | Candidate2#mining_candidate.tags] }, - Range2Exists = ar_mining_io:read_recall_range(chunk2, self(), Candidate3#mining_candidate{ tags = [compute_h2_for_peer_read_chunk2 | Candidate3#mining_candidate.tags] }, RecallRange2Start), + Candidate3 = Candidate2#mining_candidate{ cm_h1_list = [] }, + Range2Exists = ar_mining_io:read_recall_range(chunk2, self(), Candidate3, RecallRange2Start), case Range2Exists of true -> ar_mining_stats:h1_received_from_peer(Peer, length(H1List)), %% Mark second recall range as missing, just like if we didn't have the recall range %% in a regular mining setup. %% This is done because we only have a part of H1 list that passes diff checks. - State1 = mark_second_recall_range_missing(Candidate3, State, compute_for_peer), + State1 = mark_second_recall_range_missing(Candidate3, State), %% After we marked the whole second recall range as missing, we can cache the H1 list. %% During this process, we also reset the chunk2_missing flag to false for the entries %% we have H1 for. @@ -617,8 +611,8 @@ process_chunks( %% No more ChunkOffsets means no more chunks have been read. Iterate through all the %% remaining nonces and remove the full chunks from the cache. State1 = case WhichChunk of - chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, beyond_end_of_recall_range); - chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, beyond_end_of_recall_range) + chunk1 -> mark_single_chunk1_missing_or_drop(Nonce, Candidate, State); + chunk2 -> mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) end, %% Drop the reservation for the current nonce group (from Nonce to Nonce + NoncesPerChunk - 1). State2 = case ar_mining_cache:release_for_session( @@ -648,7 +642,7 @@ process_chunks( %% Nonce falls in a chunk which wasn't read from disk (for example, because there are holes %% in the recall range), e.g. the nonce is in the middle of a non-existent chunk. %% Mark single chunk1 as missing or remove it if the corresponding chunk is already read or marked as missing. - State1 = mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, missing_start_of_recall_range), + State1 = mark_single_chunk1_missing_or_drop(Nonce, Candidate, State), process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 @@ -658,7 +652,7 @@ process_chunks( %% Nonce falls in a chunk which wasn't read from disk (for example, because there are holes %% in the recall range), e.g. the nonce is in the middle of a non-existent chunk. %% Mark single chunk2 as missing or remove it if the corresponding chunk is already read and H1 is calculated. - State1 = mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, missing_start_of_recall_range), + State1 = mark_single_chunk2_missing_or_drop(Nonce, Candidate, State), process_chunks( WhichChunk, Candidate, RangeStart, Nonce + NoncesPerChunk, NoncesPerChunk, NoncesPerRecallRange, [{ChunkEndOffset, Chunk} | ChunkOffsets], SubChunkSize, Count, State1 @@ -685,7 +679,7 @@ process_all_sub_chunks(_WhichChunk, <<>>, _Candidate, _Nonce, State) -> State; process_all_sub_chunks(WhichChunk, Chunk, Candidate, Nonce, State) when Candidate#mining_candidate.packing_difficulty == 0 -> %% Spora 2.6 packing (aka difficulty 0). - Candidate2 = Candidate#mining_candidate{ nonce = Nonce, tags = [process_all_sub_chunks_spora | Candidate#mining_candidate.tags] }, + Candidate2 = Candidate#mining_candidate{ nonce = Nonce }, process_sub_chunk(WhichChunk, Candidate2, Chunk, State); process_all_sub_chunks( WhichChunk, @@ -693,7 +687,7 @@ process_all_sub_chunks( Candidate, Nonce, State ) -> %% Composite packing / replica packing (aka difficulty 1+). - Candidate2 = Candidate#mining_candidate{ nonce = Nonce, tags = [process_all_sub_chunks_replica | Candidate#mining_candidate.tags] }, + Candidate2 = Candidate#mining_candidate{ nonce = Nonce }, State1 = process_sub_chunk(WhichChunk, Candidate2, SubChunk, State), process_all_sub_chunks(WhichChunk, Rest, Candidate2, Nonce + 1, State1); process_all_sub_chunks(WhichChunk, Rest, _Candidate, Nonce, State) -> @@ -706,13 +700,13 @@ process_all_sub_chunks(WhichChunk, Rest, _Candidate, Nonce, State) -> process_sub_chunk(chunk1, Candidate, SubChunk, State) -> %% Compute h1. - ar_mining_hash:compute_h1(self(), Candidate#mining_candidate{ chunk1 = SubChunk, tags = [process_sub_chunk_chunk1_compute_h1 | Candidate#mining_candidate.tags] }), + ar_mining_hash:compute_h1(self(), Candidate#mining_candidate{ chunk1 = SubChunk }), %% Store the chunk1 in the cache. case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Candidate#mining_candidate.nonce), Candidate#mining_candidate.session_key, State#state.chunk_cache, - fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{chunk1 = SubChunk, tags = [process_chunk1 | CachedValue#ar_mining_cache_value.tags]}} end + fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{ chunk1 = SubChunk }} end ) of {ok, ChunkCache2} -> State#state{ chunk_cache = ChunkCache2 }; {error, Reason} -> @@ -724,7 +718,7 @@ process_sub_chunk(chunk1, Candidate, SubChunk, State) -> State end; process_sub_chunk(chunk2, Candidate, SubChunk, State) -> - Candidate2 = Candidate#mining_candidate{ chunk2 = SubChunk, tags = [process_sub_chunk_chunk2 | Candidate#mining_candidate.tags] }, + Candidate2 = Candidate#mining_candidate{ chunk2 = SubChunk }, case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate2#mining_candidate.cache_ref, Candidate2#mining_candidate.nonce), Candidate#mining_candidate.session_key, @@ -742,11 +736,11 @@ process_sub_chunk(chunk2, Candidate, SubChunk, State) -> {ok, drop, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; (#ar_mining_cache_value{h1 = undefined} = CachedValue) -> %% H1 is not yet calculated, cache the chunk2 for this nonce. - {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2_wait_for_h1 | CachedValue#ar_mining_cache_value.tags]}}; + {ok, CachedValue#ar_mining_cache_value{ chunk2 = SubChunk }}; (#ar_mining_cache_value{h1 = H1, chunk1 = Chunk1} = CachedValue) -> %% H1 is already calculated, compute H2 and cache the chunk2 for this nonce. - ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, chunk1 = Chunk1, tags = [process_sub_chunk_chunk2_compute_h2 | Candidate2#mining_candidate.tags] }), - {ok, CachedValue#ar_mining_cache_value{chunk2 = SubChunk, tags = [process_chunk2 | CachedValue#ar_mining_cache_value.tags]}} + ar_mining_hash:compute_h2(self(), Candidate2#mining_candidate{ h1 = H1, chunk1 = Chunk1 }), + {ok, CachedValue#ar_mining_cache_value{ chunk2 = SubChunk }} end ) of {ok, ChunkCache2} -> State#state{ chunk_cache = ChunkCache2 }; @@ -919,13 +913,13 @@ release_cache_range_space(Multiplier, SessionKey, #state{ %% @doc Mark the chunk1 as missing or drop the cache and reservation for this chunk. %% This function is called for one chunk1. -mark_single_chunk1_missing_or_drop(Nonce, Candidate, State, Reason) -> +mark_single_chunk1_missing_or_drop(Nonce, Candidate, State) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, SubChunksPerChunk = ar_block:get_nonces_per_chunk(PackingDifficulty), - mark_single_chunk1_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State, Reason). + mark_single_chunk1_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State). -mark_single_chunk1_missing_or_drop(_Nonce, 0, _Candidate, State, _Reason) -> State; -mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) -> +mark_single_chunk1_missing_or_drop(_Nonce, 0, _Candidate, State) -> State; +mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> %% Mark the chunk1 as missing. %% The cache reservation for this chunk1 will be dropped in the final (first) clause of the function. case ar_mining_cache:with_cached_value( @@ -944,27 +938,24 @@ mark_single_chunk1_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) (#ar_mining_cache_value{chunk2 = undefined} = CachedValue) -> %% Mark the chunk1 as missing. %% When the corresponding chunk2 will be read from disk, it will be dropped immediately. - {ok, CachedValue#ar_mining_cache_value{ - chunk1_missing = true, - tags = [{chunk1_missing, Reason} | CachedValue#ar_mining_cache_value.tags] - }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} + {ok, CachedValue#ar_mining_cache_value{ chunk1_missing = true }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} end ) of {ok, ChunkCache1} -> - mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); + mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); {error, Reason} -> ?LOG_ERROR([{event, mining_worker_failed_to_mark_chunk1_missing}, {reason, Reason}]), - mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) + mark_single_chunk1_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State) end. %% @doc Mark the chunk2 as missing for a single chunk. -mark_single_chunk2_missing_or_drop(Nonce, Candidate, State, Reason) -> +mark_single_chunk2_missing_or_drop(Nonce, Candidate, State) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, SubChunksPerChunk = ar_block:get_nonces_per_chunk(PackingDifficulty), - mark_single_chunk2_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State, Reason). + mark_single_chunk2_missing_or_drop(Nonce, SubChunksPerChunk, Candidate, State). -mark_single_chunk2_missing_or_drop(_Nonce, 0, _Candidate, State, _Reason) -> State; -mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) -> +mark_single_chunk2_missing_or_drop(_Nonce, 0, _Candidate, State) -> State; +mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State) -> case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Nonce), Candidate#mining_candidate.session_key, @@ -978,10 +969,7 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) %% We have the corresponding chunk1, but we didn't calculate H1 yet. %% Mark chunk2 as missing to drop the cached value after we calculate H1. %% Drop the reservation for a single subchunk. - {ok, CachedValue#ar_mining_cache_value{ - chunk2_missing = true, - tags = [{chunk2_missing, Reason} | CachedValue#ar_mining_cache_value.tags] - }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; + {ok, CachedValue#ar_mining_cache_value{ chunk2_missing = true }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)}; (#ar_mining_cache_value{h1 = H1}) when is_binary(H1) -> %% We've already calculated H1, so we can drop the cached value. %% Drop the reservation for a single subchunk. @@ -990,44 +978,38 @@ mark_single_chunk2_missing_or_drop(Nonce, NoncesLeft, Candidate, State, Reason) %% The corresponding chunk1 is not missing but we didn't read it yet, so %% we just mark the chunk2 as missing and continue. %% Drop the reservation for a single subchunk. - {ok, CachedValue#ar_mining_cache_value{ - chunk2_missing = true, - tags = [{chunk2_missing_waiting_for_h1, Reason} | CachedValue#ar_mining_cache_value.tags] - }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} + {ok, CachedValue#ar_mining_cache_value{ chunk2_missing = true }, -ar_block:get_sub_chunk_size(Candidate#mining_candidate.packing_difficulty)} end ) of {ok, ChunkCache1} -> - mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); + mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); {error, Reason} -> %% NB: this clause may cause a memory leak, because mining worker will wait for %% chunk2 to arrive. ?LOG_ERROR([{event, mining_worker_failed_to_mark_chunk2_missing}, {reason, Reason}]), - mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) + mark_single_chunk2_missing_or_drop(Nonce + 1, NoncesLeft - 1, Candidate, State) end. %% @doc Mark the chunk2 as missing for the whole recall range. -mark_second_recall_range_missing(Candidate, State, Reason) -> +mark_second_recall_range_missing(Candidate, State) -> #mining_candidate{ packing_difficulty = PackingDifficulty } = Candidate, - mark_second_recall_range_missing(0, ar_block:get_nonces_per_recall_range(PackingDifficulty), Candidate, State, Reason). + mark_second_recall_range_missing(0, ar_block:get_nonces_per_recall_range(PackingDifficulty), Candidate, State). -mark_second_recall_range_missing(_Nonce, 0, _Candidate, State, _Reason) -> State; -mark_second_recall_range_missing(Nonce, NoncesLeft, Candidate, State, Reason) -> +mark_second_recall_range_missing(_Nonce, 0, _Candidate, State) -> State; +mark_second_recall_range_missing(Nonce, NoncesLeft, Candidate, State) -> case ar_mining_cache:with_cached_value( ?CACHE_KEY(Candidate#mining_candidate.cache_ref, Nonce), Candidate#mining_candidate.session_key, State#state.chunk_cache, - fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{ - chunk2_missing = true, - tags = [{chunk2_missing_second_recall_range, Reason} | CachedValue#ar_mining_cache_value.tags] - }} end + fun(CachedValue) -> {ok, CachedValue#ar_mining_cache_value{ chunk2_missing = true }} end ) of {ok, ChunkCache1} -> - mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }, Reason); + mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State#state{ chunk_cache = ChunkCache1 }); {error, Reason} -> %% NB: this clause may cause a memory leak, because mining worker will wait for %% chunk2 to arrive. ?LOG_ERROR([{event, mining_worker_failed_to_add_chunk_to_cache}, {reason, Reason}]), - mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State, Reason) + mark_second_recall_range_missing(Nonce + 1, NoncesLeft - 1, Candidate, State) end. cache_h1_list(_Candidate, [], State) -> State; @@ -1040,11 +1022,7 @@ cache_h1_list(Candidate, [ {H1, Nonce} | H1List ], State) -> fun(CachedValue) -> %% Store the H1 received from peer, and set chunk2_missing to false, %% marking that we have a recall range for this H1 list. - {ok, CachedValue#ar_mining_cache_value{ - h1 = H1, - chunk2_missing = false, - tags = [h1_list_received | CachedValue#ar_mining_cache_value.tags] - }} + {ok, CachedValue#ar_mining_cache_value{ h1 = H1, chunk2_missing = false }} end ) of {ok, ChunkCache1} -> @@ -1088,17 +1066,17 @@ hash_computed(WhichHash, Candidate, State) -> report_and_reset_hashes(State) -> maps:foreach( - fun(Key, Value) -> - ar_mining_stats:h1_computed(Key, Value) - end, - State#state.h1_hashes - ), + fun(Key, Value) -> + ar_mining_stats:h1_computed(Key, Value) + end, + State#state.h1_hashes + ), maps:foreach( - fun(Key, Value) -> - ar_mining_stats:h2_computed(Key, Value) - end, - State#state.h2_hashes - ), + fun(Key, Value) -> + ar_mining_stats:h2_computed(Key, Value) + end, + State#state.h2_hashes + ), State#state{ h1_hashes = #{}, h2_hashes = #{} }. report_chunk_cache_metrics(#state{chunk_cache = ChunkCache, partition_number = Partition} = State) -> From 5514ed4004f9caed005b2928e7f7c017c2665bf1 Mon Sep 17 00:00:00 2001 From: Denis Farr Date: Wed, 16 Jul 2025 21:04:03 +0300 Subject: [PATCH 15/23] Remove excessive logging --- apps/arweave/src/ar_mining_cache.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/arweave/src/ar_mining_cache.erl b/apps/arweave/src/ar_mining_cache.erl index 524704567e..bf55f4d4be 100644 --- a/apps/arweave/src/ar_mining_cache.erl +++ b/apps/arweave/src/ar_mining_cache.erl @@ -321,7 +321,7 @@ maybe_search_for_anomalies(SessionId, #ar_mining_cache_session{ {event, mining_cache_anomaly}, {anomaly, reserved_size_non_zero}, {session_id, SessionId}, {actual_size, ReservedMiningCacheBytes}, {expected_size, 0}]) end, - ?LOG_INFO([{event, mining_cache_anomaly}, {anomaly, mining_cache_anomaly_search_completed}, {session_id, SessionId}]); + ?LOG_DEBUG([{event, mining_cache_anomaly}, {anomaly, mining_cache_anomaly_search_completed}, {session_id, SessionId}]); maybe_search_for_anomalies(SessionId, _InvalidSession) -> ?LOG_ERROR([{event, mining_cache_anomaly}, {anomaly, invalid_session_type}, {session_id, SessionId}]), ok. From cf8b06ae9fc7d729e2086e74d96e74c616fbcc36 Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Fri, 11 Apr 2025 18:09:41 +0200 Subject: [PATCH 16/23] Optionally base solution on sibling or uncle Useful when the previous block changes the VDF session. --- .github/workflows/test.yml | 1 + apps/arweave/src/ar_block_cache.erl | 198 +++++++++++++- apps/arweave/src/ar_mining_server.erl | 60 +---- apps/arweave/src/ar_node_worker.erl | 361 +++++++++++++++++++++++++- apps/arweave/src/ar_pool.erl | 10 +- 5 files changed, 560 insertions(+), 70 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10b54ce9f1..6c1aba8267 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -251,6 +251,7 @@ jobs: ar_node, ar_node_utils, ar_nonce_limiter, + ar_node_worker, # ar_p3, # ar_p3_config, # ar_p3_db, diff --git a/apps/arweave/src/ar_block_cache.erl b/apps/arweave/src/ar_block_cache.erl index 3779fa94f8..3f46bc100d 100644 --- a/apps/arweave/src/ar_block_cache.erl +++ b/apps/arweave/src/ar_block_cache.erl @@ -8,7 +8,8 @@ get_longest_chain_cache/1, get_block_and_status/2, remove/2, get_checkpoint_block/1, prune/2, get_by_solution_hash/5, is_known_solution_hash/2, - get_siblings/2, get_fork_blocks/2, update_timestamp/3]). + get_siblings/2, get_fork_blocks/2, update_timestamp/3, + get_validated_front/1]). -include_lib("arweave/include/ar.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -480,6 +481,16 @@ update_timestamp(Tab, H, ReceiveTimestamp) -> end end. +%% @doc Return the list of top validated blocks of the longest chains (the largest +%% cumulative difficulty blocks). +get_validated_front(Tab) -> + case ets:lookup(Tab, links) of + [] -> + []; + [{_, Set}] -> + get_validated_front(Tab, Set) + end. + %%%=================================================================== %%% Private functions. %%%=================================================================== @@ -720,6 +731,31 @@ update_longest_chain_cache(Tab) -> end, Result. +get_validated_front(Tab, Set) -> + {_MaxCDiff, Blocks} = gb_sets:fold( + fun({_Height, H}, {CurrentMaxCDiff, CurrentBlocks}) -> + case ets:lookup(Tab, {block, H}) of + [{_, {B, Status, _Timestamp, _Children}}] + when Status == validated; + Status == on_chain -> + CDiff = B#block.cumulative_diff, + case CDiff > CurrentMaxCDiff of + true -> + {CDiff, [B]}; + false when CDiff == CurrentMaxCDiff -> + {CurrentMaxCDiff, [B | CurrentBlocks]}; + false -> + {CurrentMaxCDiff, CurrentBlocks} + end; + _ -> + {CurrentMaxCDiff, CurrentBlocks} + end + end, + {0, []}, + Set + ), + Blocks. + %%%=================================================================== %%% Tests. %%%=================================================================== @@ -1760,3 +1796,163 @@ block_id(#block{ indep_hash = H }) -> on_top(B, PrevB) -> B#block{ previous_block = PrevB#block.indep_hash, height = PrevB#block.height + 1, previous_cumulative_diff = PrevB#block.cumulative_diff }. + +get_validated_front_test() -> + ets:new(bcache_test, [set, named_table]), + + %% Test empty cache + %% + %% Height Block/Status + %% + %% (empty) + ?assertEqual([], get_validated_front(bcache_test)), + + %% Test with a single on-chain block + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + new(bcache_test, B1 = random_block(0)), + ?assertEqual([B1], get_validated_front(bcache_test)), + + %% Test with a single validated block + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + add_validated(bcache_test, B2 = on_top(random_block(1), B1)), + ?assertEqual([B2], get_validated_front(bcache_test)), + + %% Test with multiple validated blocks at different heights + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + add_validated(bcache_test, B3 = on_top(random_block(2), B2)), + add_validated(bcache_test, B4 = on_top(random_block(3), B3)), + ?assertEqual([B4], get_validated_front(bcache_test)), + + %% Test with multiple blocks at the same highest cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + add_validated(bcache_test, B5 = on_top(random_block(3), B2)), + ?assertEqual(lists:sort([B4, B5]), + lists:sort(get_validated_front(bcache_test))), + + %% Test with non-validated blocks having higher cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/not_validated (cdiff=4) + add(bcache_test, B6 = on_top(random_block(4), B4)), + ?assertEqual(lists:sort([B4, B5]), + lists:sort(get_validated_front(bcache_test))), + + %% Test validating a block with higher cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/validated (cdiff=4) + add_validated(bcache_test, B6), + ?assertEqual([B6], get_validated_front(bcache_test)), + + %% Test with multiple forks at the same height and cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/validated (cdiff=4) + %% 4 B7/validated (cdiff=4) + %% 4 B8/validated (cdiff=4) + add_validated(bcache_test, B7 = on_top(random_block(4), B3)), + add_validated(bcache_test, B8 = on_top(random_block(4), B5)), + ?assertEqual(lists:sort([B6, B7, B8]), + lists:sort(get_validated_front(bcache_test))), + + %% Test with a mix of validated and non-validated blocks at different heights + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/validated (cdiff=4) + %% 4 B7/validated (cdiff=4) + %% 4 B8/validated (cdiff=4) + %% 5 B9/not_validated (cdiff=5) + %% 5 B10/not_validated (cdiff=5) + add(bcache_test, B9 = on_top(random_block(5), B6)), + add(bcache_test, B10 = on_top(random_block(5), B7)), + ?assertEqual(lists:sort([B6, B7, B8]), + lists:sort(get_validated_front(bcache_test))), + + %% Test validating blocks with higher cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/validated (cdiff=4) + %% 4 B7/validated (cdiff=4) + %% 4 B8/validated (cdiff=4) + %% 5 B9/validated (cdiff=5) + %% 5 B10/validated (cdiff=5) + add_validated(bcache_test, B9), + add_validated(bcache_test, B10), + ?assertEqual(lists:sort([B9, B10]), + lists:sort(get_validated_front(bcache_test))), + + %% Test with on_chain blocks at the highest cumulative difficulty + %% + %% Height Block/Status + %% + %% 0 B1/on_chain + %% 1 B2/validated (cdiff=1) + %% 2 B3/validated (cdiff=2) + %% 3 B4/validated (cdiff=3) + %% 3 B5/validated (cdiff=3) + %% 4 B6/validated (cdiff=4) + %% 4 B7/validated (cdiff=4) + %% 4 B8/validated (cdiff=4) + %% 5 B9/validated (cdiff=5) + %% 5 B10/validated (cdiff=5) + %% 6 B11/validated (cdiff=6) + %% 6 B12/on_chain (cdiff=7) + add(bcache_test, B11 = on_top(random_block(6), B9)), + mark_tip(bcache_test, block_id(B11)), + ?assertEqual([B11], get_validated_front(bcache_test)), + add(bcache_test, B12 = on_top(random_block(6), B9)), + mark_tip(bcache_test, block_id(B12)), + ?assertEqual(lists:sort([B11, B12]), + lists:sort(get_validated_front(bcache_test))), + + ets:delete(bcache_test). \ No newline at end of file diff --git a/apps/arweave/src/ar_mining_server.erl b/apps/arweave/src/ar_mining_server.erl index 4b9d4a1c8c..32c088c2a5 100644 --- a/apps/arweave/src/ar_mining_server.erl +++ b/apps/arweave/src/ar_mining_server.erl @@ -13,11 +13,11 @@ -export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). --include_lib("arweave/include/ar.hrl"). --include_lib("arweave/include/ar_consensus.hrl"). --include_lib("arweave/include/ar_config.hrl"). --include_lib("arweave/include/ar_data_discovery.hrl"). --include_lib("arweave/include/ar_mining.hrl"). +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). +-include("ar_data_discovery.hrl"). +-include("ar_mining.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). @@ -671,57 +671,9 @@ prepare_solution(last_step_checkpoints, Candidate, Solution) -> _ -> LastStepCheckpoints end, - prepare_solution(steps, Candidate, Solution#mining_solution{ + prepare_solution(proofs, Candidate, Solution#mining_solution{ last_step_checkpoints = LastStepCheckpoints2 }); -prepare_solution(steps, Candidate, Solution) -> - #mining_candidate{ step_number = StepNumber } = Candidate, - [{_, TipNonceLimiterInfo}] = ets:lookup(node_state, nonce_limiter_info), - #nonce_limiter_info{ global_step_number = PrevStepNumber, seed = PrevSeed, - next_seed = PrevNextSeed, - next_vdf_difficulty = PrevNextVDFDifficulty } = TipNonceLimiterInfo, - case StepNumber > PrevStepNumber of - true -> - Steps = ar_nonce_limiter:get_steps( - PrevStepNumber, StepNumber, PrevNextSeed, PrevNextVDFDifficulty), - case Steps of - not_found -> - CurrentSessionKey = ar_nonce_limiter:session_key(TipNonceLimiterInfo), - SolutionSessionKey = Candidate#mining_candidate.session_key, - LogData = [ - {current_session_key, - ar_nonce_limiter:encode_session_key(CurrentSessionKey)}, - {solution_session_key, - ar_nonce_limiter:encode_session_key(SolutionSessionKey)}, - {start_step_number, PrevStepNumber}, - {next_step_number, StepNumber}, - {seed, ar_util:safe_encode(PrevSeed)}, - {next_seed, ar_util:safe_encode(PrevNextSeed)}, - {next_vdf_difficulty, PrevNextVDFDifficulty}, - {h1, ar_util:safe_encode(Candidate#mining_candidate.h1)}, - {h2, ar_util:safe_encode(Candidate#mining_candidate.h2)}], - ?LOG_INFO([{event, found_solution_but_failed_to_find_checkpoints} - | LogData]), - may_be_leave_it_to_exit_peer( - prepare_solution(proofs, Candidate, - Solution#mining_solution{ steps = [] }), - step_checkpoints_not_found, LogData); - _ -> - prepare_solution(proofs, Candidate, - Solution#mining_solution{ steps = Steps }) - end; - false -> - log_prepare_solution_failure(Solution, stale, stale_step_number, miner, [ - {start_step_number, PrevStepNumber}, - {next_step_number, StepNumber}, - {next_seed, ar_util:safe_encode(PrevNextSeed)}, - {next_vdf_difficulty, PrevNextVDFDifficulty}, - {h1, ar_util:safe_encode(Candidate#mining_candidate.h1)}, - {h2, ar_util:safe_encode(Candidate#mining_candidate.h2)} - ]), - error - end; - prepare_solution(proofs, Candidate, Solution) -> #mining_candidate{ h0 = H0, h1 = H1, h2 = H2, nonce = Nonce, partition_number = PartitionNumber, diff --git a/apps/arweave/src/ar_node_worker.erl b/apps/arweave/src/ar_node_worker.erl index ffc59f55f2..6ed38b73c3 100644 --- a/apps/arweave/src/ar_node_worker.erl +++ b/apps/arweave/src/ar_node_worker.erl @@ -14,13 +14,13 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). -export([set_reward_addr/1]). --include("../include/ar.hrl"). --include("../include/ar_consensus.hrl"). --include("../include/ar_config.hrl"). --include("../include/ar_pricing.hrl"). --include("../include/ar_data_sync.hrl"). --include("../include/ar_vdf.hrl"). --include("../include/ar_mining.hrl"). +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). +-include("ar_pricing.hrl"). +-include("ar_data_sync.hrl"). +-include("ar_vdf.hrl"). +-include("ar_mining.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -334,9 +334,13 @@ handle_cast({found_solution, miner, _Solution, _PoACache, _PoA2Cache}, #{ automine := false, miner_2_6 := undefined } = State) -> {noreply, State}; handle_cast({found_solution, Source, Solution, PoACache, PoA2Cache}, State) -> - [{_, PrevH}] = ets:lookup(node_state, current), - PrevB = ar_block_cache:get(block_cache, PrevH), - handle_found_solution({Source, Solution, PoACache, PoA2Cache}, PrevB, State, false); + case pick_prev_block_for_solution(Solution, Source) of + not_found -> + {noreply, State}; + {PrevB, Solution2} -> + handle_found_solution( + {Source, Solution2, PoACache, PoA2Cache}, PrevB, State, false) + end; handle_cast(process_task_queue, #{ task_queue := TaskQueue } = State) -> RunTask = @@ -1902,6 +1906,95 @@ dump_mempool(TXs, MempoolSize) -> ?LOG_ERROR([{event, failed_to_dump_mempool}, {reason, Reason}]) end. +pick_prev_block_for_solution(Solution, Source) -> + FrontBlocks = ar_block_cache:get_validated_front(block_cache), + PrevBlocks = [ar_block_cache:get(block_cache, B#block.previous_block) + || B <- FrontBlocks], + pick_prev_block_from_front_blocks(Solution, Source, FrontBlocks, PrevBlocks). + +%% @doc Search for a base block among the validated front blocks. +pick_prev_block_from_front_blocks(Solution, Source, FrontBlocks, PrevBlocks) -> + #mining_solution{ mining_address = MiningAddress } = Solution, + case [B || B <- FrontBlocks, B#block.reward_addr == MiningAddress] of + [PrevB | _] -> + pick_prev_block_for_solution(Solution, Source, PrevB); + [] -> + pick_prev_block_from_front_blocks2(Solution, Source, FrontBlocks, PrevBlocks) + end. + +pick_prev_block_from_front_blocks2(Solution, Source, [PrevB | FrontBlocks], PrevBlocks) -> + #mining_solution{ step_number = StepNumber } = Solution, + MaybeSolutionOffPrevB = may_be_place_solution_on_block(PrevB, StepNumber, Solution), + case MaybeSolutionOffPrevB of + {false, _Reason} -> + pick_prev_block_from_front_blocks2(Solution, Source, FrontBlocks, PrevBlocks); + Solution2 -> + {PrevB, Solution2} + end; +pick_prev_block_from_front_blocks2(Solution, Source, [], PrevBlocks) -> + pick_prev_block_from_prev_blocks(Solution, Source, PrevBlocks). + +%% @doc Search for a base block among the previous blocks of the +%% validated front blocks. +pick_prev_block_from_prev_blocks(Solution, Source, PrevBlocks) -> + #mining_solution{ mining_address = MiningAddress } = Solution, + case [B || B <- PrevBlocks, B#block.reward_addr == MiningAddress] of + [PrevB | _] -> + pick_prev_block_for_solution(Solution, Source, PrevB); + [] -> + pick_prev_block_from_prev_blocks2(Solution, Source, PrevBlocks) + end. + +pick_prev_block_from_prev_blocks2(Solution, Source, [PrevB | PrevBlocks]) -> + #mining_solution{ step_number = StepNumber } = Solution, + MaybeSolutionOffPrevB = may_be_place_solution_on_block(PrevB, StepNumber, Solution), + case MaybeSolutionOffPrevB of + {false, _Reason} -> + pick_prev_block_from_prev_blocks2(Solution, Source, PrevBlocks); + Solution2 -> + {PrevB, Solution2} + end; +pick_prev_block_from_prev_blocks2(Solution, Source, []) -> + #mining_solution{ step_number = StepNumber, solution_hash = SolutionH } = Solution, + ar_mining_server:log_prepare_solution_failure( + Solution, stale, no_suitable_prev_block, Source, + [{step_number, StepNumber}, {solution, ar_util:encode(SolutionH)}]), + not_found. + +pick_prev_block_for_solution(Solution, Source, PrevB) -> + #mining_solution{ + step_number = StepNumber, + solution_hash = SolutionH + } = Solution, + case may_be_place_solution_on_block(PrevB, StepNumber, Solution) of + {false, Reason} -> + ar_mining_server:log_prepare_solution_failure( + Solution, stale, Reason, Source, + [{step_number, StepNumber}, + {solution, ar_util:encode(SolutionH)}, + {prev_block, ar_util:encode(PrevB#block.indep_hash)}]), + not_found; + Solution2 -> + {PrevB, Solution2} + end. + +may_be_place_solution_on_block(PrevB, StepNumber, Solution) -> + NonceLimiterInfo = PrevB#block.nonce_limiter_info, + #nonce_limiter_info{ global_step_number = PrevStepNumber, next_seed = PrevNextSeed, + next_vdf_difficulty = PrevNextVDFDifficulty } = NonceLimiterInfo, + case StepNumber > PrevStepNumber of + true -> + case ar_nonce_limiter:get_steps( + PrevStepNumber, StepNumber, PrevNextSeed, PrevNextVDFDifficulty) of + not_found -> + {false, stale_vdf_session}; + Steps -> + Solution#mining_solution{ steps = Steps } + end; + false -> + {false, stale_step_number} + end. + handle_found_solution(Args, PrevB, State, IsRebase) -> {Source, Solution, PoACache, PoA2Cache} = Args, #mining_solution{ @@ -1934,6 +2027,8 @@ handle_found_solution(Args, PrevB, State, IsRebase) -> height = PrevHeight } = PrevB, Height = PrevHeight + 1, + % TODO add CDiff check for our addresses + Now = os:system_time(second), MaxDeviation = ar_block:get_max_timestamp_deviation(), Timestamp = @@ -2364,3 +2459,249 @@ checker_test() -> ?assertEqual({3, #{ true => 3 }}, checker([true, true, true])), ?assertEqual({3, #{ true => 2, false => 1}}, checker([true, true, false])), ?assertEqual({3, #{ true => 1, false => 2}}, checker([true, false, false])). + +pick_prev_block_from_front_blocks_test_() -> + {setup, + fun() -> + %% Mock ar_nonce_limiter:get_steps/4 to return valid steps + meck:new(ar_nonce_limiter, [non_strict]), + meck:expect(ar_nonce_limiter, get_steps, fun(_, _, _, _) -> {ok, [1, 2, 3]} end) + end, + fun(_) -> + meck:unload(ar_nonce_limiter) + end, + [ + {"Test picking block with matching reward address", + fun() -> + %% Create a mining solution with step number greater than block + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 2 + }, + + %% Create front blocks with different reward addresses + FrontBlocks = [ + #block{ + reward_addr = <<"addr2">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + }, + #block{ + reward_addr = <<"addr1">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + } + ], + + %% Call the function + {Block, Solution2} = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, []), + ?assertEqual({ok, [1, 2, 3]}, Solution2#mining_solution.steps), + + %% Verify it picked the block with matching reward address + ?assertEqual(<<"addr1">>, Block#block.reward_addr) + end + }, + {"Test picking block from previous blocks when front blocks have higher step number", + fun() -> + %% Create a mining solution with step number less than front blocks + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 1 + }, + + %% Create front blocks with step numbers higher than solution + FrontBlocks = [ + #block{ + reward_addr = <<"addr2">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 2}, + previous_block = <<"prev1">> + }, + #block{ + reward_addr = <<"addr3">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 2}, + previous_block = <<"prev2">> + } + ], + + %% Create previous blocks with valid step numbers + PrevBlocks = [ + #block{ + reward_addr = <<"addr1">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 0} + } + ], + + %% Call the function + {Block, Solution2} = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, PrevBlocks), + ?assertEqual({ok, [1, 2, 3]}, Solution2#mining_solution.steps), + + %% Verify it picked the previous block with matching reward address + ?assertEqual(<<"addr1">>, Block#block.reward_addr) + end + }, + {"Test picking any suitable front block when no reward address matches", + fun() -> + %% Create a mining solution with step number greater than blocks + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 2 + }, + + %% Create front blocks with different reward addresses but valid step numbers + FrontBlocks = [ + #block{ + reward_addr = <<"addr2">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + }, + #block{ + reward_addr = <<"addr3">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + } + ], + + %% Call the function + {Block, Solution2} = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, []), + ?assertEqual({ok, [1, 2, 3]}, Solution2#mining_solution.steps), + + %% Verify it picked one of the front blocks + ?assert(lists:member(Block, FrontBlocks)) + end + }, + {"Test picking block with matching step number", + fun() -> + %% Create a mining solution with step number greater than one block + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 2 + }, + + %% Create front blocks with different step numbers + FrontBlocks = [ + #block{ + reward_addr = <<"addr1">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + }, + #block{ + reward_addr = <<"addr1">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 2} + }, + #block{ + reward_addr = <<"addr1">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 3} + } + ], + + %% Call the function + {Block, Solution2} = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, []), + ?assertEqual({ok, [1, 2, 3]}, Solution2#mining_solution.steps), + + %% Verify it picked one of the blocks with step number less than solution + ?assert(Block#block.nonce_limiter_info#nonce_limiter_info.global_step_number < Solution#mining_solution.step_number) + end + }, + {"Test picking suitable previous block when front blocks and some previous blocks are not suitable", + fun() -> + %% Create a mining solution with step number greater than some blocks + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 2 + }, + + %% Create front blocks with step numbers higher than solution + FrontBlocks = [ + #block{ + reward_addr = <<"addr2">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 3}, + previous_block = <<"prev1">> + }, + #block{ + reward_addr = <<"addr3">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 4}, + previous_block = <<"prev2">> + } + ], + + %% Create previous blocks - some with higher step numbers, one suitable + PrevBlocks = [ + #block{ + reward_addr = <<"addr4">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 3} + }, + #block{ + reward_addr = <<"addr5">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 1} + }, + #block{ + reward_addr = <<"addr6">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 4} + } + ], + + %% Call the function + {Block, Solution2} = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, PrevBlocks), + ?assertEqual({ok, [1, 2, 3]}, Solution2#mining_solution.steps), + + %% Verify it picked the previous block with step number 1 + ?assertEqual(1, Block#block.nonce_limiter_info#nonce_limiter_info.global_step_number), + ?assertEqual(<<"addr5">>, Block#block.reward_addr) + end + }, + {"Test returning not_found when no suitable blocks exist", + fun() -> + %% Create a mining solution with step number less than all blocks + Solution = #mining_solution{ + mining_address = <<"addr1">>, + step_number = 1 + }, + + %% Create front blocks with step numbers higher than solution + FrontBlocks = [ + #block{ + reward_addr = <<"addr2">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 2}, + previous_block = <<"prev1">> + }, + #block{ + reward_addr = <<"addr3">>, + height = 1, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 3}, + previous_block = <<"prev2">> + } + ], + + %% Create previous blocks with step numbers higher than solution + PrevBlocks = [ + #block{ + reward_addr = <<"addr4">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 2} + }, + #block{ + reward_addr = <<"addr5">>, + height = 0, + nonce_limiter_info = #nonce_limiter_info{global_step_number = 3} + } + ], + + %% Call the function + Result = pick_prev_block_from_front_blocks(Solution, self(), FrontBlocks, PrevBlocks), + + %% Verify it returned not_found + ?assertEqual(not_found, Result) + end + } + ]}. diff --git a/apps/arweave/src/ar_pool.erl b/apps/arweave/src/ar_pool.erl index 327f39072f..87734831a8 100644 --- a/apps/arweave/src/ar_pool.erl +++ b/apps/arweave/src/ar_pool.erl @@ -41,11 +41,11 @@ -export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). --include_lib("arweave/include/ar.hrl"). --include_lib("arweave/include/ar_config.hrl"). --include_lib("arweave/include/ar_consensus.hrl"). --include_lib("arweave/include/ar_mining.hrl"). --include_lib("arweave/include/ar_pool.hrl"). +-include("ar.hrl"). +-include("ar_config.hrl"). +-include("ar_consensus.hrl"). +-include("ar_mining.hrl"). +-include("ar_pool.hrl"). -include_lib("eunit/include/eunit.hrl"). -record(state, { From 5c79348555bc85f860c524344002ce4006fb275a Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Tue, 15 Apr 2025 12:52:32 +0200 Subject: [PATCH 17/23] Add an extra double-signing check --- apps/arweave/src/ar_block.erl | 13 ++++++- apps/arweave/src/ar_block_cache.erl | 57 ++++++++++++++++++++++++++++- apps/arweave/src/ar_node_utils.erl | 2 +- apps/arweave/src/ar_node_worker.erl | 45 +++++++++++++++++------ 4 files changed, 101 insertions(+), 16 deletions(-) diff --git a/apps/arweave/src/ar_block.erl b/apps/arweave/src/ar_block.erl index 4a73253352..965f0e585e 100644 --- a/apps/arweave/src/ar_block.erl +++ b/apps/arweave/src/ar_block.erl @@ -21,7 +21,7 @@ get_max_nonce/1, get_recall_range_size/1, get_recall_byte/3, get_sub_chunk_size/1, get_nonces_per_chunk/1, get_nonces_per_recall_range/1, get_sub_chunk_index/2, - get_chunk_padded_offset/1]). + get_chunk_padded_offset/1, get_double_signing_condition/4]). -include("../include/ar.hrl"). -include("../include/ar_consensus.hrl"). @@ -666,7 +666,16 @@ get_chunk_padded_offset(Offset) -> Offset end. - +%% @doc Return true if the given cumulative difficulty - previous cumulative difficulty +%% pairs satisfy the double signing condition. +-spec get_double_signing_condition( + CDiff1 :: non_neg_integer(), + PrevCDiff1 :: non_neg_integer(), + CDiff2 :: non_neg_integer(), + PrevCDiff2 :: non_neg_integer() +) -> boolean(). +get_double_signing_condition(CDiff1, PrevCDiff1, CDiff2, PrevCDiff2) -> + CDiff1 == CDiff2 orelse (CDiff1 > PrevCDiff2 andalso CDiff2 > PrevCDiff1). %%%=================================================================== %%% Private functions. diff --git a/apps/arweave/src/ar_block_cache.erl b/apps/arweave/src/ar_block_cache.erl index 3f46bc100d..16a9fe5d8e 100644 --- a/apps/arweave/src/ar_block_cache.erl +++ b/apps/arweave/src/ar_block_cache.erl @@ -9,7 +9,7 @@ get_block_and_status/2, remove/2, get_checkpoint_block/1, prune/2, get_by_solution_hash/5, is_known_solution_hash/2, get_siblings/2, get_fork_blocks/2, update_timestamp/3, - get_validated_front/1]). + get_validated_front/1, get_blocks_by_miner/2]). -include_lib("arweave/include/ar.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -491,6 +491,26 @@ get_validated_front(Tab) -> get_validated_front(Tab, Set) end. +%% @doc Return all blocks from the cache mined by the given address. +get_blocks_by_miner(Tab, MinerAddr) -> + case ets:lookup(Tab, links) of + [{links, Set}] -> + gb_sets:fold( + fun({_Height, H}, Acc) -> + case ets:lookup(Tab, {block, H}) of + [{_, {B, _Status, _Timestamp, _Children}}] when B#block.reward_addr == MinerAddr -> + [B | Acc]; + _ -> + Acc + end + end, + [], + Set + ); + _ -> + [] + end. + %%%=================================================================== %%% Private functions. %%%=================================================================== @@ -1955,4 +1975,37 @@ get_validated_front_test() -> ?assertEqual(lists:sort([B11, B12]), lists:sort(get_validated_front(bcache_test))), - ets:delete(bcache_test). \ No newline at end of file + ets:delete(bcache_test). + +%% @doc Test that get_blocks_by_miner returns the correct blocks for a given miner. +get_blocks_by_miner_test() -> + ets:new(bcache_test, [set, named_table]), + new(bcache_test, B0 = random_block(0)), + Tab = bcache_test, + ?assertEqual([], get_blocks_by_miner(Tab, <<"miner1">>)), + % Create some test blocks + B1 = #block{ indep_hash = <<"hash1">>, reward_addr = <<"miner1">> }, + B2 = #block{ indep_hash = <<"hash2">>, reward_addr = <<"miner2">> }, + B3 = #block{ indep_hash = <<"hash3">>, reward_addr = <<"miner1">> }, + % Add blocks to cache + add(Tab, on_top(B1, B0)), + add(Tab, on_top(B2, B0)), + add(Tab, on_top(B3, B0)), + B1_1 = B1#block{ + height = 1, + previous_block = B0#block.indep_hash, + previous_cumulative_diff = B0#block.cumulative_diff }, + B2_1 = B2#block{ + height = 1, + previous_block = B0#block.indep_hash, + previous_cumulative_diff = B0#block.cumulative_diff }, + B3_1 = B3#block{ + height = 1, + previous_block = B0#block.indep_hash, + previous_cumulative_diff = B0#block.cumulative_diff }, + + % Test getting blocks by miner + ?assertEqual([B1_1, B3_1], lists:sort(fun(A, B) -> A#block.indep_hash < B#block.indep_hash end, get_blocks_by_miner(Tab, <<"miner1">>))), + ?assertEqual([B2_1], get_blocks_by_miner(Tab, <<"miner2">>)), + ?assertEqual([], get_blocks_by_miner(Tab, <<"miner3">>)), + ets:delete(Tab). \ No newline at end of file diff --git a/apps/arweave/src/ar_node_utils.erl b/apps/arweave/src/ar_node_utils.erl index 68a4de2d77..5ff6c3ff58 100644 --- a/apps/arweave/src/ar_node_utils.erl +++ b/apps/arweave/src/ar_node_utils.erl @@ -240,7 +240,7 @@ may_be_apply_double_signing_proof(#block{ may_be_apply_double_signing_proof(B, PrevB, Accounts) -> {_Pub, _Signature1, CDiff1, PrevCDiff1, _Preimage1, _Signature2, CDiff2, PrevCDiff2, _Preimage2} = B#block.double_signing_proof, - case CDiff1 == CDiff2 orelse (CDiff1 > PrevCDiff2 andalso CDiff2 > PrevCDiff1) of + case ar_block:get_double_signing_condition(CDiff1, PrevCDiff1, CDiff2, PrevCDiff2) of false -> {error, invalid_double_signing_proof_cdiff}; true -> diff --git a/apps/arweave/src/ar_node_worker.erl b/apps/arweave/src/ar_node_worker.erl index 6ed38b73c3..246438d74a 100644 --- a/apps/arweave/src/ar_node_worker.erl +++ b/apps/arweave/src/ar_node_worker.erl @@ -1101,8 +1101,7 @@ may_be_get_double_signing_proof2(Iterator, RootHash, LockedRewards, Height) -> false -> false; true -> - CDiff1 == CDiff2 - orelse (CDiff1 > PrevCDiff2 andalso CDiff2 > PrevCDiff1) + ar_block:get_double_signing_condition(CDiff1, PrevCDiff1, CDiff2, PrevCDiff2) end, case ValidCDiffs of false -> @@ -2026,9 +2025,6 @@ handle_found_solution(Args, PrevB, State, IsRebase) -> nonce_limiter_info = PrevNonceLimiterInfo, height = PrevHeight } = PrevB, Height = PrevHeight + 1, - - % TODO add CDiff check for our addresses - Now = os:system_time(second), MaxDeviation = ar_block:get_max_timestamp_deviation(), Timestamp = @@ -2172,12 +2168,29 @@ handle_found_solution(Args, PrevB, State, IsRebase) -> {false, rebase_threshold} end end, - %% Check steps and step checkpoints. - HaveSteps = + + PrevCDiff = PrevB#block.cumulative_diff, + CDiff = ar_difficulty:next_cumulative_diff(PrevCDiff, Diff, Height), + NoDoubleSigning = case CorrectRebaseThreshold of {false, Reason5} -> + {false, Reason5}; + true -> + case check_no_double_signing(CDiff, PrevCDiff, MiningAddress) of + false -> + {false, double_signing}; + true -> + true + end + end, + + %% Check steps and step checkpoints. + HaveSteps = + case NoDoubleSigning of + {false, Reason6} -> ?LOG_WARNING([{event, ignore_mining_solution}, - {reason, Reason5}, {solution, ar_util:encode(SolutionH)}]), + {reason, Reason6}, + {solution, ar_util:encode(SolutionH)}]), false; true -> ar_nonce_limiter:get_steps(PrevStepNumber, StepNumber, PrevNextSeed, @@ -2241,8 +2254,6 @@ handle_found_solution(Args, PrevB, State, IsRebase) -> Denomination2), ScheduledPricePerGiBMinute2 = ar_pricing:redenominate(ScheduledPricePerGiBMinute, Denomination, Denomination2), - CDiff = ar_difficulty:next_cumulative_diff(PrevB#block.cumulative_diff, Diff, - Height), UnsignedB = pack_block_with_transactions(#block{ nonce = Nonce, previous_block = PrevH, @@ -2359,6 +2370,18 @@ assert_key_type(RewardKey, Height) -> end end. +check_no_double_signing(CDiff, PrevCDiff, MiningAddress) -> + Blocks = ar_block_cache:get_blocks_by_miner(block_cache, MiningAddress), + not lists:any( + fun(B) -> + ar_block:get_double_signing_condition( + B#block.cumulative_diff, + B#block.previous_cumulative_diff, + CDiff, + PrevCDiff) + end, + Blocks). + update_solution_cache(H, Args, State) -> %% Maintain a cache of mining solutions for potential reuse in rebasing. %% @@ -2401,7 +2424,7 @@ may_be_report_double_signing(B, State) -> previous_solution_hash = PrevSolutionH2, reward_key = {_, Key}, signature = Signature2 } = CacheB, - case CDiff1 == CDiff2 orelse (CDiff1 > PrevCDiff2 andalso CDiff2 > PrevCDiff1) of + case ar_block:get_double_signing_condition(CDiff1, PrevCDiff1, CDiff2, PrevCDiff2) of true -> Preimage1 = << PrevSolutionH1/binary, (ar_block:generate_signed_hash(B))/binary >>, From 84c13792136a4d1b9769829ffa5f6f7a83c0fa9e Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Wed, 9 Jul 2025 14:56:32 +0200 Subject: [PATCH 18/23] testnet: configure trusted peers for testnet-4 --- testnet/config/testnet-4.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testnet/config/testnet-4.json b/testnet/config/testnet-4.json index dbc5c03b5b..ef88236d7e 100644 --- a/testnet/config/testnet-4.json +++ b/testnet/config/testnet-4.json @@ -6,6 +6,7 @@ "1226,100000000000,hDdptPiuAlrP5RxEHwZoffm7obIyvvBi40T5PPvp57w" ], "mining_addr": "hDdptPiuAlrP5RxEHwZoffm7obIyvvBi40T5PPvp57w", + "peers": ["testnet-1.arweave.xyz", "testnet-2.arweave.xyz"], "vdf_client_peers": [ "testnet-3.arweave.xyz" ], @@ -20,4 +21,4 @@ ], "data_dir": "/arweave-data", "requests_per_minute_limit": 9000 -} \ No newline at end of file +} From d4b49a92209b8d431741c3103ed7a84374337fdb Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Wed, 9 Jul 2025 19:14:06 +0200 Subject: [PATCH 19/23] Log solution hashes of orphaned blocks --- apps/arweave/src/ar_chain_stats.erl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/arweave/src/ar_chain_stats.erl b/apps/arweave/src/ar_chain_stats.erl index 6033d583cf..9b1d1f7716 100644 --- a/apps/arweave/src/ar_chain_stats.erl +++ b/apps/arweave/src/ar_chain_stats.erl @@ -95,14 +95,22 @@ record_fork_depth([], _ForkRootB, N) -> prometheus_histogram:observe(fork_recovery_depth, N), ok; record_fork_depth([H | Orphans], ForkRootB, N) -> - ?LOG_INFO([ + SolutionHashInfo = + case ar_block_cache:get(block_cache, H) of + not_found -> + %% Should never happen, by construction. + ?LOG_ERROR([{event, block_not_found_in_cache}, {h, ar_util:encode(H)}]), + []; + #block{ hash = SolutionH } -> + [{solution_hash, ar_util:encode(SolutionH)}] + end, + LogInfo = [ {event, orphaning_block}, {block, ar_util:encode(H)}, {depth, N}, {fork_root, ar_util:encode(ForkRootB#block.indep_hash)}, - {fork_height, ForkRootB#block.height + 1} - ]), + {fork_height, ForkRootB#block.height + 1} | SolutionHashInfo], + ?LOG_INFO(LogInfo), record_fork_depth(Orphans, ForkRootB, N + 1). - %%%=================================================================== %%% Tests. %%%=================================================================== From 9096ced9967aa303c75d81fe9b8d7bce9249e5cb Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Thu, 10 Jul 2025 14:33:51 +0200 Subject: [PATCH 20/23] Fix bug where mined & orphaned block takes down watchdog Track such blocks along with confirmed_block's (block_mined_but_orphaned key). --- apps/arweave/src/ar_mining_stats.erl | 8 +++++++- apps/arweave/src/ar_watchdog.erl | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/arweave/src/ar_mining_stats.erl b/apps/arweave/src/ar_mining_stats.erl index 7ce0e1828d..04cc10a3aa 100644 --- a/apps/arweave/src/ar_mining_stats.erl +++ b/apps/arweave/src/ar_mining_stats.erl @@ -4,7 +4,7 @@ -export([start_link/0, start_performance_reports/0, pause_performance_reports/1, mining_paused/0, set_total_data_size/1, set_storage_module_data_size/6, vdf_computed/0, raw_read_rate/2, chunks_read/2, h1_computed/2, h2_computed/2, - h1_solution/0, h2_solution/0, block_found/0, + h1_solution/0, h2_solution/0, block_found/0, block_mined_but_orphaned/0, h1_sent_to_peer/2, h1_received_from_peer/2, h2_sent_to_peer/1, h2_received_from_peer/1, get_partition_data_size/2]). @@ -178,6 +178,12 @@ block_found() -> block_found(Now) -> increment_count(confirmed_block, 1, Now). +block_mined_but_orphaned() -> + block_mined_but_orphaned(erlang:monotonic_time(millisecond)). + +block_mined_but_orphaned(Now) -> + increment_count(block_mined_but_orphaned, 1, Now). + set_total_data_size(DataSize) -> try prometheus_gauge:set(v2_index_data_size, DataSize), diff --git a/apps/arweave/src/ar_watchdog.erl b/apps/arweave/src/ar_watchdog.erl index 8c05f13e4e..b1133d44ff 100644 --- a/apps/arweave/src/ar_watchdog.erl +++ b/apps/arweave/src/ar_watchdog.erl @@ -128,6 +128,10 @@ handle_cast({block_received_n_confirmations, BH, Height}, State) -> _ -> Map end; + {_BH, _Map} -> + %% The mined block was orphaned. + ar_mining_stats:block_mined_but_orphaned(), + MinedBlocks; error -> MinedBlocks end, From de90abbd7f11d7b7c489f40bafd88bde27075789 Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Thu, 10 Jul 2025 19:50:26 +0200 Subject: [PATCH 21/23] Log details about avoided double-signing --- apps/arweave/src/ar_node_worker.erl | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/arweave/src/ar_node_worker.erl b/apps/arweave/src/ar_node_worker.erl index 246438d74a..d1645f0826 100644 --- a/apps/arweave/src/ar_node_worker.erl +++ b/apps/arweave/src/ar_node_worker.erl @@ -2176,7 +2176,7 @@ handle_found_solution(Args, PrevB, State, IsRebase) -> {false, Reason5} -> {false, Reason5}; true -> - case check_no_double_signing(CDiff, PrevCDiff, MiningAddress) of + case check_no_double_signing(CDiff, PrevCDiff, MiningAddress, Height) of false -> {false, double_signing}; true -> @@ -2370,15 +2370,28 @@ assert_key_type(RewardKey, Height) -> end end. -check_no_double_signing(CDiff, PrevCDiff, MiningAddress) -> +check_no_double_signing(CDiff, PrevCDiff, MiningAddress, Height) -> Blocks = ar_block_cache:get_blocks_by_miner(block_cache, MiningAddress), not lists:any( fun(B) -> - ar_block:get_double_signing_condition( + case ar_block:get_double_signing_condition( B#block.cumulative_diff, B#block.previous_cumulative_diff, CDiff, - PrevCDiff) + PrevCDiff) of + true -> + ?LOG_WARNING([{event, avoiding_double_signing}, + {block, ar_util:encode(B#block.indep_hash)}, + {height, B#block.height}, + {new_height, Height}, + {cdiff, B#block.cumulative_diff}, + {prev_cdiff, B#block.previous_cumulative_diff}, + {new_cdiff, CDiff}, + {new_prev_cdiff, PrevCDiff}]), + true; + false -> + false + end end, Blocks). From a371366a22dc37b5f21c2a9a765d5dda5c5e5721 Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Thu, 17 Jul 2025 21:44:00 +0200 Subject: [PATCH 22/23] Split ar_data_sync_tests into several modules Put each test in a separate module to ensure clean shutdown on CI. --- .github/workflows/test.yml | 8 +- .../ar_data_sync_disk_pool_rotation_test.erl | 68 +++ .../ar_data_sync_enqueue_intervals_test.erl | 222 ++++++++ ...a_sync_mines_off_only_last_chunks_test.erl | 76 +++ ...mines_off_only_second_last_chunks_test.erl | 63 +++ ...ata_sync_recovers_from_corruption_test.erl | 22 + .../ar_data_sync_syncs_after_joining_test.erl | 44 ++ .../test/ar_data_sync_syncs_data_test.erl | 64 +++ apps/arweave/test/ar_data_sync_tests.erl | 506 ------------------ 9 files changed, 566 insertions(+), 507 deletions(-) create mode 100644 apps/arweave/test/ar_data_sync_disk_pool_rotation_test.erl create mode 100644 apps/arweave/test/ar_data_sync_enqueue_intervals_test.erl create mode 100644 apps/arweave/test/ar_data_sync_mines_off_only_last_chunks_test.erl create mode 100644 apps/arweave/test/ar_data_sync_mines_off_only_second_last_chunks_test.erl create mode 100644 apps/arweave/test/ar_data_sync_recovers_from_corruption_test.erl create mode 100644 apps/arweave/test/ar_data_sync_syncs_after_joining_test.erl create mode 100644 apps/arweave/test/ar_data_sync_syncs_data_test.erl delete mode 100644 apps/arweave/test/ar_data_sync_tests.erl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c1aba8267..576dbcbea7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -170,7 +170,13 @@ jobs: ## Long-running tests. Put these first to limit the overall runtime of the ## test suite ar_coordinated_mining_tests, - ar_data_sync_tests, + ar_data_sync_recovers_from_corruption_test, + ar_data_sync_syncs_data_test, + ar_data_sync_syncs_after_joining_test, + ar_data_sync_mines_off_only_last_chunks_test, + ar_data_sync_mines_off_only_second_last_chunks_test, + ar_data_sync_disk_pool_rotation_test, + ar_data_sync_enqueue_intervals_test, ar_fork_recovery_tests, ar_tx, ar_packing_tests, diff --git a/apps/arweave/test/ar_data_sync_disk_pool_rotation_test.erl b/apps/arweave/test/ar_data_sync_disk_pool_rotation_test.erl new file mode 100644 index 0000000000..b840d76771 --- /dev/null +++ b/apps/arweave/test/ar_data_sync_disk_pool_rotation_test.erl @@ -0,0 +1,68 @@ +-module(ar_data_sync_disk_pool_rotation_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("../include/ar.hrl"). +-include("../include/ar_consensus.hrl"). +-include("../include/ar_config.hrl"). + +-import(ar_test_node, [assert_wait_until_height/2]). + +disk_pool_rotation_test_() -> + {timeout, 120, fun test_disk_pool_rotation/0}. + +test_disk_pool_rotation() -> + ?LOG_DEBUG([{event, test_disk_pool_rotation_start}]), + Addr = ar_wallet:to_address(ar_wallet:new_keyfile()), + %% Will store the three genesis chunks. + %% The third one falls inside the "overlap" (see ar_storage_module.erl) + StorageModules = [{2 * ?DATA_CHUNK_SIZE, 0, + ar_test_node:get_default_storage_module_packing(Addr, 0)}], + Wallet = ar_test_data_sync:setup_nodes( + #{ addr => Addr, storage_modules => StorageModules }), + Chunks = [crypto:strong_rand_bytes(?DATA_CHUNK_SIZE)], + {DataRoot, DataTree} = ar_merkle:generate_tree( + ar_tx:sized_chunks_to_sized_chunk_ids( + ar_tx:chunks_to_size_tagged_chunks(Chunks) + ) + ), + {TX, Chunks} = ar_test_data_sync:tx(Wallet, {fixed_data, DataRoot, Chunks}), + ar_test_node:assert_post_tx_to_peer(main, TX), + Offset = ?DATA_CHUNK_SIZE, + DataSize = ?DATA_CHUNK_SIZE, + DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), + Proof = #{ data_root => ar_util:encode(DataRoot), + data_path => ar_util:encode(DataPath), + chunk => ar_util:encode(hd(Chunks)), + offset => integer_to_binary(Offset), + data_size => integer_to_binary(DataSize) }, + ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, + ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), + ar_test_node:mine(main), + assert_wait_until_height(main, 1), + timer:sleep(2000), + Options = #{ format => etf, random_subset => false }, + {ok, Binary1} = ar_global_sync_record:get_serialized_sync_record(Options), + {ok, Global1} = ar_intervals:safe_from_etf(Binary1), + %% 3 genesis chunks plus the two we upload here. + ?assertEqual([{1048576, 0}], ar_intervals:to_list(Global1)), + ar_test_node:mine(main), + assert_wait_until_height(main, 2), + {ok, Binary2} = ar_global_sync_record:get_serialized_sync_record(Options), + {ok, Global2} = ar_intervals:safe_from_etf(Binary2), + ?assertEqual([{1048576, 0}], ar_intervals:to_list(Global2)), + ar_test_node:mine(main), + assert_wait_until_height(main, 3), + ar_test_node:mine(main), + assert_wait_until_height(main, 4), + %% The new chunk has been confirmed but there is not storage module to take it. + ?assertEqual(3, ?SEARCH_SPACE_UPPER_BOUND_DEPTH), + true = ar_util:do_until( + fun() -> + {ok, Binary3} = ar_global_sync_record:get_serialized_sync_record(Options), + {ok, Global3} = ar_intervals:safe_from_etf(Binary3), + [{786432, 0}] == ar_intervals:to_list(Global3) + end, + 200, + 5000 + ). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_enqueue_intervals_test.erl b/apps/arweave/test/ar_data_sync_enqueue_intervals_test.erl new file mode 100644 index 0000000000..6046c0f5a9 --- /dev/null +++ b/apps/arweave/test/ar_data_sync_enqueue_intervals_test.erl @@ -0,0 +1,222 @@ +-module(ar_data_sync_enqueue_intervals_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("../include/ar.hrl"). +-include("../include/ar_consensus.hrl"). +-include("../include/ar_config.hrl"). + +enqueue_intervals_test() -> + ?LOG_DEBUG([{event, enqueue_intervals_test}]), + test_enqueue_intervals([], 2, [], [], [], "Empty Intervals"), + Peer1 = {1, 2, 3, 4, 1984}, + Peer2 = {101, 102, 103, 104, 1984}, + Peer3 = {201, 202, 203, 204, 1984}, + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])} + ], + 5, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, + {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1}, + {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer1}, + {8*?DATA_CHUNK_SIZE, 9*?DATA_CHUNK_SIZE, Peer1} + ], + "Single peer, full intervals, all chunks. Non-overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])} + ], + 2, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1} + ], + "Single peer, full intervals, 2 chunks. Non-overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])}, + {Peer2, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ])}, + {Peer3, ar_intervals:from_list([ + {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} + ])} + ], + 2, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {8*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, + {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, + {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer2}, + {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer3} + ], + "Multiple peers, overlapping, full intervals, 2 chunks. Non-overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])}, + {Peer2, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ])}, + {Peer3, ar_intervals:from_list([ + {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} + ])} + ], + 3, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {8*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, + {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, + {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1}, + {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer3} + ], + "Multiple peers, overlapping, full intervals, 3 chunks. Non-overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])} + ], + 5, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, {9*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, + {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1} + ], + "Single peer, full intervals, all chunks. Overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])}, + {Peer2, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ])}, + {Peer3, ar_intervals:from_list([ + {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} + ])} + ], + 2, + [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, {9*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE}], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, + {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, + {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer2} + ], + "Multiple peers, overlapping, full intervals, 2 chunks. Overlapping QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, trunc(5.75*?DATA_CHUNK_SIZE)} + ])} + ], + 2, + [ + {20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, + {trunc(8.5*?DATA_CHUNK_SIZE), trunc(6.5*?DATA_CHUNK_SIZE)} + ], + [ + {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, trunc(3.25*?DATA_CHUNK_SIZE), Peer1} + ], + "Single peer, partial intervals, 2 chunks. Overlapping partial QIntervals."), + + test_enqueue_intervals( + [ + {Peer1, ar_intervals:from_list([ + {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE}, + {9*?DATA_CHUNK_SIZE, trunc(5.75*?DATA_CHUNK_SIZE)} + ])}, + {Peer2, ar_intervals:from_list([ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {7*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ])}, + {Peer3, ar_intervals:from_list([ + {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} + ])} + ], + 2, + [ + {20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, + {trunc(8.5*?DATA_CHUNK_SIZE), trunc(6.5*?DATA_CHUNK_SIZE)} + ], + [ + {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, + {8*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} + ], + [ + {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, + {3*?DATA_CHUNK_SIZE, trunc(3.25*?DATA_CHUNK_SIZE), Peer1}, + {trunc(3.25*?DATA_CHUNK_SIZE), 4*?DATA_CHUNK_SIZE, Peer2}, + {6*?DATA_CHUNK_SIZE, trunc(6.5*?DATA_CHUNK_SIZE), Peer2} + ], + "Multiple peers, overlapping, full intervals, 2 chunks. Overlapping QIntervals."). + +test_enqueue_intervals(Intervals, ChunksPerPeer, QIntervalsRanges, ExpectedQIntervalRanges, ExpectedChunks, Label) -> + QIntervals = ar_intervals:from_list(QIntervalsRanges), + Q = gb_sets:new(), + {QResult, QIntervalsResult} = ar_data_sync:enqueue_intervals(Intervals, ChunksPerPeer, {Q, QIntervals}), + ExpectedQIntervals = lists:foldl(fun({End, Start}, Acc) -> + ar_intervals:add(Acc, End, Start) + end, QIntervals, ExpectedQIntervalRanges), + ?assertEqual(ar_intervals:to_list(ExpectedQIntervals), ar_intervals:to_list(QIntervalsResult), Label), + ?assertEqual(ExpectedChunks, gb_sets:to_list(QResult), Label). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_mines_off_only_last_chunks_test.erl b/apps/arweave/test/ar_data_sync_mines_off_only_last_chunks_test.erl new file mode 100644 index 0000000000..2825d4bf6e --- /dev/null +++ b/apps/arweave/test/ar_data_sync_mines_off_only_last_chunks_test.erl @@ -0,0 +1,76 @@ +-module(ar_data_sync_mines_off_only_last_chunks_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). + +-import(ar_test_node, [test_with_mocked_functions/2]). + +mines_off_only_last_chunks_test_() -> + test_with_mocked_functions([{ar_fork, height_2_6, fun() -> 0 end}, mock_reset_frequency()], + fun test_mines_off_only_last_chunks/0). + +mock_reset_frequency() -> + {ar_nonce_limiter, get_reset_frequency, fun() -> 5 end}. + +test_mines_off_only_last_chunks() -> + ?LOG_DEBUG([{event, test_mines_off_only_last_chunks_start}]), + Wallet = ar_test_data_sync:setup_nodes(), + %% Submit only the last chunks (smaller than 256 KiB) of transactions. + %% Assert the nodes construct correct proofs of access from them. + lists:foreach( + fun(Height) -> + RandomID = crypto:strong_rand_bytes(32), + Chunk = crypto:strong_rand_bytes(1023), + ChunkID = ar_tx:generate_chunk_id(Chunk), + DataSize = ?DATA_CHUNK_SIZE + 1023, + {DataRoot, DataTree} = ar_merkle:generate_tree([{RandomID, ?DATA_CHUNK_SIZE}, + {ChunkID, DataSize}]), + TX = ar_test_node:sign_tx(Wallet, #{ last_tx => ar_test_node:get_tx_anchor(main), data_size => DataSize, + data_root => DataRoot }), + ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX]), + Offset = ?DATA_CHUNK_SIZE + 1, + DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), + Proof = #{ data_root => ar_util:encode(DataRoot), + data_path => ar_util:encode(DataPath), chunk => ar_util:encode(Chunk), + offset => integer_to_binary(Offset), + data_size => integer_to_binary(DataSize) }, + ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, + ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), + case Height - ?SEARCH_SPACE_UPPER_BOUND_DEPTH of + -1 -> + %% Make sure we waited enough to have the next block use + %% the new entropy reset source. + [{_, Info}] = ets:lookup(node_state, nonce_limiter_info), + PrevStepNumber = Info#nonce_limiter_info.global_step_number, + true = ar_util:do_until( + fun() -> + ar_nonce_limiter:get_current_step_number() + > PrevStepNumber + ar_nonce_limiter:get_reset_frequency() + end, + 100, + 60000 + ); + 0 -> + %% Wait until the new chunks fall below the new upper bound and + %% remove the original big chunks. The protocol will increase the upper + %% bound based on the nonce limiter entropy reset, but ar_data_sync waits + %% for ?SEARCH_SPACE_UPPER_BOUND_DEPTH confirmations before packing the + %% chunks. + {ok, Config} = application:get_env(arweave, config), + lists:foreach( + fun(O) -> + [ar_chunk_storage:delete(O, ar_storage_module:id(Module)) + || Module <- Config#config.storage_modules] + end, + lists:seq(?DATA_CHUNK_SIZE, ar_block:strict_data_split_threshold(), + ?DATA_CHUNK_SIZE) + ); + _ -> + ok + end + end, + lists:seq(1, 6) + ). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_mines_off_only_second_last_chunks_test.erl b/apps/arweave/test/ar_data_sync_mines_off_only_second_last_chunks_test.erl new file mode 100644 index 0000000000..f380cd1a75 --- /dev/null +++ b/apps/arweave/test/ar_data_sync_mines_off_only_second_last_chunks_test.erl @@ -0,0 +1,63 @@ +-module(ar_data_sync_mines_off_only_second_last_chunks_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). + +-import(ar_test_node, [test_with_mocked_functions/2]). + +mines_off_only_second_last_chunks_test_() -> + test_with_mocked_functions([{ar_fork, height_2_6, fun() -> 0 end}, mock_reset_frequency()], + fun test_mines_off_only_second_last_chunks/0). + +mock_reset_frequency() -> + {ar_nonce_limiter, get_reset_frequency, fun() -> 5 end}. + +test_mines_off_only_second_last_chunks() -> + ?LOG_DEBUG([{event, test_mines_off_only_second_last_chunks_start}]), + Wallet = ar_test_data_sync:setup_nodes(), + %% Submit only the second last chunks (smaller than 256 KiB) of transactions. + %% Assert the nodes construct correct proofs of access from them. + lists:foreach( + fun(Height) -> + RandomID = crypto:strong_rand_bytes(32), + Chunk = crypto:strong_rand_bytes(?DATA_CHUNK_SIZE div 2), + ChunkID = ar_tx:generate_chunk_id(Chunk), + DataSize = (?DATA_CHUNK_SIZE) div 2 + (?DATA_CHUNK_SIZE) div 2 + 3, + {DataRoot, DataTree} = ar_merkle:generate_tree([{ChunkID, ?DATA_CHUNK_SIZE div 2}, + {RandomID, DataSize}]), + TX = ar_test_node:sign_tx(Wallet, #{ last_tx => ar_test_node:get_tx_anchor(main), data_size => DataSize, + data_root => DataRoot }), + ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX]), + Offset = 0, + DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), + Proof = #{ data_root => ar_util:encode(DataRoot), + data_path => ar_util:encode(DataPath), chunk => ar_util:encode(Chunk), + offset => integer_to_binary(Offset), + data_size => integer_to_binary(DataSize) }, + ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, + ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), + case Height - ?SEARCH_SPACE_UPPER_BOUND_DEPTH >= 0 of + true -> + %% Wait until the new chunks fall below the new upper bound and + %% remove the original big chunks. The protocol will increase the upper + %% bound based on the nonce limiter entropy reset, but ar_data_sync waits + %% for ?SEARCH_SPACE_UPPER_BOUND_DEPTH confirmations before packing the + %% chunks. + {ok, Config} = application:get_env(arweave, config), + lists:foreach( + fun(O) -> + [ar_chunk_storage:delete(O, ar_storage_module:id(Module)) + || Module <- Config#config.storage_modules] + end, + lists:seq(?DATA_CHUNK_SIZE, ar_block:strict_data_split_threshold(), + ?DATA_CHUNK_SIZE) + ); + _ -> + ok + end + end, + lists:seq(1, 6) + ). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_recovers_from_corruption_test.erl b/apps/arweave/test/ar_data_sync_recovers_from_corruption_test.erl new file mode 100644 index 0000000000..5b2461e97a --- /dev/null +++ b/apps/arweave/test/ar_data_sync_recovers_from_corruption_test.erl @@ -0,0 +1,22 @@ +-module(ar_data_sync_recovers_from_corruption_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("../include/ar.hrl"). +-include("../include/ar_consensus.hrl"). +-include("../include/ar_config.hrl"). + +-import(ar_test_node, [assert_wait_until_height/2]). + +recovers_from_corruption_test_() -> + {timeout, 300, fun test_recovers_from_corruption/0}. + +test_recovers_from_corruption() -> + ?LOG_DEBUG([{event, test_recovers_from_corruption_start}]), + ar_test_data_sync:setup_nodes(), + StoreID = ar_storage_module:id(hd(ar_storage_module:get_all(262144 * 3))), + ?debugFmt("Corrupting ~s...", [StoreID]), + [ar_chunk_storage:write_chunk(PaddedEndOffset, << 0:(262144*8) >>, #{}, StoreID) + || PaddedEndOffset <- lists:seq(262144, 262144 * 3, 262144)], + ar_test_node:mine(), + ar_test_node:assert_wait_until_height(main, 1). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_syncs_after_joining_test.erl b/apps/arweave/test/ar_data_sync_syncs_after_joining_test.erl new file mode 100644 index 0000000000..4517bb4ffd --- /dev/null +++ b/apps/arweave/test/ar_data_sync_syncs_after_joining_test.erl @@ -0,0 +1,44 @@ +-module(ar_data_sync_syncs_after_joining_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). + +-import(ar_test_node, [assert_wait_until_height/2, test_with_mocked_functions/2]). + +syncs_after_joining_test_() -> + ar_test_node:test_with_mocked_functions([{ar_fork, height_2_5, fun() -> 0 end}], + fun test_syncs_after_joining/0, 240). + +test_syncs_after_joining() -> + test_syncs_after_joining(original_split). + +test_syncs_after_joining(Split) -> + ?LOG_DEBUG([{event, test_syncs_after_joining}, {split, Split}]), + Wallet = ar_test_data_sync:setup_nodes(), + {TX1, Chunks1} = ar_test_data_sync:tx(Wallet, {Split, 1}, v2, ?AR(1)), + B1 = ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX1]), + Proofs1 = ar_test_data_sync:post_proofs(main, B1, TX1, Chunks1), + UpperBound = ar_node:get_partition_upper_bound(ar_node:get_block_index()), + ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs1, UpperBound), + ar_test_data_sync:wait_until_syncs_chunks(Proofs1), + ar_test_node:disconnect_from(peer1), + {MainTX2, MainChunks2} = ar_test_data_sync:tx(Wallet, {Split, 3}, v2, ?AR(1)), + MainB2 = ar_test_node:post_and_mine(#{ miner => main, await_on => main }, [MainTX2]), + MainProofs2 = ar_test_data_sync:post_proofs(main, MainB2, MainTX2, MainChunks2), + {MainTX3, MainChunks3} = ar_test_data_sync:tx(Wallet, {Split, 2}, v2, ?AR(1)), + MainB3 = ar_test_node:post_and_mine(#{ miner => main, await_on => main }, [MainTX3]), + MainProofs3 = ar_test_data_sync:post_proofs(main, MainB3, MainTX3, MainChunks3), + {PeerTX2, PeerChunks2} = ar_test_data_sync:tx(Wallet, {Split, 2}, v2, ?AR(1)), + PeerB2 = ar_test_node:post_and_mine( #{ miner => peer1, await_on => peer1 }, [PeerTX2] ), + PeerProofs2 = ar_test_data_sync:post_proofs(peer1, PeerB2, PeerTX2, PeerChunks2), + ar_test_data_sync:wait_until_syncs_chunks(peer1, PeerProofs2, infinity), + _Peer2 = ar_test_node:rejoin_on(#{ node => peer1, join_on => main }), + assert_wait_until_height(peer1, 3), + ar_test_node:connect_to_peer(peer1), + UpperBound2 = ar_node:get_partition_upper_bound(ar_node:get_block_index()), + ar_test_data_sync:wait_until_syncs_chunks(peer1, MainProofs2, UpperBound2), + ar_test_data_sync:wait_until_syncs_chunks(peer1, MainProofs3, UpperBound2), + ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs1, infinity). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_syncs_data_test.erl b/apps/arweave/test/ar_data_sync_syncs_data_test.erl new file mode 100644 index 0000000000..3e015fc0f1 --- /dev/null +++ b/apps/arweave/test/ar_data_sync_syncs_data_test.erl @@ -0,0 +1,64 @@ +-module(ar_data_sync_syncs_data_test). + +-include_lib("eunit/include/eunit.hrl"). + +-include("ar.hrl"). +-include("ar_consensus.hrl"). +-include("ar_config.hrl"). + +-import(ar_test_node, [assert_wait_until_height/2]). + +syncs_data_test_() -> + {timeout, 240, fun test_syncs_data/0}. + +test_syncs_data() -> + ?LOG_DEBUG([{event, test_syncs_data_start}]), + Wallet = ar_test_data_sync:setup_nodes(), + Records = ar_test_data_sync:post_random_blocks(Wallet), + RecordsWithProofs = lists:flatmap( + fun({B, TX, Chunks}) -> + ar_test_data_sync:get_records_with_proofs(B, TX, Chunks) end, Records), + lists:foreach( + fun({_, _, _, {_, Proof}}) -> + ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, + ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), + ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, + ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))) + end, + RecordsWithProofs + ), + Proofs = [Proof || {_, _, _, Proof} <- RecordsWithProofs], + ar_test_data_sync:wait_until_syncs_chunks(Proofs), + DiskPoolThreshold = ar_node:get_partition_upper_bound(ar_node:get_block_index()), + ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs, DiskPoolThreshold), + lists:foreach( + fun({B, #tx{ id = TXID }, Chunks, {_, Proof}}) -> + TXSize = byte_size(binary:list_to_bin(Chunks)), + TXOffset = ar_merkle:extract_note(ar_util:decode(maps:get(tx_path, Proof))), + AbsoluteTXOffset = B#block.weave_size - B#block.block_size + TXOffset, + ExpectedOffsetInfo = ar_serialize:jsonify(#{ + offset => integer_to_binary(AbsoluteTXOffset), + size => integer_to_binary(TXSize) }), + true = ar_util:do_until( + fun() -> + case ar_test_data_sync:get_tx_offset(peer1, TXID) of + {ok, {{<<"200">>, _}, _, ExpectedOffsetInfo, _, _}} -> + true; + _ -> + false + end + end, + 100, + 120 * 1000 + ), + ExpectedData = ar_util:encode(binary:list_to_bin(Chunks)), + ar_test_node:assert_get_tx_data(main, TXID, ExpectedData), + case AbsoluteTXOffset > DiskPoolThreshold of + true -> + ok; + false -> + ar_test_node:assert_get_tx_data(peer1, TXID, ExpectedData) + end + end, + RecordsWithProofs + ). \ No newline at end of file diff --git a/apps/arweave/test/ar_data_sync_tests.erl b/apps/arweave/test/ar_data_sync_tests.erl deleted file mode 100644 index 85a26e87d2..0000000000 --- a/apps/arweave/test/ar_data_sync_tests.erl +++ /dev/null @@ -1,506 +0,0 @@ --module(ar_data_sync_tests). - --include_lib("eunit/include/eunit.hrl"). - --include("../include/ar.hrl"). --include("../include/ar_consensus.hrl"). --include("../include/ar_config.hrl"). - --import(ar_test_node, [assert_wait_until_height/2, test_with_mocked_functions/2]). - -recovers_from_corruption_test_() -> - {timeout, 300, fun test_recovers_from_corruption/0}. - -test_recovers_from_corruption() -> - ?LOG_DEBUG([{event, test_recovers_from_corruption_start}]), - ar_test_data_sync:setup_nodes(), - {ok, Config} = application:get_env(arweave, config), - StoreID = ar_storage_module:id(hd(ar_storage_module:get_all(262144 * 3))), - ?debugFmt("Corrupting ~s...", [StoreID]), - [ar_chunk_storage:write_chunk(PaddedEndOffset, << 0:(262144*8) >>, #{}, StoreID) - || PaddedEndOffset <- lists:seq(262144, 262144 * 3, 262144)], - ar_test_node:mine(), - ar_test_node:assert_wait_until_height(main, 1). - -syncs_data_test_() -> - {timeout, 240, fun test_syncs_data/0}. - -test_syncs_data() -> - ?LOG_DEBUG([{event, test_syncs_data_start}]), - Wallet = ar_test_data_sync:setup_nodes(), - Records = ar_test_data_sync:post_random_blocks(Wallet), - RecordsWithProofs = lists:flatmap( - fun({B, TX, Chunks}) -> - ar_test_data_sync:get_records_with_proofs(B, TX, Chunks) end, Records), - lists:foreach( - fun({_, _, _, {_, Proof}}) -> - ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, - ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), - ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, - ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))) - end, - RecordsWithProofs - ), - Proofs = [Proof || {_, _, _, Proof} <- RecordsWithProofs], - ar_test_data_sync:wait_until_syncs_chunks(Proofs), - DiskPoolThreshold = ar_node:get_partition_upper_bound(ar_node:get_block_index()), - ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs, DiskPoolThreshold), - lists:foreach( - fun({B, #tx{ id = TXID }, Chunks, {_, Proof}}) -> - TXSize = byte_size(binary:list_to_bin(Chunks)), - TXOffset = ar_merkle:extract_note(ar_util:decode(maps:get(tx_path, Proof))), - AbsoluteTXOffset = B#block.weave_size - B#block.block_size + TXOffset, - ExpectedOffsetInfo = ar_serialize:jsonify(#{ - offset => integer_to_binary(AbsoluteTXOffset), - size => integer_to_binary(TXSize) }), - true = ar_util:do_until( - fun() -> - case ar_test_data_sync:get_tx_offset(peer1, TXID) of - {ok, {{<<"200">>, _}, _, ExpectedOffsetInfo, _, _}} -> - true; - _ -> - false - end - end, - 100, - 120 * 1000 - ), - ExpectedData = ar_util:encode(binary:list_to_bin(Chunks)), - ar_test_node:assert_get_tx_data(main, TXID, ExpectedData), - case AbsoluteTXOffset > DiskPoolThreshold of - true -> - ok; - false -> - ar_test_node:assert_get_tx_data(peer1, TXID, ExpectedData) - end - end, - RecordsWithProofs - ). - -syncs_after_joining_test_() -> - ar_test_node:test_with_mocked_functions([{ar_fork, height_2_5, fun() -> 0 end}], - fun test_syncs_after_joining/0, 240). - -test_syncs_after_joining() -> - test_syncs_after_joining(original_split). - -test_syncs_after_joining(Split) -> - ?LOG_DEBUG([{event, test_syncs_after_joining}, {split, Split}]), - Wallet = ar_test_data_sync:setup_nodes(), - {TX1, Chunks1} = ar_test_data_sync:tx(Wallet, {Split, 1}, v2, ?AR(1)), - B1 = ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX1]), - Proofs1 = ar_test_data_sync:post_proofs(main, B1, TX1, Chunks1), - UpperBound = ar_node:get_partition_upper_bound(ar_node:get_block_index()), - ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs1, UpperBound), - ar_test_data_sync:wait_until_syncs_chunks(Proofs1), - ar_test_node:disconnect_from(peer1), - {MainTX2, MainChunks2} = ar_test_data_sync:tx(Wallet, {Split, 3}, v2, ?AR(1)), - MainB2 = ar_test_node:post_and_mine(#{ miner => main, await_on => main }, [MainTX2]), - MainProofs2 = ar_test_data_sync:post_proofs(main, MainB2, MainTX2, MainChunks2), - {MainTX3, MainChunks3} = ar_test_data_sync:tx(Wallet, {Split, 2}, v2, ?AR(1)), - MainB3 = ar_test_node:post_and_mine(#{ miner => main, await_on => main }, [MainTX3]), - MainProofs3 = ar_test_data_sync:post_proofs(main, MainB3, MainTX3, MainChunks3), - {PeerTX2, PeerChunks2} = ar_test_data_sync:tx(Wallet, {Split, 2}, v2, ?AR(1)), - PeerB2 = ar_test_node:post_and_mine( #{ miner => peer1, await_on => peer1 }, [PeerTX2] ), - PeerProofs2 = ar_test_data_sync:post_proofs(peer1, PeerB2, PeerTX2, PeerChunks2), - ar_test_data_sync:wait_until_syncs_chunks(peer1, PeerProofs2, infinity), - _Peer2 = ar_test_node:rejoin_on(#{ node => peer1, join_on => main }), - assert_wait_until_height(peer1, 3), - ar_test_node:connect_to_peer(peer1), - UpperBound2 = ar_node:get_partition_upper_bound(ar_node:get_block_index()), - ar_test_data_sync:wait_until_syncs_chunks(peer1, MainProofs2, UpperBound2), - ar_test_data_sync:wait_until_syncs_chunks(peer1, MainProofs3, UpperBound2), - ar_test_data_sync:wait_until_syncs_chunks(peer1, Proofs1, infinity). - -mines_off_only_last_chunks_test_() -> - test_with_mocked_functions([{ar_fork, height_2_6, fun() -> 0 end}, mock_reset_frequency()], - fun test_mines_off_only_last_chunks/0). - -mock_reset_frequency() -> - {ar_nonce_limiter, get_reset_frequency, fun() -> 5 end}. - -test_mines_off_only_last_chunks() -> - ?LOG_DEBUG([{event, test_mines_off_only_last_chunks_start}]), - Wallet = ar_test_data_sync:setup_nodes(), - %% Submit only the last chunks (smaller than 256 KiB) of transactions. - %% Assert the nodes construct correct proofs of access from them. - lists:foreach( - fun(Height) -> - RandomID = crypto:strong_rand_bytes(32), - Chunk = crypto:strong_rand_bytes(1023), - ChunkID = ar_tx:generate_chunk_id(Chunk), - DataSize = ?DATA_CHUNK_SIZE + 1023, - {DataRoot, DataTree} = ar_merkle:generate_tree([{RandomID, ?DATA_CHUNK_SIZE}, - {ChunkID, DataSize}]), - TX = ar_test_node:sign_tx(Wallet, #{ last_tx => ar_test_node:get_tx_anchor(main), data_size => DataSize, - data_root => DataRoot }), - ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX]), - Offset = ?DATA_CHUNK_SIZE + 1, - DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), - Proof = #{ data_root => ar_util:encode(DataRoot), - data_path => ar_util:encode(DataPath), chunk => ar_util:encode(Chunk), - offset => integer_to_binary(Offset), - data_size => integer_to_binary(DataSize) }, - ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, - ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), - case Height - ?SEARCH_SPACE_UPPER_BOUND_DEPTH of - -1 -> - %% Make sure we waited enough to have the next block use - %% the new entropy reset source. - [{_, Info}] = ets:lookup(node_state, nonce_limiter_info), - PrevStepNumber = Info#nonce_limiter_info.global_step_number, - true = ar_util:do_until( - fun() -> - ar_nonce_limiter:get_current_step_number() - > PrevStepNumber + ar_nonce_limiter:get_reset_frequency() - end, - 100, - 60000 - ); - 0 -> - %% Wait until the new chunks fall below the new upper bound and - %% remove the original big chunks. The protocol will increase the upper - %% bound based on the nonce limiter entropy reset, but ar_data_sync waits - %% for ?SEARCH_SPACE_UPPER_BOUND_DEPTH confirmations before packing the - %% chunks. - {ok, Config} = application:get_env(arweave, config), - lists:foreach( - fun(O) -> - [ar_chunk_storage:delete(O, ar_storage_module:id(Module)) - || Module <- Config#config.storage_modules] - end, - lists:seq(?DATA_CHUNK_SIZE, ar_block:strict_data_split_threshold(), - ?DATA_CHUNK_SIZE) - ); - _ -> - ok - end - end, - lists:seq(1, 6) - ). - -mines_off_only_second_last_chunks_test_() -> - test_with_mocked_functions([{ar_fork, height_2_6, fun() -> 0 end}, mock_reset_frequency()], - fun test_mines_off_only_second_last_chunks/0). - -test_mines_off_only_second_last_chunks() -> - ?LOG_DEBUG([{event, test_mines_off_only_second_last_chunks_start}]), - Wallet = ar_test_data_sync:setup_nodes(), - %% Submit only the second last chunks (smaller than 256 KiB) of transactions. - %% Assert the nodes construct correct proofs of access from them. - lists:foreach( - fun(Height) -> - RandomID = crypto:strong_rand_bytes(32), - Chunk = crypto:strong_rand_bytes(?DATA_CHUNK_SIZE div 2), - ChunkID = ar_tx:generate_chunk_id(Chunk), - DataSize = (?DATA_CHUNK_SIZE) div 2 + (?DATA_CHUNK_SIZE) div 2 + 3, - {DataRoot, DataTree} = ar_merkle:generate_tree([{ChunkID, ?DATA_CHUNK_SIZE div 2}, - {RandomID, DataSize}]), - TX = ar_test_node:sign_tx(Wallet, #{ last_tx => ar_test_node:get_tx_anchor(main), data_size => DataSize, - data_root => DataRoot }), - ar_test_node:post_and_mine(#{ miner => main, await_on => peer1 }, [TX]), - Offset = 0, - DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), - Proof = #{ data_root => ar_util:encode(DataRoot), - data_path => ar_util:encode(DataPath), chunk => ar_util:encode(Chunk), - offset => integer_to_binary(Offset), - data_size => integer_to_binary(DataSize) }, - ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, - ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), - case Height - ?SEARCH_SPACE_UPPER_BOUND_DEPTH >= 0 of - true -> - %% Wait until the new chunks fall below the new upper bound and - %% remove the original big chunks. The protocol will increase the upper - %% bound based on the nonce limiter entropy reset, but ar_data_sync waits - %% for ?SEARCH_SPACE_UPPER_BOUND_DEPTH confirmations before packing the - %% chunks. - {ok, Config} = application:get_env(arweave, config), - lists:foreach( - fun(O) -> - [ar_chunk_storage:delete(O, ar_storage_module:id(Module)) - || Module <- Config#config.storage_modules] - end, - lists:seq(?DATA_CHUNK_SIZE, ar_block:strict_data_split_threshold(), - ?DATA_CHUNK_SIZE) - ); - _ -> - ok - end - end, - lists:seq(1, 6) - ). - -disk_pool_rotation_test_() -> - {timeout, 120, fun test_disk_pool_rotation/0}. - -test_disk_pool_rotation() -> - ?LOG_DEBUG([{event, test_disk_pool_rotation_start}]), - Addr = ar_wallet:to_address(ar_wallet:new_keyfile()), - %% Will store the three genesis chunks. - %% The third one falls inside the "overlap" (see ar_storage_module.erl) - StorageModules = [{2 * ?DATA_CHUNK_SIZE, 0, - ar_test_node:get_default_storage_module_packing(Addr, 0)}], - Wallet = ar_test_data_sync:setup_nodes( - #{ addr => Addr, storage_modules => StorageModules }), - Chunks = [crypto:strong_rand_bytes(?DATA_CHUNK_SIZE)], - {DataRoot, DataTree} = ar_merkle:generate_tree( - ar_tx:sized_chunks_to_sized_chunk_ids( - ar_tx:chunks_to_size_tagged_chunks(Chunks) - ) - ), - {TX, Chunks} = ar_test_data_sync:tx(Wallet, {fixed_data, DataRoot, Chunks}), - ar_test_node:assert_post_tx_to_peer(main, TX), - Offset = ?DATA_CHUNK_SIZE, - DataSize = ?DATA_CHUNK_SIZE, - DataPath = ar_merkle:generate_path(DataRoot, Offset, DataTree), - Proof = #{ data_root => ar_util:encode(DataRoot), - data_path => ar_util:encode(DataPath), - chunk => ar_util:encode(hd(Chunks)), - offset => integer_to_binary(Offset), - data_size => integer_to_binary(DataSize) }, - ?assertMatch({ok, {{<<"200">>, _}, _, _, _, _}}, - ar_test_node:post_chunk(main, ar_serialize:jsonify(Proof))), - ar_test_node:mine(main), - assert_wait_until_height(main, 1), - timer:sleep(2000), - Options = #{ format => etf, random_subset => false }, - {ok, Binary1} = ar_global_sync_record:get_serialized_sync_record(Options), - {ok, Global1} = ar_intervals:safe_from_etf(Binary1), - %% 3 genesis chunks plus the two we upload here. - ?assertEqual([{1048576, 0}], ar_intervals:to_list(Global1)), - ar_test_node:mine(main), - assert_wait_until_height(main, 2), - {ok, Binary2} = ar_global_sync_record:get_serialized_sync_record(Options), - {ok, Global2} = ar_intervals:safe_from_etf(Binary2), - ?assertEqual([{1048576, 0}], ar_intervals:to_list(Global2)), - ar_test_node:mine(main), - assert_wait_until_height(main, 3), - ar_test_node:mine(main), - assert_wait_until_height(main, 4), - %% The new chunk has been confirmed but there is not storage module to take it. - ?assertEqual(3, ?SEARCH_SPACE_UPPER_BOUND_DEPTH), - true = ar_util:do_until( - fun() -> - {ok, Binary3} = ar_global_sync_record:get_serialized_sync_record(Options), - {ok, Global3} = ar_intervals:safe_from_etf(Binary3), - [{786432, 0}] == ar_intervals:to_list(Global3) - end, - 200, - 5000 - ). - -enqueue_intervals_test() -> - ?LOG_DEBUG([{event, enqueue_intervals_test}]), - test_enqueue_intervals([], 2, [], [], [], "Empty Intervals"), - Peer1 = {1, 2, 3, 4, 1984}, - Peer2 = {101, 102, 103, 104, 1984}, - Peer3 = {201, 202, 203, 204, 1984}, - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])} - ], - 5, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, - {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1}, - {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer1}, - {8*?DATA_CHUNK_SIZE, 9*?DATA_CHUNK_SIZE, Peer1} - ], - "Single peer, full intervals, all chunks. Non-overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])} - ], - 2, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1} - ], - "Single peer, full intervals, 2 chunks. Non-overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])}, - {Peer2, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ])}, - {Peer3, ar_intervals:from_list([ - {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} - ])} - ], - 2, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {8*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, - {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, - {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer2}, - {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer3} - ], - "Multiple peers, overlapping, full intervals, 2 chunks. Non-overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])}, - {Peer2, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ])}, - {Peer3, ar_intervals:from_list([ - {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} - ])} - ], - 3, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {8*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, - {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, - {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1}, - {7*?DATA_CHUNK_SIZE, 8*?DATA_CHUNK_SIZE, Peer3} - ], - "Multiple peers, overlapping, full intervals, 3 chunks. Non-overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])} - ], - 5, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, {9*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, - {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer1} - ], - "Single peer, full intervals, all chunks. Overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])}, - {Peer2, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ])}, - {Peer3, ar_intervals:from_list([ - {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} - ])} - ], - 2, - [{20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, {9*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE}], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 5*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, 4*?DATA_CHUNK_SIZE, Peer1}, - {5*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE, Peer2}, - {6*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE, Peer2} - ], - "Multiple peers, overlapping, full intervals, 2 chunks. Overlapping QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, trunc(5.75*?DATA_CHUNK_SIZE)} - ])} - ], - 2, - [ - {20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, - {trunc(8.5*?DATA_CHUNK_SIZE), trunc(6.5*?DATA_CHUNK_SIZE)} - ], - [ - {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, trunc(3.25*?DATA_CHUNK_SIZE), Peer1} - ], - "Single peer, partial intervals, 2 chunks. Overlapping partial QIntervals."), - - test_enqueue_intervals( - [ - {Peer1, ar_intervals:from_list([ - {trunc(3.25*?DATA_CHUNK_SIZE), 2*?DATA_CHUNK_SIZE}, - {9*?DATA_CHUNK_SIZE, trunc(5.75*?DATA_CHUNK_SIZE)} - ])}, - {Peer2, ar_intervals:from_list([ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {7*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ])}, - {Peer3, ar_intervals:from_list([ - {8*?DATA_CHUNK_SIZE, 7*?DATA_CHUNK_SIZE} - ])} - ], - 2, - [ - {20*?DATA_CHUNK_SIZE, 10*?DATA_CHUNK_SIZE}, - {trunc(8.5*?DATA_CHUNK_SIZE), trunc(6.5*?DATA_CHUNK_SIZE)} - ], - [ - {4*?DATA_CHUNK_SIZE, 2*?DATA_CHUNK_SIZE}, - {8*?DATA_CHUNK_SIZE, 6*?DATA_CHUNK_SIZE} - ], - [ - {2*?DATA_CHUNK_SIZE, 3*?DATA_CHUNK_SIZE, Peer1}, - {3*?DATA_CHUNK_SIZE, trunc(3.25*?DATA_CHUNK_SIZE), Peer1}, - {trunc(3.25*?DATA_CHUNK_SIZE), 4*?DATA_CHUNK_SIZE, Peer2}, - {6*?DATA_CHUNK_SIZE, trunc(6.5*?DATA_CHUNK_SIZE), Peer2} - ], - "Multiple peers, overlapping, full intervals, 2 chunks. Overlapping QIntervals."). - -test_enqueue_intervals(Intervals, ChunksPerPeer, QIntervalsRanges, ExpectedQIntervalRanges, ExpectedChunks, Label) -> - QIntervals = ar_intervals:from_list(QIntervalsRanges), - Q = gb_sets:new(), - {QResult, QIntervalsResult} = ar_data_sync:enqueue_intervals(Intervals, ChunksPerPeer, {Q, QIntervals}), - ExpectedQIntervals = lists:foldl(fun({End, Start}, Acc) -> - ar_intervals:add(Acc, End, Start) - end, QIntervals, ExpectedQIntervalRanges), - ?assertEqual(ar_intervals:to_list(ExpectedQIntervals), ar_intervals:to_list(QIntervalsResult), Label), - ?assertEqual(ExpectedChunks, gb_sets:to_list(QResult), Label). - From e6608d67ec756d3f358e8fd9870c4161806d7d8e Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Thu, 17 Jul 2025 22:26:59 +0200 Subject: [PATCH 23/23] Increase base mining difficulty in sync tests --- apps/arweave/test/ar_test_data_sync.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/arweave/test/ar_test_data_sync.erl b/apps/arweave/test/ar_test_data_sync.erl index 00c01295ee..3a48b68fd4 100644 --- a/apps/arweave/test/ar_test_data_sync.erl +++ b/apps/arweave/test/ar_test_data_sync.erl @@ -34,7 +34,7 @@ setup_nodes2(#{ peer_addr := PeerAddr } = Options) -> {B0, Options2} = case maps:get(b0, Options, not_set) of not_set -> - [Genesis] = ar_weave:init([{ar_wallet:to_address(Pub), ?AR(200000), <<>>}]), + [Genesis] = ar_weave:init([{ar_wallet:to_address(Pub), ?AR(200000), <<>>}], ar_retarget:switch_to_linear_diff(2)), {Genesis, Options#{ b0 => Genesis }}; Value -> {Value, Options}