diff --git a/libwild/src/elf_writer.rs b/libwild/src/elf_writer.rs index 968f78125..2249c1dac 100644 --- a/libwild/src/elf_writer.rs +++ b/libwild/src/elf_writer.rs @@ -1467,28 +1467,11 @@ fn write_object<'data, A: Arch>( let _span = debug_span!("write_file", filename = %object.input).entered(); let _file_span = layout.args().common().trace_span_for_file(object.file_id); - let Some(crate::layout::FileLayout::Epilogue(epilogue)) = - layout.group_layouts.last().and_then(|g| g.files.last()) - else { - unreachable!("Epilogue is broken and must be the last file in the final layout group"); - }; - for (i, sec) in object.sections.iter().enumerate() { let section_index = object::SectionIndex(i); match sec { SectionSlot::Loaded(sec) => { - let is_harvested = epilogue - .script_sorted_sections - .binary_search_by_key(&(object.file_id, section_index.0), |h| { - (h.file_id, h.section_index.0) - }) - .is_ok(); - - if is_harvested { - continue; - } - write_object_section::( object, layout, @@ -2134,6 +2117,7 @@ fn write_symbols<'data>( if let Some(section_index) = object.object.symbol_section(sym, sym_index)? { match &object.sections[section_index.0] { SectionSlot::Loaded(_) + | SectionSlot::Sorted(_) | SectionSlot::LoadedDebugInfo(_) | SectionSlot::MergeStrings(_) => object .section_part_id(section_index, &layout.symbol_db.section_part_ids) @@ -4052,18 +4036,18 @@ fn write_epilogue>( let build_id_buffer = buffers.get_mut(part_id::NOTE_GNU_BUILD_ID); build_id_buffer.fill(0); - for harvested in &epilogue.script_sorted_sections { - let crate::layout::FileLayout::Object(object) = layout.file_layout(harvested.file_id) + for sorted_section in &layout.script_sorted_sections { + let crate::layout::FileLayout::Object(object) = layout.file_layout(sorted_section.file_id) else { - continue; + unreachable!(); }; - if let SectionSlot::Loaded(sec) = &object.sections[harvested.section_index.0] { + if let SectionSlot::Sorted(sec) = &object.sections[sorted_section.section_index.0] { write_object_section::( object, layout, - *sec, - harvested.section_index, + sec.section, + sorted_section.section_index, buffers, table_writer, trace, diff --git a/libwild/src/layout.rs b/libwild/src/layout.rs index 8798c3e32..5c9905181 100644 --- a/libwild/src/layout.rs +++ b/libwild/src/layout.rs @@ -159,19 +159,18 @@ pub fn compute<'data, P: Platform, A: Arch>( let mut dynamic_symbol_definitions = merge_dynamic_symbol_definitions(&group_states, &symbol_db)?; - let script_sorted_sections = harvest_and_sort_script_sections( + + let mut script_sorted_sections = harvest_and_sort_script_sections( &mut group_states, &output_sections, &symbol_db.section_part_ids, ); - let num_sorted_sections = script_sorted_sections.len(); group_states.push(GroupState { files: vec![FileLayoutState::Epilogue(EpilogueLayoutState::new( symbol_db.args, symbol_db.output_kind, &mut dynamic_symbol_definitions, - script_sorted_sections, &group_states, ))], queue: LocalWorkQueue::new(epilogue_file_id.group()), @@ -187,6 +186,7 @@ pub fn compute<'data, P: Platform, A: Arch>( symbol_db: &symbol_db, merged_strings: &merged_strings, format_specific: &finalise_sizes_ext, + script_sorted_sections: &script_sorted_sections, }; finalise_all_sizes( @@ -356,15 +356,10 @@ pub fn compute<'data, P: Platform, A: Arch>( &merged_strings, ); - // At this stage, sections marked for sorting have been harvested but lack concrete memory - // addresses. We perform a two-step finalization: - // Sort the harvested sections according to the requested criteria. - // Linearize them in memory, starting from the base offset of their respective output section - // part, and advancing by the size of each section. - let harvested_sections_registry = assign_addresses_to_sorted_sections( + assign_addresses_to_sorted_sections( &mut group_states, &starting_mem_offsets_by_group, - num_sorted_sections, + &mut script_sorted_sections, ); let mut symbol_resolutions = SymbolResolutions { @@ -385,7 +380,6 @@ pub fn compute<'data, P: Platform, A: Arch>( let resources = FinaliseLayoutResources { symbol_db: &symbol_db, output_sections: &output_sections, - harvested_sections_registry: &harvested_sections_registry, output_order: &output_order, section_layouts: §ion_layouts, merged_string_start_addresses: &merged_string_start_addresses, @@ -397,6 +391,7 @@ pub fn compute<'data, P: Platform, A: Arch>( format_specific: &finalise_sizes_ext, thunk_blocks: &thunk_blocks, thunk_block_addresses: &thunk_block_addresses_out, + script_sorted_sections: &script_sorted_sections, }; let group_layouts = compute_symbols_and_layouts( @@ -471,6 +466,7 @@ pub fn compute<'data, P: Platform, A: Arch>( thunk_block_addresses, compressed_debug_sections: OutputSectionMap::with_size(num_sections), gdb_index_data, + script_sorted_sections, }; P::maybe_compress_debug_sections::(&mut layout)?; @@ -485,6 +481,7 @@ pub(crate) struct FinaliseSizesResources<'data, 'scope, P: Platform> { pub(crate) symbol_db: &'scope SymbolDb<'data, P>, merged_strings: &'scope OutputSectionMap>, pub(crate) format_specific: &'scope P::FinaliseSizesExt<'data>, + script_sorted_sections: &'scope [InputSortedSection], } /// Update resolutions for symbol redirects. @@ -732,6 +729,7 @@ pub struct Layout<'data, P: Platform> { pub(crate) compressed_debug_sections: OutputSectionMap>, pub(crate) gdb_index_data: Option>, + pub(crate) script_sorted_sections: Vec, } #[derive(Debug, Default)] @@ -848,7 +846,6 @@ pub(crate) struct SyntheticSymbolsLayoutState<'data, P: Platform> { pub(crate) struct EpilogueLayoutState { format_specific: P::EpilogueLayoutExt, - pub(crate) script_sorted_sections: Vec, } #[derive(Debug)] @@ -881,7 +878,6 @@ pub(crate) struct SyntheticSymbolsLayout<'data, P: Platform> { pub(crate) struct EpilogueLayout { pub(crate) format_specific: P::EpilogueLayoutExt, pub(crate) dynsym_start_index: u32, - pub(crate) script_sorted_sections: Vec, } #[derive(Debug)] @@ -1280,18 +1276,6 @@ impl<'data, P: Platform> CommonGroupState<'data, P> { self.mem_sizes.increment(part_id, size); } - /// Allocate resources and update attributes based on a section having been loaded. - fn section_loaded( - &mut self, - part_id: PartId, - header: &P::SectionHeader, - section: Section, - output_sections: &OutputSections

, - ) { - self.allocate(part_id, section.capacity(part_id, output_sections)); - self.store_section_attributes(part_id, header); - } - fn store_section_attributes(&mut self, part_id: PartId, header: &P::SectionHeader) { let existing_attributes = self.section_attributes.get_mut(part_id.output_section_id()); @@ -1371,6 +1355,12 @@ pub(crate) struct Section { pub(crate) size: u64, } +#[derive(Debug, Clone, Copy)] +pub(crate) struct SortedSection { + pub(crate) address: u64, + pub(crate) section: Section, +} + #[derive(Debug)] pub(crate) struct GroupLayout<'data, P: Platform> { pub(crate) files: Vec>, @@ -1453,12 +1443,12 @@ pub(crate) struct FinaliseLayoutResources<'scope, 'data, P: Platform> { output_sections: &'scope OutputSections<'data, P>, output_order: &'scope OutputOrder<'data>, pub(crate) section_layouts: &'scope OutputSectionMap, - pub(crate) harvested_sections_registry: &'scope [HarvestedSortedSection], merged_string_start_addresses: &'scope MergedStringStartAddresses, merged_strings: &'scope OutputSectionMap>, dynamic_symbol_definitions: &'scope Vec>, segment_layouts: &'scope SegmentLayouts, program_segments: &'scope ProgramSegments, + script_sorted_sections: &'scope [InputSortedSection], pub(crate) format_specific: &'scope P::FinaliseSizesExt<'data>, pub(crate) thunk_blocks: &'scope [crate::thunks::ThunkBlock], @@ -3791,7 +3781,6 @@ impl<'data, P: Platform> EpilogueLayoutState

{ args: &P::Args, output_kind: OutputKind, dynamic_symbol_definitions: &mut [DynamicSymbolDefinition<'data, P>], - script_sorted_sections: Vec, group_states: &[GroupState<'data, P>], ) -> Self { EpilogueLayoutState { @@ -3801,7 +3790,6 @@ impl<'data, P: Platform> EpilogueLayoutState

{ dynamic_symbol_definitions, group_states, ), - script_sorted_sections, } } @@ -3812,7 +3800,7 @@ impl<'data, P: Platform> EpilogueLayoutState

{ resources: &FinaliseSizesResources<'data, '_, P>, ) -> Result { let mut extra_sizes = OutputSectionPartMap::with_size(common.mem_sizes.num_parts()); - for sec in &self.script_sorted_sections { + for sec in resources.script_sorted_sections { extra_sizes.increment(sec.part_id, sec.size); } P::apply_late_size_adjustments_epilogue( @@ -3868,16 +3856,14 @@ impl<'data, P: Platform> EpilogueLayoutState

{ dynsym_start_index, resources.dynamic_symbol_definitions, )?; - for sec in &mut self.script_sorted_sections { + for sec in resources.script_sorted_sections { let offset = memory_offsets.get_mut(sec.part_id); *offset = sec.alignment.align_up(*offset); - sec.mem_offset = *offset; *offset += sec.size; } Ok(EpilogueLayout { format_specific: self.format_specific, dynsym_start_index, - script_sorted_sections: self.script_sorted_sections, }) } } @@ -4056,6 +4042,7 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { ); } SectionSlot::Loaded(_) + | SectionSlot::Sorted(_) | SectionSlot::FrameData(..) | SectionSlot::LoadedDebugInfo(..) | SectionSlot::NoteGnuProperty(..) @@ -4109,7 +4096,24 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { tracing::debug!(loaded_section = %self.object.section_display_name(section_index), file = %self.input); - common.section_loaded(part_id, header, section, resources.output_sections); + self.sections[section_index.0] = if unloaded.needs_sorting { + self.script_sorted_sections.push(ScriptSortedSectionDetail { + index: section_index, + }); + SectionSlot::Sorted(SortedSection { + // Filled in later. + address: 0, + section, + }) + } else { + common.allocate( + part_id, + section.capacity(part_id, resources.output_sections), + ); + SectionSlot::Loaded(section) + }; + + common.store_section_attributes(part_id, header); if let Some(config) = A::thunk_config() && resources.thunk_layout_builder.is_some() @@ -4126,8 +4130,6 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { resources.keep_section(section_id); } - self.sections[section_index.0] = SectionSlot::Loaded(section); - Ok(()) } @@ -4150,7 +4152,11 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { // live sections. tracing::debug!(loaded_debug_section = %self.object.section_display_name(section_index),); - common.section_loaded(part_id, header, section, resources.output_sections); + common.allocate( + part_id, + section.capacity(part_id, resources.output_sections), + ); + common.store_section_attributes(part_id, header); self.sections[section_index.0] = SectionSlot::LoadedDebugInfo(section); Ok(()) @@ -4217,11 +4223,10 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { let section_id_range = self.section_id_range; let object_part_ids = &resources.symbol_db.section_part_ids[section_id_range.as_usize()]; - for (sec_idx, (slot, &part_id)) in self.sections.iter_mut().zip(object_part_ids).enumerate() - { + for (slot, &part_id) in self.sections.iter_mut().zip(object_part_ids) { let resolution = match slot { SectionSlot::Loaded(sec) => { - let mut address = *memory_offsets.get(part_id); + let address = *memory_offsets.get(part_id); // TODO: We probably need to be able to handle sections that are ifuncs and // sections that need a TLS GOT struct. @@ -4235,18 +4240,13 @@ impl<'data, P: Platform> ObjectLayoutState<'data, P> { sframe_ranges.push(offset..offset + len); } - if let Ok(idx) = resources - .harvested_sections_registry - .binary_search_by_key(&(self.file_id, sec_idx), |s| { - (s.file_id, s.section_index.0) - }) - { - address = resources.harvested_sections_registry[idx].mem_offset; - } - SectionResolution { address } } + SectionSlot::Sorted(sec) => SectionResolution { + address: sec.address, + }, + &mut SectionSlot::LoadedDebugInfo(sec) => { let address = *memory_offsets.get(part_id); *memory_offsets.get_mut(part_id) += @@ -5828,21 +5828,21 @@ impl<'data, P: Platform> Drop for Layout<'data, P> { fn drop(&mut self) {} } -#[derive(Clone, Debug)] -pub(crate) struct HarvestedSortedSection { +/// An input section that needs to be sorted due to a SORT_BY_NAME directive or equivalent. +#[derive(Copy, Clone, Debug)] +pub(crate) struct InputSortedSection { pub(crate) file_id: FileId, pub(crate) section_index: object::SectionIndex, pub(crate) part_id: PartId, pub(crate) size: u64, pub(crate) alignment: Alignment, - pub(crate) mem_offset: u64, } fn harvest_and_sort_script_sections<'data, P: Platform>( group_states: &mut [GroupState<'data, P>], output_sections: &OutputSections

, section_part_ids: &[PartId], -) -> Vec { +) -> Vec { timing_phase!("Harvest and sort script sections"); let has_any_sorting = group_states.iter().any(|g| { @@ -5864,20 +5864,19 @@ fn harvest_and_sort_script_sections<'data, P: Platform>( for file in &mut group.files { if let FileLayoutState::Object(obj) = file { for sorted_section in &obj.script_sorted_sections { - if let SectionSlot::Loaded(sec) = &obj.sections[sorted_section.index.0] { + if let SectionSlot::Sorted(sec) = &obj.sections[sorted_section.index.0] { let part_id = obj.section_part_id(sorted_section.index, section_part_ids); - let capacity = sec.capacity(part_id, output_sections); + let capacity = sec.section.capacity(part_id, output_sections); sections_out.push(( obj.object .section_name(sorted_section.index) .unwrap_or_default(), - HarvestedSortedSection { + InputSortedSection { file_id: obj.file_id, section_index: sorted_section.index, part_id, size: capacity, alignment: part_id.alignment(output_sections), - mem_offset: 0, }, )); } @@ -5892,29 +5891,29 @@ fn harvest_and_sort_script_sections<'data, P: Platform>( .map(|(_, harvested)| harvested) .collect() } -// Assigning memory addresses to script sorted sections and returning the finalized registry. + fn assign_addresses_to_sorted_sections( group_states: &mut [GroupState

], starting_mem_offsets_by_group: &[OutputSectionPartMap], - num_sorted_sections: usize, -) -> Vec { - let mut harvested_sections_registry = Vec::with_capacity(num_sorted_sections); + sorted_sections: &mut [InputSortedSection], +) { let mut epilogue_offsets = starting_mem_offsets_by_group.last().unwrap().clone(); - if let FileLayoutState::Epilogue(epilogue_state) = - &mut group_states.last_mut().unwrap().files[0] - { - for sec in &mut epilogue_state.script_sorted_sections { - let offset = epilogue_offsets.get_mut(sec.part_id); - *offset = sec.alignment.align_up(*offset); - sec.mem_offset = *offset; - *offset += sec.size; + for sec in sorted_sections { + let offset = epilogue_offsets.get_mut(sec.part_id); + *offset = sec.alignment.align_up(*offset); - harvested_sections_registry.push(sec.clone()); - } - } + let FileLayoutState::Object(obj) = + &mut group_states[sec.file_id.group()].files[sec.file_id.file()] + else { + unreachable!(); + }; - harvested_sections_registry.sort_unstable_by_key(|s| (s.file_id, s.section_index.0)); + let SectionSlot::Sorted(slot) = &mut obj.sections[sec.section_index.0] else { + unreachable!(); + }; - harvested_sections_registry + slot.address = *offset; + *offset += sec.size; + } } diff --git a/libwild/src/layout_rules.rs b/libwild/src/layout_rules.rs index 029ac7439..88a17f355 100644 --- a/libwild/src/layout_rules.rs +++ b/libwild/src/layout_rules.rs @@ -105,13 +105,13 @@ pub(crate) enum SectionRuleOutcome { DebugIndex, RiscVAttribute, SortedSection(SectionOutputInfo), - ScriptSortedSection(SectionOutputInfo), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct SectionOutputInfo { pub(crate) section_id: OutputSectionId, pub(crate) must_keep: bool, + pub(crate) sorted: bool, } impl SectionOutputInfo { @@ -119,6 +119,7 @@ impl SectionOutputInfo { Self { section_id, must_keep: false, + sorted: false, } } @@ -126,6 +127,7 @@ impl SectionOutputInfo { Self { section_id, must_keep: true, + sorted: false, } } } @@ -262,17 +264,13 @@ impl<'data> LayoutRulesBuilder<'data> { let output_info = SectionOutputInfo { section_id, must_keep: matcher.must_keep, + sorted: pattern.sorted, }; - // If the script requested sorting, tag it for the - // global Harvester instead of standard placement. - let outcome = if pattern.sorted { - SectionRuleOutcome::ScriptSortedSection(output_info) - } else { + let outcome = crate::layout_rules::SectionRuleOutcome::Section( output_info, - ) - }; + ); self.add_section_rule(SectionRule::new( pattern.name, @@ -649,7 +647,8 @@ fn test_section_mapping() { lookup_name(".comment"), SectionRuleOutcome::Section(SectionOutputInfo { section_id: output_section_id::COMMENT, - must_keep: true + must_keep: true, + sorted: false, }) ); } diff --git a/libwild/src/linker_script.rs b/libwild/src/linker_script.rs index 579382c03..b31fcd248 100644 --- a/libwild/src/linker_script.rs +++ b/libwild/src/linker_script.rs @@ -1245,7 +1245,6 @@ fn parse_pattern<'input>(input: &mut &'input BStr) -> winnow::Result( must_load |= output_info.must_keep; unloaded_section = UnloadedSection::new(); + unloaded_section.needs_sorting = output_info.sorted; } SectionRuleOutcome::SortedSection(output_info) => { part_id = if output_info.section_id.is_regular() { @@ -1360,22 +1368,6 @@ fn resolve_section<'data, P: Platform>( unloaded_section = UnloadedSection::new(); } - SectionRuleOutcome::ScriptSortedSection(output_info) => { - part_id = if output_info.section_id.is_regular() { - output_info.section_id.part_id_with_alignment(alignment) - } else { - output_info.section_id.base_part_id() - }; - - obj.script_sorted_sections.push(ScriptSortedSectionDetail { - index: input_section_index, - }); - - must_load |= output_info.must_keep; - - unloaded_section = UnloadedSection::new(); - } - SectionRuleOutcome::Discard => return Ok((SectionSlot::Discard, part_id::UNMAPPED)), SectionRuleOutcome::NoteGnuStack => { P::validate_stack_section(input_section, obj, args)?; diff --git a/wild/tests/sources/elf/script-sort/script-sort.c b/wild/tests/sources/elf/script-sort/script-sort.c index 74a1c8d9b..a40410472 100644 --- a/wild/tests/sources/elf/script-sort/script-sort.c +++ b/wild/tests/sources/elf/script-sort/script-sort.c @@ -1,25 +1,26 @@ //#LinkArgs: -T tests/sources/elf/script-sort/script-sort.ld //#Object:runtime.c +//#Object:ptr_black_box.c //#Object:script-sort-2.c //#Object:script-sort-3.c //#EnableLinker:lld //#DiffMatchAny:true +//#ExpectSym:func_kept +//#NoSym:func_drop +#include "../common/ptr_black_box.h" #include "../common/runtime.h" extern int func_a(); extern int func_b(); extern int func_c(); -#if defined(__x86_64__) -__attribute__((force_align_arg_pointer)) -#endif void _start(void) { runtime_init(); - unsigned long a_addr = (unsigned long)&func_a; - unsigned long b_addr = (unsigned long)&func_b; - unsigned long c_addr = (unsigned long)&func_c; + size_t a_addr = ptr_to_int(&func_a); + size_t b_addr = ptr_to_int(&func_b); + size_t c_addr = ptr_to_int(&func_c); if (a_addr >= b_addr) { exit_syscall(101); diff --git a/wild/tests/sources/elf/script-sort/script-sort.ld b/wild/tests/sources/elf/script-sort/script-sort.ld index de7e19298..3b2b48da3 100644 --- a/wild/tests/sources/elf/script-sort/script-sort.ld +++ b/wild/tests/sources/elf/script-sort/script-sort.ld @@ -12,7 +12,7 @@ SECTIONS *(SORT(.text.drop.*)) } - /* plz ensure valid alignment for subsequent sections */ + /* Ensure valid alignment for subsequent sections */ . = 0x410000;