diff --git a/verilog/analysis/BUILD b/verilog/analysis/BUILD index a95be9756..6bfc08e3f 100644 --- a/verilog/analysis/BUILD +++ b/verilog/analysis/BUILD @@ -74,6 +74,34 @@ cc_library( ], ) +cc_library( + name = "flow_tree", + srcs = ["flow_tree.cc"], + hdrs = ["flow_tree.h"], + deps = [ + "//common/lexer:token_stream_adapter", + "//verilog/parser:verilog_token_enum", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/status", + ], +) + +cc_test( + name = "flow_tree_test", + srcs = ["flow_tree_test.cc"], + deps = [ + ":flow_tree", + "//common/util:logging", + "//common/lexer:token_stream_adapter", + "//verilog/parser:verilog_lexer", + "//verilog/parser:verilog_token_enum", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + + cc_library( name = "lint_rule_registry", srcs = ["lint_rule_registry.cc"], diff --git a/verilog/analysis/flow_tree.cc b/verilog/analysis/flow_tree.cc new file mode 100644 index 000000000..64b397040 --- /dev/null +++ b/verilog/analysis/flow_tree.cc @@ -0,0 +1,359 @@ +// Copyright 2017-2022 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "verilog/analysis/flow_tree.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/lexer/token_stream_adapter.h" +#include "verilog/parser/verilog_token_enum.h" + +namespace verilog { + +// Adds edges within a conditonal block. +// Such that the first edge represents the condition being true, +// and the second edge represents the condition being false. +absl::Status FlowTree::AddBlockEdges(const ConditionalBlock &block) { + bool contains_elsif = !block.elsif_locations.empty(); + bool contains_else = block.else_location != source_sequence_.end(); + + // Handling `ifdef/ifndef. + + // Assuming the condition is true. + edges_[block.if_location].push_back(block.if_location + 1); + + // Assuming the condition is false. + // Checking if there is an `elsif. + if (contains_elsif) { + // Add edge to the first `elsif in the block. + edges_[block.if_location].push_back(block.elsif_locations[0]); + } else if (contains_else) { + // Checking if there is an `else. + edges_[block.if_location].push_back(block.else_location); + } else { + // `endif exists. + edges_[block.if_location].push_back(block.endif_location); + } + + // Handling `elsif. + if (contains_elsif) { + for (auto iter = block.elsif_locations.begin(); + iter != block.elsif_locations.end(); iter++) { + // Assuming the condition is true. + edges_[*iter].push_back((*iter) + 1); + + // Assuming the condition is false. + if (iter + 1 != block.elsif_locations.end()) + edges_[*iter].push_back(*(iter + 1)); + else if (contains_else) + edges_[*iter].push_back(block.else_location); + else + edges_[*iter].push_back(block.endif_location); + } + } + + // Handling `else. + if (contains_else) { + edges_[block.else_location].push_back(block.else_location + 1); + } + + // For edges that are generated assuming the conditons are true, + // We need to add an edge from the end of the condition group of lines to + // `endif, e.g. `ifdef + // + // + // ... + // + // `else + // + // `endif + // Edge to be added: from to `endif. + edges_[block.endif_location - 1].push_back(block.endif_location); + if (contains_elsif) { + for (auto iter : block.elsif_locations) + edges_[iter - 1].push_back(block.endif_location); + } + if (contains_else) { + edges_[block.else_location - 1].push_back(block.endif_location); + } + + // Connecting `endif to the next token directly (if not EOF). + auto next_iter = block.endif_location + 1; + if (next_iter != source_sequence_.end() && + next_iter->token_enum() != PP_else && + next_iter->token_enum() != PP_elsif && + next_iter->token_enum() != PP_endif) { + edges_[block.endif_location].push_back(next_iter); + } + + return absl::OkStatus(); +} + +// Checks if the iterator is pointing to a conditional directive. +bool FlowTree::IsConditional(TokenSequenceConstIterator iterator) { + auto current_node = iterator->token_enum(); + return current_node == PP_ifndef || current_node == PP_ifdef || + current_node == PP_elsif || current_node == PP_else || + current_node == PP_endif; +} + +// Checks if after the conditional_iterator (`ifdef/`ifndef... ) there exists +// a macro identifier. +absl::Status FlowTree::MacroFollows( + TokenSequenceConstIterator conditional_iterator) { + if (conditional_iterator->token_enum() != PP_ifdef && + conditional_iterator->token_enum() != PP_ifndef && + conditional_iterator->token_enum() != PP_elsif) { + return absl::InvalidArgumentError("Error macro name can't be extracted."); + } + auto macro_iterator = conditional_iterator + 1; + if (macro_iterator->token_enum() != PP_Identifier) + return absl::InvalidArgumentError("Expected identifier for macro name."); + else + return absl::OkStatus(); +} + +// Adds a conditional macro to conditional_macros_ if not added before, +// And gives it a new ID, then saves the ID in conditional_macro_id_ map. +absl::Status FlowTree::AddMacroOfConditional( + TokenSequenceConstIterator conditional_iterator) { + auto status = MacroFollows(conditional_iterator); + if (!status.ok()) { + return absl::InvalidArgumentError( + "Error no macro follows the conditional directive."); + } + auto macro_iterator = conditional_iterator + 1; + auto macro_identifier = macro_iterator->text(); + if (conditional_macro_id_.find(macro_identifier) == + conditional_macro_id_.end()) { + conditional_macro_id_[macro_identifier] = conditional_macros_counter_; + conditional_macros_.push_back(macro_iterator); + conditional_macros_counter_++; + } + return absl::OkStatus(); +} + +// Gets the conditonal macro ID from the conditional_macro_id_. +// Note: conditional_iterator is pointing to the conditional. +int FlowTree::GetMacroIDOfConditional( + TokenSequenceConstIterator conditional_iterator) { + auto status = MacroFollows(conditional_iterator); + if (!status.ok()) { + // TODO(karimtera): add a better error handling. + return -1; + } + auto macro_iterator = conditional_iterator + 1; + auto macro_identifier = macro_iterator->text(); + // It is always assumed that the macro already exists in the map. + return conditional_macro_id_[macro_identifier]; +} + +// An API that provides a callback function to receive variants. +absl::Status FlowTree::GenerateVariants(const VariantReceiver &receiver) { + auto status = GenerateControlFlowTree(); + if (!status.ok()) { + return status; + } + return DepthFirstSearch(receiver, source_sequence_.begin()); +} + +// Constructs the control flow tree, which determines the edge from each node +// (token index) to the next possible childs, And save edge_from_iterator in +// edges_. +absl::Status FlowTree::GenerateControlFlowTree() { + // Adding edges for if blocks. + int current_token_enum = 0; + ConditionalBlock empty_block; + empty_block.if_location = source_sequence_.end(); + empty_block.else_location = source_sequence_.end(); + empty_block.endif_location = source_sequence_.end(); + + for (TokenSequenceConstIterator iter = source_sequence_.begin(); + iter != source_sequence_.end(); iter++) { + current_token_enum = iter->token_enum(); + + if (IsConditional(iter)) { + switch (current_token_enum) { + case PP_ifdef: { + if_blocks_.push_back(empty_block); + if_blocks_.back().if_location = iter; + if_blocks_.back().positive_condition = 1; + auto status = AddMacroOfConditional(iter); + if (!status.ok()) { + return absl::InvalidArgumentError( + "ERROR: couldn't give a macro an ID."); + } + break; + } + case PP_ifndef: { + if_blocks_.push_back(empty_block); + if_blocks_.back().if_location = iter; + if_blocks_.back().positive_condition = 0; + auto status = AddMacroOfConditional(iter); + if (!status.ok()) { + return absl::InvalidArgumentError( + "ERROR: couldn't give a macro an ID."); + } + break; + } + case PP_elsif: { + if (if_blocks_.empty()) { + return absl::InvalidArgumentError("ERROR: Unmatched `elsif."); + } + if_blocks_.back().elsif_locations.push_back(iter); + auto status = AddMacroOfConditional(iter); + if (!status.ok()) { + return absl::InvalidArgumentError( + "ERROR: couldn't give a macro an ID."); + } + break; + } + case PP_else: { + if (if_blocks_.empty()) { + return absl::InvalidArgumentError("ERROR: Unmatched `else."); + } + if_blocks_.back().else_location = iter; + break; + } + case PP_endif: { + if (if_blocks_.empty()) { + return absl::InvalidArgumentError("ERROR: Unmatched `endif."); + } + if_blocks_.back().endif_location = iter; + auto status = AddBlockEdges(if_blocks_.back()); + if (!status.ok()) return status; + // TODO(karimtera): add an error message. + if_blocks_.pop_back(); + break; + } + } + + } else { + // Only add normal edges if the next token is not `else/`elsif/`endif. + auto next_iter = iter + 1; + if (next_iter != source_sequence_.end() && + next_iter->token_enum() != PP_else && + next_iter->token_enum() != PP_elsif && + next_iter->token_enum() != PP_endif) { + edges_[iter].push_back(next_iter); + } + } + } + + // Checks for uncompleted conditionals. + if (!if_blocks_.empty()) + return absl::InvalidArgumentError( + "ERROR: Uncompleted conditional is found."); + + return absl::OkStatus(); +} + +// Traveses the control flow tree in a depth first manner, appending the visited +// tokens to current_variant_, then provide the completed variant to the user +// using a callback function (VariantReceiver). +absl::Status FlowTree::DepthFirstSearch( + const VariantReceiver &receiver, TokenSequenceConstIterator current_node) { + if (!wants_more_) return absl::OkStatus(); + + // Skips directives so that current_variant_ doesn't contain any. + if (current_node->token_enum() != PP_Identifier && + current_node->token_enum() != PP_ifndef && + current_node->token_enum() != PP_ifdef && + current_node->token_enum() != PP_define && + current_node->token_enum() != PP_define_body && + current_node->token_enum() != PP_elsif && + current_node->token_enum() != PP_else && + current_node->token_enum() != PP_endif) { + current_variant_.sequence.push_back(*current_node); + } + + // Checks if the current token is a `ifdef/`ifndef/`elsif. + if (current_node->token_enum() == PP_ifdef || + current_node->token_enum() == PP_ifndef || + current_node->token_enum() == PP_elsif) { + int macro_id = GetMacroIDOfConditional(current_node); + bool negated = (current_node->token_enum() == PP_ifndef); + // Checks if this macro is already visited (either defined/undefined). + if (current_variant_.visited.test(macro_id)) { + bool assume_condition_is_true = + (negated ^ current_variant_.macros_mask.test(macro_id)); + if (auto status = DepthFirstSearch( + receiver, edges_[current_node][!assume_condition_is_true]); + !status.ok()) { + std::cerr << "ERROR: DepthFirstSearch fails."; + return status; + } + } else { + current_variant_.visited.flip(macro_id); + // This macro wans't visited before, then we can check both edges. + // Assume the condition is true. + if (negated) + current_variant_.macros_mask.reset(macro_id); + else + current_variant_.macros_mask.set(macro_id); + if (auto status = DepthFirstSearch(receiver, edges_[current_node][0]); + !status.ok()) { + std::cerr << "ERROR: DepthFirstSearch fails."; + return status; + } + + // Assume the condition is false. + if (!negated) + current_variant_.macros_mask.reset(macro_id); + else + current_variant_.macros_mask.set(macro_id); + if (auto status = DepthFirstSearch(receiver, edges_[current_node][1]); + !status.ok()) { + std::cerr << "ERROR: DepthFirstSearch fails."; + return status; + } + // Undo the change to allow for backtracking. + current_variant_.visited.flip(macro_id); + } + } else { + // Do recursive search through every possible edge. + // Expected to be only one edge in this case. + for (auto next_node : edges_[current_node]) { + if (auto status = FlowTree::DepthFirstSearch(receiver, next_node); + !status.ok()) { + std::cerr << "ERROR: DepthFirstSearch fails\n"; + return status; + } + } + } + // If the current node is the last one, push the completed current_variant_ + // then it is ready to be sent. + if (current_node == source_sequence_.end() - 1) { + wants_more_ &= receiver(current_variant_); + } + if (current_node->token_enum() != PP_Identifier && + current_node->token_enum() != PP_ifndef && + current_node->token_enum() != PP_ifdef && + current_node->token_enum() != PP_define && + current_node->token_enum() != PP_define_body && + current_node->token_enum() != PP_elsif && + current_node->token_enum() != PP_else && + current_node->token_enum() != PP_endif) { + // Remove tokens to back track into other variants. + current_variant_.sequence.pop_back(); + } + return absl::OkStatus(); +} + +} // namespace verilog diff --git a/verilog/analysis/flow_tree.h b/verilog/analysis/flow_tree.h new file mode 100644 index 000000000..f9b0141f1 --- /dev/null +++ b/verilog/analysis/flow_tree.h @@ -0,0 +1,158 @@ +// Copyright 2017-2022 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VERIBLE_VERILOG_FLOW_TREE_H_ +#define VERIBLE_VERILOG_FLOW_TREE_H_ + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "common/lexer/token_stream_adapter.h" +#include "verilog/parser/verilog_token_enum.h" + +namespace verilog { + +// FlowTree class builds the control flow graph of a tokenized System-Verilog +// source code. Furthermore, enabling doing the following queries on the graph: +// - Generating all the possible variants (provided via a callback function). +class FlowTree { + private: + // 'kMaxDistinctMacros' shows the maximum number of distinct macros that can + // be considered in conditonal directives. + static constexpr int kMaxDistinctMacros = 128; + + public: + using BitSet = std::bitset; + using TokenSequenceConstIterator = verible::TokenSequence::const_iterator; + + // "ConditionalBlock" saves locations of conditionals in a "TokenSequence". + // All locations should point inside this specific "TokenSequence". + // Since it is only used in conjunction with a "TokenSequence", + // It should be initialized to last location in this "TokenSequence", + // For example in "GenerateControlFlowTree" is initialized to + // 'source_sequence_.end()'. + + struct Variant { + // Contains the token sequence of the variant. + verible::TokenSequence sequence; + + // The i-th bit in "macros_mask" is 1 when the macro (with ID = i) is + // assumed to be defined, otherwise it is assumed to be undefined. + BitSet macros_mask; + + // The i-th bit in "visited" is 1 when the macro (with ID = i) is visited or + // assumed (either defined or not), otherwise it is not visited (its value + // doesn't affect this variant). + // + // e.g.: + // `ifdef A + // `ifdef B + // ... + // `endif + // `endif + // + // Consider the variant in which A is undefined, + // we notice that B doesn't affect the variant. + // Then the bit corresponding to B in "visited" is 0. + BitSet visited; + }; + + // Receive a complete token sequence of one variant. + // variant_sequence: the generated variant token sequence. + using VariantReceiver = std::function; + + explicit FlowTree(verible::TokenSequence source_sequence) + : source_sequence_(std::move(source_sequence)){}; + + // Generates all possible variants. + absl::Status GenerateVariants(const VariantReceiver &receiver); + + // Returns all the used macros in conditionals, ordered with the same ID as + // used in BitSets. + const std::vector &GetUsedMacros() { + return conditional_macros_; + } + + private: + struct ConditionalBlock { + // "if_location" points to `ifdef or `ifndef. + TokenSequenceConstIterator if_location; + + // When "positive_condition" equals 1, then "if_location" points to `ifdef, + // Otherwise it points to `ifndef. + bool positive_condition; + std::vector elsif_locations; + TokenSequenceConstIterator else_location; + TokenSequenceConstIterator endif_location; + }; + + // Constructs the control flow tree by adding the tree edges in edges_. + absl::Status GenerateControlFlowTree(); + + // Traveses the tree in a depth first manner. + absl::Status DepthFirstSearch(const VariantReceiver &receiver, + TokenSequenceConstIterator current_node); + + // Checks if the iterator points to a conditonal directive (`ifdef/ifndef...). + static bool IsConditional(TokenSequenceConstIterator iterator); + + // Adds all edges withing a conditional block. + absl::Status AddBlockEdges(const ConditionalBlock &block); + + // The tree edges which defines the possible next childs of each token in + // source_sequence_. + std::map> + edges_; + + // Extracts the conditional macro checked. + static absl::Status MacroFollows( + TokenSequenceConstIterator conditional_iterator); + + // Adds macro to conditional_macros_ vector, and save its ID in + // conditional_macro_id_ map. + absl::Status AddMacroOfConditional( + TokenSequenceConstIterator conditional_iterator); + + int GetMacroIDOfConditional(TokenSequenceConstIterator conditional_iterator); + + // Holds all of the conditional blocks. + std::vector if_blocks_; + + // The original source code lexed token seqouence. + const verible::TokenSequence source_sequence_; + + // Current variant being generated by DepthFirstSearch. + Variant current_variant_; + + // A flag that determines if the VariantReceiver returned 'false'. + // By default: it assumes VariantReceiver wants more variants. + bool wants_more_ = 1; + + // Mapping each conditional macro to an integer ID, + // to use it later as a bit offset. + std::map conditional_macro_id_; + + // A vector containing all the macros used placed by their given ID. + std::vector conditional_macros_; + + // Number of macros appeared in `ifdef/`ifndef/`elsif. + int conditional_macros_counter_ = 0; +}; + +} // namespace verilog + +#endif // VERIBLE_VERILOG_FLOW_TREE_H_ diff --git a/verilog/analysis/flow_tree_test.cc b/verilog/analysis/flow_tree_test.cc new file mode 100644 index 000000000..a34f90807 --- /dev/null +++ b/verilog/analysis/flow_tree_test.cc @@ -0,0 +1,361 @@ +// Copyright 2017-2022 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "verilog/analysis/flow_tree.h" + +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "common/lexer/token_stream_adapter.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "verilog/parser/verilog_lexer.h" +#include "verilog/parser/verilog_token_enum.h" + +namespace verilog { +namespace { + +using testing::StartsWith; + +// Lexes a SystemVerilog source code, and returns a TokenSequence. +verible::TokenSequence LexToSequence(absl::string_view source_contents) { + verible::TokenSequence lexed_sequence; + VerilogLexer lexer(source_contents); + for (lexer.DoNextToken(); !lexer.GetLastToken().isEOF(); + lexer.DoNextToken()) { + if (verilog::VerilogLexer::KeepSyntaxTreeTokens(lexer.GetLastToken())) { + lexed_sequence.push_back(lexer.GetLastToken()); + } + } + return lexed_sequence; +} + +TEST(FlowTree, MultipleConditionalsSameMacro) { + const absl::string_view test_case = + R"( + `ifdef A + A_TRUE_1 + `else + A_FALSE_1 + `endif + + `ifdef A + A_TRUE_2 + `else + A_FALSE_2 + `endif + + `ifndef A + A_FALSE_3 + `else + A_TRUE_3 + `endif)"; + + FlowTree tree_test(LexToSequence(test_case)); + std::vector variants; + auto status = + tree_test.GenerateVariants([&variants](const FlowTree::Variant& variant) { + variants.push_back(variant); + return true; + }); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(variants.size(), 2); + + const auto& used_macros = tree_test.GetUsedMacros(); + EXPECT_EQ(used_macros.size(), 1); + EXPECT_THAT(used_macros[0]->text(), "A"); + + // First variant: A is defined. + EXPECT_TRUE(variants[0].macros_mask.test(0)); + EXPECT_TRUE(variants[0].visited.test(0)); + EXPECT_THAT(variants[0].sequence[0].text(), "A_TRUE_1"); + EXPECT_THAT(variants[0].sequence[1].text(), "A_TRUE_2"); + EXPECT_THAT(variants[0].sequence[2].text(), "A_TRUE_3"); + + // Second variant: A is undefined. + EXPECT_FALSE(variants[1].macros_mask.test(0)); + EXPECT_TRUE(variants[1].visited.test(0)); + EXPECT_THAT(variants[1].sequence[0].text(), "A_FALSE_1"); + EXPECT_THAT(variants[1].sequence[1].text(), "A_FALSE_2"); + EXPECT_THAT(variants[1].sequence[2].text(), "A_FALSE_3"); +} + +TEST(FlowTree, UnmatchedElses) { + const absl::string_view test_cases[] = { + R"( + `elsif A + A_TRUE + `endif + )", + R"( + `ifdef A + A_TRUE + `endif + `elsif B + B_TRUE + `endif + )", + R"( + `else + A_TRUE + `endif + )", + R"( + `ifdef A + `elsif B + B_TRUE + `endif + `endif + )", + R"( + `endif + )"}; + + for (auto test : test_cases) { + FlowTree tree_test(LexToSequence(test)); + auto status = tree_test.GenerateVariants( + [](const FlowTree::Variant& variant) { return true; }); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.message(), StartsWith("ERROR: Unmatched")); + } +} + +TEST(FlowTree, UnvalidConditionals) { + const absl::string_view test_cases[] = { + R"( + `ifdef A + A_TRUE + `elsif + A_FALSE + )", + R"( + `ifdef + A_TRUE + `else + A_FALSE + )", + R"( + `ifndef + A_TRUE + `else + A_FALSE + `endif + )"}; + + for (auto test : test_cases) { + FlowTree tree_test(LexToSequence(test)); + auto status = tree_test.GenerateVariants( + [](const FlowTree::Variant& variant) { return true; }); + EXPECT_FALSE(status.ok()); + } +} + +TEST(FlowTree, UncompletedConditionals) { + const absl::string_view test_cases[] = { + R"( + `ifdef A + A_TRUE + `else + A_FALSE + )", + R"( + `ifdef A + A_TRUE + `ifndef B + B_FALSE + `else + B_TRUE + `endif + )"}; + + for (auto test : test_cases) { + FlowTree tree_test(LexToSequence(test)); + auto status = tree_test.GenerateVariants( + [](const FlowTree::Variant& variant) { return true; }); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.message(), StartsWith("ERROR: Uncompleted")); + } +} + +TEST(FlowTree, NestedConditionals) { + const absl::string_view test_cases[] = { + R"( + `ifdef A + `ifdef B + A_B + `else + A_nB + `endif + `else + nA_B_or_nA_nB + `endif)"}; + + for (auto test : test_cases) { + FlowTree tree_test(LexToSequence(test)); + std::vector variants; + auto status = tree_test.GenerateVariants( + [&variants](const FlowTree::Variant& variant) { + variants.push_back(variant); + return true; + }); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(variants.size(), 3); + for (const auto& variant : variants) { + EXPECT_EQ(variant.sequence.size(), 1); + if (variant.macros_mask.test(0) == 0) { + // Check that if A is undefined, then B is not visited. + EXPECT_FALSE(variant.visited.test(1)); + } else { + // Check that if A is defined, then B is visited. + EXPECT_TRUE(variant.visited.test(1)); + } + } + const auto& used_macros = tree_test.GetUsedMacros(); + EXPECT_EQ(used_macros.size(), 2); + EXPECT_THAT(used_macros[0]->text(), "A"); + EXPECT_THAT(used_macros[1]->text(), "B"); + } +} + +TEST(FlowTree, MultipleElseIfs) { + const absl::string_view test_case = + R"( + `ifdef A + A_TRUE + `elsif B + B_TRUE + `elsif EMPTY + `elsif C + C_TRUE + `endif)"; + + FlowTree tree_test(LexToSequence(test_case)); + std::vector variants; + auto status = + tree_test.GenerateVariants([&variants](const FlowTree::Variant& variant) { + variants.push_back(variant); + return true; + }); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(variants.size(), 5); + + const auto& used_macros = tree_test.GetUsedMacros(); + EXPECT_EQ(used_macros.size(), 4); + EXPECT_THAT(used_macros[0]->text(), "A"); + EXPECT_THAT(used_macros[1]->text(), "B"); + EXPECT_THAT(used_macros[2]->text(), "EMPTY"); + EXPECT_THAT(used_macros[3]->text(), "C"); + + // A is defined. + EXPECT_TRUE(variants[0].macros_mask.test(0)); + EXPECT_THAT(variants[0].sequence[0].text(), "A_TRUE"); + + // B is defined. + EXPECT_TRUE(variants[1].macros_mask.test(1)); + EXPECT_THAT(variants[1].sequence[0].text(), "B_TRUE"); + + // EMPTY is defined. + EXPECT_TRUE(variants[2].macros_mask.test(2)); + EXPECT_TRUE(variants[2].sequence.empty()); + + // C is defined. + EXPECT_TRUE(variants[3].macros_mask.test(3)); + EXPECT_THAT(variants[3].sequence[0].text(), "C_TRUE"); +} + +TEST(FlowTree, SwappedNegatedIfs) { + const absl::string_view test_case = + R"( + `ifndef A + A_FALSE + `elsif B + B_TRUE + `endif + + `ifndef B + B_FALSE + `elsif A + A_TRUE + `endif)"; + + FlowTree tree_test(LexToSequence(test_case)); + std::vector variants; + auto status = + tree_test.GenerateVariants([&variants](const FlowTree::Variant& variant) { + variants.push_back(variant); + return true; + }); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(variants.size(), 4); + + const auto& used_macros = tree_test.GetUsedMacros(); + EXPECT_EQ(used_macros.size(), 2); + EXPECT_THAT(used_macros[0]->text(), "A"); + EXPECT_THAT(used_macros[1]->text(), "B"); + + EXPECT_THAT(variants[0].sequence[0].text(), "A_FALSE"); + EXPECT_THAT(variants[0].sequence[1].text(), "B_FALSE"); + + EXPECT_THAT(variants[1].sequence[0].text(), "A_FALSE"); + + EXPECT_THAT(variants[2].sequence[0].text(), "B_TRUE"); + EXPECT_THAT(variants[2].sequence[1].text(), "A_TRUE"); + + EXPECT_THAT(variants[3].sequence[0].text(), "B_FALSE"); +} + +TEST(FlowTree, CompleteConditional) { + const absl::string_view test_case = + R"( + `ifdef A + A_TRUE + `elsif B + B_TRUE + `elsif C + C_TRUE + `else + ALL_FALSE + `endif)"; + + FlowTree tree_test(LexToSequence(test_case)); + std::vector variants; + auto status = + tree_test.GenerateVariants([&variants](const FlowTree::Variant& variant) { + variants.push_back(variant); + return true; + }); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(variants.size(), 4); + + const auto& used_macros = tree_test.GetUsedMacros(); + EXPECT_EQ(used_macros.size(), 3); + EXPECT_THAT(used_macros[0]->text(), "A"); + EXPECT_THAT(used_macros[1]->text(), "B"); + EXPECT_THAT(used_macros[2]->text(), "C"); + + EXPECT_TRUE(variants[0].macros_mask.test(0)); + EXPECT_THAT(variants[0].sequence[0].text(), "A_TRUE"); + + EXPECT_TRUE(variants[1].macros_mask.test(1)); + EXPECT_THAT(variants[1].sequence[0].text(), "B_TRUE"); + + EXPECT_TRUE(variants[2].macros_mask.test(2)); + EXPECT_THAT(variants[2].sequence[0].text(), "C_TRUE"); + + EXPECT_THAT(variants[3].sequence[0].text(), "ALL_FALSE"); +} + +} // namespace +} // namespace verilog diff --git a/verilog/tools/preprocessor/BUILD b/verilog/tools/preprocessor/BUILD index 4216fe9e8..3df875b7b 100644 --- a/verilog/tools/preprocessor/BUILD +++ b/verilog/tools/preprocessor/BUILD @@ -14,6 +14,10 @@ cc_binary( "//common/util:init_command_line", "//common/util:subcommand", "//verilog/transform:strip_comments", + "//verilog/preprocessor:verilog_preprocess", + "//verilog/parser:verilog_lexer", + "//verilog/analysis:verilog_analyzer", + "//verilog/analysis:flow_tree", "@com_google_absl//absl/flags:usage", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", diff --git a/verilog/tools/preprocessor/verilog_preprocessor.cc b/verilog/tools/preprocessor/verilog_preprocessor.cc index 6e0121e83..b37e4084b 100644 --- a/verilog/tools/preprocessor/verilog_preprocessor.cc +++ b/verilog/tools/preprocessor/verilog_preprocessor.cc @@ -1,4 +1,4 @@ -// Copyright 2017-2020 The Verible Authors. +// Copyright 2017-2022 The Verible Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,18 +16,27 @@ #include #include +#include "absl/flags/flag.h" #include "absl/flags/usage.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "common/lexer/token_stream_adapter.h" #include "common/util/file_util.h" #include "common/util/init_command_line.h" #include "common/util/subcommand.h" +#include "verilog/analysis/flow_tree.h" +#include "verilog/analysis/verilog_analyzer.h" +#include "verilog/parser/verilog_lexer.h" +#include "verilog/parser/verilog_token_enum.h" +#include "verilog/preprocessor/verilog_preprocess.h" #include "verilog/transform/strip_comments.h" using verible::SubcommandArgsRange; using verible::SubcommandEntry; +ABSL_FLAG(int, limit_variants, 20, "Maximum number of variants printed"); + static absl::Status StripComments(const SubcommandArgsRange& args, std::istream&, std::ostream& outs, std::ostream&) { @@ -63,25 +72,149 @@ static absl::Status StripComments(const SubcommandArgsRange& args, return absl::OkStatus(); } +static absl::Status PreprocessSingleFile(absl::string_view source_file, + std::ostream& outs, + std::ostream& message_stream) { + std::string source_contents; + if (auto status = verible::file::GetContents(source_file, &source_contents); + !status.ok()) { + message_stream << source_file << status; + return status; + } + verilog::VerilogPreprocess::Config config; + config.filter_branches = 1; + // config.expand_macros=1; + verilog::VerilogPreprocess preprocessor(config); + verilog::VerilogLexer lexer(source_contents); + verible::TokenSequence lexed_sequence; + for (lexer.DoNextToken(); !lexer.GetLastToken().isEOF(); + lexer.DoNextToken()) { + // For now we will store the syntax tree tokens only, ignoring all the + // white-space characters. however that should be stored to output the + // source code just like it was, but with conditionals filtered. + if (verilog::VerilogLexer::KeepSyntaxTreeTokens(lexer.GetLastToken())) + lexed_sequence.push_back(lexer.GetLastToken()); + } + verible::TokenStreamView lexed_streamview; + // Initializing the lexed token stream view. + InitTokenStreamView(lexed_sequence, &lexed_streamview); + verilog::VerilogPreprocessData preprocessed_data = + preprocessor.ScanStream(lexed_streamview); + auto& preprocessed_stream = preprocessed_data.preprocessed_token_stream; + for (auto u : preprocessed_stream) outs << *u << '\n'; + for (auto& u : preprocessed_data.errors) outs << u.error_message << '\n'; + + return absl::OkStatus(); +} + +static absl::Status MultipleCU(const SubcommandArgsRange& args, std::istream&, + std::ostream& outs, + std::ostream& message_stream) { + if (args.empty()) { + return absl::InvalidArgumentError("Missing file arguments."); + } + for (const char* source_file : args) { + message_stream << source_file << ":\n"; + auto status = PreprocessSingleFile(source_file, outs, message_stream); + if (!status.ok()) return status; + outs << '\n'; + } + return absl::OkStatus(); +} + +static absl::Status GenerateVariants(const SubcommandArgsRange& args, + std::istream&, std::ostream& outs, + std::ostream& message_stream) { + const int limit_variants = absl::GetFlag(FLAGS_limit_variants); + if (args.size() > 1) { + return absl::InvalidArgumentError( + "ERROR: generate-variants only works on one file."); + } + const char* source_file = args[0]; + std::string source_contents; + if (auto status = verible::file::GetContents(source_file, &source_contents); + !status.ok()) { + message_stream << source_file << status; + return status; + } + + // Lexing the input SV source code. + verilog::VerilogLexer lexer(source_contents); + verible::TokenSequence lexed_sequence; + for (lexer.DoNextToken(); !lexer.GetLastToken().isEOF(); + lexer.DoNextToken()) { + // For now we will store the syntax tree tokens only, ignoring all the + // white-space characters. however that should be stored to output the + // source code just like it was. + if (verilog::VerilogLexer::KeepSyntaxTreeTokens(lexer.GetLastToken())) { + lexed_sequence.push_back(lexer.GetLastToken()); + } + } + + // Control flow tree constructing. + verilog::FlowTree control_flow_tree(lexed_sequence); + int counter = 0; + auto status = control_flow_tree.GenerateVariants( + [limit_variants, &outs, &message_stream, + &counter](const verilog::FlowTree::Variant& variant) { + if (counter == limit_variants) return false; + counter++; + message_stream << "Variant number " << counter << ":\n"; + for (auto token : variant.sequence) outs << token << '\n'; + // TODO(karimtera): Consider creating an output file per vairant, + // Such that the files naming reflects which defines are + // defined/undefined. + return true; + }); + if (!status.ok()) { + return status; + } + + return absl::OkStatus(); +} + static const std::pair kCommands[] = { - {"strip-comments", // - {&StripComments, // + {"strip-comments", + {&StripComments, R"(strip-comments file [replacement-char] - Inputs: 'file' is a Verilog or SystemVerilog source file. Use '-' to read from stdin. - 'replacement-char' is a character to replace comments with. If not given, or given as a single space character, the comment contents and delimiters are replaced with spaces. If an empty string, the comment contents and delimiters are deleted. Newlines are not deleted. If a single character, the comment contents are replaced with the character. - Output: (stdout) Contents of original file with // and /**/ comments removed. )"}}, + {"multiple-compilation-unit", + {&MultipleCU, + R"(multiple-compilation-unit file [more_files] +Inputs: + 'file' is a Verilog or SystemVerilog source file. + There can be multiple SystemVerilog source files. + Each one of them will be prepropcessed separatly which means that declarations + scoopes will end by the end of each file, and won't be seen from other files. +Output: (stdout) + The preprocessed files content (same contents with directives interpreted). +)"}}, + {"generate-variants", + {&GenerateVariants, + R"(generate-variants file [-limit_variants number] +Inputs: + 'file' is a Verilog or SystemVerilog source file. + '-limit_variants' flag limits variants to 'number' (20 by default). +Output: (stdout) + Generates every possible variant considering the conditional directives. +)"}}, + // TODO(karimtera): We can add another argument to `generate-variants`, + // Which allows us to set some defines, as if we are only interested + // in the variants in which these defines are set. + + // TODO(karimtera): Another candidate subcommand is `list-defines`, + // Which would be the output of `GetUsedMacros()`. }; int main(int argc, char* argv[]) { diff --git a/verilog/tools/preprocessor/verilog_preprocessor_test.sh b/verilog/tools/preprocessor/verilog_preprocessor_test.sh index d13b89795..8373d087d 100755 --- a/verilog/tools/preprocessor/verilog_preprocessor_test.sh +++ b/verilog/tools/preprocessor/verilog_preprocessor_test.sh @@ -97,7 +97,6 @@ cat > "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_OUTPUT_FILE" + +status="$?" +[[ $status == 0 ]] || { + "Expected exit code 0, but got $status" + exit 0 +} + +diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { + exit 1 +} + +################################################################################ +echo "=== Test generate-variants: on a source file with nested conditionals" + +cat > "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 0 ]] || { + "Expected exit code 0, but got $status" + exit 0 +} + +diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { + exit 1 +} + +################################################################################ +echo "=== Test generate-variants: with limited variants" + +cat > "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 0 ]] || { + "Expected exit code 0, but got $status" + exit 0 +} + +diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { + exit 1 +} + +################################################################################ +echo "=== Test generate-variants: with invalid conditionals" + +cat > "$MY_INPUT_FILE" < "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 1 ]] || { + "Expected exit code 1, but got $status" + exit 0 +} + +################################################################################ +echo "=== Test generate-variants: with multiple files" + +cat > "$MY_INPUT_FILE" < "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 1 ]] || { + "Expected exit code 1, but got $status" + exit 0 +} + +################################################################################ +echo "=== Test multiple-compilation-unit: with missing files" + +"$preprocessor" multiple-compilation-unit > "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 1 ]] || { + "Expected exit code 1, but got $status" + exit 0 +} + +################################################################################ +echo "=== Test generate-variants: with multiple files" + +cat > "$MY_INPUT_FILE" < "$MY_OUTPUT_FILE" 2>&1 + +status="$?" +[[ $status == 1 ]] || { + "Expected exit code 1, but got $status" + exit 0 +} + +################################################################################ +echo "=== Test multiple-compilation-unit: with multiple files" + +cat > "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_OUTPUT_FILE" + +status="$?" +[[ $status == 0 ]] || { + "Expected exit code 0, but got $status" + exit 0 +} + +diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { + exit 1 +} ################################################################################ echo "PASS"