2023-20262-release-incremental-compilation-mark-packing-dirty-on-sta_p_v_t_corner-parameter-change

src/Compiler/CommandWrapper.cpp

@@ -27,5 +27,6 @@ std::filesystem::path ScriptRenderer::s_projectPath;
 
 std::filesystem::path CommandWrapper::s_projectPath;
 std::unordered_set<std::string> CommandWrapper::s_bigFilesSet = {};
+bool CommandWrapper::s_enableLog = false;
 
 }  // namespace FOEDAG

src/Compiler/CommandWrapper.h

@@ -34,6 +34,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <regex>
 #include <memory>
 
+#include <QDebug>
 // #include <iostream>
 
 // #define DEBUG_ENABLE_PARANOIC_CHECK
@@ -463,6 +464,7 @@ CompilationFilesScopedSession& operator=(CompilationFilesScopedSession&&) = dele
 class CommandWrapper {
   static std::filesystem::path s_projectPath;
   static std::unordered_set<std::string> s_bigFilesSet;
+  static bool s_enableLog;
 
 public:
   CommandWrapper()=default;
@@ -484,6 +486,13 @@ class CommandWrapper {
   static const std::filesystem::path& projectPath() { return s_projectPath; }
   static void addBigFileName(const std::string& fileName) { s_bigFilesSet.insert(fileName); }
 
+  // Developer toggle for verbose CommandWrapper debug logging. Enabled once at
+  // startup from the AURORA_INCR_COMPILATOR_LOG environment variable (see
+  // TaskCompilationStateManager), so debugging can be turned on at runtime
+  // without recompiling the app.
+  static void setLogEnabled(bool enabled) { s_enableLog = enabled; }
+  static bool isLogEnabled() { return s_enableLog; }
+
   DiffCommandPtr collectDiff(const CommandWrapper& old) {
     DiffCommandPtr diff = std::make_shared<DiffCommand>();
     compareArguments(old.arguments(), arguments(), diff);
@@ -511,15 +520,15 @@ class CommandWrapper {
     appendArgument(parameter, value);
   }
 
-  void appendFile(const std::string& filePath)=delete;
-  void appendFile(const std::filesystem::path& file, const std::string& mask = "") {
+  void appendPath(const std::string& filePath)=delete;
+  void appendPath(const std::filesystem::path& file, const std::string& mask = "") {
     appendArgument(file.string(), "", mask);
-    handleFile(file, mask);
+    handlePath(file, mask);
   }
-  void appendFile(const std::string& parameter, const std::string& file)=delete;
-  void appendFile(const std::string& parameter, const std::filesystem::path& file, const std::string& mask = "") {
+  void appendPath(const std::string& parameter, const std::string& file)=delete;
+  void appendPath(const std::string& parameter, const std::filesystem::path& file, const std::string& mask = "") {
     appendArgument(parameter, file.string(), mask);
-    handleFile(file, mask);
+    handlePath(file, mask);
   }
 
   void prepend(const std::string& parameter, const std::string& value = "") {
@@ -529,13 +538,13 @@ class CommandWrapper {
   void prependFile(const std::string& file)=delete;
   void prependFile(const std::filesystem::path& file, const std::string& mask = "") {
     prependArgument(file.string(), "", mask);
-    handleFile(file, mask);
+    handlePath(file, mask);
   }
 
   void prependFile(const std::string& parameter, const std::string& file)=delete;
   void prependFile(const std::string& parameter, const std::filesystem::path& file, const std::string& mask) {
     prependArgument(parameter, file.string(), mask);
-    handleFile(file, mask);
+    handlePath(file, mask);
   }
 
 private:
@@ -605,30 +614,57 @@ class CommandWrapper {
     }
   }
 
-  void handleFile(const std::filesystem::path& file, const std::string& mask) {
-    std::filesystem::path resolvedFilePath(file);
-    if (file.is_relative() && !s_projectPath.empty()) {
-      resolvedFilePath = s_projectPath / file;
+  void resolveProjectRelativePath(std::filesystem::path& path) {
+    if (path.is_relative() && !s_projectPath.empty()) {
+      path = s_projectPath / path;
     }
+  }
 
-    if (!std::filesystem::exists(resolvedFilePath)) {
+  void handlePath(const std::filesystem::path& file, const std::string& mask) {
+    std::filesystem::path resolvedPath(file);
+    resolveProjectRelativePath(resolvedPath);
+
+    if (!std::filesystem::exists(resolvedPath)) {
       return;
     }
 
+    if (std::filesystem::is_directory(resolvedPath)) {
+      handeDir(resolvedPath);
+    } else {
+      handleFile(resolvedPath, mask);
+    }
+  }
+
+  void handeDir(const std::filesystem::path& dir) {
+    if (isLogEnabled()) {
+      qInfo() << "commandwrapper::handleDir" << dir.string().c_str();
+    }
+
+    std::vector<std::filesystem::path> files = FileUtils::FindAbsoluteFilePathsRecursively(dir);
+    for (const auto& file: files) {
+      handleFile(file);
+    }
+  }
+
+  void handleFile(const std::filesystem::path& file, const std::string& mask = "") {
+    if (isLogEnabled()) {
+      qInfo() << "commandwrapper::handleFile" << file.string().c_str();
+    }
+
     bool skipHashCheck = false;
-    auto it = s_bigFilesSet.find(resolvedFilePath.filename().string());
+    auto it = s_bigFilesSet.find(file.filename().string());
     if (it != s_bigFilesSet.end()) {
       skipHashCheck = true;
     }
-    if (resolvedFilePath.extension() == ".bin") {
+    if (file.extension() == ".bin") {
       skipHashCheck = true; // calc md5 sum for big bin files takes a lot of time
     }
 
-    std::string key = mask.empty()? resolvedFilePath.string(): mask;
+    std::string key = mask.empty()? file.string(): mask;
     if (FilesIdentityCache::instance().isEnabled()) {
-      m_files[key] = FilesIdentityCache::instance().get(resolvedFilePath, mask, skipHashCheck);
+      m_files[key] = FilesIdentityCache::instance().get(file, mask, skipHashCheck);
     } else {
-      m_files[key] = std::make_shared<FileIdentity>(resolvedFilePath, mask, skipHashCheck);
+      m_files[key] = std::make_shared<FileIdentity>(file, mask, skipHashCheck);
     }
   }
 

src/Compiler/CompilerOpenFPGA_ql.cpp

@@ -2567,9 +2567,9 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
           QLSettingsManager::getStringValue("vpr", "filename", "net_file") + " , specified in vpr>filename>net_file setting. \n");
         return nullptr;
     }
-    command->appendFile("--net_file", QLSettingsManager::getPathValue("vpr", "filename", "net_file"));
+    command->appendPath("--net_file", QLSettingsManager::getPathValue("vpr", "filename", "net_file"));
   } else {
-    command->appendFile("--net_file", std::filesystem::path{netlistFilePrefix + std::string(".net")});
+    command->appendPath("--net_file", std::filesystem::path{netlistFilePrefix + std::string(".net")});
   }
 
   if (cfg.use_place_file) {
@@ -2579,9 +2579,9 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
           QLSettingsManager::getStringValue("vpr", "filename", "place_file") + " , specified in vpr>filename>place_file setting. \n");
           return nullptr;
       }
-      command->appendFile("--place_file", QLSettingsManager::getPathValue("vpr", "filename", "place_file"));
+      command->appendPath("--place_file", QLSettingsManager::getPathValue("vpr", "filename", "place_file"));
     } else {
-      command->appendFile("--place_file", std::filesystem::path{netlistFilePrefix + std::string(".place")});
+      command->appendPath("--place_file", std::filesystem::path{netlistFilePrefix + std::string(".place")});
     }
   }
 
@@ -2592,9 +2592,9 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
           QLSettingsManager::getStringValue("vpr", "filename", "route_file") + " , specified in vpr>filename>route_file setting. \n");
           return nullptr;
         }
-      command->appendFile("--route_file", QLSettingsManager::getPathValue("vpr", "filename", "route_file"));
+      command->appendPath("--route_file", QLSettingsManager::getPathValue("vpr", "filename", "route_file"));
     } else {
-      command->appendFile("--route_file", std::filesystem::path{netlistFilePrefix + std::string(".route")});
+      command->appendPath("--route_file", std::filesystem::path{netlistFilePrefix + std::string(".route")});
     }
   }
 
@@ -2611,7 +2611,7 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
   // if we have a valid sdc_file_path at this point, pass it on to vpr:
   if(!sdc_file_path.empty()) {
     Message(std::string("SDC file found: ") + sdc_file_path.string());
-    command->appendFile("--sdc_file", sdc_file_path);
+    command->appendPath("--sdc_file", sdc_file_path);
   }
   else {
     Message(std::string("SDC file not found, no constraints passed to vpr."));
@@ -2620,7 +2620,7 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
 
 
   if( !QLSettingsManager::getStringValue("vpr", "filename", "write_rr_graph").empty() ) {
-    command->appendFile("--write_rr_graph", QLSettingsManager::getPathValue("vpr", "filename", "write_rr_graph"));
+    command->appendPath("--write_rr_graph", QLSettingsManager::getPathValue("vpr", "filename", "write_rr_graph"));
   }
 
   // parse vpr netlist options
@@ -2750,8 +2750,8 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
       QLDeviceManager::getInstance()->deviceVPRRouterLookaheadFile(device_target);
 
   if(!rr_graph_file_path.empty() && !router_lookahead_file_path.empty()) {
-    command->appendFile("--read_rr_graph", rr_graph_file_path);
-    command->appendFile("--read_router_lookahead", router_lookahead_file_path);
+    command->appendPath("--read_rr_graph", rr_graph_file_path);
+    command->appendPath("--read_router_lookahead", router_lookahead_file_path);
   }
   else {
     // no rr_graph available to use, try to use dynamic rr_graph generation.
@@ -2766,8 +2766,8 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
 
       // Plaintext and encrypted CRR flows pass --sb_maps/--sb_templates
       // identically; encryption only adds --crypt_key_db.
-      command->appendFile("--sb_maps", m_SBMapsFile);
-      command->appendFile("--sb_templates", m_SBTemplatesDir);
+      command->appendPath("--sb_maps", m_SBMapsFile);
+      command->appendPath("--sb_templates", m_SBTemplatesDir);
 
       // VPR keys CRR encryption solely off the ".en" suffix on --sb_maps and
       // then requires --crypt_key_db (see VTR read_options.cpp). Keep FOEDAG in
@@ -2787,7 +2787,7 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
           ErrorMessage("Cannot find key database for encrypted device data: " + crypt_key_db.string());
           return nullptr;
         }
-        command->appendFile("--crypt_key_db", crypt_key_db);
+        command->appendPath("--crypt_key_db", crypt_key_db);
       }
 
       command->append("--preserve_input_pin_connections off");
@@ -2841,7 +2841,7 @@ CommandWrapperPtr CompilerOpenFPGA_ql::BaseVprCommand(QLDeviceTarget device_targ
     std::filesystem::path fp_constraint_filepath = ProjManager()->projectName() + "_constraints.xml";
     std::filesystem::path fp_constraint_filepath_absolute = std::filesystem::path(ProjManager()->projectPath()) / fp_constraint_filepath;
     if (fs::exists(fp_constraint_filepath_absolute)) {
-      command->appendFile("--read_vpr_constraints", std::filesystem::path{ProjManager()->projectName() + "_constraints.xml"});
+      command->appendPath("--read_vpr_constraints", std::filesystem::path{ProjManager()->projectName() + "_constraints.xml"});
     }
   }
   else { //IO floorplanning generation failed, must stop the flow
@@ -4757,7 +4757,7 @@ bool CompilerOpenFPGA_ql::TimingAnalysisHelper(const QLDeviceTarget& current_dev
         std::filesystem::is_regular_file(sdfFileName) &&
         std::filesystem::is_regular_file(sdcFileName)) {
       taCommand = BaseStaCommand();
-      taCommand->appendFile(BaseStaScript(libFileName, netlistFileName, sdfFileName, sdcFileName));
+      taCommand->appendPath(BaseStaScript(libFileName, netlistFileName, sdfFileName, sdcFileName));
 
       FileUtils::WriteToFile(sta_cmd_filepath, taCommand->string());
     } else {
@@ -9975,7 +9975,7 @@ CommandWrapperPtr CompilerOpenFPGA_ql::getPlacementCommand() {
 
 
   // if (!filepath_fpga_fix_pins_place_str.empty()) {
-  //   command->appendFile("--fix_clusters", std::filesystem::path(filepath_fpga_fix_pins_place_str));
+  //   command->appendPath("--fix_clusters", std::filesystem::path(filepath_fpga_fix_pins_place_str));
   // }
   // else
   // {

src/Compiler/TaskCompilationStateManager.h

@@ -30,14 +30,20 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #include <unordered_map>
 #include <iostream>
+#include <cstdlib>
 
 namespace FOEDAG {
 
 constexpr const char* COMPILATION_CACHE_FILENAME = "compilation_cache.json";
 class TaskCompilationStateManager {
 public:
   TaskCompilationStateManager(Compiler* compiler): m_compiler(compiler) {
-
+    // Read the developer log toggle once and propagate it to CommandWrapper, so
+    // verbose logging can be enabled at runtime via the environment variable
+    // without recompiling the app.
+    static const bool logEnabled =
+        std::getenv("AURORA_INCR_COMPILATOR_LOG") != nullptr;
+    CommandWrapper::setLogEnabled(logEnabled);
   }
 
   bool isCompilationRequired(int taskId, const CommandWrapperPtr& command) const {
@@ -90,7 +96,6 @@ class TaskCompilationStateManager {
   }
 
 private:
-  bool m_isLogEnabled = false;
   Compiler* m_compiler = nullptr; // used for log messages
   std::unordered_map<std::string, CommandWrapperPtr> m_taskCommandsMap;
   std::filesystem::path m_filePath{COMPILATION_CACHE_FILENAME};
@@ -117,15 +122,15 @@ class TaskCompilationStateManager {
 
     auto it = m_taskCommandsMap.find(id);
     if (it == m_taskCommandsMap.end()) {
-      if (m_isLogEnabled) {
+      if (CommandWrapper::isLogEnabled()) {
         m_compiler->Message(logPrefix() + "task [" + id + "] wasn't previously compiled");
       }
       return true;
     }
     const CommandWrapper& commandOld = *it->second;
     DiffCommandPtr diff = command->collectDiff(commandOld);
     if (!diff->isEmpty()) {
-      if (m_isLogEnabled) {
+      if (CommandWrapper::isLogEnabled()) {
         for (const std::string& msg: diff->messages()) {
           m_compiler->Message(logPrefix() + "task(" + id + "), detects " + msg);
         }

src/Utils/FileUtils.cpp

@@ -244,6 +244,25 @@ std::filesystem::path candidate = searchPath / filename;
   return result;
 }
 
+std::vector<std::filesystem::path> FileUtils::FindAbsoluteFilePathsRecursively(
+    const std::filesystem::path& searchPath) {
+  std::vector<std::filesystem::path> results{};
+  if (!FileUtils::FileIsDirectory(searchPath)) {
+    return results;
+  }
+  std::error_code ec;
+  std::filesystem::recursive_directory_iterator it(
+      searchPath, std::filesystem::directory_options::skip_permission_denied,
+      ec);
+  const std::filesystem::recursive_directory_iterator end;
+  for (; !ec && it != end; it.increment(ec)) {
+    if (FileUtils::FileIsRegular(it->path())) {
+      results.push_back(std::filesystem::absolute(it->path()));
+    }
+  }
+  return results;
+}
+
 // This will search the given paths (non-recursively) for a child file.
 // All matches will be returned in a vector
 std::vector<std::filesystem::path> FileUtils::FindFileInDirs(

src/Utils/FileUtils.h

@@ -69,6 +69,12 @@ class FileUtils final {
   static std::filesystem::path LocateFileRecursive(
       const std::filesystem::path& searchPath, const std::string filename);
 
+  // Recursively collect every regular file under searchPath as an absolute
+  // path. Returns an empty vector if searchPath does not exist or is not a
+  // directory.
+  static std::vector<std::filesystem::path> FindAbsoluteFilePathsRecursively(
+      const std::filesystem::path& searchPath);
+
   static std::vector<std::filesystem::path> FindFileInDirs(
       const std::string& filename,
       const std::vector<std::filesystem::path>& searchPaths,

tests/unittest/CMakeLists.txt

@@ -49,6 +49,7 @@ set (CPP_LIST
     Compiler/CompilerDefines_test.cpp
     Compiler/BlifParser_test.cpp
     Compiler/CommandWrapper_test.cpp
+    Compiler/TaskCompilationStateManager_test.cpp
     PinAssignment/PortsModel_test.cpp
     PinAssignment/PinAssignmentBaseView_test.cpp
     Simulation/Simulation_test.cpp