Add automated Sitemap creation

.gitignore

@@ -26,7 +26,8 @@ yarn-debug.log*
 /config/puma.rb
 
 # Ignore public uploaded files
-public/uploads
+public/uploads/
+public/sitemaps/
 
 # Ignore public/assets as assets are generated in each server when deploying
 public/assets
@@ -41,9 +42,10 @@ public/sw.js
 spec/decidim_dummy_app/
 .rspec-failures
 
-# Ignore idea config files
+# Ignore IDE config files
 .idea
 *.iml
+.vscode/
 
 coverage/
 storage/

Gemfile

@@ -36,6 +36,7 @@ gem "daemons"
 gem "deface"
 gem "delayed_job_active_record"
 
+gem "sitemap_generator", "~> 7.0"
 gem "whenever", require: false
 
 gem "recaptcha"

Gemfile.lock

@@ -860,6 +860,8 @@ GEM
       simplecov (~> 0.19)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
+    sitemap_generator (7.0.1)
+      builder (~> 3.0)
     smart_properties (1.17.0)
     snaky_hash (2.0.3)
       hashie (>= 0.1.0, < 6)
@@ -974,6 +976,7 @@ DEPENDENCIES
   rubocop-faker
   rubocop-rspec
   rubocop-rspec_rails
+  sitemap_generator (~> 7.0)
   soda-ruby
   stringio (~> 3.1.7)
   web-console

config/schedule.rb

@@ -22,6 +22,10 @@
   rake "tmp:clear"
 end
 
+every 1.day, at: "3:15 am" do
+  rake "sitemap:generate"
+end
+
 every 5.minutes do
   rake "participatory_processes_phases:enqueue_change_active_step"
 end

config/sitemap.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+# ---------------------------------------------------------------------------
+# Sitemap configuration for participa.gencat.cat
+#
+# Groups and their content:
+#   processes      – participatory process landing pages + component pages
+#   regulations    – /regulations index + regulation process + component pages
+#   assemblies     – assembly landing pages + component pages
+#   meetings       – individual meeting show pages (all spaces)
+#   proposals      – individual proposal show pages (all spaces)
+#   pages          – homepage, listing pages, static CMS pages
+#   blogs          – individual blog post show pages (all spaces)
+#   debates        – individual debate show pages (all spaces)
+#   budgets        – individual budget project show pages (all spaces)
+#   accountability – individual accountability result show pages (all spaces)
+#   attachments    – PDF and image file blob URLs
+#
+# Generated files land in public/sitemaps/.
+# The master index is public/sitemaps/sitemap.xml.gz.
+# ---------------------------------------------------------------------------
+
+# Make Rails route helpers (incl. rails_blob_path) available inside the
+# create block and all group blocks.
+SitemapGenerator::Interpreter.send(:include, Rails.application.routes.url_helpers)
+
+# Helper: returns the URL prefix for a participatory space, or nil for
+# space types that are not publicly routable.
+SitemapGenerator::Interpreter.class_eval do
+  def space_url_prefix(space)
+    if space.is_a?(Decidim::ParticipatoryProcess)
+      "/processes"
+    elsif space.is_a?(Decidim::Assembly)
+      "/assemblies"
+    end
+  end
+end
+
+organization = Decidim::Organization.first
+host = "https://#{organization.host}"
+regulation_group_id = Rails.application.config.regulation
+
+SitemapGenerator::Sitemap.default_host = host
+SitemapGenerator::Sitemap.sitemaps_path = "sitemaps/"
+SitemapGenerator::Sitemap.create_index = true
+SitemapGenerator::Sitemap.compress = true
+SitemapGenerator::Sitemap.include_root = false
+SitemapGenerator::Sitemap.include_index = false
+
+SitemapGenerator::Sitemap.create do
+  # ── Participatory Processes (excluding regulations) ────────────────────────
+  group(filename: :processes) do
+    Decidim::ParticipatoryProcess
+      .where(organization:)
+      .published
+      .where("decidim_participatory_process_group_id IS NULL OR decidim_participatory_process_group_id != ?", regulation_group_id)
+      .preload(:components)
+      .find_each do |process|
+        next if process.slug.blank?
+
+        add "/processes/#{process.slug}",
+            lastmod: process.updated_at, changefreq: "weekly", priority: 0.8
+
+        process.components.select { |c| c.published_at.present? }.each do |component|
+          add "/processes/#{process.slug}/f/#{component.id}",
+              lastmod: component.updated_at, changefreq: "weekly", priority: 0.6
+        end
+      end
+  end
+
+  # ── Regulations ─────────────────────────────────────────────────────────────
+  group(filename: :regulations) do
+    add "/regulations", changefreq: "weekly", priority: 0.7
+
+    Decidim::ParticipatoryProcess
+      .where(organization:, decidim_participatory_process_group_id: regulation_group_id)
+      .published
+      .preload(:components)
+      .find_each do |process|
+        next if process.slug.blank?
+
+        add "/processes/#{process.slug}",
+            lastmod: process.updated_at, changefreq: "monthly", priority: 0.7
+
+        process.components.select { |c| c.published_at.present? }.each do |component|
+          add "/processes/#{process.slug}/f/#{component.id}",
+              lastmod: component.updated_at, changefreq: "monthly", priority: 0.5
+        end
+      end
+  end
+
+  # ── Assemblies ──────────────────────────────────────────────────────────────
+  group(filename: :assemblies) do
+    Decidim::Assembly
+      .where(organization:)
+      .published
+      .preload(:components)
+      .find_each do |assembly|
+        next if assembly.slug.blank?
+
+        add "/assemblies/#{assembly.slug}",
+            lastmod: assembly.updated_at, changefreq: "weekly", priority: 0.8
+
+        assembly.components.select { |c| c.published_at.present? }.each do |component|
+          add "/assemblies/#{assembly.slug}/f/#{component.id}",
+              lastmod: component.updated_at, changefreq: "weekly", priority: 0.6
+        end
+      end
+  end
+
+  # ── Meetings ────────────────────────────────────────────────────────────────
+  group(filename: :meetings) do
+    Decidim::Meetings::Meeting
+      .joins(:component)
+      .where.not(decidim_components: { published_at: nil })
+      .where.not(published_at: nil)
+      .preload(component: :participatory_space)
+      .find_each do |meeting|
+        space = meeting.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{meeting.decidim_component_id}/meetings/#{meeting.id}",
+            lastmod: meeting.updated_at, changefreq: "weekly", priority: 0.5
+      end
+  end
+
+  # ── Proposals ───────────────────────────────────────────────────────────────
+  group(filename: :proposals) do
+    Decidim::Proposals::Proposal
+      .joins(:component)
+      .where.not(decidim_components: { published_at: nil })
+      .where.not(published_at: nil)
+      .preload(component: :participatory_space)
+      .find_each do |proposal|
+        space = proposal.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{proposal.decidim_component_id}/proposals/#{proposal.id}",
+            lastmod: proposal.updated_at, changefreq: "monthly", priority: 0.5
+      end
+  end
+
+  # ── Static Pages ────────────────────────────────────────────────────────────
+  group(filename: :pages) do
+    add "/", changefreq: "daily", priority: 1.0
+    add "/processes", changefreq: "daily", priority: 0.7
+    add "/assemblies", changefreq: "daily", priority: 0.7
+
+    Decidim::StaticPage
+      .where(organization:)
+      .find_each do |page|
+        add "/pages/#{page.slug}",
+            lastmod: page.updated_at, changefreq: "monthly", priority: 0.4
+      end
+  end
+
+  # ── Blog Posts ──────────────────────────────────────────────────────────────
+  group(filename: :blogs) do
+    Decidim::Blogs::Post
+      .joins(:component)
+      .where.not(decidim_components: { published_at: nil })
+      .preload(component: :participatory_space)
+      .find_each do |post|
+        space = post.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{post.decidim_component_id}/posts/#{post.id}",
+            lastmod: post.updated_at, changefreq: "monthly", priority: 0.5
+      end
+  end
+
+  # ── Debates ─────────────────────────────────────────────────────────────────
+  group(filename: :debates) do
+    Decidim::Debates::Debate
+      .joins(:component)
+      .where.not(decidim_components: { published_at: nil })
+      .preload(component: :participatory_space)
+      .find_each do |debate|
+        space = debate.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{debate.decidim_component_id}/debates/#{debate.id}",
+            lastmod: debate.updated_at, changefreq: "weekly", priority: 0.5
+      end
+  end
+
+  # ── Budget Projects ─────────────────────────────────────────────────────────
+  group(filename: :budgets) do
+    Decidim::Budgets::Project
+      .joins(budget: :component)
+      .where.not(decidim_components: { published_at: nil })
+      .preload(budget: { component: :participatory_space })
+      .find_each do |project|
+        space = project.budget.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{project.budget.decidim_component_id}" \
+            "/budgets/#{project.decidim_budgets_budget_id}/projects/#{project.id}",
+            lastmod: project.updated_at, changefreq: "monthly", priority: 0.4
+      end
+  end
+
+  # ── Accountability Results ───────────────────────────────────────────────────
+  group(filename: :accountability) do
+    Decidim::Accountability::Result
+      .joins(:component)
+      .where.not(decidim_components: { published_at: nil })
+      .preload(component: :participatory_space)
+      .find_each do |result|
+        space = result.component.participatory_space
+        prefix = space_url_prefix(space)
+        next unless prefix && space&.slug.present?
+
+        add "#{prefix}/#{space.slug}/f/#{result.decidim_component_id}/results/#{result.id}",
+            lastmod: result.updated_at, changefreq: "monthly", priority: 0.4
+      end
+  end
+
+  # ── Attachments (PDFs and images) ───────────────────────────────────────────
+  # Adds blob redirect URLs so search engines can index uploaded documents and
+  # images.  The ActiveStorage redirect path is stable (the signed_id does not
+  # expire).
+  group(filename: :attachments) do
+    Decidim::Attachment
+      .where("content_type LIKE 'image/%' OR content_type = 'application/pdf'")
+      .find_each do |attachment|
+        next unless attachment.file.attached?
+
+        blob_path = rails_blob_path(attachment.file, only_path: true)
+
+        add blob_path,
+            changefreq: "yearly",
+            priority: attachment.content_type == "application/pdf" ? 0.4 : 0.3
+      end
+  end
+end

lib/tasks/sitemap.rake

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+namespace :sitemap do
+  desc "Generate sitemaps, purge stale files, and update robots.txt"
+  task generate: :environment do
+    unless Rails.env.production?
+      puts "Sitemap generation is intended for production environments only. Aborting."
+      exit(-1)
+    end
+
+    sitemaps_dir = Rails.public_path.join("sitemaps")
+
+    # ── Step 1: Generate ────────────────────────────────────────────────────
+    generation_started_at = Time.current
+
+    SitemapGenerator::Sitemap.verbose = true
+    load Rails.root.join("config", "sitemap.rb")
+
+    # ── Step 2: Purge stale files ────────────────────────────────────────────
+    # Any sitemap file whose mtime predates this run was produced by a previous
+    # generation and is no longer valid (e.g. a group that produced N numbered
+    # files before now only produces N-1).
+    stale = Dir.glob(sitemaps_dir.join("*.xml.gz")).select do |file|
+      File.mtime(file) < generation_started_at
+    end
+
+    stale.each do |file|
+      File.delete(file)
+      puts "Removed stale sitemap: #{File.basename(file)}"
+    end
+
+    # ── Step 3: Rebuild robots.txt ───────────────────────────────────────────
+    organization = Decidim::Organization.first
+    host = "https://#{organization.host}"
+
+    new_sitemap_files = Dir.glob(sitemaps_dir.join("*.xml.gz"))
+    sitemap_directives = new_sitemap_files.map do |file|
+      "Sitemap: #{host}/sitemaps/#{File.basename(file)}"
+    end
+
+    robots_path = Rails.public_path.join("robots.txt")
+    current_content = File.read(robots_path)
+
+    # Strip any existing Sitemap: lines (including the trailing newline of each)
+    cleaned_content = current_content.gsub(/^Sitemap:.*\n?/, "").rstrip
+
+    new_content = "#{cleaned_content}\n\n#{sitemap_directives.join("\n")}\n"
+    File.write(robots_path, new_content)
+
+    puts "robots.txt updated with #{sitemap_directives.size} Sitemap directives."
+  end
+end

spec/factories.rb

@@ -4,6 +4,11 @@
 require "decidim/participatory_processes/test/factories"
 require "decidim/proposals/test/factories"
 require "decidim/meetings/test/factories"
+require "decidim/assemblies/test/factories"
+require "decidim/blogs/test/factories"
+require "decidim/debates/test/factories"
+require "decidim/budgets/test/factories"
+require "decidim/accountability/test/factories"
 
 FactoryBot.define do
   factory :external_author, class: "Decidim::ExternalAuthor" do