|
| 1 | +# Copyright 2026 APSL-Nagarro Antoni Marroig |
| 2 | +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). |
| 3 | + |
| 4 | +import gc |
| 5 | +import logging |
| 6 | + |
| 7 | +from odoo import models |
| 8 | + |
| 9 | +_logger = logging.getLogger(__name__) |
| 10 | + |
| 11 | + |
| 12 | +class FsFileGc(models.Model): |
| 13 | + _inherit = "fs.file.gc" |
| 14 | + |
| 15 | + # S3 bulk delete limit is 1000 objects per request |
| 16 | + _GC_BATCH_SIZE = 1000 |
| 17 | + |
| 18 | + def _gc_files_unsafe(self) -> None: |
| 19 | + """ |
| 20 | + Overrides to optimize S3 storage using bulk deletes and |
| 21 | + delegates other storage types to the super() method. |
| 22 | + """ |
| 23 | + # 1. HEAVY LIFTING: Clean up S3 storages first using mass deletion |
| 24 | + self._gc_s3_bulk_delete() |
| 25 | + |
| 26 | + # 2. DELEGATION: Call super() for any remaining storage types |
| 27 | + # Since S3 records have already been removed from fs_file_gc, |
| 28 | + # super() will only process remaining types (Filestore, SFTP, etc.) |
| 29 | + # without hitting timeouts. |
| 30 | + return super()._gc_files_unsafe() |
| 31 | + |
| 32 | + def _gc_s3_bulk_delete(self): |
| 33 | + """ |
| 34 | + Specific logic for S3 using Boto3 Batch. |
| 35 | + Filters storages in memory since autovacuum_gc is not a stored field. |
| 36 | + """ |
| 37 | + # Search for all storages and filter by the computed field in Python |
| 38 | + all_storages = self.env["fs.storage"].search([]) |
| 39 | + # We filter those with autovacuum active and verify it's an S3-based path |
| 40 | + storages_to_gc = all_storages.filtered( |
| 41 | + lambda s: s.autovacuum_gc and s.is_s3_storage |
| 42 | + ) |
| 43 | + |
| 44 | + for storage in storages_to_gc: |
| 45 | + code = storage.code |
| 46 | + |
| 47 | + try: |
| 48 | + # directory_path usually contains the bucket name in S3 configurations |
| 49 | + bucket_name = storage.get_directory_path().strip("/") |
| 50 | + root_fs = storage._get_root_filesystem() |
| 51 | + |
| 52 | + s3_client = root_fs.s3 |
| 53 | + |
| 54 | + if not s3_client or not bucket_name: |
| 55 | + _logger.warning( |
| 56 | + "GC: Could not retrieve S3 client or Bucket name for %s", code |
| 57 | + ) |
| 58 | + continue |
| 59 | + |
| 60 | + _logger.info( |
| 61 | + "GC: Starting optimized S3 cleanup for %s in bucket %s", |
| 62 | + code, |
| 63 | + bucket_name, |
| 64 | + ) |
| 65 | + |
| 66 | + while True: |
| 67 | + # Select the next batch of orphaned files |
| 68 | + self._cr.execute( |
| 69 | + """ |
| 70 | + SELECT store_fname |
| 71 | + FROM fs_file_gc |
| 72 | + WHERE fs_storage_code = %s |
| 73 | + AND NOT EXISTS ( |
| 74 | + SELECT 1 FROM ir_attachment |
| 75 | + WHERE store_fname = fs_file_gc.store_fname |
| 76 | + ) |
| 77 | + LIMIT %s |
| 78 | + """, |
| 79 | + (code, self._GC_BATCH_SIZE), |
| 80 | + ) |
| 81 | + |
| 82 | + rows = self._cr.fetchall() |
| 83 | + if not rows: |
| 84 | + break |
| 85 | + |
| 86 | + fnames = [row[0] for row in rows] |
| 87 | + # Prepare the keys by removing the protocol prefix (e.g., s3://) |
| 88 | + objects_to_delete = [{"Key": f.partition("://")[2]} for f in fnames] |
| 89 | + |
| 90 | + try: |
| 91 | + # Perform mass deletion (1 HTTP request per 1000 files) |
| 92 | + _logger.info( |
| 93 | + "GC: Sending bulk delete request to S3 (%s files)", |
| 94 | + len(objects_to_delete), |
| 95 | + ) |
| 96 | + self.env["fs.storage"]._s3_call_delete_objects( |
| 97 | + s3_client, |
| 98 | + Bucket=bucket_name, |
| 99 | + Delete={"Objects": objects_to_delete}, |
| 100 | + ) |
| 101 | + # Mass delete from database |
| 102 | + self._cr.execute( |
| 103 | + "DELETE FROM fs_file_gc WHERE store_fname = ANY(%s)", |
| 104 | + (fnames,), |
| 105 | + ) |
| 106 | + |
| 107 | + # Commit per batch: releases locks and ensures progress is saved |
| 108 | + self._cr.commit() # pylint: disable=invalid-commit |
| 109 | + except Exception as e: |
| 110 | + _logger.error( |
| 111 | + "GC Error: Failed S3 delete_objects for %s: %s", code, e |
| 112 | + ) |
| 113 | + # We break the loop for this storage |
| 114 | + # to prevent DB deletion if S3 fails |
| 115 | + break |
| 116 | + |
| 117 | + # Explicit memory cleanup |
| 118 | + gc.collect() |
| 119 | + |
| 120 | + except Exception as e: |
| 121 | + _logger.error( |
| 122 | + "GC Error: Could not initialize S3 storage %s: %s", code, e |
| 123 | + ) |
| 124 | + continue |
| 125 | + |
| 126 | + _logger.info("GC: Optimized S3 cleanup process finished.") |
0 commit comments