Merge branch 'master' into replace-efs-with-objectstore

Author: munishchouhan

config.yml

@@ -11,7 +11,4 @@ wave:
     195996028523.dkr.ecr.eu-west-1.amazonaws.com:
       username : "${AWS_ACCESS_KEY_ID:}"
       password : "${AWS_SECRET_ACCESS_KEY:}"
-    seqeralabs.azurecr.io:
-      username : "${AZURECR_USER:}"
-      password : "${AZURECR_PAT:}"
 ...

docs/configuration.mdx

@@ -1,6 +1,6 @@
-# Wave Application Configuration
+# Wave application configuration
 
-Set Wave configuration values using environment variables or in [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) configuration file
+Set Wave configuration values using environment variables or in the [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) configuration file.
 
 ### config.yml configuration
 
@@ -16,15 +16,15 @@ Environment variables for various attributes has been listed in third column, ei
 
 ## General configuration
 
-General configuration options such as wave application name, port, whether to allow anonymous access (without tower token), wave and Seqera platform url.
+General configuration options such as Wave application name, port, whether to allow anonymous access (without tower token), Wave and Seqera platform url.
 
 - **`micronaut.application.name`**: the name of the Wave application. By default, it is set to `wave-app`. *Optional*.
 
 - **`micronaut.server.port`**: the port used by the Wave server. The default port is `8080`. *Optional*.
 
-- **`wave.allowAnonymous`**: specifies whether anonymous access to the Wave server is permitted. By default, it is set to `false`, meaning authentication is required. Change this option based on your security requirements. *Mandatory*.
+- **`wave.allowAnonymous`**: specifies whether anonymous access to the Wave server is permitted. By default, it is set to `false`, meaning authentication is required. Change this option based on your security requirements.  *Mandatory*.
 
-- **`wave.server.url`**: the URL for the Wave server. You can also set it using `${WAVE_SERVER_URL}` environment variable. *Optional*.
+- **`wave.server.url`**: the URL for the Wave server. It can be set using `${WAVE_SERVER_URL}` environment variable. *Optional*.
 
 - **`wave.tokens.cache.duration`**: the duration for cached tokens generated by Wave. The default is `1h`, and you can modify it according to your token caching preferences. *Optional*.
 
@@ -36,10 +36,13 @@ General configuration options such as wave application name, port, whether to al
 ## Container registry configuration
 
 The generic format for the attributes is `wave.registries.<registry_name>.username` and `wave.registries.<registry_name>.password`.
-You need to specify all the repositories you will use in the respective wave installation.
+You need to specify all the repositories you will use in the respective Wave installation.
 Below are the standard format for known registries, but you can change registry name `(azurecr.io)` to specific one like `seqeralabs.azurecr.io `.
 
-**Note**: Container registry credentials can be defined in [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) too. These configurations are important for the wave authentication to the repositories used to push or pull artifacts.
+:::note
+Container registry credentials can be defined in [`config.yml`](https://github.com/seqeralabs/wave/blob/master/config.yml) too.
+These configurations are important for the Wave authentication to the repositories used to push or pull artifacts.
+:::
 
 - **`wave.registries.default`**: the default Docker registry for Wave. The default is `docker.io`, and it represents the Docker Hub. *Optional*.
 
@@ -55,10 +58,6 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.registries.<AWS ECR Repo name>.password`**: the AWS ECR password for authentication. An example AWS ECR Repo name is `195996028523.dkr.ecr.eu-west-1.amazonaws.com`. It can be set using `${AWS_SECRET_ACCESS_KEY}` environment variable. *Optional*.
 
-- **`wave.registries.<azurecr Repo name>.username`**: the Azure Container Registry username for authentication. An example of an Azure Container Registry Repo name is `seqeralabs.azurecr.io`. It can be set using `${AZURECR_USER}` environment variable. *Optional*.
-
-- **`wave.registries.<azurecr Repo name>.password`**: the Azure Container Registry password or PAT for authentication. An example of an Azure Container Registry Repo name is `seqeralabs.azurecr.io`. It can be set using `${AZURECR_PAT}` environment variable. *Optional*.
-
 ## HTTP client configuration
 
 - **`wave.httpclient.connectTimeout`**: sets the connection timeout for the HTTP client. Its default value is `20s`. *Optional*.
@@ -73,7 +72,7 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.httpclient.retry.multiplier`**: defines the multiplier for HTTP client retries. Its default value is `1.0`. *Optional*.
 
-- **`micronaut.http.services.stream-client.read-timeout`**: sets the read timeout for the streaming HTTP client. Its default value is `30s`. *Optional*.
+- **`micronaut.http.services.stream-client.read-timeout`**: sets the read timeout for the streaming HTTP client. Its default value is `30s`.  *Optional*.
 
 - **`micronaut.http.services.stream-client.read-idle-timeout`**: configures the read idle timeout for the streaming HTTP client. Its default value is `120s`. *Optional*.
 
@@ -93,15 +92,15 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.build.repo`**: specifies the Docker container repository for the Docker images built by Wave. This setting is required to define where the images will be stored. *Mandatory*.
 
-- **`wave.build.cache`**: determines the Docker container repository used to cache layers of images built by Wave. *Mandatory*.
+- **`wave.build.cache`**: determines the Docker container repository used to cache layers of images built by Wave.  *Mandatory*.
 
-- **`wave.build.status.delay`**: sets the delay between build status checks. Its default value is `5s`, providing a reasonable interval for status checks. *Optional*.
+- **`wave.build.status.delay`**: sets the delay between build status checks. Its default value is `5s`, providing a reasonable interval for status checks.  *Optional*.
 
-- **`wave.build.status.duration`**: defines the duration for build status checks. Its default value is `1d` (1 day), indicating how long the system should check the build status. *Optional*.
+- **`wave.build.status.duration`**: defines the duration for build status checks. Its default value is `1d` (1 day), indicating how long the system should check the build status.  *Optional*.
 
 - **`wave.build.public`**: indicates whether the Docker container repository is public. If set to true, Wave freeze will prefer this public repository over `wave.build.repo`. *Optional*.
 
-- **`wave.build.oci-mediatypes`**: defines whether to use OCI mediatypes in exported manifests. its default value is `true`. *Optional*.
+- **`wave.build.oci-mediatypes`**: defines whether to use OCI media types in exported manifests. Its default value is `true`. *Optional*.
 
 - **`wave.build.compression`**: defines which type of compression will be applied to cache layers. its default value is `gzip` and other options are `uncompressed|estargz|zstd`. *Optional*.
 
@@ -113,6 +112,10 @@ This configuration specifies attributes for the persistence of the logs fetched
 
 - **`wave.build.logs.bucket`**: the AWS S3 bucket where Wave will store build process logs. *Mandatory*.
 
+- **`wave.build.logs.path`**: the path inside  `wave.build.logs.bucket`, where build logs will be stored. *Mandatory*.
+
+- **`wave.build.locks.path`**: the path inside `wave.build.logs.bucket`, where conda lock files will be stored. *Mandatory*.
+
 - **`wave.build.logs.prefix`**: sets the prefix to be used for build process log files in the specified S3 bucket. *Optional*.
 
 - **`wave.build.logs.maxLength`**: determines the maximum number of bytes that can be read from a log file. If a log file exceeds this limit, it will be truncated. The default value is `100000` (100 KB). *Optional*.
@@ -122,7 +125,9 @@ This configuration specifies attributes for the persistence of the logs fetched
 
 Kubernetes configuration has options specific for k8s, and most of them, except CPU and memory, are the same for the build and scan process.
 
-**Note**: only applies when using Kubernetes.
+:::note
+These settings apply only when running on Kubernetes.
+:::
 
 - **`wave.build.k8s.namespace`**: the Kubernetes namespace where Wave will run its build pods. This is a required setting, specifying the namespace to isolate and manage the Wave build processes within the Kubernetes cluster. *Mandatory*.
 
@@ -156,7 +161,9 @@ Scan process configuration lets the user provide a [Trivy docker image](https://
 
 Wave scan process uses the same k8s configuration of the build process except for CPU and memory requirements for the k8s pod.
 
-**Note**: only applies when using Kubernetes.
+:::note
+These settings apply only when using Kubernetes.
+:::
 
 - **`wave.scan.k8s.resources.requests.cpu`**: the amount of [CPU resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes) allocated to Wave scan processes. For instance, you can set it to `2` or `1500Mi` (1.5 CPU cores). *Optional*.
 
@@ -166,7 +173,9 @@ Wave scan process uses the same k8s configuration of the build process except fo
 
 Rate limit configuration controls the limits of anonymous and authenticated users' access to Wave.
 
-**Note**: Change these properties to tweak rate limits in wave.
+:::note
+Change these properties to tweak rate limits in Wave.
+:::
 
 - **`rate-limit.build.anonymous`**: the rate limit for anonymous build requests. By default, it allows up to 25 build requests per hour (`25/1d`) from anonymous users. This setting controls the frequency at which anonymous users can trigger build processes in Wave. *Mandatory*.
 
@@ -180,19 +189,23 @@ Rate limit configuration controls the limits of anonymous and authenticated user
 
 ## Database and cache configuration
 
+### Redis configuration
+
 - **`redis.uri`**: the Uniform Resource Identifier (URI) for connecting to Redis, a popular in-memory data store. By default, it uses the format `redis://${REDIS_HOST:redis}:${REDIS_PORT:6379}`, allowing customization of the Redis host and port through environment variables. *Mandatory*.
 
 - **`redis.pool.enabled`**: whether to enable the Redis pool. It is set to `true` by default, enabling the use of a connection pool for efficient management of connections to the Redis server. *Optional*.
 
-- **`redis.pool.minIdle`**: Specifies the minimum number of idle connections to maintain in the Redis connection pool. The default value is `0`. This ensures that connections are readily available for use.  *Optional*.
+- **`redis.pool.minIdle`**: specifies the minimum number of idle connections to maintain in the Redis connection pool. The default value is `0`. This ensures that connections are readily available for use.  *Optional*.
 
-- **`redis.pool.maxIdle`**: Specifies the maximum number of idle connections to maintain in the Redis connection pool. The default value is `10`.  *Optional*.
+- **`redis.pool.maxIdle`**: specifies the maximum number of idle connections to maintain in the Redis connection pool. The default value is `10`.  *Optional*.
 
-- **`redis.pool.maxTotal`**: Specifies the maximum number of connections that can be maintained in the Redis connection pool. The default value is `50`. This helps to manage resource usage efficiently while supporting high demand.  *Optional*.
+- **`redis.pool.maxTotal`**: specifies the maximum number of connections that can be maintained in the Redis connection pool. The default value is `50`. This helps to manage resource usage efficiently while supporting high demand.  *Optional*.
 
-- **`redis.client.timeout`**: Defines the timeout duration (in milliseconds) for Redis client operations. The default value is `5000` (5 seconds).  *Optional*.
+- **`redis.client.timeout`**: defines the timeout duration (in milliseconds) for Redis client operations. The default value is `5000` (5 seconds).  *Optional*.
 
-- **`redis.password`**: Specifies the password used to authenticate with the Redis server. This is needed when redis authentication is enabled.  *Optional*.
+- **`redis.password`**: specifies the password used to authenticate with the Redis server. This is needed when redis authentication is enabled.  *Optional*.
+
+### SurrealDB configuration
 
 - **`surreal.default.ns`**: the namespace for the Surreal database. It can be set using `${SURREALDB_NS}` environment variable. *Mandatory*.
 
@@ -206,6 +219,14 @@ Rate limit configuration controls the limits of anonymous and authenticated user
 
 - **`surreal.default.init-db`**: whether to create database tables, records and indices at application startup  and `db`. *Optional*.
 
+### PostgreSQL configuration
+
+- **`wave.db.uri`**: JDBC connection string for the PostgreSQL database. For example, `jdbc:postgresql://localhost:5432/wave`. It can be set using `${WAVE_DB_URI}` environment variable.  *Mandatory*.
+
+- **`wave.db.user`**: Username for authenticating with the PostgreSQL database. It can be set using `${WAVE_DB_USER}` environment variable. *Mandatory*.
+
+- **`wave.db.password`**: Password for the PostgreSQL database user. It can be set using `${WAVE_DB_PASSWORD}` environment variable. *Mandatory*.
+
 ## Blob Cache configuration
 
 Wave offers a feature to provide a cache for Docker blobs, which improves the performance of supplying blobs to the client. If you use Kubernetes, Wave can also use the k8s pod to delegate the transfer task for scalability.
@@ -222,7 +243,7 @@ Wave offers a feature to provide a cache for Docker blobs, which improves the pe
 
 - **`wave.blobCache.baseUrl`**: the URL, which will override the base URL (part of URL before the blob path) of blobs sent to the end client. *Optional*.
 
-- **`wave.blobCache.signing-strategy`**: the URL signing strategy for different services. Currently, Wave offers it for AWS S3 and Cloudflare and you can use the respective values to enable them `aws-presigned-url` and `cloudflare-waf-token`. *Mandatory*. 
+- **`wave.blobCache.signing-strategy`**: the URL signing strategy for different services. Currently, Wave offers it for AWS S3 and Cloudflare and you can use the respective values to enable them `aws-presigned-url` and `cloudflare-waf-token`. *Mandatory*.
 
 - **`wave.blobCache.cloudflare.lifetime`**: the validity of the cloud flare WAF token. *Mandatory*.
 
@@ -246,20 +267,20 @@ Wave offers a feature to provide a cache for Docker blobs, which improves the pe
 
 ## Email configuration
 
-Email id to send emails from on the behalf of the Wave service.
+Email ID to send emails from on the behalf of the Wave service.
 
-- **`mail.from`**: specifies the sender's email address for Wave notifications. This email address serves as the point of origin for any emails sent by Wave, providing a recognizable source for notifications. This setting is crucial for configuring the sender identity when Wave sends email notifications. *Mandatory*.
+- **`mail.from`**: specifies the sender's email address for Wave notifications. This email address serves as the point of origin for any emails sent by Wave, providing a recognizable source for notifications. This setting is crucial for configuring the sender identity when Wave sends email notifications.  *Mandatory*.
 
 ## Metrics configuration
 
 Wave Metrics service provides data about container builds and pulls per ip, container image, and user.
 
-- **`wave.metrics.enabled`**: whether to enable the wave metrics. It is `false` by default. *Optional*.
+- **`wave.metrics.enabled`**: whether to enable the Wave metrics. It is `false` by default. *Optional*.
 
 ## Accounts configuration
 
-- **`wave.accounts`**: Specifies a list of credentials to access authenticated wave APIs  like metrics APIs. *Mandatory*.<br/>
+- **`wave.accounts`**: specifies a list of credentials to access authenticated Wave APIs like metrics APIs. *Mandatory*.<br/>
   Format of the credential list
     ```
-    - <USERNAME>:<PASSWORD CHECKSUM>
+    - <USERNAME>:<PASSWORD_CHECKSUM>
     ```

docs/db-migration.mdx

@@ -0,0 +1,63 @@
+# Wave Database Migration from SurrealDB to PostgreSQL
+
+## Pre-requisites
+- Install [wave-cli] (https://github.com/seqeralabs/wave-cli)
+- Ensure you have a PostgreSQL database set up and accessible from wave.
+
+## Migration Steps
+1. Create verification data, run some builds, scans, augmentation and mirror operations in Wave and note down buildId, scanId, token and mirrorId respectively for verification.
+#### commands:
+- ```bash
+    wave --conda-package bwa --wave-endpoint <WAVE_URL>
+    ```
+- ```bash
+    curl <WAVE_URL>/view/builds/<buildId>
+    ```
+- ```bash
+    curl <WAVE_URL>/view/scans/<scanId>
+    ```
+- ```bash
+    wave -i ubuntu --config-file <path_to_config_file> --wave-endpoint <WAVE_URL>
+    ```
+- ```bash
+    curl <WAVE_URL>/view/containers/<token>
+    ```
+- ```bash
+    wave --mirror  -i ubuntu --build-repo <repo_where_you_w>--wave-endpoint <WAVE_URL>
+    ```
+- ```bash
+    curl <WAVE_URL>/view/mirrors/<mirrorId>
+    ```
+1. Ensure you have all necessary credentials for both [SurrealDB](configuration.mdx#SurrealDB configuration) and [PostgreSQL](configuration.mdx#PostgreSQL configuration).
+2. Enable the following environment by adding them in `MICRONAUT_ENVIRONMENTS`:
+  - `postgres`
+  - `surrealdb`
+  - `migrate`
+  - `redis`
+3. Start Wave application and check whether the migration is successful by checking the logs.
+   ```
+   INFO  i.s.w.s.p.m.DataMigrationService - Data migration service initialized
+   ```
+4. check for the following logs:
+   ```
+   INFO  i.s.w.s.p.m.DataMigrationService - All wave_request records migrated.
+   INFO  i.s.w.s.p.m.DataMigrationService - All wave_scan records migrated.
+   INFO  i.s.w.s.p.m.DataMigrationService - All wave_build records migrated.
+   INFO  i.s.w.s.p.m.DataMigrationService - All wave_mirror records migrated.
+   ```
+5. Remove the `migrate` and `surrealdb`environment from `MICRONAUT_ENVIRONMENTS` to stop the migration process and restart wave.
+
+6. Verify the migration by checking the PostgreSQL database for the migrated data and also run same curl from step one.
+- ```bash
+  curl <WAVE_URL>/view/builds/<buildId>
+    ```
+- ```bash
+  curl <WAVE_URL>/view/scans/<scanId>
+    ```
+- ```bash
+  curl <WAVE_URL>/view/containers/<token>
+    ```
+- ```bash
+  curl <WAVE_URL>/view/mirrors/<mirrorId>
+    ```
+

docs/get-started.mdx

@@ -39,7 +39,7 @@ Nextflow can use the container that Seqera Containers built in the previous sect
 
 1. Create a `main.nf` file with the following contents:
 
-    ```groovy
+    ```nextflow
     process SAMTOOLS {
       container '<container_uri>'
       debug true
@@ -164,7 +164,7 @@ Nextflow can use Wave to seamlessly build a container directly from a Dockerfile
 
 1. Create a `wave.nf` file with the following contents:
 
-    ```groovy
+    ```nextflow
     include { HELLO } from './modules/gamma'
 
     workflow {
@@ -329,7 +329,7 @@ Nextflow can use the container that Wave froze to the build repository that you
 
 1. Create a `main.nf` file with the following contents:
 
-    ```groovy
+    ```nextflow
     process FAKER {
       container 'docker.io/example-user/repo:faker--2aa7a4d826a76301'
       debug true

src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy

@@ -54,10 +54,10 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis
     String quayPassword
 
     @Shared
-    @Value('${wave.registries.seqeralabs.azurecr.io.username}')
+    @Value('${wave.registries.wavetest.azurecr.io.username}')
     String azureUsername
     @Shared
-    @Value('${wave.registries.seqeralabs.azurecr.io.password}')
+    @Value('${wave.registries.wavetest.azurecr.io.password}')
     String azurePassword
 
     @Shared
@@ -98,7 +98,7 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis
     @IgnoreIf({!System.getenv('AZURECR_USER')})
     void 'test valid azure login'() {
         given:
-        def REGISTRY_URL = 'seqeralabs.azurecr.io'
+        def REGISTRY_URL = 'wavetest.azurecr.io'
 
         expect:
         loginService.login(REGISTRY_URL, azureUsername, azurePassword)