Artifactory integration for integration tests

.github/workflows/lumigator_pipeline.yaml

@@ -229,12 +229,15 @@ jobs:
           mkdir -p /tmp/ray_pip_cache
           chmod -R 777 /tmp/ray_pip_cache
 
-      - name: Spinning up containers for the tests
+      - name: Add Artifactory configuration
         if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')
-        run:  make start-lumigator-build
+        run: |
+          echo HF_TOKEN=${{ secrets.JF_ACCESS_TOKEN }} >> .default.conf
+          echo HF_ENDPOINT=${{ secrets.JF_URL_HF }} >> .default.conf
 
-      - name: Configure 'default' test environment variables
-        run: ./scripts/set_env_vars.sh
+      - name: Spinning up containers for the tests
+        if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')
+        run: CI=true make start-lumigator-build
 
       - name: Run ${{ matrix.test }} integration tests
         if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')
@@ -310,12 +313,15 @@ jobs:
           mkdir -p ${HOME}/.cache/huggingface
           chmod -R 777 ${HOME}/.cache/huggingface
 
-      - name: Spinning up containers for the tests
+      - name: Add Artifactory configuration
         if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')
-        run:  make start-lumigator-build-postgres
+        run: |
+          echo HF_TOKEN=${{ secrets.JF_ACCESS_TOKEN }} >> .default.conf
+          echo HF_ENDPOINT=${{ secrets.JF_URL_HF }} >> .default.conf
 
-      - name: Configure 'default' test environment variables
-        run: ./scripts/set_env_vars.sh
+      - name: Spinning up containers for the tests
+        if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')
+        run: CI=true make start-lumigator-build-postgres
 
       - name: Run ${{ matrix.test }} integration tests
         if: steps.filter.outputs.test == 'true' || contains(github.ref, 'refs/tags/')

Makefile

@@ -54,6 +54,11 @@ ifeq ($(shell test $(RAY_WORKER_GPUS) -ge 1; echo $$?) , 0)
 	COMPUTE_TYPE := -gpu
 	GPU_COMPOSE := -f docker-compose.gpu.override.yaml
 endif
+CI ?= false
+
+ifeq ($(CI), true)
+	CI_COMPOSE := -f docker-compose.ci.override.yaml
+endif
 
 # lumigator runs on a set of containers (backend, ray, minio, etc).
 # The following allows one to start all of them before calling a target
@@ -138,11 +143,11 @@ start-lumigator: config-generate-env
 
 # Launches lumigator with no code mounted in, and forces build of containers (used in CI for integration tests)
 start-lumigator-build: config-generate-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) ARCH=${ARCH} COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env"  --profile local $(GPU_COMPOSE)  -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d --build
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) ARCH=${ARCH} COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env"  --profile local $(CI_COMPOSE) $(GPU_COMPOSE)  -f $(LOCAL_DOCKERCOMPOSE_FILE) up -d --build
 
 # Launches lumigator with no code mounted in, and forces build of containers (used in CI for integration tests)
 start-lumigator-build-postgres: config-generate-env
-	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) ARCH=${ARCH} COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(GPU_COMPOSE)  -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d --build
+	RAY_ARCH_SUFFIX=$(RAY_ARCH_SUFFIX) ARCH=${ARCH} COMPUTE_TYPE=$(COMPUTE_TYPE) docker compose --env-file "$(CONFIG_BUILD_DIR)/.env" --profile local $(CI_COMPOSE) $(GPU_COMPOSE)  -f $(LOCAL_DOCKERCOMPOSE_FILE) -f $(POSTGRES_DOCKER_COMPOSE_FILE) up -d --build
 
 # Launches lumigator without local dependencies (ray, S3)
 start-lumigator-external-services: config-generate-env

docker-compose.ci.override.yaml

@@ -0,0 +1,10 @@
+services:
+  ray:
+    environment:
+      - HF_TOKEN
+      - HF_ENDPOINT
+
+  backend:
+    environment:
+      - HF_TOKEN
+      - HF_ENDPOINT