Add script to run pipeline locally

.github/workflows/pipeline.yml

@@ -2,71 +2,26 @@ name: Implementation process pipeline
 run-name: ${{ github.actor }} is running the pipeline
 on: [push]
 
-env:
-  ap_url:
-  shacl_url:
-
 jobs:
   Set-Up-Repo:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
         uses: actions/checkout@v5
-      - name: Create temp file if AP and SHACL url exist
-        run: ./scripts/create-env-temp-file.sh $ap_url $shacl_url
-      - name: Check if tmp file exists
-        id: check_env_temp
-        uses: andstor/file-existence-action@v3
+      - uses: actions/setup-node@v6
         with:
-          files: "env.txt"
-      - name: Download SHACL if needed
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/download-shacl.sh $shacl_url
+          node-version: 22
+      - name: Set up repo
+        run: ./scripts/set-up-repo.sh
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: in-shacl/shacl.ttl
           message: "Add SHACL"
-      - name: Add AP and SHACL links to README
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/update-readme.sh $ap_url $shacl_url
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: README.md original-README.md
           message: "Add AP and SHACL links to README"
-      - name: Check if template Excel exists
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        id: check_template
-        uses: andstor/file-existence-action@v3
-        with:
-          files: "template.xlsx"
-      - name: Clone ap-data-to-dashboard to generate template Excel
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        uses: actions/checkout@v5
-        with:
-          repository: RMLio/ap-data-to-dashboard
-          path: ap-data-to-dashboard
-      - uses: actions/setup-node@v6
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        with:
-          node-version: 22
-      - name: "Copy shacl file in-shacl directory"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: cp in-shacl/* ap-data-to-dashboard/in-shacl
-      - name: "Install ap-data-in-dashboard"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/install-ap-data-to-dashboard.sh
-      - name: "Generate template Excel, template schema, and example data"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: | 
-          cd ap-data-to-dashboard && node src/shacl-to-template.js
-          cp in-shacl/template.xlsx ../template.xlsx
-          cp in-shacl/template.schema.json ../in-shacl/template.schema.json
-          rm -rf ../example-data && mkdir ../example-data
-          cp in-shacl/dummy* ../example-data
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: template.xlsx in-shacl/template.schema.json example-data
           message: "Add template Excel, template schema, and example data"
@@ -77,59 +32,34 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref.name }}
       - name: Check if there is any data available
         id: check_data
         uses: andstor/file-existence-action@v3
         with:
           files: data/*.xlsx
-      - name: Clone ap-data-to-dashboard
-        if: steps.check_data.outputs.files_exists == 'true'
-        uses: actions/checkout@v5
-        with:
-          repository: RMLio/ap-data-to-dashboard
-          path: ap-data-to-dashboard
       - uses: actions/setup-node@v6
         if: steps.check_data.outputs.files_exists == 'true'
         with:
           node-version: 22
-      - name: "Copy input data to in directory"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: | 
-          rm -rf ap-data-to-dashboard/in
-          mkdir ap-data-to-dashboard/in
-          cp data/* ap-data-to-dashboard/in
-      - name: "Copy in-shacl directory content to ap-data-to-dashboard"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: cp in-shacl/* ap-data-to-dashboard/in-shacl
-      - name: "Install ap-data-in-dashboard"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: ./scripts/install-ap-data-to-dashboard.sh
-      - name: Copy dashboard config
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: ./scripts/copy-dashboard-config.sh
       - name: "Generate RDF and build Miravi"
         if: steps.check_data.outputs.files_exists == 'true'
-        run: |
-          cd ap-data-to-dashboard
-          ./run.sh -u https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/
+        run: ./scripts/generate-rdf-build-miravi.sh https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/
       - name: Clone gh-pages branch of this repo
         uses: actions/checkout@v5
         with:
           ref: gh-pages
           path: gh-pages
-      - uses: actions/setup-node@v6
       - name: Move Miravi dist directory to correct folder in gh-pages
-        run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }}
-      - name: Move generated RDF to output directory
         if: steps.check_data.outputs.files_exists == 'true'
+        run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }}
+      - name: Move output and mappings to root of repo
         run: |
-          rm -rf output
-          mkdir output
-          rm -rf mappings
-          mkdir mappings
-          mv ap-data-to-dashboard/out/serve-me/* output/
-          mv ap-data-to-dashboard/out/*.rml.ttl mappings/
-          mv ap-data-to-dashboard/out/*.yml mappings/
+          rm -rf output mappings
+          mv dist/output .
+          mv dist/mappings .
       - uses: EndBug/add-and-commit@v9
         if: steps.check_data.outputs.files_exists == 'true'
         with:

.gitignore

@@ -1 +1,3 @@
 node_modules
+dist
+tmp

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Script to run pipeline locally
+
+### Changed
+
+- GitHub actions uses scripts that are introduced to run pipeline locally
+
 ## [v2.0.0] - 2025-11-25
 
 ### Added

README.md

@@ -11,7 +11,8 @@ This template repo helps with setting up a pipeline for an OSLO implementation p
 - [Implementation process pipeline](#implementation-process-pipeline)
   - [Table of contents](#table-of-contents)
   - [Usage](#usage)
-    - [Setting up this repository](#setting-up-this-repository)
+    - [Setting up this repository with GitHub Actions and Pages](#setting-up-this-repository-with-github-actions-and-pages)
+    - [Setting up this repository for local execution](#setting-up-this-repository-for-local-execution)
     - [Configure the dashboard](#configure-the-dashboard)
     - [Adding data via an Excel file](#adding-data-via-an-excel-file)
     - [Updating data in an Excel file](#updating-data-in-an-excel-file)
@@ -23,27 +24,69 @@ This template repo helps with setting up a pipeline for an OSLO implementation p
 
 ## Usage
 
-### Setting up this repository
+### Setting up this repository with GitHub Actions and Pages
 
 You only have to do these steps once.
 
 1. [Create a new repository based on this repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template).
-   Make sure check the option "Include all branches".
+   Make sure to check the option "Include all branches".
 2. Remove all the data from the `gh-pages` branch.
 3. Navigate to Settings > Actions > General > Workflow permissions and
    make sure that "Read and write permissions" is checked.
 4. [Set up GitHub pages](https://docs.github.com/en/pages/getting-started-with-github-pages/creating-a-github-pages-site#creating-your-site)
    and let it point to the root directory on the `gh-pages` branch.
-5. Set the link to the application profile at `env.ap_url` in the file `.github/workflows/pipeline.yml`.
-6. Set the link to the SHACL at `env.shacl_url` in the file `.github/workflows/pipeline.yml`.
+5. Set the link to the application profile as value for `AP_URL` in the file `urls.env`.
+6. Set the link to the SHACL as value for `SHACL_URL` in the file `urls.env`.
 7. Read and, if necessary, update [the license](LICENSE).
 8. Commit and push your changes to GitHub.
    This will trigger the workflows, which will also add example data to the [example-data](./example-data) directory.
 9. Navigate to Actions and make sure that there are no errors in the workflows.
 10. Copy the Excel files in the `example-data` directory to the [data](./data) directory.
 11. Once the workflows have finished, the dashboard is available via your GitHub pages.
 12. If everything works, empty the [data](./data) directory.
-    You can now start [adding data](#adding-data-via-an-excel-file).
+    You can now start [adding your own data](#adding-data-via-an-excel-file).
+13. [Configure the dashboard](#configure-the-dashboard).
+
+### Setting up this repository for local execution
+
+You only have to do these steps once.
+
+1. [Create a new repository based on this repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-repository-from-a-template).
+   Make sure to check the option "Include all branches".
+2. Remove all the data from the `gh-pages` branch.
+3. Set the link to the application profile as value for `AP_URL` in the file `urls.env`.
+4. Set the link to the SHACL as value for `SHACL_URL` in the file `urls.env`.
+5. Read and, if necessary, update [the license](LICENSE).
+6. Run the pipeline locally via
+
+   ```bash
+   ./scripts/run-pipeline-locally.sh
+   ```
+
+   This script will finish with an error message saying that it didn't find any data,
+   which is what we expect considering we didn't add any data yet.
+7. Copy the Excel files in the `example-data` directory to the [data](./data) directory.
+8. Run the pipeline again via
+
+   ```bash
+   ./scripts/run-pipeline-locally.sh
+   ```
+
+9. Host the RDF files via
+
+   ```shell
+   npx http-server dist/output -p 5500 --cors
+   ```
+
+10. Host the dashboard via
+
+    ```shell
+    npx http-server dist/docs -p 8080
+    ```
+
+11. Browse to <http://localhost:8080>.
+12. If everything works, empty the [data](./data) directory.
+    You can now start [adding your own data](#adding-data-via-an-excel-file).
 13. [Configure the dashboard](#configure-the-dashboard).
 
 ### Configure the dashboard
@@ -168,10 +211,15 @@ Note that the dashboard will still rely on the RDF in this repository.
   ./scripts/reset-repo.sh
   ```
 
-- Some example env values are
+- Some example values for `urls.env` are
 
    ```yaml
-   env:
-     ap_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/
-     shacl_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl
+   AP_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/
+   SHACL_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl
+   ```
+
+- You can run the pipeline locally via
+
+   ```bash
+   ./scripts/run-pipeline-locally.sh
    ```

scripts/README.md

@@ -0,0 +1,21 @@
+# Scripts
+
+- `run-pipeline-locally.sh`: this script runs the pipeline locally.
+- `set-up-repo.sh`: this script sets up the repo, including downloading the SHACL file, generating a template Excel file,
+   and generating example data.
+- `generate-rdf-build-miravi-locally.sh`: this script generates the RDF and builds Miravi for local testing.
+   It relies on `generate-rdf-build-miravi.sh`.
+- `clone-ap-data-to-dashboard.sh`: this script clones the `ap-data-to-dashboard` repo in the `tmp` folder,
+  if it doesn't exist yet.
+- `copy-dashboard-config.sh`: this script copies the content of the `dashboard-config` folder to
+  the corresponding folder in the `ap-data-to-dashboard` folder.
+- `download-shacl.sh`: this script downloads the SHACL file,
+  if it doesn't exist yet.
+- `generate-rdf-build-miravi.sh`: this script generates the RDF and builds Miravi.
+   The script `generate-rdf-build-miravi-locally.sh` and `/.github/workflows/pipeline.yml` use this script.
+- `install-ap-data-to-dashboard.sh`: this script installs `ap-data-to-dashboard` and runs its setup script.
+- `move-miravi-dist.sh`: this script moves the `dist` folder created by Miravi
+  to correct subfolder in the `gh-pages` folder.
+- `reset-readme.sh`: this script reset `README.md` to the state it was in when the repository was created.
+- `reset-repo.sh`: this script reset the repository to the state it was in when it was created.
+- `update-readme.sh`: this script updates `README.md` with the application profile and SHACL urls.

scripts/clone-ap-data-to-dashboard.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+WORK_DIR="$SCRIPTS_DIR/../tmp"
+DIRECTORY="$WORK_DIR/ap-data-to-dashboard"
+
+if [ -d "$DIRECTORY" ]; then
+  echo "Folder $DIRECTORY already exists, so no cloning"
+else
+  echo "Cloning ap-data-to-dashboard"
+  git clone --depth 1 https://github.com/RMLio/ap-data-to-dashboard.git $DIRECTORY &> /dev/null
+fi
+

scripts/copy-dashboard-config.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
-DIRECTORY=dashboard-config
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+WORK_DIR="$SCRIPTS_DIR/../tmp"
+DIRECTORY="$SCRIPTS_DIR/../dashboard-config"
 
 if [ -d "$DIRECTORY" ]; then
   echo "$DIRECTORY exists."
-  rm -rf ap-data-to-dashboard/miravi-initial-config
-  mv $DIRECTORY ap-data-to-dashboard/miravi-initial-config
+  rm -rf $WORK_DIR/ap-data-to-dashboard/miravi-initial-config
+  mv $DIRECTORY $WORK_DIR/ap-data-to-dashboard/miravi-initial-config
 else
   echo "$DIRECTORY doesn't exist. Using initial config."
 fi

scripts/download-shacl.sh

@@ -4,7 +4,7 @@ DIR=in-shacl
 FILE="$DIR/shacl.ttl"
 
 if [ -d "$DIR" ]; then
-  echo "DIR already exists."
+  echo "$DIR already exists."
 else
   mkdir $DIR
 fi

scripts/generate-rdf-build-miravi-locally.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+OUTPUT_DIR=$SCRIPTS_DIR"/../dist"
+WORK_DIR=$SCRIPTS_DIR"/../tmp"
+BASE_URL="http://localhost:5500/"
+
+if compgen -G "data/*.xlsx" > /dev/null; then
+  $SCRIPTS_DIR/generate-rdf-build-miravi.sh $BASE_URL
+
+  echo "Moving Miravi dist to docs"
+  rm -rf $OUTPUT_DIR/docs && mkdir $OUTPUT_DIR/docs
+  mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* $OUTPUT_DIR/docs
+
+  echo ""
+  echo ""
+  echo "1. Host RDF files via \"npx http-server $OUTPUT_DIR/output -p 5500 --cors\""
+  echo "2. Host Miravi via \"npx http-server $OUTPUT_DIR/docs -p 8080\""
+  echo "3. Browse to http://localhost:8080"
+else
+  echo "No Excel files found in the data folder."
+  exit 1
+fi