Skip to content

Commit 24cea51

Browse files
vladd-bitvladd-bit
authored
Merge pull request #30 from CogStack/nifi-2.0
Nifi 2.0
2 parents 4c18a8c + 996fc5e commit 24cea51

177 files changed

Lines changed: 2990 additions & 14525 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@
2323
*.webp binary
2424
*.psd binary
2525

26+
# certificates
27+
*.jks binary
28+
*.p12 binary
29+
*.pfx binary
30+
*.pem -text -diff
31+
*.csr -text -diff
32+
*.key -text -diff
33+
*.crt -text -diff
34+
2635
# Fonts
2736
*woff2 -text diff
2837

‎.github/workflows/docker-elasticsearch-stack.yml‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,19 @@ jobs:
2121

2222
build:
2323

24-
runs-on: ubuntu-latest
24+
runs-on: ${{ matrix.runner }}
25+
26+
strategy:
27+
matrix:
28+
platform: [amd64, arm64]
29+
include:
30+
- platform: amd64
31+
runner: ubuntu-22.04
32+
- platform: arm64
33+
runner: ubuntu-22.04-arm
2534

2635
env:
27-
working-directory: ./services/
36+
working-directory: ./services/elasticsearch
2837

2938
# Steps represent a sequence of tasks that will be executed as part of the job
3039
steps:
@@ -44,7 +53,7 @@ jobs:
4453
id: meta
4554
uses: docker/metadata-action@v5
4655
with:
47-
images: cogstacksystems/cogstack-kibana:latest
56+
images: cogstacksystems/kibana:latest
4857

4958
- uses: actions/checkout@v3
5059
- name: Get and set release version env var
@@ -73,26 +82,14 @@ jobs:
7382
echo $RELEASE_VERSION
7483
7584
- name: Build and push Docker Kibana image
76-
uses: docker/build-push-action@v5
77-
with:
78-
platforms: linux/amd64
79-
context : ./services/kibana
80-
file : "./services/kibana/Dockerfile_native"
81-
allow: network.host
82-
github-token: ${{ github.token }}
83-
tags: cogstacksystems/cogstack-kibana:${{ env.RELEASE_VERSION }}
84-
push: true
85-
85+
run: |
86+
docker build --platform linux/${{ matrix.platform }} -t cogstacksystems/kibana:${{ env.RELEASE_VERSION }}-${{ matrix.platform }} -f "./services/kibana/Dockerfile_native" --build-arg CPU_ARCHITECTURE=${{ matrix.platform }} .
87+
docker push cogstacksystems/kibana:${{ env.RELEASE_VERSION }}-${{ matrix.platform }}
88+
8689
- name: Build and push Docker Elasticsearch image
87-
uses: docker/build-push-action@v5
88-
with:
89-
platforms: linux/amd64
90-
context : ./services/elasticsearch
91-
file : "./services/elasticsearch/Dockerfile_native"
92-
allow: network.host
93-
github-token: ${{ github.token }}
94-
tags: cogstacksystems/cogstack-elasticsearch:${{ env.RELEASE_VERSION }}
95-
push: true
90+
run: |
91+
docker build --platform linux/${{ matrix.platform }} -t cogstacksystems/elasticsearch:${{ env.RELEASE_VERSION }}-${{ matrix.platform }} -f "./services/elasticsearch/Dockerfile_native" --build-arg CPU_ARCHITECTURE=${{ matrix.platform }} .
92+
docker push cogstacksystems/elasticsearch:${{ env.RELEASE_VERSION }}-${{ matrix.platform }}
9693
9794
- name: Image digest
9895
run: echo ${{ steps.docker_build.outputs.digest }}

‎.github/workflows/docker-nginx.yml‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,16 @@ jobs:
2121

2222
build:
2323

24-
runs-on: ubuntu-latest
24+
runs-on: ${{ matrix.runner }}
25+
26+
strategy:
27+
matrix:
28+
platform: [amd64, arm64]
29+
include:
30+
- platform: amd64
31+
runner: ubuntu-22.04
32+
- platform: arm64
33+
runner: ubuntu-22.04-arm
2534

2635
env:
2736
working-directory: ./services/nginx
@@ -44,7 +53,7 @@ jobs:
4453
id: meta
4554
uses: docker/metadata-action@v5
4655
with:
47-
images: cogstacksystems/cogstack-nifi-nginx:latest
56+
images: cogstacksystems/nifi-nginx:latest
4857

4958
- uses: actions/checkout@v3
5059
- name: Get and set release version env var
@@ -73,15 +82,9 @@ jobs:
7382
echo $RELEASE_VERSION
7483
7584
- name: Build and push Docker NiFi-Nginx image
76-
uses: docker/build-push-action@v6
77-
with:
78-
platforms: linux/amd64,linux/arm64
79-
context : ./services/nginx
80-
file : "./services/nginx/Dockerfile"
81-
allow: network.host
82-
github-token: ${{ github.token }}
83-
tags: cogstacksystems/nifi-nginx:${{ env.RELEASE_VERSION }}
84-
push: true
85+
run: |
86+
docker build --platform linux/${{ matrix.platform }} -t cogstacksystems/nifi-nginx:${{ env.RELEASE_VERSION }}-${{ matrix.platform }} -f "./services/nginx/Dockerfile" --build-arg CPU_ARCHITECTURE=${{ matrix.platform }} .
87+
docker push cogstacksystems/nifi-nginx:${{ env.RELEASE_VERSION }}-${{ matrix.platform }}
8588
8689
- name: Image digest
8790
run: echo ${{ steps.docker_build.outputs.digest }}

‎.github/workflows/docker-nifi.yml‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,16 @@ jobs:
1717

1818
build:
1919

20-
runs-on: ubuntu-latest
20+
runs-on: ${{ matrix.runner }}
21+
22+
strategy:
23+
matrix:
24+
platform: [amd64, arm64]
25+
include:
26+
- platform: amd64
27+
runner: ubuntu-22.04
28+
- platform: arm64
29+
runner: ubuntu-22.04-arm
2130

2231
env:
2332
working-directory: ./nifi
@@ -68,16 +77,9 @@ jobs:
6877
run: echo "RELEASE_VERSION=latest" >> $GITHUB_ENV
6978

7079
- name: Build and push Docker NiFi image
71-
id: docker_build
72-
uses: docker/build-push-action@v6
73-
with:
74-
platforms: linux/amd64,linux/arm64
75-
context : ./nifi
76-
file : "./nifi/Dockerfile"
77-
allow: network.host
78-
github-token: ${{ github.token }}
79-
tags: cogstacksystems/cogstack-nifi:${{ env.RELEASE_VERSION }}
80-
push: true
81-
80+
run: |
81+
docker build --platform linux/${{ matrix.platform }} -t cogstacksystems/cogstack-nifi:${{ env.RELEASE_VERSION }}-${{ matrix.platform }} -f "./nifi/Dockerfile" --build-arg CPU_ARCHITECTURE=${{ matrix.platform }} .
82+
docker push cogstacksystems/cogstack-nifi:${{ env.RELEASE_VERSION }}-${{ matrix.platform }}
83+
8284
- name: Image digest
8385
run: echo ${{ steps.docker_build.outputs.digest }}

‎.gitignore‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
# IDE envs and system folders
2+
.DS_Store
3+
**/.DS_Store
4+
.idea
5+
.vscode
6+
.venv
7+
**__pycache__
8+
19
# keys and certificates
210
*.pem
311
*.key
@@ -15,18 +23,10 @@
1523
*.env
1624
security/es_roles/*
1725
security/es_certificates/*
26+
security/root_certificates/*
1827
security/nifi_certificates/*
1928
security/nifi_toolkit
2029

21-
# other
22-
scripts/data_processing/anonymistion_hw_clean/anonymised
23-
2430
# docs
2531
docs/build/*
2632

27-
# IDE envs and system folders
28-
.DS_Store
29-
**/.DS_Store
30-
.idea
31-
.vscode
32-
.venv

‎.gitmodules‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
11
[submodule "services/jupyter-hub"]
22
path = services/jupyter-hub
33
url = https://github.com/CogStack/cogstack-jupyter-hub.git
4+
[submodule "services/nlp-services/medcat-service"]
5+
path = services/nlp-services/medcat-service
6+
url = https://github.com/CogStack/MedCATservice.git
7+
branch = main
8+
[submodule "services/ocr-service"]
9+
path = services/ocr-service
10+
url = https://github.com/CogStack/ocr-service.git
11+
branch = main
12+
[submodule "services/cogstack-cohort"]
13+
path = services/cogstack-cohort
14+
url = https://github.com/CogStack/CogStackCohort.git
15+
branch = main
16+
[submodule "services/medcat-trainer"]
17+
path = services/medcat-trainer
18+
url = https://github.com/CogStack/MedCATtrainer.git
19+
branch = main

‎deploy/database.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ POSTGRES_DATABANK_DB=cogstack
44
POSTGRES_DB_MAX_CONNECTIONS=100
55

66
# Prefix of file names to load the DB schema for in /services/cogstack-db/(pgsql/mssql)/schemas/ folder
7-
DB_SCHEMA_PREFIX="cogstack_db"
7+
POSTGRES_DB_SCHEMA_PREFIX="cogstack_db"

‎deploy/elasticsearch.env‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch
99
# possible values :
1010
# - elasticsearch : docker.elastic.co/elasticsearch/elasticsearch:8.18.1
1111
# - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest
12-
# - opensearch : opensearchproject/opensearch:2.19.0
12+
# - opensearch : opensearchproject/opensearch:3.0.0
1313
# the custom cogstack image is always based on the last image of ES native
1414

15-
ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:2.19.0
15+
ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.0.0
1616

1717
ELASTICSEARCH_LOG_LEVEL=INFO
1818

@@ -159,10 +159,10 @@ KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
159159
# possible values:
160160
# - elasticsearch : docker.elastic.co/kibana/kibana:8.18.1
161161
# - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
162-
# - opensearch : opensearchproject/opensearch-dashboards:2.19.0
162+
# - opensearch : opensearchproject/opensearch-dashboards:3.0.0
163163
# the custom cogstack image is always based on the last image of ES native
164164

165-
ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:2.19.0
165+
ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.0.0
166166

167167
KIBANA_SERVER_NAME="cogstack-kibana"
168168
KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601"

‎deploy/export_env_vars.sh‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,29 @@ set -o allexport
55
current_dir=$(pwd)
66
security_dir="../security/"
77

8-
env_files=("general.env"
8+
env_files=(
9+
$security_dir"certificates_nifi.env"
10+
$security_dir"certificates_general.env"
11+
$security_dir"certificates_elasticsearch.env"
12+
$security_dir"users_elasticsearch.env"
13+
$security_dir"users_database.env"
14+
$security_dir"users_nifi.env"
15+
$security_dir"users_nginx.env"
16+
"general.env"
917
"nifi.env"
1018
"elasticsearch.env"
1119
"database.env"
1220
"jupyter.env"
1321
"nlp_service.env"
14-
"ocr_service.env"
1522
"network_settings.env"
1623
"project.env"
17-
$security_dir"certificates_nifi.env"
18-
$security_dir"certificates_general.env"
19-
$security_dir"certificates_elasticsearch.env"
20-
$security_dir"elasticsearch_users.env"
21-
$security_dir"database_users.env"
2224
)
2325

24-
set -a
25-
2626
for env_file in ${env_files[@]}; do
2727
source $env_file
2828
done
2929

3030
# for nginx vars
3131
export DOLLAR="$"
3232

33-
set +a
34-
3533
set +o allexport

‎deploy/general.env‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,10 @@
44
# set only CPU_ARCHITECTURE to amd/arm64
55
# remove if this causes issues on any other platform
66
# possible values: amd64, arm64
7-
CPU_ARCHITECTURE=amd64
8-
DOCKER_DEFAULT_PLATFORM=linux/${CPU_ARCHITECTURE:-amd64}
7+
CPU_ARCHITECTURE=arm64
8+
DOCKER_DEFAULT_PLATFORM=linux/${CPU_ARCHITECTURE:-amd64}
9+
10+
COGSTACK_ADMIN_EMAIL=admin@cogstack.net
11+
12+
# used in NiFi for the timezone
13+
TZ="Europe/London"

0 commit comments

Comments
 (0)