cdmtaskservice_config.toml.jinja

[Authentication]

# The URL of a KBase auth2 service
url = "{{ KBCTS_KBASE_AUTH2_URL or "https://ci.kbase.us/services/auth" }}"

# KBase auth2 custom roles that signify that a user is an admin for the CDM task service
# This is a comma delimited list, e.g. "KBASE_ADMIN, CDM_TASK_SERVICE_ADMIN"
admin_roles_full = "{{ KBCTS_KBASE_AUTH2_ADMIN_ROLES or "" }}"

# KBase auth2 custom roles that authorize a user to use the CDM task service.
# This is a comma delimited list, e.g. "KBASE_STAFF, CDM_TASK_SERVICE_USER"
# Defaults to "KBASE_STAFF" if not set.
user_roles = "{{ KBCTS_CTS_USER_ROLES or "KBASE_STAFF" }}"

# KBase auth2 custom role that signifies a user possesses a NERSC account.
# This role is required to run jobs at NERSC and LRC.
has_nersc_account_role = "{{ KBCTS_HAS_NERSC_ACCOUNT_ROLE or "HAS_NERSC_ACCOUNT" }}"

# KBase auth2 custom role that signifies a request is coming from a user representing an
# external job executor. Used by executors running on HTCondor workers to get
# and update job state.
external_executor_role = "{{ KBCTS_EXTERNAL_EXECUTOR_ROLE or "CTS_EXTERNAL_EXECUTOR" }}"

# KBase auth2 custom role that signifies a request is coming from a user representing a
# refdata service. Used by refdata servers to get and update refdata state.
refdata_service_role = "{{ KBCTS_REFDATA_SERVICE_ROLE or "CTS_REFDATA_SERVICE" }}"

[NERSC_JAWS]

# The username of the NERSC user that will interact with the Superfacility API and JAWS.
# This user must match the SFAPI credentials in the NERSC block below and the token in the JAWS
# block.
# The NERSC user's default shell must be bash.
# If the SFAPI client credentials are updated but the user doesn't match they will not be accepted.
# It is advised to create a collaboration user for the service.
# All the services in a service group must use the same user.
user = "{{ KBCTS_NERSC_JAWS_USER or "" }}"

# The NERSC refdata directory root for the JAWS installation the service is using.
# The JAWS installation is expected to be configured to use this directory to look up refdata.
refdata_root_dir = "{{ KBCTS_NERSC_JAWS_REFDATA_DIR or "" }}"

# The JAWS site staging directory at NERSC on a Data Transfer Node.
# Note that currently the CTS is hard coded to use the `kbase` site at NERSC Perlmutter.
# This is typically something like
# `/global/pscratch/sd/<JAWS site user first letter>/<JAWS site user>/<site>-prod/`
jaws_staging_dir_dtn = "{{
	 KBCTS_NERSC_JAWS_DTN_STAGING_DIR or "/global/pscratch/sd/k/kbjaws/kbase-prod/"
 }}"

# The JAWS site staging directory at NERSC on the Perlmutter system.
# This is the equivalent directory to the DTN staging directory.
# Note that currently the CTS is hard coded to use the `kbase` site at NERSC Perlmutter.
# This is typically something like
# `/pscratch/sd/<JAWS site user first letter>/<JAWS site user>/<site>-prod/`
jaws_staging_dir_perlmutter = "{{
	KBCTS_NERSC_JAWS_PERMUTTER_STAGING_DIR or "/pscratch/sd/k/kbjaws/kbase-prod/"
}}"

[NERSC]

# The path to a NERSC Superfacility API (SFAPI) credential file. The file is expected to have
# the client ID as the first line and the client private key in PEM format as the remaining lines.
# Client credentials can be obtained from
# https://iris.nersc.gov -> Profile -> Superfacility API Clients
# Clients must have "red" level permissions.
# Since clients expire every 2 or 30 days, depending on whether extended lifetime clients have
# been approved for the user, the service periodically checks for updates to this file and
# reloads the contents if changed.
sfapi_cred_path = "{{ KBCTS_SFAPI_CRED_PATH or "" }}"

# Where to store remote code at NERSC. This must be writeable by the service account.
remote_code_dir = "{{ KBCTS_NERSC_REMOTE_CODE_DIR or "/global/cfs/cdirs/kbase/cdm_task_service" }}"

[JAWS]

# The JAWS Central server URL.
url = "{{ KBCTS_JAWS_URL or "https://jaws-api.jgi.doe.gov/" }}"

# The JGI JAWS token to use to run jobs. This can be obtained from a JAWS representative.
# Note that all the services in a service group (see the Services block) should have the same
# token, as they use the same JAWS config file, and will otherwise overwrite each other's
# tokens.
token = "{{ KBCTS_JAWS_TOKEN or "" }}"

# The JAWS group to use to run jobs.
group = "{{ KBCTS_JAWS_GROUP or "kbase" }}"

[HTCondor]

# The initial job directory on the condor SCHEDD daemon host and locally. Equivalent to the
# "initialdir" argument in a submit file or to Submit(). Must exist or be createable by the
# service locally and must exist on the SCHEDD host.
initialdir = "{{ KBCTS_HTC_INITIAL_DIR or "/condor_workdir" }}"

# The client group to submit to. This is a ClassAd on HTC workers with the name CLIENTGROUP.
# If omitted, the service will not distinguish between workers.
clientgroup = "{{ KBCTS_HTC_CLIENT_GROUP or "" }}"

# The client group to submit to for jobs that request > 0 GPUs, instead of `clientgroup`.
# This is a ClassAd on HTC workers with the name CLIENTGROUP.
# If omitted, the service will not distinguish between workers for GPU jobs.
gpu_clientgroup = "{{ KBCTS_HTC_GPU_CLIENT_GROUP or "" }}"

# The file path on the HTCondor workers that will contain a KBase token for
# the job executor to use to contact the service. This token must have the external executor role
# defined in the Authentication section above.
token_path = "{{ KBCTS_HTC_TOKEN_PATH or "/cts/cts_token" }}"

# The file path on the HTCondor workers that will contain the S3 access secret
# for the S3 system defined in the S3 section below.
s3_access_secret_path = "{{ KBCTS_HTC_S3_ACCESS_SECRET_PATH or "/cts/s3_secret" }}"

# Additional elements to prepend to the $PATH variable in the HTCondor worker.
additional_path = "{{ KBCTS_HTC_ADDL_PATH or ""}}"

# A directory appropriate for cached data on the condor workers; for example the uv cache.
cache_dir = "{{ KBCTS_HTC_CACHE_DIR or "/tmp"}}"

# Whether to use the external S3 URL when contacting S3 from the Condor workers as opposed to
# the standard URL (see the S3 section below). "True", case insensitive, will cause the external
# url to be used. Anything else will cause the standard URL to be used.
use_S3_external_url = "{{ KBCTS_HTC_USE_S3_EXTERNAL_URL or "false" }}"

# The local path, accessible to the server, to the executable file to run on the HTCondor workers.
# If using the provided Dockerfile, this value is set there.
executable_path = "{{ KBCTS_HTC_EXE_PATH or "" }}"

# A URL that will override the default location for HTCondor to download the executable file
# for the worker. By default this is constructed from the service root url (below) and a
# default path.
# If set, it becomes the user's responsibility to keep the executable file up to date with the
# service.
executable_url_override = "{{ KBCTS_HTC_EXE_URL_OVERRIDE or "" }}"

[ExternalExecution]

# The url of the reference data server.
refserver_url = "{{ KBCTS_REFDATA_SERVER_URL or "" }}"

# The token to use when communicating with the refdata server. The token must have the role
# defined in the Authentication section of the refdata server configuration that allows use
# of the refdata server.
refserver_token = "{{ KBCTS_REFDATA_SERVER_TOKEN or "" }}"

# The root path local to the job container host where reference data is stored, e.g.
# reference data will be stored in <reference_data_path>/<refdata_id>. This *must* be an
# absolute path. This will be used to mount the reference data into the job container.
refdata_host_path = "{{ KBCTS_REFDATA_HOST_PATH or "" }}"

# The time in minutes after which an external executor should give up and fail rather than
# continue to try and update a job's state in the service if the service can't be contacted.
# The default is 6 hours and minimum 1 minute.
job_update_timeout_min = {{ KBCTS_EXTERNAL_JOB_UPDATE_TIMEOUT_MIN or 360 }}

# A prefix override for the input and output data mount path for the job container.
# This is not a mount string, although it looks like one. By default, the external job
# executor mounts subdirectories in the current working directory into the job container
# for input and output. If the external executor is running in a container itself and the
# job container will be run as a sibling container, that will obviously fail. This argument
# replaces the portion of the executor's current working directory in the argument prior
# to the colon with the portion of the argument after the colon.
# For example, if the working directory of the executor in HTCondor is
# /var/lib/condor/execute/dir_1469 (and dir_1469 can change from run ro run but the remainder of
# the path is static), and the directory on the host where data can be shared between
# containers is /home/user1/condor_stuff, the the argument would be
# /var/lib/condor/execute:/home/user1/condor_stuff
# The host directory /home/user1/condor_stuff must be mounted into the condor container at
# /var/lib/condor/execute, making all job files available for mounting into other containers.
# The executor will then mount the appropriate directories into the job container, using
# /home/user1/condor_stuff as the host mount prefix and appending the remainder of the appropriate
# directories, e.g. dir_1469 and subdirectories.
# Generally speaking, the argument is the reverse of the condor container mount string
# /home/user1/condor_stuff:/var/lib/condor/execute
mount_prefix_override = "{{ KBCTS_EXTERNAL_JOB_MOUNT_PREFIX_OVERRIDE or "" }}"

# The local path, accessible to the server, to the external execution code tgz archive file.
# If using the provided Dockerfile, this value is set there.
archive_path = "{{ KBCTS_CODE_ARCHIVE_PATH or  "" }}"

# A URL that will override the default location to download the external execution code
# archive tgz file.
# By default this is constructed from the service root url (below) and a default path.
# If set, it becomes the user's responsibility to keep the archive file up to date with the
# service.
archive_url_override = "{{ KBCTS_CODE_ARCHIVE_URL_OVERRIDE or "" }}"

[S3]

# The connection parameters for the S3 instance to use for data source and storage.
# See S3 requirements in the README.

# The URL of the S3 instance accessible to the service.
url = "{{ KBCTS_S3_URL or "" }}"
# The URL of the same S3 instance accessible to external, remote code. This may not be accessible
# to the service.
external_url = "{{ KBCTS_S3_EXTERNAL_URL or "" }}"
# Set to `false` to skip connection verification of the external URL at service startup.
# If the external url is accessible to the service set to `true` or any other string to verify
# the URL at startup.
verify_external_url = "{{ KBCTS_VERIFY_S3_EXTERNAL_URL or "true"}}"
# The S3 access key.
access_key = "{{ KBCTS_S3_ACCESS_KEY or "" }}"
# The S3 secret key.
access_secret = "{{ KBCTS_S3_ACCESS_SECRET or "" }}"
# Setting allow_insecure = true (and only the exact string "true") skips checking the validity of
# the SSL certificate for the internal url, leaving the service vulnerable to Man in the Middle
# attacks.
allow_insecure = "{{ KBCTS_S3_ALLOW_INSECURE or "false"}}"
# Setting allow_insecure_external = true (and only the exact string "true") skips checking the
# validity of the SSL certificate for the external url, leaving the service vulnerable to Man in
# the Middle attacks.
allow_insecure_external = "{{ KBCTS_S3_ALLOW_INSECURE_EXTERNAL or "false"}}"

[MongoDB]

# The connection parameters for MongoDB. User and password are optional. Retry writes
# sets the Mongo DB retrywrites parameter on only if the value of the key is the
# case-insensitive string "true". See https://www.mongodb.com/docs/manual/core/retryable-writes/

mongo_host = "{{ KBCTS_MONGO_HOST or "mongodb://localhost:27017" }}"
mongo_db = "{{ KBCTS_MONGO_DB or "cdmtaskservice" }}"
mongo_user = "{{ KBCTS_MONGO_USER or "" }}"
mongo_pwd = "{{ KBCTS_MONGO_PWD or "" }}"
mongo_retrywrites = "{{ KBCTS_MONGO_RETRYWRITES or "" }}"

[Jobs]

# S3 paths where users are allowed to read / write job files, starting with the bucket.
# Bucket only paths are allowed.
# If omitted, users can read and write to any locations where the service can read and write
# except for the log directory (configured below).
# Paths may not be prefixes or suffixes of each other or the log directory.
# To specify multiple paths, separate them with a comma. This means that commas
# are not allowed in the path names.
# Paths will be appended with a / if absent.
allowed_s3_paths = "{{ KBCTS_ALLOWED_S3_PATHS or "" }}"

# An S3 path, starting with the bucket, to where container log files should be uploaded from jobs.
# Must contain a key. Must be writable by the service and readable by users who need to see
# their job logs.
# Path will be appended with a / if absent.
container_s3_log_dir = "{{ KBCTS_CONTAINER_S3_LOG_DIR or "" }}"

# The maximum CPU hours allowed per job, calculated as
# the number of CPUs * the number of containers * the time per container.
max_cpu_hours = {{ KBCTS_JOB_MAX_CPU_HOURS or 100 }}

# How often the cleaning code that removes transient files from completed jobs and refdata
# should be run in hours.
# Currently this cleans up various manifests and results at NERSC.
clean_frequency_hours = {{ KBCTS_JOB_CLEAN_FREQUENCY_HOURS or 24 }}

# How old jobs / refdata must be before they're cleaned in days.
clean_age_days = {{ KBCTS_CLEAN_AGE_DAYS or 30 }}

[Kafka]

# The Kafka boostrap servers in standard Kafka format
bootstrap_servers = "{{ KBCTS_KAFKA_BOOTSTRAP_SERVERS or "" }}"

# The topic where job status updates will be published.
# Legal characters are alphanumerics and the hyphen.
topic_jobs = "{{ KBCTS_KAFKA_TOPIC_JOBS or "cts-jobs" }}"

# WARNING: read this documentation fully before enabling this setting
#
# On service startup, check for unsent Kafka job updates older than the specified time in
# minutes and send them.
#
# Note that it is possible to cause messages to be sent multiple times by enabling this setting.
# For instance, if two services are brought up at the same time and there are unsent
# messages in the database, it's possible both of them could process the messages, leading
# to duplicates. Similarly, if there is a large message backlog and a new CTS service is
# brought up, it might process the backlogged messages, leading to duplicates.

# It it safe to enable this setting when:
# * all other CTS servers are down and Kafka is up, or
# * other CTS servers and Kafka are up, and both have been up long enough such that
# there is a minimal update queue in the CTS servers; e.g. the CTS servers should send
# messages before the specified time delay expires.
  
# It is not safe to run this code when updates are not sent to Kafka within the time delay; in
# that case running this code may cause duplicate messages.

# A number less than 1 indicates that unsent messages will not be checked on startup.
check_unsent_messages_on_startup_delay_min = {{ KBCTS_KAFKA_STARTUP_UNSENT_MSG_DELAY_MIN or -1 }}

[Images]

# The path to a crane executable. See the README for details.
# If using the provided Dockerfile, this value is set there.
crane_path = "{{ KBCTS_CRANE_PATH or "" }}"

[Service]

# The root URL of the service. This is used to construct callback URLs for remote code.
root_url = "{{ KBCTS_SERVICE_ROOT_URL or "https://ci.kbase.us/services/cts" }}"

# If the service is behind a reverse proxy that rewrites the service path, that path needs
# to be defined here in order for the OpenAPI documentation to function.
root_path = "{{ KBCTS_SERVICE_ROOT_PATH or "" }}"

# The service group to which this instance of the service belongs. This is used to separate files
# on remote services so files from different S3 instances (say production and development)
# don't collide.
# MongoDB databases should not be split among different service groups or errors accessing
# job files may occur.
group_id = "{{ KBCTS_SERVICE_GROUP or "dev" }}"