diff --git a/.github/workflows/sandpaper-main.yaml b/.github/workflows/sandpaper-main.yaml index b3d1de8c..ff447dd7 100644 --- a/.github/workflows/sandpaper-main.yaml +++ b/.github/workflows/sandpaper-main.yaml @@ -5,8 +5,6 @@ on: branches: - main - master - schedule: - - cron: '0 0 * * 2' workflow_dispatch: inputs: name: diff --git a/.github/workflows/sandpaper-version.txt b/.github/workflows/sandpaper-version.txt index 084c7bd0..f2f6c138 100644 --- a/.github/workflows/sandpaper-version.txt +++ b/.github/workflows/sandpaper-version.txt @@ -1 +1 @@ -0.16.10 +0.16.11 diff --git a/.gitignore b/.gitignore index 9b846a3c..14ea8afe 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ site/* /*.Rcheck/ # RStudio files .Rproj.user/ +*.Rproj # produced vignettes vignettes/*.html vignettes/*.pdf diff --git a/MAINTENANCE.md b/MAINTENANCE.md index b70c98c4..57278b8d 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -1,4 +1,6 @@ -# Site Structure and Maintenance +--- +title: "Site Structure and Maintenance" +--- This site depends strongly on GitHub infrastructure, and a number of operational decisions have been made during the development process diff --git a/README.md b/README.md index 331faa7d..e2421275 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,12 @@ This lesson teaches the basics of interacting with high-performance computing ## Using this material -NOTE: This is *not* Carpentries boilerplate! Please read carefully. +NOTE: This lesson is being transitioned to Carpentries Workbench but is still *not* Carpentries boilerplate! +Instructions below still need to be edited to reflect the transition. Proceed with caution. -1. Follow the instructions found in The Carpentries' [example lesson][ex-lesson] +1. ~~Follow the instructions found in The Carpentries' [example lesson][ex-lesson] to create a repository for your lesson. Install Ruby, Make, and Jekyll - following the instructions [here][setup]. + following the instructions [here][setup].~~ 2. For easier portability, we use snippets of text and code to capture inputs and outputs that are host- or site-specific and cannot be scripted. These @@ -37,17 +38,17 @@ NOTE: This is *not* Carpentries boilerplate! Please read carefully. or omit episodes, by editing the configuration block under `episode_names` in this file. -4. Set the environment variable `HPC_JEKYLL_CONFIG` to the relative path of the - configuration file in your snippets folder: +4. ~~Set the environment variable `HPC_JEKYLL_CONFIG` to the relative path of the + configuration file in your snippets folder:~~ ```bash export HPC_JEKYLL_CONFIG=_includes/snippets_library/.../_config_options.yml ``` -5. Preview the lesson locally, by running `make serve`. You can then view the +5. ~~Preview the lesson locally, by running `make serve`. You can then view the website in your browser, following the links in the output (usually, ). Pages will be automatically regenerated every - time you write to them. + time you write to them.~~ 6. If there are discrepancies in the output, edit the snippet file containing it, or create a new one and customize. @@ -67,6 +68,7 @@ NOTE: This is *not* Carpentries boilerplate! Please read carefully. future will probably not be accepted. ## Deploying a Customized Lesson +**While this lesson is in transition, we do not recommend customizing** The steps above will help you port the default HPC Intro lesson to your specific cluster, but the changes will only be visible on your local machine. diff --git a/config.yaml b/config.yaml index 8c7be715..5aa74027 100644 --- a/config.yaml +++ b/config.yaml @@ -20,7 +20,7 @@ carpentry: 'cp' title: 'Introduction to High-Performance Computing' # Date the lesson was created (YYYY-MM-DD, this is empty by default) -created: 2017-03-24 +created: 2017-03-24 # Comma-separated list of keywords for the lesson keywords: 'software, data, lesson, The Carpentries' @@ -33,7 +33,7 @@ life_cycle: 'alpha' license: 'CC-BY 4.0' # Link to the source repository for this lesson -source: 'https://github.com/tobyhodges/probable-pancake' +source: 'https://github.com/ocaisa/probable-pancake' # Default branch of your lesson branch: 'main' @@ -64,26 +64,29 @@ contact: 'team@carpentries.org' # - another-learner.md # Order of episodes in your lesson -episodes: -- 10-hpc-intro.md -- 11-connecting.md -- 12-cluster.md -- 13-scheduler.md -- 14-environment-variables.md -- 15-modules.md -- 16-transferring-files.md -- 17-parallel.md -- 18-resources.md -- 19-responsibility.md +episodes: + - 10-hpc-intro.Rmd + - 11-connecting.Rmd + - 12-cluster.Rmd + - 13-scheduler.Rmd + - 14-environment-variables.Rmd + - 15-modules.Rmd + - 16-transferring-files.Rmd + - 17-parallel.Rmd + - 18-resources.Rmd + - 19-responsibility.Rmd # Information for Learners -learners: +learners: + - setup.md # Information for Instructors -instructors: +instructors: + - instructor-notes.Rmd # Learner Profiles -profiles: +profiles: + - learner-profiles.md # Customisation --------------------------------------------- # @@ -92,6 +95,6 @@ profiles: carpentry_description: Lesson Description -url: 'https://tkphd.github.io/hpc-intro-online' +url: 'https://ocaisa.github.io/probable_pancake' analytics: carpentries lang: en diff --git a/episodes/10-hpc-intro.md b/episodes/10-hpc-intro.Rmd similarity index 77% rename from episodes/10-hpc-intro.md rename to episodes/10-hpc-intro.Rmd index 5b48859c..6059c5a7 100644 --- a/episodes/10-hpc-intro.md +++ b/episodes/10-hpc-intro.Rmd @@ -4,6 +4,11 @@ teaching: 15 exercises: 5 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Describe what an HPC system is @@ -22,7 +27,7 @@ Frequently, research problems that use computing can outgrow the capabilities of the desktop or laptop computer where they started: - A statistics student wants to cross-validate a model. This involves running - the model 1000 times -- but each run takes an hour. Running the model on + the model 1000 times — but each run takes an hour. Running the model on a laptop will take over a month! In this research problem, final results are calculated after all 1000 models have run, but typically only one model is run at a time (in **serial**) on the laptop. Since each of the 1000 runs is @@ -30,7 +35,7 @@ of the desktop or laptop computer where they started: possible to run them all at once (in **parallel**). - A genomics researcher has been using small datasets of sequence data, but soon will be receiving a new type of sequencing data that is 10 times as - large. It's already challenging to open the datasets on a computer -- + large. It's already challenging to open the datasets on a computer — analyzing these larger datasets will probably crash it. In this research problem, the calculations required might be impossible to parallelize, but a computer with **more memory** would be required to analyze the much larger @@ -54,7 +59,7 @@ problems in parallel**. ## Jargon Busting Presentation -Open the [HPC Jargon Buster]({{ site.url }}files/jargon#p1) +Open the [HPC Jargon Buster](files/jargon.html#p1) in a new tab. To present the content, press `C` to open a **c**lone in a separate window, then press `P` to toggle **p**resentation mode. @@ -71,48 +76,44 @@ results. ## Some Ideas - Checking email: your computer (possibly in your pocket) contacts a remote - machine, authenticates, and downloads a list of new messages; it also - uploads changes to message status, such as whether you read, marked as - junk, or deleted the message. Since yours is not the only account, the - mail server is probably one of many in a data center. -- Searching for a phrase online involves comparing your search term against - a massive database of all known sites, looking for matches. This "query" + machine, authenticates, and downloads a list of new messages; it also uploads + changes to message status, such as whether you read, marked as junk, or + deleted the message. Since yours is not the only account, the mail server is + probably one of many in a data center. +- Searching for a phrase online involves comparing your search term against a + massive database of all known sites, looking for matches. This "query" operation can be straightforward, but building that database is a [monumental task][mapreduce]! Servers are involved at every step. -- Searching for directions on a mapping website involves connecting your - (A) starting and (B) end points by [traversing a graph][dijkstra] in - search of the "shortest" path by distance, time, expense, or another - metric. Converting a map into the right form is relatively simple, but - calculating all the possible routes between A and B is expensive. +- Searching for directions on a mapping website involves connecting your (A) + starting and (B) end points by [traversing a graph][dijkstra] in search of + the "shortest" path by distance, time, expense, or another metric. Converting + a map into the right form is relatively simple, but calculating all the + possible routes between A and B is expensive. Checking email could be serial: your machine connects to one server and exchanges data. Searching by querying the database for your search term (or -endpoints) could also be serial, in that one machine receives your query -and returns the result. However, assembling and storing the full database -is far beyond the capability of any one machine. Therefore, these functions -are served in parallel by a large, ["hyperscale"][hyperscale] collection of -servers working together. - - +endpoints) could also be serial, in that one machine receives your query and +returns the result. However, assembling and storing the full database is far +beyond the capability of any one machine. Therefore, these functions are served +in parallel by a large, ["hyperscale"][hyperscale] collection of servers +working together. ::::::::::::::::::::::::: :::::::::::::::::::::::::::::::::::::::::::::::::: - - [mapreduce]: https://en.wikipedia.org/wiki/MapReduce [dijkstra]: https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm [hyperscale]: https://en.wikipedia.org/wiki/Hyperscale_computing - :::::::::::::::::::::::::::::::::::::::: keypoints -- High Performance Computing (HPC) typically involves connecting to very large computing systems elsewhere in the world. -- These other systems can be used to do work that would either be impossible or much slower on smaller systems. +- High Performance Computing (HPC) typically involves connecting to very large + computing systems elsewhere in the world. +- These other systems can be used to do work that would either be impossible or + much slower on smaller systems. - HPC resources are shared by multiple users. -- The standard method of interacting with such systems is via a command line interface. +- The standard method of interacting with such systems is via a command line + interface. :::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/11-connecting.md b/episodes/11-connecting.Rmd similarity index 83% rename from episodes/11-connecting.md rename to episodes/11-connecting.Rmd index 1d756e82..428ac5e6 100644 --- a/episodes/11-connecting.md +++ b/episodes/11-connecting.Rmd @@ -4,6 +4,11 @@ teaching: 25 exercises: 10 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Configure secure access to a remote HPC system. @@ -33,15 +38,17 @@ results. If you have ever opened the Windows Command Prompt or macOS Terminal, you have seen a CLI. If you have already taken The Carpentries' courses on the UNIX Shell or Version Control, you have used the CLI on your *local machine* -extensively. The only leap to be made here is to open a CLI on a *remote machine*, -while taking some precautions so that other folks on the network can't see (or -change) the commands you're running or the results the remote machine sends -back. We will use the Secure SHell protocol (or SSH) to open an encrypted -network connection between two machines, allowing you to send \& receive text -and data without having to worry about prying eyes. - -![](/fig/connect-to-remote.svg){max-width="50%" alt="Connect to cluster"} - +extensively. The only leap to be made here is to open a CLI on a *remote +machine*, while taking some precautions so that other folks on the network +can't see (or change) the commands you're running or the results the remote +machine sends back. We will use the Secure SHell protocol (or SSH) to open an +encrypted network connection between two machines, allowing you to send \& +receive text and data without having to worry about prying eyes. + +![connect-to-remote.svg](fig/connect-to-remote.svg){ + max-width="50%" + alt="Connect to cluster. " +} SSH clients are usually command-line tools, where you provide the remote machine address as the only required argument. If your username on the remote @@ -56,7 +63,7 @@ When logging in to a laptop, tablet, or other personal device, a username, password, or pattern are normally required to prevent unauthorized access. In these situations, the likelihood of somebody else intercepting your password is low, since logging your keystrokes requires a malicious exploit or physical -access. For systems like {{ site.remote.host }} running an SSH server, anybody +access. For systems like ``r config$remote$host`` running an SSH server, anybody on the network can log in, or try to. Since usernames are often public or easy to guess, your password is often the weakest link in the security chain. Many clusters therefore forbid password-based login, requiring instead that you @@ -80,16 +87,18 @@ In this section you will create a pair of SSH keys: - a private key which you keep on your own computer, and - a public key which can be placed on any remote system you will access. -> ## Private keys are your secure digital passport -> -> A private key that is visible to anyone but you should be considered -> compromised, and must be destroyed. This includes having improper permissions -> on the directory it (or a copy) is stored in, traversing any network that is -> not secure (encrypted), attachment on unencrypted email, and even displaying -> the key on your terminal window. -> -> Protect this key as if it unlocks your front door. In many ways, it does. -> {: .caution} +::::::::::::::::::::::::::::::::::::::::: caution + +## Private keys are your secure digital passport + +A private key that is visible to anyone but you should be considered +compromised, and must be destroyed. This includes having improper permissions +on the directory it (or a copy) is stored in, traversing any network that is +not secure (encrypted), attachment on unencrypted email, and even displaying +the key on your terminal window. + +Protect this key as if it unlocks your front door. In many ways, it does. +:::::::::::::::::::::::::::::::::::::::::::::::::: Regardless of the software or operating system you use, *please* choose a strong password or passphrase to act as another layer of protection for your @@ -111,8 +120,6 @@ common approaches to this: 3. Nothing is *less* secure than a private key with no password. If you skipped password entry by accident, go back and generate a new key pair *with* a strong password. - - :::::::::::::::::::::::::::::::::::::::::::::::::: #### SSH Keys on Linux, Mac, MobaXterm, and Windows Subsystem for Linux @@ -121,7 +128,7 @@ Once you have opened a terminal, check for existing SSH keys and filenames since existing SSH keys are overwritten. ```bash -{{ site.local.prompt }} ls ~/.ssh/ +`r config$local$prompt` ls ~/.ssh/ ``` If `~/.ssh/id_ed25519` already exists, you will need to specify @@ -140,7 +147,7 @@ produce a stronger key than the `ssh-keygen` default by invoking these flags: extension added. ```bash -{{ site.local.prompt }} ssh-keygen -a 100 -f ~/.ssh/id_ed25519 -t ed25519 +`r config$local$prompt` ssh-keygen -a 100 -f ~/.ssh/id_ed25519 -t ed25519 ``` When prompted, enter a strong password with the @@ -165,7 +172,7 @@ If key generation failed because ed25519 is not available, try using the older check for an existing key: ```bash -{{ site.local.prompt }} ls ~/.ssh/ +`r config$local$prompt` ls ~/.ssh/ ``` If `~/.ssh/id_rsa` already exists, you will need to specify choose a different @@ -177,7 +184,7 @@ name for the new key-pair. Generate it as above, with the following extra flags: rather than PEM. ```bash -{{ site.local.prompt }} ssh-keygen -a 100 -b 4096 -f ~/.ssh/id_rsa -o -t rsa +`r config$local$prompt` ssh-keygen -a 100 -b 4096 -f ~/.ssh/id_rsa -o -t rsa ``` When prompted, enter a strong password with the @@ -233,7 +240,7 @@ type it in again. Open your terminal application and check if an agent is running: ```bash -{{ site.local.prompt }} ssh-add -l +`r config$local$prompt` ssh-add -l ``` - If you get an error like this one, @@ -245,7 +252,7 @@ Open your terminal application and check if an agent is running: ... then you need to launch the agent as follows: ```bash - {{ site.local.prompt }} eval $(ssh-agent) + `r config$local$prompt` eval $(ssh-agent) ``` ::::::::::::::::::::::::::::::::::::::::: callout @@ -258,7 +265,7 @@ Open your terminal application and check if an agent is running: shell commands that can be used to reach it -- but *does not execute them!* ```bash - {{ site.local.prompt }} ssh-agent + `r config$local$prompt` ssh-agent ``` ```output @@ -283,7 +290,7 @@ Open your terminal application and check if an agent is running: Add your key to the agent, with session expiration after 8 hours: ```bash -{{ site.local.prompt }} ssh-add -t 8h ~/.ssh/id_ed25519 +`r config$local$prompt` ssh-add -t 8h ~/.ssh/id_ed25519 ``` ```output @@ -302,27 +309,40 @@ See the [PuTTY documentation][putty-agent]. ### Transfer Your Public Key -{% if site.remote.portal %} -Visit [{{ site.remote.portal }}]({{ site.remote.portal }}) to upload your SSH -public key. (Remember, it's the one ending in `.pub`!) - -{% else %} -Use the **s**ecure **c**o**p**y tool to send your public key to the cluster. - -```bash -{{ site.local.prompt }} scp ~/.ssh/id_ed25519.pub {{ site.remote.user }}@{{ site.remote.login }}:~/ +```{r, eval=is.null(config$remote$portal), echo=FALSE, results="asis"} +cat( + paste( + "```bash\n", + config$local$prompt, + " scp ~/.ssh/id_ed25519.pub ", + config$remote$user, + "@", + config$remote$login, + ":~/\n```", + sep="") +) ``` -{% endif %} +```{r, eval=!is.null(config$remote$portal), echo=FALSE, results="asis"} +cat( + paste( + "Visit [", + config$remote$portal, + "](", + config$remote$portal, + ") to upload your SSH public key. (Remember, it's the one ending in `.pub`!)", + sep="") +) +``` ## Log In to the Cluster Go ahead and open your terminal or graphical SSH client, then log in to the -cluster. Replace `{{ site.remote.user }}` with your username or the one +cluster. Replace ``r config$remote$user`` with your username or the one supplied by the instructors. ```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` ``` You may be asked for your password. Watch out: the characters you type after @@ -340,9 +360,9 @@ connected to the local system and the remote system will typically be different for every user. We still need to indicate which system we are entering commands on though so we will adopt the following convention: -- `{{ site.local.prompt }}` when the command is to be entered on a terminal +- ``r config$local$prompt`` when the command is to be entered on a terminal connected to your local computer -- `{{ site.remote.prompt }}` when the command is to be entered on a +- ``r config$remote$prompt`` when the command is to be entered on a terminal connected to the remote system - `$` when it really doesn't matter which system the terminal is connected to. @@ -356,28 +376,28 @@ computer we are logged onto can be checked with the `hostname` command. (You may also notice that the current hostname is also part of our prompt!) ```bash -{{ site.remote.prompt }} hostname +`r config$remote$prompt` hostname ``` ```output -{{ site.remote.host }} +`r config$remote$host` ``` So, we're definitely on the remote machine. Next, let's find out where we are by running `pwd` to **p**rint the **w**orking **d**irectory. ```bash -{{ site.remote.prompt }} pwd +`r config$remote$prompt` pwd ``` ```output -{{ site.remote.homedir }}/{{ site.remote.user }} +`r config$remote$homedir`/`r config$remote$user` ``` Great, we know where we are! Let's see what's in our current directory: ```bash -{{ site.remote.prompt }} ls +`r config$remote$prompt` ls ``` ```output @@ -390,7 +410,7 @@ other filesystems. If they did not, your home directory may appear empty. To double-check, include hidden files in your directory listing: ```bash -{{ site.remote.prompt }} ls -a +`r config$remote$prompt` ls -a ``` ```output @@ -399,13 +419,11 @@ double-check, include hidden files in your directory listing: ``` In the first column, `.` is a reference to the current directory and `..` a -reference to its parent (`{{ site.remote.homedir }}`). You may or may not see +reference to its parent (``r config$remote$homedir``). You may or may not see the other files, or files like them: `.bashrc` is a shell configuration file, which you can edit with your preferences; and `.ssh` is a directory storing SSH keys and a record of authorized connections. -{% unless site.remote.portal %} - ### Install Your SSH Key ::::::::::::::::::::::::::::::::::::::::: callout @@ -416,8 +434,6 @@ Policies and practices for handling SSH keys vary between HPC clusters: follow any guidance provided by the cluster administrators or documentation. In particular, if there is an online portal for managing SSH keys, use that instead of the directions outlined here. - - :::::::::::::::::::::::::::::::::::::::::::::::::: If you transferred your SSH public key with `scp`, you should see @@ -428,14 +444,14 @@ If the `.ssh` folder was not listed above, then it does not yet exist: create it. ```bash -{{ site.remote.prompt }} mkdir ~/.ssh +`r config$remote$prompt` mkdir ~/.ssh ``` Now, use `cat` to print your public key, but redirect the output, appending it to the `authorized_keys` file: ```bash -{{ site.remote.prompt }} cat ~/id_ed25519.pub >> ~/.ssh/authorized_keys +`r config$remote$prompt` cat ~/id_ed25519.pub >> ~/.ssh/authorized_keys ``` That's all! Disconnect, then try to log back into the remote: if your key and @@ -443,17 +459,13 @@ agent have been configured correctly, you should not be prompted for the password for your SSH key. ```bash -{{ site.remote.prompt }} logout +`r config$remote$prompt` logout ``` ```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` ``` -{% endunless %} - - - [gh-ssh]: https://docs.github.com/en/authentication/connecting-to-github-with-ssh [keepass]: https://keepass.info [bitwarden]: https://bitwarden.com @@ -463,14 +475,12 @@ password for your SSH key. [ssh-agent]: https://www.ssh.com/academy/ssh/agent [putty-agent]: https://tartarus.org/~simon/putty-prerel-snapshots/htmldoc/Chapter9.html#pageant - :::::::::::::::::::::::::::::::::::::::: keypoints - An HPC system is a set of networked machines. - HPC systems typically provide login nodes and a set of worker nodes. -- The resources found on independent (worker) nodes can vary in volume and type (amount of RAM, processor architecture, availability of network mounted filesystems, etc.). +- The resources found on independent (worker) nodes can vary in volume and type + (amount of RAM, processor architecture, availability of network mounted + filesystems, etc.). - Files saved on one node are available on all nodes. - :::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/12-cluster.Rmd b/episodes/12-cluster.Rmd new file mode 100644 index 00000000..8a6d7e66 --- /dev/null +++ b/episodes/12-cluster.Rmd @@ -0,0 +1,355 @@ +--- +title: "Working on a remote HPC system" +teaching: 25 +exercises: 10 +--- + +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + +::: questions +- "What is an HPC system?" +- "How does an HPC system work?" +- "How do I log on to a remote HPC system?" +::: + +::: objectives +- "Connect to a remote HPC system." +- "Understand the general HPC system architecture." +::: + +## What Is an HPC System? + +The words "cloud", "cluster", and the phrase "high-performance computing" or +"HPC" are used a lot in different contexts and with various related meanings. +So what do they mean? And more importantly, how do we use them in our work? + +The *cloud* is a generic term commonly used to refer to computing resources +that are a) *provisioned* to users on demand or as needed and b) represent real +or *virtual* resources that may be located anywhere on Earth. For example, a +large company with computing resources in Brazil, Zimbabwe and Japan may manage +those resources as its own *internal* cloud and that same company may also +utilize commercial cloud resources provided by Amazon or Google. Cloud +resources may refer to machines performing relatively simple tasks such as +serving websites, providing shared storage, providing web services (such as +e-mail or social media platforms), as well as more traditional compute +intensive tasks such as running a simulation. + +The term *HPC system*, on the other hand, describes a stand-alone resource for +computationally intensive workloads. They are typically comprised of a +multitude of integrated processing and storage elements, designed to handle +high volumes of data and/or large numbers of floating-point operations +([FLOPS](https://en.wikipedia.org/wiki/FLOPS)) with the highest possible +performance. For example, all of the machines on the +[Top-500](https://www.top500.org) list are HPC systems. To support these +constraints, an HPC resource must exist in a specific, fixed location: +networking cables can only stretch so far, and electrical and optical signals +can travel only so fast. + +The word "cluster" is often used for small to moderate scale HPC resources less +impressive than the [Top-500](https://www.top500.org). Clusters are often +maintained in computing centers that support several such systems, all sharing +common networking and storage to support common compute intensive tasks. + +## Logging In + +The first step in using a cluster is to establish a connection from our laptop +to the cluster. When we are sitting at a computer (or standing, or holding it +in our hands or on our wrists), we have come to expect a visual display with +icons, widgets, and perhaps some windows or applications: a graphical user +interface, or GUI. Since computer clusters are remote resources that we connect +to over often slow or laggy interfaces (WiFi and VPNs especially), it is more +practical to use a command-line interface, or CLI, in which commands and +results are transmitted via text, only. Anything other than text (images, for +example) must be written to disk and opened with a separate program. + +If you have ever opened the Windows Command Prompt or macOS Terminal, you have +seen a CLI. If you have already taken The Carpentries' courses on the UNIX +Shell or Version Control, you have used the CLI on your local machine somewhat +extensively. The only leap to be made here is to open a CLI on a *remote* +machine, while taking some precautions so that other folks on the network can't +see (or change) the commands you're running or the results the remote machine +sends back. We will use the Secure SHell protocol (or SSH) to open an encrypted +network connection between two machines, allowing you to send & receive text +and data without having to worry about prying eyes. + +![Connect to cluster](fig/connect-to-remote.svg){alt-text="Connect to cluster"} + +Make sure you have a SSH client installed on your laptop. Refer to the +[setup](../index.md) section for more details. SSH clients are +usually command-line tools, where you provide the remote machine address as the +only required argument. If your username on the remote system differs from what +you use locally, you must provide that as well. If your SSH client has a +graphical front-end, such as PuTTY or MobaXterm, you will set these arguments +before clicking "connect." From the terminal, you'll write something like `ssh +userName@hostname`, where the "@" symbol is used to separate the two parts of a +single argument. + +Go ahead and open your terminal or graphical SSH client, then log in to the +cluster using your username and the remote computer you can reach from the +outside world, `r config$remote$location`. + +```bash +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` +``` + +Remember to replace ``r config$remote$user`` with your username or the one +supplied by the instructors. You may be asked for your password. Watch out: the +characters you type after the password prompt are not displayed on the screen. +Normal output will resume once you press `Enter`. + +## Where Are We? + +Very often, many users are tempted to think of a high-performance computing +installation as one giant, magical machine. Sometimes, people will assume that +the computer they've logged onto is the entire computing cluster. So what's +really happening? What computer have we logged on to? The name of the current +computer we are logged onto can be checked with the `hostname` command. (You +may also notice that the current hostname is also part of our prompt!) + +```bash +`r config$remote$prompt` hostname +``` + +```output +`r config$remote$host` +``` + +::: challenge + +## What's in Your Home Directory? + +The system administrators may have configured your home directory with some +helpful files, folders, and links (shortcuts) to space reserved for you on +other filesystems. Take a look around and see what you can find. +*Hint:* The shell commands `pwd` and `ls` may come in handy. +Home directory contents vary from user to user. Please discuss any +differences you spot with your neighbors. + +:::: solution + +## It's a Beautiful Day in the Neighborhood + +The deepest layer should differ: ``r config$remote$user`` is uniquely yours. +Are there differences in the path at higher levels? + +If both of you have empty directories, they will look identical. If you +or your neighbor has used the system before, there may be differences. What +are you working on? + +Use `pwd` to **p**rint the **w**orking **d**irectory path: + +```bash +`r config$remote$prompt` pwd +``` + +You can run `ls` to **l**i**s**t the directory contents, though it's +possible nothing will show up (if no files have been provided). To be sure, +use the `-a` flag to show hidden files, too. + +```bash +`r config$remote$prompt` ls -a +``` + +At a minimum, this will show the current directory as `.`, and the parent +directory as `..`. + +:::: +::: + +## Nodes + +Individual computers that compose a cluster are typically called *nodes* +(although you will also hear people call them *servers*, *computers* and +*machines*). On a cluster, there are different types of nodes for different +types of tasks. The node where you are right now is called the *head node*, +*login node*, *landing pad*, or *submit node*. A login node serves as an access +point to the cluster. + +As a gateway, it is well suited for uploading and downloading files, setting up +software, and running quick tests. Generally speaking, the login node should +not be used for time-consuming or resource-intensive tasks. You should be alert +to this, and check with your site's operators or documentation for details of +what is and isn't allowed. In these lessons, we will avoid running jobs on the +head node. + +::: callout + +## Dedicated Transfer Nodes + +If you want to transfer larger amounts of data to or from the cluster, some +systems offer dedicated nodes for data transfers only. The motivation for +this lies in the fact that larger data transfers should not obstruct +operation of the login node for anybody else. Check with your cluster's +documentation or its support team if such a transfer node is available. As a +rule of thumb, consider all transfers of a volume larger than 500 MB to 1 GB +as large. But these numbers change, e.g., depending on the network connection +of yourself and of your cluster or other factors. +::: + +The real work on a cluster gets done by the *worker* (or *compute*) *nodes*. +Worker nodes come in many shapes and sizes, but generally are dedicated to long +or hard tasks that require a lot of computational resources. + +All interaction with the worker nodes is handled by a specialized piece of +software called a scheduler (the scheduler used in this lesson is called +**`r config$sched$name`**). We'll learn more about how to use the +scheduler to submit jobs next, but for now, it can also tell us more +information about the worker nodes. + +For example, we can view all of the worker nodes by running the command +``r config$sched$info``. + +```bash +`r config$remote$prompt` `r config$sched$info` +``` + +```{r, child=paste(snippets, '/cluster/queue-info.Rmd', sep='')} +``` + +There are also specialized machines used for managing disk storage, user +authentication, and other infrastructure-related tasks. Although we do not +typically logon to or interact with these machines directly, they enable a +number of key features like ensuring our user account and files are available +throughout the HPC system. + +## What\'s in a Node? + +All of the nodes in an HPC system have the same components as your own laptop +or desktop: *CPUs* (sometimes also called *processors* or *cores*), *memory* +(or *RAM*), and *disk* space. CPUs are a computer's tool for actually running +programs and calculations. Information about a current task is stored in the +computer's memory. Disk refers to all storage that can be accessed like a file +system. This is generally storage that can hold data permanently, i.e. data is +still there even if the computer has been restarted. While this storage can be +local (a hard drive installed inside of it), it is more common for nodes to +connect to a shared, remote fileserver or cluster of servers. + +![Node anatomy](fig/node_anatomy.png){max-width="20%" alt="Node anatomy" caption=""} + +::: challenge + +## Explore Your Computer + +Try to find out the number of CPUs and amount of memory available on your +personal computer. +Note that, if you're logged in to the remote computer cluster, you need to +log out first. To do so, type `Ctrl+d` or `exit`: + +```bash +`r config$remote$prompt` exit +`r config$local$prompt` +``` + +:::: solution + +There are several ways to do this. Most operating systems have a graphical +system monitor, like the Windows Task Manager. More detailed information can +sometimes be found on the command line. For example, some of the commands used +on a Linux system are: + +Run system utilities + +```bash +`r config$local$prompt` nproc --all +`r config$local$prompt` free -m +``` + +Read from `/proc` + +```bash +`r config$local$prompt` cat /proc/cpuinfo +`r config$local$prompt` cat /proc/meminfo +``` + +Use a system monitor + +```bash +`r config$local$prompt` htop +``` + +:::: +::: + +::: challenge + +## Explore the login node + +Now compare the resources of your computer with those of the head node. + +:::: solution + +```bash +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` +`r config$remote$prompt` nproc --all +`r config$remote$prompt` free -m +``` + +You can get more information about the processors using `lscpu`, +and a lot of detail about the memory by reading the file `/proc/meminfo`: + +```bash +`r config$remote$prompt` less /proc/meminfo +``` + +You can also explore the available filesystems using `df` to show **d**isk +**f**ree space. The `-h` flag renders the sizes in a human-friendly format, +i.e., GB instead of B. The **t**ype flag `-T` shows what kind of filesystem +each resource is. + +```bash +`r config$remote$prompt` df -Th +``` +:::: +::: + +::: discussion +The local filesystems (ext, tmp, xfs, zfs) will depend on whether you're +on the same login node (or compute node, later on). Networked filesystems +(beegfs, cifs, gpfs, nfs, pvfs) will be similar --- but may include +`r config$remote$user`, depending on how it is [mounted]( +https://en.wikipedia.org/wiki/Mount_(computing)). +::: + +::: callout +## Shared Filesystems + +This is an important point to remember: files saved on one node +(computer) are often available everywhere on the cluster! + +::: + +```{r, child=paste(snippets, '/cluster/specific-node-info.Rmd', sep=''), eval=TRUE} +``` + +::: discussion +## Compare Your Computer, the login node and the compute node +Compare your laptop's number of processors and memory with the numbers you +see on the cluster head node and worker node. Discuss the differences with +your neighbor. + +What implications do you think the differences might have on running your +research work on the different systems and nodes? +::: + +::: callout +## Differences Between Nodes + +Many HPC clusters have a variety of nodes optimized for particular workloads. +Some nodes may have larger amount of memory, or specialized resources such as +Graphical Processing Units (GPUs). +::: + +With all of this in mind, we will now cover how to talk to the cluster's +scheduler, and use it to start running our scripts and programs! + +::: keypoints + - "An HPC system is a set of networked machines." + - "HPC systems typically provide login nodes and a set of worker nodes." + - "The resources found on independent (worker) nodes can vary in volume and + type (amount of RAM, processor architecture, availability of network mounted + filesystems, etc.)." + - "Files saved on one node are available on all nodes." +::: diff --git a/episodes/12-cluster.md b/episodes/12-cluster.md deleted file mode 100644 index c83dad32..00000000 --- a/episodes/12-cluster.md +++ /dev/null @@ -1,379 +0,0 @@ ---- -title: Exploring Remote Resources -teaching: 25 -exercises: 10 ---- - -::::::::::::::::::::::::::::::::::::::: objectives - -- Survey system resources using `nproc`, `free`, and the queuing system -- Compare \& contrast resources on the local machine, login node, and worker nodes -- Learn about the various filesystems on the cluster using `df` -- Find out `who` else is logged in -- Assess the number of idle and occupied nodes - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::: questions - -- How does my local computer compare to the remote systems? -- How does the login node compare to the compute nodes? -- Are all compute nodes alike? - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -## Look Around the Remote System - -If you have not already connected to {{ site.remote.name }}, please do so now: - -```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -``` - -Take a look at your home directory on the remote system: - -```bash -{{ site.remote.prompt }} ls -``` - -:::::::::::::::::::::::::::::::::::::: challenge - -## What's different between your machine and the remote? - -Open a second terminal window on your local computer and run the `ls` command -(without logging in to {{ site.remote.name }}). What differences do you see? - -::::::::::::::: solution - -## Solution - -You would likely see something more like this: - -```bash -{{ site.local.prompt }} ls -``` - -```output -Applications Documents Library Music Public -Desktop Downloads Movies Pictures -``` - -The remote computer's home directory shares almost nothing in common with -the local computer: they are completely separate systems! - - - -::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -Most high-performance computing systems run the Linux operating system, which -is built around the UNIX [Filesystem Hierarchy Standard][fshs]. Instead of -having a separate root for each hard drive or storage medium, all files and -devices are anchored to the "root" directory, which is `/`: - -```bash -{{ site.remote.prompt }} ls / -``` - -```output -bin etc lib64 proc sbin sys var -boot {{ site.remote.homedir | replace: "/", "" }} mnt root scratch tmp working -dev lib opt run srv usr -``` - -The "{{ site.remote.homedir | replace: "/", "" }}" directory is the one where -we generally want to keep all of our files. Other folders on a UNIX OS contain -system files and change as you install new software or upgrade your OS. - -::::::::::::::::::::::::::::::::::::::::: callout - -## Using HPC filesystems - -On HPC systems, you have a number of places where you can store your files. -These differ in both the amount of space allocated and whether or not they -are backed up. - -- **Home** -- often a *network filesystem*, data stored here is available - throughout the HPC system, and often backed up periodically. Files stored - here are typically slower to access, the data is actually stored on another - computer and is being transmitted and made available over the network! -- **Scratch** -- typically faster than the networked Home directory, but not - usually backed up, and should not be used for long term storage. -- **Work** -- sometimes provided as an alternative to Scratch space, Work is - a fast file system accessed over the network. Typically, this will have - higher performance than your home directory, but lower performance than - Scratch; it may not be backed up. It differs from Scratch space in that - files in a work file system are not automatically deleted for you: you must - manage the space yourself. - - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -## Nodes - -Individual computers that compose a cluster are typically called *nodes* -(although you will also hear people call them *servers*, *computers* and -*machines*). On a cluster, there are different types of nodes for different -types of tasks. The node where you are right now is called the *login node*, -*head node*, *landing pad*, or *submit node*. A login node serves as an access -point to the cluster. - -As a gateway, the login node should not be used for time-consuming or -resource-intensive tasks. You should be alert to this, and check with your -site's operators or documentation for details of what is and isn't allowed. It -is well suited for uploading and downloading files, setting up software, and -running tests. Generally speaking, in these lessons, we will avoid running jobs -on the login node. - -Who else is logged in to the login node? - -```bash -{{ site.remote.prompt }} who -``` - -This may show only your user ID, but there are likely several other people -(including fellow learners) connected right now. - -::::::::::::::::::::::::::::::::::::::::: callout - -## Dedicated Transfer Nodes - -If you want to transfer larger amounts of data to or from the cluster, some -systems offer dedicated nodes for data transfers only. The motivation for -this lies in the fact that larger data transfers should not obstruct -operation of the login node for anybody else. Check with your cluster's -documentation or its support team if such a transfer node is available. As a -rule of thumb, consider all transfers of a volume larger than 500 MB to 1 GB -as large. But these numbers change, e.g., depending on the network connection -of yourself and of your cluster or other factors. - - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -The real work on a cluster gets done by the *compute* (or *worker*) *nodes*. -compute nodes come in many shapes and sizes, but generally are dedicated to long -or hard tasks that require a lot of computational resources. - -All interaction with the compute nodes is handled by a specialized piece of -software called a scheduler (the scheduler used in this lesson is called -{{ site.sched.name }}). We'll learn more about how to use the -scheduler to submit jobs next, but for now, it can also tell us more -information about the compute nodes. - -For example, we can view all of the compute nodes by running the command -`{{ site.sched.info }}`. - -```bash -{{ site.remote.prompt }} {{ site.sched.info }} -``` - -```output -PARTITION AVAIL TIMELIMIT NODES STATE NODELIST -cpubase_bycore_b1* up infinite 4 idle node[1-2],smnode[1-2] -node up infinite 2 idle node[1-2] -smnode up infinite 2 idle smnode[1-2] -``` - -A lot of the nodes are busy running work for other users: we are not alone -here! - -There are also specialized machines used for managing disk storage, user -authentication, and other infrastructure-related tasks. Although we do not -typically logon to or interact with these machines directly, they enable a -number of key features like ensuring our user account and files are available -throughout the HPC system. - -## What's in a Node? - -All of the nodes in an HPC system have the same components as your own laptop -or desktop: *CPUs* (sometimes also called *processors* or *cores*), *memory* -(or *RAM*), and *disk* space. CPUs are a computer's tool for actually running -programs and calculations. Information about a current task is stored in the -computer's memory. Disk refers to all storage that can be accessed like a file -system. This is generally storage that can hold data permanently, i.e. data is -still there even if the computer has been restarted. While this storage can be -local (a hard drive installed inside of it), it is more common for nodes to -connect to a shared, remote fileserver or cluster of servers. - -![](/fig/node\_anatomy.png){max-width="40%" alt="Node anatomy"} - -::::::::::::::::::::::::::::::::::::::: challenge - -## Explore Your Computer - -Try to find out the number of CPUs and amount of memory available on your -personal computer. - -Note that, if you're logged in to the remote computer cluster, you need to -log out first. To do so, type `Ctrl+d` or `exit`: - -```bash -{{ site.remote.prompt }} exit -{{ site.local.prompt }} -``` - -::::::::::::::: solution - -## Solution - -There are several ways to do this. Most operating systems have a graphical -system monitor, like the Windows Task Manager. More detailed information -can be found on the command line: - -- Run system utilities - - ```bash - {{ site.local.prompt }} nproc --all - {{ site.local.prompt }} free -m - ``` - -- Read from `/proc` - - ```bash - {{ site.local.prompt }} cat /proc/cpuinfo - {{ site.local.prompt }} cat /proc/meminfo - ``` - -- Run system monitor - - ```bash - {{ site.local.prompt }} htop - ``` - -::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::::::::::::::::: challenge - -## Explore the Login Node - -Now compare the resources of your computer with those of the login node. - -::::::::::::::: solution - -## Solution - -```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -{{ site.remote.prompt }} nproc --all -{{ site.remote.prompt }} free -m -``` - -You can get more information about the processors using `lscpu`, -and a lot of detail about the memory by reading the file `/proc/meminfo`: - -```bash -{{ site.remote.prompt }} less /proc/meminfo -``` - -You can also explore the available filesystems using `df` to show **d**isk -**f**ree space. The `-h` flag renders the sizes in a human-friendly format, -i.e., GB instead of B. The **t**ype flag `-T` shows what kind of filesystem -each resource is. - -```bash -{{ site.remote.prompt }} df -Th -``` - -:::::::::::::::::::::::::::::::::::::: discussion - -## Different results from `df` - -- The local filesystems (ext, tmp, xfs, zfs) will depend on whether - you're on the same login node (or compute node, later on). -- Networked filesystems (beegfs, cifs, gpfs, nfs, pvfs) will be similar - \-- but may include {{ site.remote.user }}, depending on how it - is [mounted][mount]. - - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::::::::::::::::::: callout - -## Shared Filesystems - -This is an important point to remember: files saved on one node -(computer) are often available everywhere on the cluster! - - - - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::::::::::::::::: challenge - -## Explore a Worker Node - -Finally, let's look at the resources available on the worker nodes where your -jobs will actually run. Try running this command to see the name, CPUs and -memory available on the worker nodes: - -```bash -{{ site.remote.prompt }} sinfo -o "%n %c %m" | column -t -``` - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::: challenge - -## Compare Your Computer, the Login Node and the Compute Node - -Compare your laptop's number of processors and memory with the numbers you -see on the cluster login node and compute node. What implications do -you think the differences might have on running your research work on the -different systems and nodes? - -::::::::::::::: solution - -## Solution - -Compute nodes are usually built with processors that have *higher -core-counts* than the login node or personal computers in order to support -highly parallel tasks. Compute nodes usually also have substantially *more -memory (RAM)* installed than a personal computer. More cores tends to help -jobs that depend on some work that is easy to perform in *parallel*, and -more, faster memory is key for large or *complex numerical tasks*. - - - -::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -::::::::::::::::::::::::::::::::::::::::: callout - -## Differences Between Nodes - -Many HPC clusters have a variety of nodes optimized for particular workloads. -Some nodes may have larger amount of memory, or specialized resources such as -Graphics Processing Units (GPUs or "video cards"). - - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -With all of this in mind, we will now cover how to talk to the cluster's -scheduler, and use it to start running our scripts and programs! - - - -[fshs]: https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard -[mount]: https://en.wikipedia.org/wiki/Mount_\(computing\) - - -:::::::::::::::::::::::::::::::::::::::: keypoints - -- An HPC system is a set of networked machines. -- HPC systems typically provide login nodes and a set of compute nodes. -- The resources found on independent (worker) nodes can vary in volume and type (amount of RAM, processor architecture, availability of network mounted filesystems, etc.). -- Files saved on shared storage are available on all nodes. -- The login node is a shared machine: be considerate of other users. - -:::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/13-scheduler.md b/episodes/13-scheduler.Rmd similarity index 73% rename from episodes/13-scheduler.md rename to episodes/13-scheduler.Rmd index eebee36e..1d8bc45d 100644 --- a/episodes/13-scheduler.md +++ b/episodes/13-scheduler.Rmd @@ -1,9 +1,14 @@ --- -title: Scheduler Fundamentals +title: "Scheduler Fundamentals" teaching: 45 exercises: 30 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Submit a simple script to the cluster. @@ -34,10 +39,10 @@ in a restaurant. If you can relate to an instance where you had to wait for a while in a queue to get in to a popular restaurant, then you may now understand why sometimes your job do not start instantly as in your laptop. -![](/fig/restaurant\_queue\_manager.svg){alt="Compare a job scheduler to a waiter in a restaurant" max-width="75%"} +![](fig/restaurant_queue_manager.svg){alt="Compare a job scheduler to a waiter in a restaurant" max-width="75%"} -The scheduler used in this lesson is {{ site.sched.name }}. Although -{{ site.sched.name }} is not used everywhere, running jobs is quite similar +The scheduler used in this lesson is `r config$sched$name`. Although +`r config$sched$name` is not used everywhere, running jobs is quite similar regardless of what software is being used. The exact syntax might change, but the concepts remain the same. @@ -52,7 +57,7 @@ In this case, the job we want to run is a shell script -- essentially a text file containing a list of UNIX commands to be executed in a sequential manner. Our shell script will have three parts: -- On the very first line, add `{{ site.remote.bash_shebang }}`. The `#!` +- On the very first line, add ``r config$remote$shebang``. The `#!` (pronounced "hash-bang" or "shebang") tells the computer what program is meant to process the contents of this file. In this case, we are telling it that the commands that follow are written for the command-line shell (what @@ -66,11 +71,11 @@ manner. Our shell script will have three parts: name of the machine the script is run on. ```bash -{{ site.remote.prompt }} nano example-job.sh +`r config$remote$prompt` nano example-job.sh ``` -```output -{{ site.remote.bash_shebang }} +```bash +`r config$remote$shebang` echo -n "This script is running on " hostname @@ -87,11 +92,11 @@ Run the script. Does it execute on the cluster or just our login node? ## Solution ```bash -{{ site.remote.prompt }} bash example-job.sh +`r config$remote$prompt` bash example-job.sh ``` ```output -This script is running on {{ site.remote.host }} +This script is running on `r config$remote$host` ``` ::::::::::::::::::::::::: @@ -103,27 +108,26 @@ the compute nodes: we need the scheduler to queue up `example-job.sh` to run on a compute node. To submit this task to the scheduler, we use the -`{{ site.sched.submit.name }}` command. +``r config$sched$submit$name`` command. This creates a *job* which will run the *script* when *dispatched* to a compute node which the queuing system has identified as being available to perform the work. ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh ``` -```output -Submitted batch job 9 +```{r, child=paste(snippets, '/scheduler/basic-job-script.Rmd', sep=''), eval=TRUE} ``` And that's all we need to do to submit a job. Our work is done -- now the scheduler takes over and tries to run the job for us. While the job is waiting to run, it goes into a list of jobs called the *queue*. To check on our job's status, we check the queue using the command -`{{ site.sched.status }} {{ site.sched.flag.user }}`. +``r config$sched$status` `r config$sched$flag$user``. ```bash -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` ```output @@ -140,13 +144,12 @@ or `RUNNING` state. Sometimes our jobs might need to wait in a queue ## Where's the Output? On the login node, this script printed output to the terminal -- but -now, when `{{ site.sched.status }}` shows the job has finished, +now, when ``r config$sched$status`` shows the job has finished, nothing was printed to the terminal. Cluster job output is typically redirected to a file in the directory you launched it from. Use `ls` to find and `cat` to read the file. - :::::::::::::::::::::::::::::::::::::::::::::::::: ## Customising a Job @@ -160,24 +163,24 @@ resources we must customize our job script. Comments in UNIX shell scripts (denoted by `#`) are typically ignored, but there are exceptions. For instance the special `#!` comment at the beginning of scripts specifies what program should be used to run it (you'll typically see -`{{ site.local.bash_shebang }}`). Schedulers like {{ site.sched.name }} also +``r config$local$shebang``). Schedulers like `r config$sched$name` also have a special comment used to denote special scheduler-specific options. Though these comments differ from scheduler to scheduler, -{{ site.sched.name }}'s special comment is `{{ site.sched.comment }}`. Anything -following the `{{ site.sched.comment }}` comment is interpreted as an +`r config$sched$name`'s special comment is ``r config$sched$comment``. Anything +following the ``r config$sched$comment`` comment is interpreted as an instruction to the scheduler. Let's illustrate this by example. By default, a job's name is the name of the -script, but the `{{ site.sched.flag.name }}` option can be used to change the +script, but the ``r config$sched$flag$name`` option can be used to change the name of a job. Add an option to the script: ```bash -{{ site.remote.prompt }} cat example-job.sh +`r config$remote$prompt` cat example-job.sh ``` -```output -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} hello-world +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` hello-world echo -n "This script is running on " hostname @@ -186,8 +189,8 @@ hostname Submit the job and monitor its status: ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` ```output @@ -221,7 +224,7 @@ The following are several key resource requests: - `--nodes=` or `-N `: How many separate machines does your job need to run on? Note that if you set `ntasks` to a number greater than what - one machine can offer, {{ site.sched.name }} will set this value + one machine can offer, `r config$sched$name` will set this value automatically. Note that just *requesting* these resources does not make your job run faster, @@ -246,12 +249,12 @@ for it on the cluster. ## Solution ```bash -{{ site.remote.prompt }} cat example-job.sh +`r config$remote$prompt` cat example-job.sh ``` -```output -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.time }} 00:01 # timeout in HH:MM +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$time` 00:01 # timeout in HH:MM echo -n "This script is running on " sleep 20 # time in seconds @@ -259,10 +262,10 @@ hostname ``` ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh ``` -Why are the {{ site.sched.name }} runtime and `sleep` time not identical? +Why are the `r config$sched$name` runtime and `sleep` time not identical? @@ -275,13 +278,13 @@ killed. Let's use wall time as an example. We will request 1 minute of wall time, and attempt to run a job for two minutes. ```bash -{{ site.remote.prompt }} cat example-job.sh +`r config$remote$prompt` cat example-job.sh ``` -```output -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} long_job -{{ site.sched.comment }} {{ site.sched.flag.time }} 00:01 # timeout in HH:MM +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` long_job +`r config$sched$comment` `r config$sched$flag$time` 00:01 # timeout in HH:MM echo "This script is running on ... " sleep 240 # time in seconds @@ -292,12 +295,12 @@ Submit the job and wait for it to finish. Once it is has finished, check the log file. ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` ```bash -{{ site.remote.prompt }} cat slurm-12.out +`r config$remote$prompt` cat slurm-12.out ``` ```output @@ -311,7 +314,7 @@ this appears harsh, this is actually a feature. Strict adherence to resource requests allows the scheduler to find the best possible place for your jobs. Even more importantly, it ensures that another user cannot use more resources than they've been given. If another user messes up and accidentally attempts to -use all of the cores or memory on a node, {{ site.sched.name }} will either +use all of the cores or memory on a node, `r config$sched$name` will either restrain their job to the requested resources or kill the job outright. Other jobs on the node will be unaffected. This means that one user cannot mess up the experience of others, the only jobs affected by a mistake in scheduling @@ -320,13 +323,13 @@ will be their own. ## Cancelling a Job Sometimes we'll make a mistake and need to cancel a job. This can be done with -the `{{ site.sched.del }}` command. Let's submit a job and then cancel it using +the ``r config$sched$del`` command. Let's submit a job and then cancel it using its job number (remember to change the walltime so that it runs long enough for you to cancel it before it is killed!). ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` ```output @@ -341,9 +344,9 @@ return of your command prompt indicates that the request to cancel the job was successful. ```bash -{{ site.remote.prompt }} {{site.sched.del }} 38759 +`r config$remote$prompt` `r config$sched$del` 38759 # It might take a minute for the job to disappear from the queue... -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` ```output @@ -367,15 +370,15 @@ Try submitting multiple jobs and then cancelling them all. First, submit a trio of jobs: ```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh ``` Then, cancel them all: ```bash -{{ site.remote.prompt }} {{ site.sched.del }} -u {{ site.remote.user }} +`r config$remote$prompt` `r config$sched$del` -u `r config$remote$user` ``` ::::::::::::::::::::::::: @@ -385,34 +388,34 @@ Then, cancel them all: ## Other Types of Jobs Up to this point, we've focused on running jobs in batch mode. -{{ site.sched.name }} also provides the ability to start an interactive session. +``r config$sched$name`` also provides the ability to start an interactive session. There are very frequently tasks that need to be done interactively. Creating an entire job script might be overkill, but the amount of resources required is too much for a login node to handle. A good example of this might be building a genome index for alignment with a tool like [HISAT2][hisat]. Fortunately, we -can run these types of tasks as a one-off with `{{ site.sched.interactive }}`. +can run these types of tasks as a one-off with ``r config$sched$interactive``. -`{{ site.sched.interactive }}` runs a single command on the cluster and then +``r config$sched$interactive`` runs a single command on the cluster and then exits. Let's demonstrate this by running the `hostname` command with -`{{ site.sched.interactive }}`. (We can cancel an `{{ site.sched.interactive }}` +``r config$sched$interactive``. (We can cancel an ``r config$sched$interactive`` job with `Ctrl-c`.) ```bash -{{ site.remote.prompt }} {{ site.sched.interactive }} hostname +`r config$remote$prompt` `r config$sched$interactive` hostname ``` ```output -{{ site.remote.node }} +`r config$remote$node` ``` -`{{ site.sched.interactive }}` accepts all of the same options as -`{{ site.sched.submit.name }}`. However, instead of specifying these in a script, +``r config$sched$interactive`` accepts all of the same options as +``r config$sched$submit$name``. However, instead of specifying these in a script, these options are specified on the command-line when starting a job. To submit a job that uses 2 CPUs for instance, we could use the following command: ```bash -{{ site.remote.prompt }} {{ site.sched.interactive }} -n 2 echo "This job will use 2 CPUs." +`r config$remote$prompt` `r config$sched$interactive` -n 2 echo "This job will use 2 CPUs." ``` ```output @@ -421,17 +424,17 @@ This job will use 2 CPUs. ``` Typically, the resulting shell environment will be the same as that for -`{{ site.sched.submit.name }}`. +``r config$sched$submit$name``. ### Interactive jobs Sometimes, you will need a lot of resources for interactive use. Perhaps it's our first time running an analysis or we are attempting to debug something that -went wrong with a previous job. Fortunately, {{ site.sched.name }} makes it -easy to start an interactive job with `{{ site.sched.interactive }}`: +went wrong with a previous job. Fortunately, `r config$sched$name` makes it +easy to start an interactive job with ``r config$sched$interactive``: ```bash -{{ site.remote.prompt }} {{ site.sched.interactive }} --pty bash +`r config$remote$prompt` `r config$sched$interactive` --pty bash ``` You should be presented with a bash prompt. Note that the prompt will likely @@ -444,7 +447,7 @@ logged on. You can also verify this with `hostname`. To see graphical output inside your jobs, you need to use X11 forwarding. To connect with this feature enabled, use the `-Y` option when you login with -the `ssh` command, e.g., `ssh -Y {{ site.remote.user }}@{{ site.remote.login }}`. +the `ssh` command, e.g., `ssh -Y `r config$remote$user`@`r config$remote$login``. To demonstrate what happens when you create a graphics window on the remote node, use the `xeyes` command. A relatively adorable pair of eyes should pop @@ -454,19 +457,15 @@ XQuartz (and restarted your computer) for this to work. If your cluster has the [slurm-spank-x11](https://github.com/hautreux/slurm-spank-x11) plugin installed, you can ensure X11 forwarding within interactive jobs by using the -`--x11` option for `{{ site.sched.interactive }}` with the command -`{{ site.sched.interactive }} --x11 --pty bash`. - +`--x11` option for ``r config$sched$interactive`` with the command +``r config$sched$interactive` --x11 --pty bash`. :::::::::::::::::::::::::::::::::::::::::::::::::: When you are done with the interactive job, type `exit` to quit your session. - - [hisat]: https://daehwankimlab.github.io/hisat2/ - :::::::::::::::::::::::::::::::::::::::: keypoints - The scheduler handles how compute resources are shared between users. diff --git a/episodes/14-environment-variables.md b/episodes/14-environment-variables.Rmd similarity index 94% rename from episodes/14-environment-variables.md rename to episodes/14-environment-variables.Rmd index 84ccdec9..39bc80d3 100644 --- a/episodes/14-environment-variables.md +++ b/episodes/14-environment-variables.Rmd @@ -4,6 +4,11 @@ teaching: 10 exercises: 5 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Understand how variables are implemented in the shell @@ -27,8 +32,7 @@ exercises: 5 This episode has been remixed from the [Shell Extras episode on Shell Variables](https://github.com/carpentries-incubator/shell-extras/blob/gh-pages/_episodes/08-environment-variables.md) -and the [HPC Shell episode on scripts](https://github.com/hpc-carpentry/hpc-shell/blob/gh-pages/_episodes/05-scripts.md) - +and the [HPC Shell episode on scripts](https://github.com/hpc-carpentry/hpc-shell/blob/gh-pages/_episodes/05-scripts.md). :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -191,7 +195,7 @@ run**. ## Job environment variables -When {{ site.sched.name }} runs a job, it sets a number of environment +When ``r config$sched$name`` runs a job, it sets a number of environment variables for the job. One of these will let us check what directory our job script was submitted from. The `SLURM_SUBMIT_DIR` variable is set to the directory from which our job was submitted. Using the `SLURM_SUBMIT_DIR` @@ -203,13 +207,13 @@ job was submitted. ## Solution ```bash -{{ site.remote.prompt }} nano example-job.sh -{{ site.remote.prompt }} cat example-job.sh +`r config$remote$prompt` nano example-job.sh +`r config$remote$prompt` cat example-job.sh ``` ```output -{{ site.remote.bash_shebang }} -#SBATCH -t 00:00:30 +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$time` 00:00:30 echo -n "This script is running on " hostname @@ -275,7 +279,7 @@ unless we type in the full path to the program, since the directory `/users/vlad` isn't in `PATH`. This means that I can have executables in lots of different places as long as -I remember that I need to to update my `PATH` so that my shell can find them. +I remember that I need to update my `PATH` so that my shell can find them. What if I want to run two different versions of the same program? Since they share the same name, if I add them both to my `PATH` the first one @@ -285,8 +289,6 @@ runtime environment to make that possible without us needing to do a lot of bookkeeping on what the value of `PATH` (and other important environment variables) is or should be. - - :::::::::::::::::::::::::::::::::::::::: keypoints - Shell variables are by default treated as strings @@ -295,5 +297,3 @@ variables) is or should be. - The `PATH` variable defines the shell's search path :::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/15-modules.Rmd b/episodes/15-modules.Rmd new file mode 100644 index 00000000..f02b43a7 --- /dev/null +++ b/episodes/15-modules.Rmd @@ -0,0 +1,232 @@ +--- +title: Accessing software via Modules +teaching: 30 +exercises: 15 +--- + +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + +::::::::::::::::::::::::::::::::::::::: objectives + +- Load and use a software package. +- Explain how the shell environment changes when the module mechanism loads or unloads packages. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do we load and unload software packages? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +On a high-performance computing system, it is seldom the case that the software +we want to use is available when we log in. It is installed, but we will need +to "load" it before it can run. + +Before we start using individual software packages, however, we should +understand the reasoning behind this approach. The three biggest factors are: + +- software incompatibilities +- versioning +- dependencies + +Software incompatibility is a major headache for programmers. Sometimes the +presence (or absence) of a software package will break others that depend on +it. Two well known examples are Python and C compiler versions. +Python 3 famously provides a `python` command that conflicts with that provided +by Python 2. Software compiled against a newer version of the C libraries and +then run on a machine that has older C libraries installed will result in an +opaque `'GLIBCXX_3.4.20' not found` error. + +Software versioning is another common issue. A team might depend on a certain +package version for their research project -- if the software version was to +change (for instance, if a package was updated), it might affect their results. +Having access to multiple software versions allows a set of researchers to +prevent software versioning issues from affecting their results. + +Dependencies are where a particular software package (or even a particular +version) depends on having access to another software package (or even a +particular version of another software package). For example, the VASP +materials science software may require a particular version of the +FFTW (Fastest Fourier Transform in the West) software library available for it +to work. + +## Environment Modules + +Environment modules are the solution to these problems. A *module* is a +self-contained description of a software package -- it contains the +settings required to run a software package and, usually, encodes required +dependencies on other software packages. + +There are a number of different environment module implementations commonly +used on HPC systems: the two most common are *TCL modules* and *Lmod*. Both of +these use similar syntax and the concepts are the same so learning to use one +will allow you to use whichever is installed on the system you are using. In +both implementations the `module` command is used to interact with environment +modules. An additional subcommand is usually added to the command to specify +what you want to do. For a list of subcommands you can use `module -h` or +`module help`. As for all commands, you can access the full help on the *man* +pages with `man module`. + +On login you may start out with a default set of modules loaded or you may +start out with an empty environment; this depends on the setup of the system +you are using. + +### Listing Available Modules + +To see available software modules, use `module avail`: + +```{r, child=paste(snippets, '/modules/available-modules.Rmd', sep=''), eval=TRUE} +``` + +Note that piping the output through `less` allows us to search within the output using the / key. + +### Listing Currently Loaded Modules + +You can use the `module list` command to see which modules you currently have +loaded in your environment. If you have no modules loaded, you will see a +message telling you so. + +```bash +`r config$remote$prompt` module list +``` + +```output +No Modulefiles Currently Loaded. +``` + +## Loading and Unloading Software + +To load a software module, use `module load`. + +In this example we will use Python 3. Initially, it is not loaded. +We can test this by using the `which` command. `which` looks for +programs the same way that Bash does, so we can use it to tell us +where a particular piece of software is stored. + +```bash +`r config$remote$prompt` which python3 +``` + +```{r, child=paste(snippets, '/modules/missing-python.Rmd', sep=''), eval=TRUE} +``` + +We can load the `python3` command with `module load`: + +```{r, child=paste(snippets, '/modules/module-load-python.Rmd', sep=''), eval=TRUE} +``` + +So, what just happened? + +To understand the output, first we need to understand the nature of the `$PATH` +environment variable. `$PATH` is a special environment variable that controls +where a UNIX system looks for software. Specifically `$PATH` is a list of +directories (separated by `:`) that the OS searches through for a command +before giving up and telling us it can't find it. As with all environment +variables we can print it out using `echo`. + +```bash +`r config$remote$prompt` echo $PATH +``` + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/SQLite/3.31.1-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Tcl/8.6.10-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/GCCcore/x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin:/opt/software/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/puppetlabs/bin:/home/user01/.local/bin:/home/user01/bin +``` + +You'll notice a similarity to the output of the `which` command. In this case, +there's only one difference: the different directory at the beginning. When we +ran the `module load` command, it added a directory to the beginning of our +`$PATH` -- or "prepended to PATH". Let's examine what's there: + +```{r, child=paste(snippets, '/modules/python-executable-dir.Rmd', sep=''), eval=TRUE} +``` + +Taking this to its conclusion, `module load` will add software to your `$PATH`. +It "loads" software. A special note on this - depending on which version of the +`module` program that is installed at your site, `module load` will also load +required software dependencies. + +```{r, child=paste(snippets, '/modules/software-dependencies.Rmd', sep=''), eval=TRUE} +``` + +Note that this module loading process happens principally through +the manipulation of environment variables like `$PATH`. There +is usually little or no data transfer involved. + +The module loading process manipulates other special environment +variables as well, including variables that influence where the +system looks for software libraries, and sometimes variables which +tell commercial software packages where to find license servers. + +The module command also restores these shell environment variables +to their previous state when a module is unloaded. + +## Software Versioning + +So far, we've learned how to load and unload software packages. This is very +useful. However, we have not yet addressed the issue of software versioning. At +some point or other, you will run into issues where only one particular version +of some software will be suitable. Perhaps a key bugfix only happened in a +certain version, or version X broke compatibility with a file format you use. +In either of these example cases, it helps to be very specific about what +software is loaded. + +Let's examine the output of `module avail` more closely, using the pager since +there may be reams of output: + +```{r, child=paste(snippets, '/modules/available-modules.Rmd', sep=''), eval=TRUE} +``` + +If the software your Slurm script runs requires on a specific version +of a dependency, make sure you use the full name of the module, rather +than the _default_ loaded when you give only its name (up to the first +slash). + +::::::::::::::::::::::::::::::::::::::: challenge + +## Using Software Modules in Scripts + +Create a job that is able to run `python3 --version`. Remember, no software +is loaded by default! Running a job is just like logging on to the system +(you should not assume a module loaded on the login node is loaded on a +compute node). + +::::::::::::::: solution + +## Solution + +```bash +`r config$remote$prompt` nano python-module.sh +`r config$remote$prompt` cat python-module.sh +``` + +```output +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$partition` +`r if (!is.null(config$sched$flag$qos)) {print(paste(config$sched$comment,config$sched$flag$qos))} +`r config$sched$comment` `r config$sched$flag$time` 00:00:30 + +module load `r config$remote$modules$python` + +python3 --version +``` + +```bash +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` python-module.sh +``` + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Load software with `module load softwareName`. +- Unload software with `module unload` +- The module system handles software versioning and package conflicts for you automatically. + +:::::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/episodes/15-modules.md b/episodes/15-modules.md deleted file mode 100644 index 4b77091d..00000000 --- a/episodes/15-modules.md +++ /dev/null @@ -1,394 +0,0 @@ ---- -title: Accessing software via Modules -teaching: 30 -exercises: 15 ---- - -::::::::::::::::::::::::::::::::::::::: objectives - -- Load and use a software package. -- Explain how the shell environment changes when the module mechanism loads or unloads packages. - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::: questions - -- How do we load and unload software packages? - -:::::::::::::::::::::::::::::::::::::::::::::::::: - -On a high-performance computing system, it is seldom the case that the software -we want to use is available when we log in. It is installed, but we will need -to "load" it before it can run. - -Before we start using individual software packages, however, we should -understand the reasoning behind this approach. The three biggest factors are: - -- software incompatibilities -- versioning -- dependencies - -Software incompatibility is a major headache for programmers. Sometimes the -presence (or absence) of a software package will break others that depend on -it. Two well known examples are Python and C compiler versions. -Python 3 famously provides a `python` command that conflicts with that provided -by Python 2. Software compiled against a newer version of the C libraries and -then run on a machine that has older C libraries installed will result in a -nasty `'GLIBCXX_3.4.20' not found` error. - -Software versioning is another common issue. A team might depend on a certain -package version for their research project - if the software version was to -change (for instance, if a package was updated), it might affect their results. -Having access to multiple software versions allows a set of researchers to -prevent software versioning issues from affecting their results. - -Dependencies are where a particular software package (or even a particular -version) depends on having access to another software package (or even a -particular version of another software package). For example, the VASP -materials science software may depend on having a particular version of the -FFTW (Fastest Fourier Transform in the West) software library available for it -to work. - -## Environment Modules - -Environment modules are the solution to these problems. A *module* is a -self-contained description of a software package -- it contains the -settings required to run a software package and, usually, encodes required -dependencies on other software packages. - -There are a number of different environment module implementations commonly -used on HPC systems: the two most common are *TCL modules* and *Lmod*. Both of -these use similar syntax and the concepts are the same so learning to use one -will allow you to use whichever is installed on the system you are using. In -both implementations the `module` command is used to interact with environment -modules. An additional subcommand is usually added to the command to specify -what you want to do. For a list of subcommands you can use `module -h` or -`module help`. As for all commands, you can access the full help on the *man* -pages with `man module`. - -On login you may start out with a default set of modules loaded or you may -start out with an empty environment; this depends on the setup of the system -you are using. - -### Listing Available Modules - -To see available software modules, use `module avail`: - -```bash -{{ site.remote.prompt }} module avail -``` - -```output -~~~ /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/modules/all ~~~ - Bazel/3.6.0-GCCcore-x.y.z NSS/3.51-GCCcore-x.y.z - Bison/3.5.3-GCCcore-x.y.z Ninja/1.10.0-GCCcore-x.y.z - Boost/1.72.0-gompi-2020a OSU-Micro-Benchmarks/5.6.3-gompi-2020a - CGAL/4.14.3-gompi-2020a-Python-3.x.y OpenBLAS/0.3.9-GCC-x.y.z - CMake/3.16.4-GCCcore-x.y.z OpenFOAM/v2006-foss-2020a - -[removed most of the output here for clarity] - - Where: - L: Module is loaded - Aliases: Aliases exist: foo/1.2.3 (1.2) means that "module load foo/1.2" - will load foo/1.2.3 - D: Default Module - -Use "module spider" to find all possible modules and extensions. -Use "module keyword key1 key2 ..." to search for all possible modules matching -any of the "keys". -``` - -### Listing Currently Loaded Modules - -You can use the `module list` command to see which modules you currently have -loaded in your environment. If you have no modules loaded, you will see a -message telling you so - -```bash -{{ site.remote.prompt }} module list -``` - -```output -No Modulefiles Currently Loaded. -``` - -## Loading and Unloading Software - -To load a software module, use `module load`. In this example we will use -Python 3. - -Initially, Python 3 is not loaded. We can test this by using the `which` -command. `which` looks for programs the same way that Bash does, so we can use -it to tell us where a particular piece of software is stored. - -```bash -{{ site.remote.prompt }} which python3 -``` - -If the `python3` command was unavailable, we would see output like - -```output -/usr/bin/which: no python3 in (/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin:/opt/software/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/puppetlabs/bin:/home/{{site.remote.user}}/.local/bin:/home/{{site.remote.user}}/bin) -``` - -Note that this wall of text is really a list, with values separated -by the `:` character. The output is telling us that the `which` command -searched the following directories for `python3`, without success: - -```output -/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin -/opt/software/slurm/bin -/usr/local/bin -/usr/bin -/usr/local/sbin -/usr/sbin -/opt/puppetlabs/bin -/home/{{site.remote.user}}/.local/bin -/home/{{site.remote.user}}/bin -``` - -However, in our case we do have an existing `python3` available so we see - -```output -/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin/python3 -``` - -We need a different Python than the system provided one though, so let us load -a module to access it. - -We can load the `python3` command with `module load`: - -```bash -{{ site.remote.prompt }} module load {{ site.remote.module_python3 }} -{{ site.remote.prompt }} which python3 -``` - -```output -/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin/python3 -``` - -So, what just happened? - -To understand the output, first we need to understand the nature of the `$PATH` -environment variable. `$PATH` is a special environment variable that controls -where a UNIX system looks for software. Specifically `$PATH` is a list of -directories (separated by `:`) that the OS searches through for a command -before giving up and telling us it can't find it. As with all environment -variables we can print it out using `echo`. - -```bash -{{ site.remote.prompt }} echo $PATH -``` - -```output -/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/SQLite/3.31.1-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Tcl/8.6.10-GCCcore-x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/GCCcore/x.y.z/bin:/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin:/opt/software/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/puppetlabs/bin:/home/user01/.local/bin:/home/user01/bin -``` - -You'll notice a similarity to the output of the `which` command. In this case, -there's only one difference: the different directory at the beginning. When we -ran the `module load` command, it added a directory to the beginning of our -`$PATH`. Let's examine what's there: - -```bash -{{ site.remote.prompt }} ls /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin -``` - -```output -2to3 nosetests-3.8 python rst2s5.py -2to3-3.8 pasteurize python3 rst2xetex.py -chardetect pbr python3.8 rst2xml.py -cygdb pip python3.8-config rstpep2html.py -cython pip3 python3-config runxlrd.py -cythonize pip3.8 rst2html4.py sphinx-apidoc -easy_install pybabel rst2html5.py sphinx-autogen -easy_install-3.8 __pycache__ rst2html.py sphinx-build -futurize pydoc3 rst2latex.py sphinx-quickstart -idle3 pydoc3.8 rst2man.py tabulate -idle3.8 pygmentize rst2odt_prepstyles.py virtualenv -netaddr pytest rst2odt.py wheel -nosetests py.test rst2pseudoxml.py -``` - -Taking this to its conclusion, `module load` will add software to your `$PATH`. -It "loads" software. A special note on this - depending on which version of the -`module` program that is installed at your site, `module load` will also load -required software dependencies. - -To demonstrate, let's use `module list`. `module list` shows all loaded -software modules. - -```bash -{{ site.remote.prompt }} module list -``` - -```output -Currently Loaded Modules: - 1) GCCcore/x.y.z 4) GMP/6.2.0-GCCcore-x.y.z - 2) Tcl/8.6.10-GCCcore-x.y.z 5) libffi/3.3-GCCcore-x.y.z - 3) SQLite/3.31.1-GCCcore-x.y.z 6) Python/3.x.y-GCCcore-x.y.z -``` - -```bash -{{ site.remote.prompt }} module load GROMACS -{{ site.remote.prompt }} module list -``` - -```output -Currently Loaded Modules: - 1) GCCcore/x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z - 2) Tcl/8.6.10-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z - 3) SQLite/3.31.1-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z - 4) GMP/6.2.0-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z - 5) libffi/3.3-GCCcore-x.y.z 18) gompi/2020a - 6) Python/3.x.y-GCCcore-x.y.z 19) FFTW/3.3.8-gompi-2020a - 7) GCC/x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a - 8) numactl/2.0.13-GCCcore-x.y.z 21) foss/2020a - 9) libxml2/2.9.10-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... - 10) libpciaccess/0.16-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... - 11) hwloc/2.2.0-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.8... - 12) libevent/2.1.11-GCCcore-x.y.z 25) GROMACS/2020.1-foss-2020a-Python-3... - 13) UCX/1.8.0-GCCcore-x.y.z -``` - -So in this case, loading the `GROMACS` module (a bioinformatics software -package), also loaded `GMP/6.2.0-GCCcore-x.y.z` and -`SciPy-bundle/2020.03-foss-2020a-Python-3.x.y` as well. Let's try unloading the -`GROMACS` package. - -```bash -{{ site.remote.prompt }} module unload GROMACS -{{ site.remote.prompt }} module list -``` - -```output -Currently Loaded Modules: - 1) GCCcore/x.y.z 13) UCX/1.8.0-GCCcore-x.y.z - 2) Tcl/8.6.10-GCCcore-x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z - 3) SQLite/3.31.1-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z - 4) GMP/6.2.0-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z - 5) libffi/3.3-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z - 6) Python/3.x.y-GCCcore-x.y.z 18) gompi/2020a - 7) GCC/x.y.z 19) FFTW/3.3.8-gompi-2020a - 8) numactl/2.0.13-GCCcore-x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a - 9) libxml2/2.9.10-GCCcore-x.y.z 21) foss/2020a - 10) libpciaccess/0.16-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... - 11) hwloc/2.2.0-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... - 12) libevent/2.1.11-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.x.y -``` - -So using `module unload` "un-loads" a module, and depending on how a site is -configured it may also unload all of the dependencies (in our case it does -not). If we wanted to unload everything at once, we could run `module purge` -(unloads everything). - -```bash -{{ site.remote.prompt }} module purge -{{ site.remote.prompt }} module list -``` - -```output -No modules loaded -``` - -Note that `module purge` is informative. It will also let us know if a default -set of "sticky" packages cannot be unloaded (and how to actually unload these -if we truly so desired). - -Note that this module loading process happens principally through -the manipulation of environment variables like `$PATH`. There -is usually little or no data transfer involved. - -The module loading process manipulates other special environment -variables as well, including variables that influence where the -system looks for software libraries, and sometimes variables which -tell commercial software packages where to find license servers. - -The module command also restores these shell environment variables -to their previous state when a module is unloaded. - -## Software Versioning - -So far, we've learned how to load and unload software packages. This is very -useful. However, we have not yet addressed the issue of software versioning. At -some point or other, you will run into issues where only one particular version -of some software will be suitable. Perhaps a key bugfix only happened in a -certain version, or version X broke compatibility with a file format you use. -In either of these example cases, it helps to be very specific about what -software is loaded. - -Let's examine the output of `module avail` more closely. - -```bash -{{ site.remote.prompt }} module avail -``` - -```output -~~~ /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/modules/all ~~~ - Bazel/3.6.0-GCCcore-x.y.z NSS/3.51-GCCcore-x.y.z - Bison/3.5.3-GCCcore-x.y.z Ninja/1.10.0-GCCcore-x.y.z - Boost/1.72.0-gompi-2020a OSU-Micro-Benchmarks/5.6.3-gompi-2020a - CGAL/4.14.3-gompi-2020a-Python-3.x.y OpenBLAS/0.3.9-GCC-x.y.z - CMake/3.16.4-GCCcore-x.y.z OpenFOAM/v2006-foss-2020a - -[removed most of the output here for clarity] - - Where: - L: Module is loaded - Aliases: Aliases exist: foo/1.2.3 (1.2) means that "module load foo/1.2" - will load foo/1.2.3 - D: Default Module - -Use "module spider" to find all possible modules and extensions. -Use "module keyword key1 key2 ..." to search for all possible modules matching -any of the "keys". -``` - -::::::::::::::::::::::::::::::::::::::: challenge - -## Using Software Modules in Scripts - -Create a job that is able to run `python3 --version`. Remember, no software -is loaded by default! Running a job is just like logging on to the system -(you should not assume a module loaded on the login node is loaded on a -compute node). - -::::::::::::::: solution - -## Solution - -```bash -{{ site.remote.prompt }} nano python-module.sh -{{ site.remote.prompt }} cat python-module.sh -``` - -```output -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.partition }}{% if site.sched.flag.qos %} -{{ site.sched.comment }} {{ site.sched.flag.qos }} -{% endif %}{{ site.sched.comment }} {{ site.sched.flag.time }} 00:00:30 - -module load {{ site.remote.module_python3 }} - -python3 --version -``` - -```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} {% if site.sched.submit.options != '' %}{{ site.sched.submit.options }} {% endif %}python-module.sh -``` - -::::::::::::::::::::::::: - -:::::::::::::::::::::::::::::::::::::::::::::::::: - - - -:::::::::::::::::::::::::::::::::::::::: keypoints - -- Load software with `module load softwareName`. -- Unload software with `module unload` -- The module system handles software versioning and package conflicts for you automatically. - -:::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/16-transferring-files.md b/episodes/16-transferring-files.Rmd similarity index 90% rename from episodes/16-transferring-files.md rename to episodes/16-transferring-files.Rmd index 824489f0..2108df8a 100644 --- a/episodes/16-transferring-files.md +++ b/episodes/16-transferring-files.Rmd @@ -4,6 +4,11 @@ teaching: 15 exercises: 15 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Transfer files to and from a computing cluster. @@ -62,9 +67,9 @@ Use one of the above commands to save the tarball as `amdahl.tar.gz`. ## `wget` and `curl` Commands ```bash -{{ site.local.prompt }} wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main +`r config$local$prompt` wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main # or -{{ site.local.prompt }} curl -o amdahl.tar.gz -L https://github.com/hpc-carpentry/amdahl/tarball/main +`r config$local$prompt` curl -o amdahl.tar.gz -L https://github.com/hpc-carpentry/amdahl/tarball/main ``` The `-L` option to `curl` tells it to follow URL redirects (which `wget` does by default). @@ -78,7 +83,7 @@ The `-L` option to `curl` tells it to follow URL redirects (which `wget` does by After downloading the file, use `ls` to see it in your working directory: ```bash -{{ site.local.prompt }} ls +`r config$local$prompt` ls ``` ## Archiving Files @@ -115,7 +120,7 @@ However, the argument following `-f` must be a filename, so writing `-ft` will *not* work. ```bash -{{ site.local.prompt }} tar -tf amdahl.tar.gz +`r config$local$prompt` tar -tf amdahl.tar.gz hpc-carpentry-amdahl-46c9b4b/ hpc-carpentry-amdahl-46c9b4b/.github/ hpc-carpentry-amdahl-46c9b4b/.github/workflows/ @@ -150,7 +155,7 @@ Using the flags above, unpack the source code tarball into a new directory named "amdahl" using `tar`. ```bash -{{ site.local.prompt }} tar -xvzf amdahl.tar.gz +`r config$local$prompt` tar -xvzf amdahl.tar.gz ``` ```output @@ -181,16 +186,16 @@ The folder has an unfortunate name, so let's change that to something more convenient. ```bash -{{ site.local.prompt }} mv hpc-carpentry-amdahl-46c9b4b amdahl +`r config$local$prompt` mv hpc-carpentry-amdahl-46c9b4b amdahl ``` Check the size of the extracted directory and compare to the compressed file size, using `du` for "**d**isk **u**sage". ```bash -{{ site.local.prompt }} du -sh amdahl.tar.gz +`r config$local$prompt` du -sh amdahl.tar.gz 8.0K amdahl.tar.gz -{{ site.local.prompt }} du -sh amdahl +`r config$local$prompt` du -sh amdahl 48K amdahl ``` @@ -202,7 +207,7 @@ extracting it -- set a `c` flag instead of `x`, set the archive filename, then provide a directory to compress: ```bash -{{ site.local.prompt }} tar -cvzf compressed_code.tar.gz amdahl +`r config$local$prompt` tar -cvzf compressed_code.tar.gz amdahl ``` ```output @@ -264,7 +269,7 @@ mechanism. To *upload to* another computer, the template command is ```bash -{{ site.local.prompt }} scp local_file {{ site.remote.user }}@{{ site.remote.login }}:remote_destination +`r config$local$prompt` scp local_file `r config$remote$user`@`r config$remote$login`:remote_destination ``` in which `@` and `:` are field separators and `remote_destination` is a path @@ -283,12 +288,12 @@ for `local_file`. Upload the lesson material to your remote home directory like so: ```bash -{{ site.local.prompt }} scp amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` scp amdahl.tar.gz `r config$remote$user`@`r config$remote$login`: ``` ::::::::::::::::::::::::::::::::::::::: challenge -## Why Not Download on {{ site.remote.name }} Directly? +## Why Not Download on `r config$remote$name` Directly? Most computer clusters are protected from the open internet by a *firewall*. For enhanced security, some are configured to allow traffic *inbound*, but @@ -305,10 +310,10 @@ OK! ## Commands ```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -{{ site.remote.prompt }} wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` +`r config$remote$prompt` wget -O amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main # or -{{ site.remote.prompt }} curl -o amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main +`r config$remote$prompt` curl -o amdahl.tar.gz https://github.com/hpc-carpentry/amdahl/tarball/main ``` ::::::::::::::::::::::::: @@ -327,7 +332,7 @@ until it reaches the bottom of the directory tree rooted at the folder name you provided. ```bash -{{ site.local.prompt }} scp -r amdahl {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` scp -r amdahl `r config$remote$user`@`r config$remote$login`: ``` ::::::::::::::::::::::::::::::::::::::::: callout @@ -378,7 +383,7 @@ The syntax is similar to `scp`. To transfer *to* another computer with commonly used options: ```bash -{{ site.local.prompt }} rsync -avP amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` rsync -avP amdahl.tar.gz `r config$remote$user`@`r config$remote$login`: ``` The options are: @@ -392,7 +397,7 @@ The options are: To recursively copy a directory, we can use the same options: ```bash -{{ site.local.prompt }} rsync -avP amdahl {{ site.remote.user }}@{{ site.remote.login }}:~/ +`r config$local$prompt` rsync -avP amdahl `r config$remote$user`@`r config$remote$login`:~/ ``` As written, this will place the local directory and its contents under your @@ -404,7 +409,7 @@ copied directly into the destination directory. To download a file, we simply change the source and destination: ```bash -{{ site.local.prompt }} rsync -avP {{ site.remote.user }}@{{ site.remote.login }}:amdahl ./ +`r config$local$prompt` rsync -avP `r config$remote$user`@`r config$remote$login`:amdahl ./ ``` :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -423,7 +428,7 @@ Say we have to connect `rsync` through port 768 instead of 22. How would we modify this command? ```bash -{{ site.local.prompt }} rsync amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` rsync amdahl.tar.gz `r config$remote$user`@`r config$remote$login`: ``` *Hint:* check the `man` page or "help" for `rsync`. @@ -433,11 +438,11 @@ modify this command? ## Solution ```bash -{{ site.local.prompt }} man rsync -{{ site.local.prompt }} rsync --help | grep port +`r config$local$prompt` man rsync +`r config$local$prompt` rsync --help | grep port --port=PORT specify double-colon alternate port number See http://rsync.samba.org/ for updates, bug reports, and answers -{{ site.local.prompt }} rsync --port=768 amdahl.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` rsync --port=768 amdahl.tar.gz `r config$remote$user`@`r config$remote$login`: ``` (Note that this command will fail, as the correct port in this case is the @@ -466,7 +471,7 @@ side. To connect to the cluster, we'll just need to enter our credentials at the top of the screen: -- Host: `sftp://{{ site.remote.login }}` +- Host: `sftp://`r config$remote$login`` - User: Your cluster username - Password: Your cluster password - Port: (leave blank to use the default port) @@ -482,12 +487,9 @@ will be more efficient than using FileZilla (or related applications) that would copy from the source to your local machine, then to the destination machine. - - [git-swc]: https://swcarpentry.github.io/git-novice/ [rsync]: https://rsync.samba.org/ - :::::::::::::::::::::::::::::::::::::::: keypoints - `wget` and `curl -O` download a file from the internet. @@ -495,5 +497,3 @@ machine. - You can use an SFTP client like FileZilla to transfer files through a GUI. :::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/17-parallel.md b/episodes/17-parallel.Rmd similarity index 65% rename from episodes/17-parallel.md rename to episodes/17-parallel.Rmd index 7d587c36..f12ebe49 100644 --- a/episodes/17-parallel.md +++ b/episodes/17-parallel.Rmd @@ -4,6 +4,11 @@ teaching: 30 exercises: 60 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Install a Python package using `pip` @@ -29,7 +34,7 @@ we have to improve the performance of computational tasks. If you disconnected, log back in to the cluster. ```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` ``` ## Install the Amdahl Program @@ -40,8 +45,8 @@ Move into the extracted directory, then use the Package Installer for Python, or `pip`, to install it in your ("user") home directory: ```bash -{{ site.remote.prompt }} cd amdahl -{{ site.remote.prompt }} python3 -m pip install --user . +`r config$remote$prompt` cd amdahl +`r config$remote$prompt` python3 -m pip install --user . ``` ::::::::::::::::::::::::::::::::::::::::: callout @@ -73,20 +78,20 @@ retrieve the tarball from then `rsync` it to the cluster, extract, and install: ```bash -{{ site.local.prompt }} wget -O mpi4py.tar.gz https://github.com/mpi4py/mpi4py/releases/download/3.1.4/mpi4py-3.1.4.tar.gz -{{ site.local.prompt }} scp mpi4py.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` wget -O mpi4py.tar.gz https://github.com/mpi4py/mpi4py/releases/download/3.1.4/mpi4py-3.1.4.tar.gz +`r config$local$prompt` scp mpi4py.tar.gz `r config$remote$user`@`r config$remote$login`: # or -{{ site.local.prompt }} rsync -avP mpi4py.tar.gz {{ site.remote.user }}@{{ site.remote.login }}: +`r config$local$prompt` rsync -avP mpi4py.tar.gz `r config$remote$user`@`r config$remote$login`: ``` ```bash -{{ site.local.prompt }} ssh {{ site.remote.user }}@{{ site.remote.login }} -{{ site.remote.prompt }} tar -xvzf mpi4py.tar.gz # extract the archive -{{ site.remote.prompt }} mv mpi4py* mpi4py # rename the directory -{{ site.remote.prompt }} cd mpi4py -{{ site.remote.prompt }} python3 -m pip install --user . -{{ site.remote.prompt }} cd ../amdahl -{{ site.remote.prompt }} python3 -m pip install --user . +`r config$local$prompt` ssh `r config$remote$user`@`r config$remote$login` +`r config$remote$prompt` tar -xvzf mpi4py.tar.gz # extract the archive +`r config$remote$prompt` mv mpi4py* mpi4py # rename the directory +`r config$remote$prompt` cd mpi4py +`r config$remote$prompt` python3 -m pip install --user . +`r config$remote$prompt` cd ../amdahl +`r config$remote$prompt` python3 -m pip install --user . ``` :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -107,7 +112,7 @@ To check whether this warning is a problem, use `which` to search for the `amdahl` program: ```bash -{{ site.remote.prompt }} which amdahl +`r config$remote$prompt` which amdahl ``` If the command returns no output, displaying a new prompt, it means the file @@ -117,15 +122,15 @@ Edit your shell configuration file as follows, then log off the cluster and back on again so it takes effect. ```bash -{{ site.remote.prompt }} nano ~/.bashrc -{{ site.remote.prompt }} tail ~/.bashrc +`r config$remote$prompt` nano ~/.bashrc +`r config$remote$prompt` tail ~/.bashrc ``` ```output export PATH=${PATH}:${HOME}/.local/bin ``` -After logging back in to {{ site.remote.login }}, `which` should be able to +After logging back in to `r config$remote$login`, `which` should be able to find `amdahl` without difficulties. If you had to load a Python module, load it again. @@ -137,7 +142,7 @@ If you had to load a Python module, load it again. Many command-line programs include a "help" message. Try it with `amdahl`: ```bash -{{ site.remote.prompt }} amdahl --help +`r config$remote$prompt` amdahl --help ``` ```output @@ -162,34 +167,14 @@ tell us the important flags we might want to use when launching it. Create a submission file, requesting one task on a single node, then launch it. -```bash -{{ site.remote.prompt }} nano serial-job.sh -{{ site.remote.prompt }} cat serial-job.sh -``` - -```bash -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} solo-job -{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} -{{ site.sched.comment }} -N 1 -{{ site.sched.comment }} -n 1 - -# Load the computing environment we need -module load {{ site.remote.module_python3 }} - -# Execute the task -amdahl +```{r, child=paste(snippets, '/parallel/one-task.Rmd', sep=''), eval=TRUE} ``` -```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} serial-job.sh -``` - -As before, use the {{ site.sched.name }} status commands to check whether your job +As before, use the `r config$sched$name` status commands to check whether your job is running and when it ends: ```bash -{{ site.remote.prompt }} {{ site.sched.status }} {{ site.sched.flag.user }} +`r config$remote$prompt` `r config$sched$status` `r config$sched$flag$user` ``` Use `ls` to locate the output file. The `-t` flag sorts in @@ -203,7 +188,7 @@ The cluster output should be written to a file in the folder you launched the job from. For example, ```bash -{{ site.remote.prompt }} ls -t +`r config$remote$prompt` ls -t ``` ```output @@ -211,15 +196,15 @@ slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt ``` ```bash -{{ site.remote.prompt }} cat slurm-347087.out +`r config$remote$prompt` cat slurm-347087.out ``` ```output Doing 30.000 seconds of 'work' on 1 processor, which should take 30.000 seconds with 0.850 parallel proportion of the workload. - Hello, World! I am process 0 of 1 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. - Hello, World! I am process 0 of 1 on {{ site.remote.node }}. I will do parallel 'work' for 25.500 seconds. + Hello, World! I am process 0 of 1 on `r config$remote$node`. I will do all the serial 'work' for 4.500 seconds. + Hello, World! I am process 0 of 1 on `r config$remote$node`. I will do parallel 'work' for 25.500 seconds. Total execution time (according to rank 0): 30.033 seconds ``` @@ -239,7 +224,7 @@ for 25.5 seconds, and no time was saved. The cluster can do better, if we ask. ## Running the Parallel Job The `amdahl` program uses the Message Passing Interface (MPI) for parallelism -\-- this is a common tool on HPC systems. +-- this is a common tool on HPC systems. ::::::::::::::::::::::::::::::::::::::::: callout @@ -278,66 +263,11 @@ In the context of a queuing system, however, it is frequently the case that MPI run-time will obtain the necessary parameters from the queuing system, by examining the environment variables set when the job is launched. - :::::::::::::::::::::::::::::::::::::::::::::::::: Let's modify the job script to request more cores and use the MPI run-time. -```bash, bash -{{ site.remote.prompt }} cp serial-job.sh parallel-job.sh -{{ site.remote.prompt }} nano parallel-job.sh -{{ site.remote.prompt }} cat parallel-job.sh -``` - -```bash -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} parallel-job -{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} -{{ site.sched.comment }} -N 1 -{{ site.sched.comment }} -n 4 - -# Load the computing environment we need -# (mpi4py and numpy are in SciPy-bundle) -module load {{ site.remote.module_python3 }} -module load SciPy-bundle - -# Execute the task -mpiexec amdahl -``` - -Then submit your job. Note that the submission command has not really changed -from how we submitted the serial job: all the parallel settings are in the -batch file rather than the command line. - -```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} parallel-job.sh -``` - -As before, use the status commands to check when your job runs. - -```bash -{{ site.remote.prompt }} ls -t -``` - -```output -slurm-347178.out parallel-job.sh slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt -``` - -```bash -{{ site.remote.prompt }} cat slurm-347178.out -``` - -```output -Doing 30.000 seconds of 'work' on 4 processors, -which should take 10.875 seconds with 0.850 parallel proportion of the workload. - - Hello, World! I am process 0 of 4 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. - Hello, World! I am process 2 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 1 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 3 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - Hello, World! I am process 0 of 4 on {{ site.remote.node }}. I will do parallel 'work' for 6.375 seconds. - -Total execution time (according to rank 0): 10.888 seconds +```{r, child=paste(snippets, '/parallel/four-tasks.Rmd', sep=''), eval=TRUE} ``` ::::::::::::::::::::::::::::::::::::::: challenge @@ -369,8 +299,6 @@ This is the basic principle behind [Amdahl's Law][amdahl], which is one way of predicting improvements in execution time for a **fixed** workload that can be subdivided and run in parallel to some extent. - - ::::::::::::::::::::::::: :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -402,63 +330,7 @@ versus the number of CPUs *n* would give a straight line, *S* = *n*. Let's run one more job, so we can see how close to a straight line our `amdahl` code gets. -```bash -{{ site.remote.prompt }} nano parallel-job.sh -{{ site.remote.prompt }} cat parallel-job.sh -``` - -```bash -{{ site.remote.bash_shebang }} -{{ site.sched.comment }} {{ site.sched.flag.name }} parallel-job -{{ site.sched.comment }} {{ site.sched.flag.queue }} {{ site.sched.queue.testing }} -{{ site.sched.comment }} -N 1 -{{ site.sched.comment }} -n 8 - -# Load the computing environment we need -# (mpi4py and numpy are in SciPy-bundle) -module load {{ site.remote.module_python3 }} -module load SciPy-bundle - -# Execute the task -mpiexec amdahl -``` - -Then submit your job. Note that the submission command has not really changed -from how we submitted the serial job: all the parallel settings are in the -batch file rather than the command line. - -```bash -{{ site.remote.prompt }} {{ site.sched.submit.name }} parallel-job.sh -``` - -As before, use the status commands to check when your job runs. - -```bash -{{ site.remote.prompt }} ls -t -``` - -```output -slurm-347271.out parallel-job.sh slurm-347178.out slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt -``` - -```bash -{{ site.remote.prompt }} cat slurm-347178.out -``` - -```output -which should take 7.688 seconds with 0.850 parallel proportion of the workload. - - Hello, World! I am process 4 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 0 of 8 on {{ site.remote.node }}. I will do all the serial 'work' for 4.500 seconds. - Hello, World! I am process 2 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 1 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 3 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 5 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 6 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 7 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - Hello, World! I am process 0 of 8 on {{ site.remote.node }}. I will do parallel 'work' for 3.188 seconds. - -Total execution time (according to rank 0): 7.697 seconds +```{r, child=paste(snippets, '/parallel/eight-tasks.Rmd', sep=''), eval=TRUE} ``` :::::::::::::::::::::::::::::::::::::: discussion @@ -473,28 +345,32 @@ With 8 workers, this is not the case: since the parallel workers take less time than the serial work, it is hard to say which process will write its output first, except that it will *not* be process 0! - :::::::::::::::::::::::::::::::::::::::::::::::::: Now, let's summarize the amount of time it took each job to run: -| Number of CPUs | Runtime (sec) | +| Number of CPUs | Runtime (sec) | | -------------- | ------------- | -| 1 | 30\.033 | -| 4 | 10\.888 | -| 8 | 7\.697 | +| 1 | 30\.033 | +| 4 | 10\.888 | +| 8 | 7\.697 | + +Then, use the first row to compute speedups $S$, using Python as a command-line +calculator and the formula -Then, use the first row to compute speedups *S*, using Python as a command-line calculator: +$$ +S(t_{n}) = \frac{t_{1}}{t_{n}} +$$ ```bash -{{ site.remote.prompt }} for n in 30.033 10.888 7.697; do python3 -c "print(30.033 / $n)"; done +`r config$remote$prompt` for n in 30.033 10.888 7.697; do python3 -c "print(30.033 / $n)"; done ``` -| Number of CPUs | Speedup | Ideal | +| Number of CPUs | Speedup | Ideal | | -------------- | ------------- | ----- | -| 1 | 1\.0 | 1 | -| 4 | 2\.75 | 4 | -| 8 | 3\.90 | 8 | +| 1 | 1\.0 | 1 | +| 4 | 2\.75 | 4 | +| 8 | 3\.90 | 8 | The job output files have been telling us that this program is performing 85% of its work in parallel, leaving 15% to run in serial. This seems reasonably @@ -518,7 +394,6 @@ thousands of CPUs into solving a single problem. To learn more about parallelization, see the [parallel novice lesson][parallel-novice] lesson. - [amdahl]: https://en.wikipedia.org/wiki/Amdahl\'s_law [parallel-novice]: https://www.hpc-carpentry.org/hpc-parallel-novice/ @@ -530,5 +405,3 @@ parallelization, see the [parallel novice lesson][parallel-novice] lesson. - Performance improvements from parallel execution do not scale linearly. :::::::::::::::::::::::::::::::::::::::::::::::::: - - diff --git a/episodes/18-resources.md b/episodes/18-resources.Rmd similarity index 73% rename from episodes/18-resources.md rename to episodes/18-resources.Rmd index 99d01e3b..86901f40 100644 --- a/episodes/18-resources.md +++ b/episodes/18-resources.Rmd @@ -4,10 +4,16 @@ teaching: 10 exercises: 20 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Look up job statistics. -- Make more accurate resource requests in job scripts based on data describing past performance. +- Make more accurate resource requests in job scripts based on data + describing past performance. :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -25,8 +31,8 @@ might matter. ## Estimating Required Resources Using the Scheduler -Although we covered requesting resources from the scheduler earlier with the -π code, how do we know what type of resources the software will need in +Although we covered requesting resources from the scheduler earlier, +how do we know what type of resources the software will need in the first place, and its demand for each? In general, unless the software documentation or user testimonials provide some idea, we won't know how much memory or compute time a program will need. @@ -40,12 +46,11 @@ document sent along when you register for an account. Take a look at these resources, and search for the software you plan to use: somebody might have written up guidance for getting the most out of it. - :::::::::::::::::::::::::::::::::::::::::::::::::: A convenient way of figuring out the resources required for a job to run successfully is to submit a test job, and then ask the scheduler about its -impact using `{{ site.sched.hist }}`. You can use this knowledge to set up the +impact using ``r config$sched$hist``. You can use this knowledge to set up the next job with a closer estimate of its load on the system. A good general rule is to ask the scheduler for 20% to 30% more time and memory than you expect the job to need. This ensures that minor fluctuations in run time or memory use @@ -58,24 +63,13 @@ finish and free up the resources needed to match what you asked for. Since we already submitted `amdahl` to run on the cluster, we can query the scheduler to see how long our job took and what resources were used. We will -use `{{ site.sched.hist }}` to get statistics about `parallel-job.sh`. +use ``r config$sched$hist`` to get statistics about `parallel-job.sh`. ```bash -{{ site.remote.prompt }} {{ site.sched.hist }} +`r config$remote$prompt` `r config$sched$hist` ``` -```output - JobID JobName Partition Account AllocCPUS State ExitCode ------------- ---------- ---------- ---------- ---------- ---------- -------- -7 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 -7.batch batch def-spons+ 1 COMPLETED 0:0 -7.extern extern def-spons+ 1 COMPLETED 0:0 -8 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 -8.batch batch def-spons+ 1 COMPLETED 0:0 -8.extern extern def-spons+ 1 COMPLETED 0:0 -9 example-j+ cpubase_b+ def-spons+ 1 COMPLETED 0:0 -9.batch batch def-spons+ 1 COMPLETED 0:0 -9.extern extern def-spons+ 1 COMPLETED 0:0 +```{r, child=paste(snippets, '/resources/account-history.Rmd', sep=''), eval=TRUE} ``` This shows all the jobs we ran today (note that there are multiple entries per @@ -84,7 +78,7 @@ To get info about a specific job (for example, 347087), we change command slightly. ```bash -{{ site.remote.prompt }} {{ site.sched.hist }} {{ site.sched.flag.histdetail }} 347087 +`r config$remote$prompt` `r config$sched$hist` `r config$sched$flag$histdetail` 347087 ``` It will show a lot of info; in fact, every single piece of info collected on @@ -93,7 +87,7 @@ information to `less` to make it easier to view (use the left and right arrow keys to scroll through fields). ```bash -{{ site.remote.prompt }} {{ site.sched.hist }} {{ site.sched.flag.histdetail }} 347087 | less -S +`r config$remote$prompt` `r config$sched$hist` `r config$sched$flag$histdetail` 347087 | less -S ``` :::::::::::::::::::::::::::::::::::::: discussion @@ -129,17 +123,17 @@ get your job dispatched earlier. Edit `parallel_job.sh` to set a better time estimate. How close can you get? -Hint: use `{{ site.sched.flag.time }}`. +Hint: use ``r config$sched$flag$time``. ::::::::::::::: solution ## Solution -The following line tells {{ site.sched.name }} that our job should +The following line tells `r config$sched$name` that our job should finish within 2 minutes: ```bash -{{ site.sched.comment }} {{ site.sched.flag.time }}{% if site.sched.name == "Slurm" %} {% else %}={% endif %}00:02:00 +`r config$sched$comment` `r config$sched$flag$time` `r if(config$sched$name == "Slurm") {print(" ")} else {print("=")}`00:02:00 ``` ::::::::::::::::::::::::: @@ -147,11 +141,9 @@ finish within 2 minutes: :::::::::::::::::::::::::::::::::::::::::::::::::: - :::::::::::::::::::::::::::::::::::::::: keypoints - Accurate job scripts help the queuing system efficiently allocate shared resources. :::::::::::::::::::::::::::::::::::::::::::::::::: - diff --git a/episodes/19-responsibility.md b/episodes/19-responsibility.Rmd similarity index 94% rename from episodes/19-responsibility.md rename to episodes/19-responsibility.Rmd index e214fd19..72252418 100644 --- a/episodes/19-responsibility.md +++ b/episodes/19-responsibility.Rmd @@ -4,6 +4,11 @@ teaching: 15 exercises: 5 --- +```{r, echo=FALSE} +# Source the external configuration script +source("load_config.R") +``` + ::::::::::::::::::::::::::::::::::::::: objectives - Describe how the actions of a single user can affect the experience of others on a shared system. @@ -262,30 +267,17 @@ data. Say you have a "data" folder containing 10,000 or so files, a healthy mix of small and large ASCII and binary data. Which of the following would be the -best way to transfer them to {{ site.remote.name }}? - -1. - ```bash - {{ site.local.prompt }} scp -r data {{ site.remote.user }}@{{ site.remote.login }}:~/ - ``` -2. - ```bash - {{ site.local.prompt }} rsync -ra data {{ site.remote.user }}@{{ site.remote.login }}:~/ - ``` -3. - ```bash - {{ site.local.prompt }} rsync -raz data {{ site.remote.user }}@{{ site.remote.login }}:~/ - ``` -4. - ```bash - {{ site.local.prompt }} tar -cvf data.tar data - {{ site.local.prompt }} rsync -raz data.tar {{ site.remote.user }}@{{ site.remote.login }}:~/ - ``` -5. - ```bash - {{ site.local.prompt }} tar -cvzf data.tar.gz data - {{ site.local.prompt }} rsync -ra data.tar.gz {{ site.remote.user }}@{{ site.remote.login }}:~/ - ``` +best way to transfer them to `r config$remote$name`? + +1. `scp -r data `r config$remote$user`@`r config$remote$login`:~/` +2. `rsync -ra data `r config$remote$user`@`r config$remote$login`:~/` +3. `rsync -raz data `r config$remote$user`@`r config$remote$login`:~/` +4. `tar -cvf data.tar data;` + + `rsync -raz data.tar `r config$remote$user`@`r config$remote$login`:~/` +5. `tar -cvzf data.tar.gz data;` + + `rsync -ra data.tar.gz `r config$remote$user`@`r config$remote$login`:~/` ::::::::::::::: solution @@ -306,8 +298,6 @@ best way to transfer them to {{ site.remote.name }}? transfer it. This may perform similarly to #4, but in most cases (for large datasets), it's the best combination of high throughput and low latency (making the most of your time and network connection). - - ::::::::::::::::::::::::: diff --git a/episodes/files/jargon.html b/episodes/files/jargon.html new file mode 100644 index 00000000..11d35eeb --- /dev/null +++ b/episodes/files/jargon.html @@ -0,0 +1,178 @@ + + + + + HPC Jargon Buster + + + + + + + + + + diff --git a/episodes/files/snippets/Ghastly_Mistakes/_config_options.yml b/episodes/files/snippets/Ghastly_Mistakes/_config_options.yml new file mode 100644 index 00000000..a6d78a72 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/_config_options.yml @@ -0,0 +1,66 @@ +# ------------------------ +# Ghastly Mistakes Cluster +# ------------------------ +# +# This is not a real cluster, and its options are meant to +# highlight configuration changes while we develop the config +# chain-loader, per @tobyhodges' suggestion. +# +# Compute irresponsibly. +--- + +snippets: "Ghastly_Mistakes" + +local: + prompt: "luigi@mushroomkingdom:~$" + bash_shebang: "#!/bin/bash" + +remote: + name: "Bowser's Castle" + login: "castle.bowser.org" + portal: "https://pipe.bowser.org" + host: "castle" + node: "turtle" + location: "World 8-4" + homedir: "/darkland" + user: "luigi" + module_python3: "Boa" + prompt: "luigi@castle:~$" + bash_shebang: "#!/bin/no" + +sched: + name: "Slurp" + submit: + name: "smash" + options: "" + queue: + debug: "doom" + testing: "gloom" + status: "skew" + flag: + user: "-u luigi" + interactive: "" + histdetail: "-l -j" + name: "-J" + time: "-t" + queue: "-p" + del: "scandle" + interactive: "spun" + info: "sink" + comment: "#SMASH" + hist: "sacked -u luigi" + hist_filter: "" + +episode_order: + - 10-hpc-intro + - 11-connecting + - 12-cluster + - 13-scheduler + - 14-environment-variables + - 15-modules + - 16-transferring-files + - 17-parallel + - 18-resources + - 19-responsibility + +... diff --git a/episodes/files/snippets/Ghastly_Mistakes/cluster/queue-info.Rmd b/episodes/files/snippets/Ghastly_Mistakes/cluster/queue-info.Rmd new file mode 100644 index 00000000..2487fc02 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/cluster/queue-info.Rmd @@ -0,0 +1,6 @@ +```output +PARTITION AVAIL TIMELIMIT NODES STATE NODELIST +cpubase_bycore_b1* up infinite 4 idle node[1-2],smnode[1-2] +node up infinite 2 idle node[1-2] +smnode up infinite 2 idle smnode[1-2] +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/cluster/specific-node-info.Rmd b/episodes/files/snippets/Ghastly_Mistakes/cluster/specific-node-info.Rmd new file mode 100644 index 00000000..b9f6c8dc --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/cluster/specific-node-info.Rmd @@ -0,0 +1,12 @@ +::: challenge + +## Explore a Worker Node + +Finally, let's look at the resources available on the worker nodes +where your jobs will actually run. Try running this command to see +the name, CPUs and memory available on one of the worker nodes: + +```bash +`r config$remote$prompt` sinfo -o "%n %c %m" | column -t +``` +::: diff --git a/episodes/files/snippets/Ghastly_Mistakes/modules/available-modules.Rmd b/episodes/files/snippets/Ghastly_Mistakes/modules/available-modules.Rmd new file mode 100644 index 00000000..8402eaa0 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/modules/available-modules.Rmd @@ -0,0 +1,24 @@ +```bash +`r config$remote$prompt` module avail | less +``` + +```output +~~~ /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/modules/all ~~~ + Bazel/3.6.0-GCCcore-x.y.z NSS/3.51-GCCcore-x.y.z + Bison/3.5.3-GCCcore-x.y.z Ninja/1.10.0-GCCcore-x.y.z + Boost/1.72.0-gompi-2020a OSU-Micro-Benchmarks/5.6.3-gompi-2020a + CGAL/4.14.3-gompi-2020a-Python-3.x.y OpenBLAS/0.3.9-GCC-x.y.z + CMake/3.16.4-GCCcore-x.y.z OpenFOAM/v2006-foss-2020a + +[removed most of the output here for clarity] + + Where: + L: Module is loaded + D: Default Module + Aliases exist: foo/1.2.3 (1.2) means that + "module load foo/1.2" will load foo/1.2.3 + +Use "module spider" to find all possible modules and extensions. +Use "module keyword key1 key2 ..." to search for all possible modules matching +any of the "keys". +``` \ No newline at end of file diff --git a/episodes/files/snippets/Ghastly_Mistakes/modules/missing-python.Rmd b/episodes/files/snippets/Ghastly_Mistakes/modules/missing-python.Rmd new file mode 100644 index 00000000..f06ed919 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/modules/missing-python.Rmd @@ -0,0 +1,30 @@ +If the `python3` command was unavailable, we would see output like + +```output +/usr/bin/which: no python3 in (/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin:/opt/software/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/puppetlabs/bin:/home/`r config$remote$user`/.local/bin:/home/`r config$remote$user`/bin) +``` + +Note that this wall of text is really a list, with values separated +by the `:` character. The output is telling us that the `which` command +searched the following directories for `python3`, without success: + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin +/opt/software/slurm/bin +/usr/local/bin +/usr/bin +/usr/local/sbin +/usr/sbin +/opt/puppetlabs/bin +/home/`r config$remote$user`/.local/bin +/home/`r config$remote$user`/bin +``` + +However, in our case we do have an existing `python3` available so we see + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin/python3 +``` + +We need a different Python than the system provided one though, so let us load +a module to access it. diff --git a/episodes/files/snippets/Ghastly_Mistakes/modules/module-load-python.Rmd b/episodes/files/snippets/Ghastly_Mistakes/modules/module-load-python.Rmd new file mode 100644 index 00000000..81d3be34 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/modules/module-load-python.Rmd @@ -0,0 +1,8 @@ +```bash +`r config$remote$prompt` module load `r config$remote$modules$python` +`r config$remote$prompt` which python3 +``` + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin/python3 +``` \ No newline at end of file diff --git a/episodes/files/snippets/Ghastly_Mistakes/modules/python-executable-dir.Rmd b/episodes/files/snippets/Ghastly_Mistakes/modules/python-executable-dir.Rmd new file mode 100644 index 00000000..13031bf9 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/modules/python-executable-dir.Rmd @@ -0,0 +1,19 @@ +```bash +`r config$remote$prompt` ls /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin +``` + +```output +2to3 nosetests-3.8 python rst2s5.py +2to3-3.8 pasteurize python3 rst2xetex.py +chardetect pbr python3.8 rst2xml.py +cygdb pip python3.8-config rstpep2html.py +cython pip3 python3-config runxlrd.py +cythonize pip3.8 rst2html4.py sphinx-apidoc +easy_install pybabel rst2html5.py sphinx-autogen +easy_install-3.8 __pycache__ rst2html.py sphinx-build +futurize pydoc3 rst2latex.py sphinx-quickstart +idle3 pydoc3.8 rst2man.py tabulate +idle3.8 pygmentize rst2odt_prepstyles.py virtualenv +netaddr pytest rst2odt.py wheel +nosetests py.test rst2pseudoxml.py +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/modules/software-dependencies.Rmd b/episodes/files/snippets/Ghastly_Mistakes/modules/software-dependencies.Rmd new file mode 100644 index 00000000..01234436 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/modules/software-dependencies.Rmd @@ -0,0 +1,79 @@ +To demonstrate, let's use `module list`. `module list` shows all loaded +software modules. + +```bash +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 4) GMP/6.2.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 5) libffi/3.3-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 6) Python/3.x.y-GCCcore-x.y.z +``` + +```bash +`r config$remote$prompt` module load GROMACS +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z + 4) GMP/6.2.0-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z + 5) libffi/3.3-GCCcore-x.y.z 18) gompi/2020a + 6) Python/3.x.y-GCCcore-x.y.z 19) FFTW/3.3.8-gompi-2020a + 7) GCC/x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a + 8) numactl/2.0.13-GCCcore-x.y.z 21) foss/2020a + 9) libxml2/2.9.10-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... + 10) libpciaccess/0.16-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... + 11) hwloc/2.2.0-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.8... + 12) libevent/2.1.11-GCCcore-x.y.z 25) GROMACS/2020.1-foss-2020a-Python-3... + 13) UCX/1.8.0-GCCcore-x.y.z +``` + +So in this case, loading the `GROMACS` module (a bioinformatics software +package), also loaded `GMP/6.2.0-GCCcore-x.y.z` and +`SciPy-bundle/2020.03-foss-2020a-Python-3.x.y` as well. Let's try unloading the +`GROMACS` package. + +```bash +`r config$remote$prompt` module unload GROMACS +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 13) UCX/1.8.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z + 4) GMP/6.2.0-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z + 5) libffi/3.3-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z + 6) Python/3.x.y-GCCcore-x.y.z 18) gompi/2020a + 7) GCC/x.y.z 19) FFTW/3.3.8-gompi-2020a + 8) numactl/2.0.13-GCCcore-x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a + 9) libxml2/2.9.10-GCCcore-x.y.z 21) foss/2020a + 10) libpciaccess/0.16-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... + 11) hwloc/2.2.0-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... + 12) libevent/2.1.11-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.x.y +``` + +So using `module unload` "un-loads" a module, and depending on how a site is +configured it may also unload all of the dependencies (in our case it does +not). If we wanted to unload everything at once, we could run `module purge` +(unloads everything). + +```bash +`r config$remote$prompt` module purge +`r config$remote$prompt` module list +``` + +```output +No modules loaded +``` + +Note that `module purge` is informative. It will also let us know if a default +set of "sticky" packages cannot be unloaded (and how to actually unload these +if we truly so desired). diff --git a/episodes/files/snippets/Ghastly_Mistakes/parallel/eight-tasks.Rmd b/episodes/files/snippets/Ghastly_Mistakes/parallel/eight-tasks.Rmd new file mode 100644 index 00000000..dcbe379a --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/parallel/eight-tasks.Rmd @@ -0,0 +1,58 @@ +```bash +`r config$remote$prompt` nano parallel-job.sh +`r config$remote$prompt` cat parallel-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` parallel-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 8 + +# Load the computing environment we need +# (mpi4py and numpy are in SciPy-bundle) +module load `r config$remote$modules$python` +module load SciPy-bundle + +# Execute the task +mpiexec amdahl +``` + +Then submit your job. Note that the submission command has not really changed +from how we submitted the serial job: all the parallel settings are in the +batch file rather than the command line. + +```bash +`r config$remote$prompt` `r config$sched$submit$name` parallel-job.sh +``` + +As before, use the status commands to check when your job runs. + +```bash +`r config$remote$prompt` ls -t +``` + +```output +slurm-347271.out parallel-job.sh slurm-347178.out slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt +``` + +```bash +`r config$remote$prompt` cat slurm-347178.out +``` + +```output +which should take 7.688 seconds with 0.850 parallel proportion of the workload. + + Hello, World! I am process 4 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 0 of 8 on `r config$remote$node`. I will do all the serial 'work' for 4.500 seconds. + Hello, World! I am process 2 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 1 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 3 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 5 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 6 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 7 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 0 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + +Total execution time (according to rank 0): 7.697 seconds +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/parallel/four-tasks.Rmd b/episodes/files/snippets/Ghastly_Mistakes/parallel/four-tasks.Rmd new file mode 100644 index 00000000..2e885505 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/parallel/four-tasks.Rmd @@ -0,0 +1,56 @@ +```bash +`r config$remote$prompt` cp serial-job.sh parallel-job.sh +`r config$remote$prompt` nano parallel-job.sh +`r config$remote$prompt` cat parallel-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` parallel-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 4 + +# Load the computing environment we need +# (mpi4py and numpy are in SciPy-bundle) +module load `r config$remote$modules$python` +module load SciPy-bundle + +# Execute the task +mpiexec amdahl +``` + +Then submit your job. Note that the submission command has not really changed +from how we submitted the serial job: all the parallel settings are in the +batch file rather than the command line. + +```bash +`r config$remote$prompt` `r config$sched$submit$name` parallel-job.sh +``` + +As before, use the status commands to check when your job runs. + +```bash +`r config$remote$prompt` ls -t +``` + +```output +slurm-347178.out parallel-job.sh slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt +``` + +```bash +`r config$remote$prompt` cat slurm-347178.out +``` + +```output +Doing 30.000 seconds of 'work' on 4 processors, +which should take 10.875 seconds with 0.850 parallel proportion of the workload. + + Hello, World! I am process 0 of 4 on `r config$remote$node`. I will do all the serial 'work' for 4.500 seconds. + Hello, World! I am process 2 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 1 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 3 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 0 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + +Total execution time (according to rank 0): 10.888 seconds +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/parallel/one-task.Rmd b/episodes/files/snippets/Ghastly_Mistakes/parallel/one-task.Rmd new file mode 100644 index 00000000..ceefdfdb --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/parallel/one-task.Rmd @@ -0,0 +1,22 @@ +```bash +`r config$remote$prompt` nano serial-job.sh +`r config$remote$prompt` cat serial-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` solo-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 1 + +# Load the computing environment we need +module load `r config$remote$modules$python` + +# Execute the task +amdahl +``` + +```bash +`r config$remote$prompt` `r config$sched$submit$name` serial-job.sh +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/resources/account-history.Rmd b/episodes/files/snippets/Ghastly_Mistakes/resources/account-history.Rmd new file mode 100644 index 00000000..5e8a53f2 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/resources/account-history.Rmd @@ -0,0 +1,13 @@ +```output + JobID JobName Partition Account AllocCPUS State ExitCode +------------ ---------- ---------- ---------- ---------- ---------- -------- +7 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 +7.batch batch def-spons+ 1 COMPLETED 0:0 +7.extern extern def-spons+ 1 COMPLETED 0:0 +8 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 +8.batch batch def-spons+ 1 COMPLETED 0:0 +8.extern extern def-spons+ 1 COMPLETED 0:0 +9 example-j+ cpubase_b+ def-spons+ 1 COMPLETED 0:0 +9.batch batch def-spons+ 1 COMPLETED 0:0 +9.extern extern def-spons+ 1 COMPLETED 0:0 +``` \ No newline at end of file diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-script.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-script.Rmd new file mode 100644 index 00000000..5f76fb2c --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-script.Rmd @@ -0,0 +1,3 @@ +```output +Submitted batch job 7 +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-status.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-status.Rmd new file mode 100644 index 00000000..9011f5b4 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/basic-job-status.Rmd @@ -0,0 +1,8 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 9 cpubase_b example- user01 R 0:05 1 node1 +``` + +We can see all the details of our job, most importantly that it is in the `R` +or `RUNNING` state. Sometimes our jobs might need to wait in a queue +(`PENDING`) or have an error (`E`). diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/job-with-name-status.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/job-with-name-status.Rmd new file mode 100644 index 00000000..815f3b9d --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/job-with-name-status.Rmd @@ -0,0 +1,4 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 10 cpubase_b hello-wo user01 R 0:02 1 node1 +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/option-flags-list.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/option-flags-list.Rmd new file mode 100644 index 00000000..5e80b164 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/option-flags-list.Rmd @@ -0,0 +1,15 @@ +* `--ntasks=` or `-n `: How many CPU cores does your job need, + in total? + +* `--time ` or `-t `: + How much real-world time (walltime) will your job take to run? The `` + part can be omitted. + +* `--mem=`: How much memory on a node does your job need in + megabytes? You can also specify gigabytes using by adding a little "g" + afterwards (example: `--mem=5g`) + +* `--nodes=` or `-N `: How many separate machines does your job + need to run on? Note that if you set `ntasks` to a number greater than what + one machine can offer, {{ site.sched.name }} will set this value + automatically. diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/print-sched-variables.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/print-sched-variables.Rmd new file mode 100644 index 00000000..21fec883 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/print-sched-variables.Rmd @@ -0,0 +1,32 @@ +::: challenge + +## Job environment variables + +When {{ site.sched.name }} runs a job, it sets a number of environment +variables for the job. One of these will let us check what directory our job +script was submitted from. The `SLURM_SUBMIT_DIR` variable is set to the +directory from which our job was submitted. Using the `SLURM_SUBMIT_DIR` +variable, modify your job so that it prints out the location from which the +job was submitted. + +:::: solution + +## Solution + +```bash +{{ site.remote.prompt }} nano example-job.sh +{{ site.remote.prompt }} cat example-job.sh +``` + +```bash +`r site$remote$bash_shebang` +#SBATCH -t 00:00:30 + +echo -n "This script is running on " +hostname + +echo "This job was launched in the following directory:" +echo ${SLURM_SUBMIT_DIR} +``` +:::: +::: diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-job.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-job.Rmd new file mode 100644 index 00000000..7dde2ada --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-job.Rmd @@ -0,0 +1,3 @@ +```bash +`r config$remote$prompt` cat slurm-12.out +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-output.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-output.Rmd new file mode 100644 index 00000000..106d9ecc --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/runtime-exceeded-output.Rmd @@ -0,0 +1,5 @@ +```output +This script is running on ... +slurmstepd: error: *** JOB 12 ON node1 CANCELLED AT 2021-02-19T13:55:57 +DUE TO TIME LIMIT *** +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-begin.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-begin.Rmd new file mode 100644 index 00000000..3cfb8c95 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-begin.Rmd @@ -0,0 +1,6 @@ +```output +Submitted batch job 13 + +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 13 cpubase_b long_job user01 R 0:02 1 node1 +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-cancel.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-cancel.Rmd new file mode 100644 index 00000000..ccb9cd24 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-job-cancel.Rmd @@ -0,0 +1,3 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) +``` diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-multiple-jobs.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-multiple-jobs.Rmd new file mode 100644 index 00000000..5ee30f8d --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/terminate-multiple-jobs.Rmd @@ -0,0 +1,29 @@ +::: challenge +## Cancelling multiple jobs + +We can also cancel all of our jobs at once using the `-u` option. This will +delete all jobs for a specific user (in this case, yourself). Note that you +can only delete your own jobs. + +Try submitting multiple jobs and then cancelling them all. + +:::: solution + +## Solution + +First, submit a trio of jobs: + +```bash +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +``` + +Then, cancel them all: + +```bash +`r config$remote$prompt` `r config$sched$del` -u `r config$remote$user` +``` + +:::: +::: diff --git a/episodes/files/snippets/Ghastly_Mistakes/scheduler/using-nodes-interactively.Rmd b/episodes/files/snippets/Ghastly_Mistakes/scheduler/using-nodes-interactively.Rmd new file mode 100644 index 00000000..0a305412 --- /dev/null +++ b/episodes/files/snippets/Ghastly_Mistakes/scheduler/using-nodes-interactively.Rmd @@ -0,0 +1,66 @@ +The `r config$sched$interactive` command runs a single command on the cluster +and then exits. Let's demonstrate this by running the `hostname` command with +`r config$sched$interactive`. (We can cancel an `r config$sched$interactive` +job with `Ctrl-c`.) + +```bash +`r config$remote.prompt` `r config$sched$interactive` hostname +``` + +```output +{{ config$remote.node` +``` + +`r config$sched$interactive` accepts all of the same options as +`r config$sched$submit.name`. However, instead of specifying these in a script, +these options are specified on the command-line when starting a job. To submit +a job that uses 2 CPUs for instance, we could use the following command: + +```bash +`r config$remote.prompt` `r config$sched$interactive` -n 2 echo "This job will use 2 CPUs." +``` + +```output +This job will use 2 CPUs. +This job will use 2 CPUs. +``` + +Typically, the resulting shell environment will be the same as that for +`r config$sched$submit.name`. + +### Interactive jobs + +Sometimes, you will need a lot of resources for interactive use. Perhaps it's +our first time running an analysis or we are attempting to debug something that +went wrong with a previous job. Fortunately, {{ config$sched$name` makes it +easy to start an interactive job with `r config$sched$interactive`: + +```bash +`r config$remote.prompt` `r config$sched$interactive` --pty bash +``` + +You should be presented with a bash prompt. Note that the prompt will likely +change to reflect your new location, in this case the compute node we are +logged on. You can also verify this with `hostname`. + +::: callout + +## Creating remote graphics + +To see graphical output inside your jobs, you need to use X11 forwarding. To +connect with this feature enabled, use the `-Y` option when you login with +the `ssh` command, e.g., `r paste("ssh -Y ", config$remote$user, "@", config$remote$login, sep="")`. + +To demonstrate what happens when you create a graphics window on the remote +node, use the `xeyes` command. A relatively adorable pair of eyes should pop +up (press `Ctrl-C` to stop). If you are using a Mac, you must have installed +XQuartz (and restarted your computer) for this to work. + +If your cluster has the +[slurm-spank-x11](https://github.com/hautreux/slurm-spank-x11) plugin +installed, you can ensure X11 forwarding within interactive jobs by using the +`--x11` option for `r config$sched$interactive` with the command +`r paste(config$sched$interactive, " --x11", " --pty bash", sep="")`. +::: + +When you are done with the interactive job, type `exit` to quit your session. diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/_config_options.yml b/episodes/files/snippets/HPCC_MagicCastle_slurm/_config_options.yml new file mode 100644 index 00000000..524b6807 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/_config_options.yml @@ -0,0 +1,69 @@ +# --------------------------------------------------------------- +# HPC Carpentries in the Cloud: Slurm + Software Stack from EESSI +# --------------------------------------------------------------- +# +# The HPC Carpentry Cluster in the Cloud is provided as a public +# service by volunteers. It is provisioned with Magic Castle +# using the EESSI +# software stack. If you need an +# account, please visit . +# +# Compute responsibly. +--- + +snippets: "HPCC_MagicCastle_slurm" + +local: + prompt: "[you@laptop:~]$" + shebang: "#!/bin/bash" + +remote: + name: "HPC Carpentry's Cloud Cluster" + login: "cluster.hpc-carpentry.org" + portal: "https://mokey.cluster.hpc-carpentry.org" + host: "login1" + node: "smnode1" + location: "cluster.hpc-carpentry.org" + homedir: "/home" + user: "yourUsername" + module_python3: "Python" + prompt: "[yourUsername@login1 ~]$" + shebang: "#!/bin/bash" + modules: + python: "Python" + + +sched: + name: "Slurm" + submit: + name: "sbatch" + options: "" + queue: + debug: "smnode" + testing: "cpubase_bycore_b1" + status: "squeue" + flag: + user: "-u yourUsername" + interactive: "" + histdetail: "-l -j" + name: "-J" + time: "-t" + queue: "-p" + del: "scancel" + interactive: "srun" + info: "sinfo" + comment: "#SBATCH" + hist: "sacct -u yourUsername" + hist_filter: "" + +episode_order: + - 10-hpc-intro + - 11-connecting + - 12-cluster + - 13-scheduler + - 14-environment-variables + - 15-modules + - 16-transferring-files + - 17-parallel + - 18-resources + - 19-responsibility diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/queue-info.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/queue-info.Rmd new file mode 100644 index 00000000..2487fc02 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/queue-info.Rmd @@ -0,0 +1,6 @@ +```output +PARTITION AVAIL TIMELIMIT NODES STATE NODELIST +cpubase_bycore_b1* up infinite 4 idle node[1-2],smnode[1-2] +node up infinite 2 idle node[1-2] +smnode up infinite 2 idle smnode[1-2] +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/specific-node-info.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/specific-node-info.Rmd new file mode 100644 index 00000000..b9f6c8dc --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/cluster/specific-node-info.Rmd @@ -0,0 +1,12 @@ +::: challenge + +## Explore a Worker Node + +Finally, let's look at the resources available on the worker nodes +where your jobs will actually run. Try running this command to see +the name, CPUs and memory available on one of the worker nodes: + +```bash +`r config$remote$prompt` sinfo -o "%n %c %m" | column -t +``` +::: diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/available-modules.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/available-modules.Rmd new file mode 100644 index 00000000..8402eaa0 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/available-modules.Rmd @@ -0,0 +1,24 @@ +```bash +`r config$remote$prompt` module avail | less +``` + +```output +~~~ /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/modules/all ~~~ + Bazel/3.6.0-GCCcore-x.y.z NSS/3.51-GCCcore-x.y.z + Bison/3.5.3-GCCcore-x.y.z Ninja/1.10.0-GCCcore-x.y.z + Boost/1.72.0-gompi-2020a OSU-Micro-Benchmarks/5.6.3-gompi-2020a + CGAL/4.14.3-gompi-2020a-Python-3.x.y OpenBLAS/0.3.9-GCC-x.y.z + CMake/3.16.4-GCCcore-x.y.z OpenFOAM/v2006-foss-2020a + +[removed most of the output here for clarity] + + Where: + L: Module is loaded + D: Default Module + Aliases exist: foo/1.2.3 (1.2) means that + "module load foo/1.2" will load foo/1.2.3 + +Use "module spider" to find all possible modules and extensions. +Use "module keyword key1 key2 ..." to search for all possible modules matching +any of the "keys". +``` \ No newline at end of file diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/missing-python.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/missing-python.Rmd new file mode 100644 index 00000000..f06ed919 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/missing-python.Rmd @@ -0,0 +1,30 @@ +If the `python3` command was unavailable, we would see output like + +```output +/usr/bin/which: no python3 in (/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin:/opt/software/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/puppetlabs/bin:/home/`r config$remote$user`/.local/bin:/home/`r config$remote$user`/bin) +``` + +Note that this wall of text is really a list, with values separated +by the `:` character. The output is telling us that the `which` command +searched the following directories for `python3`, without success: + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin +/opt/software/slurm/bin +/usr/local/bin +/usr/bin +/usr/local/sbin +/usr/sbin +/opt/puppetlabs/bin +/home/`r config$remote$user`/.local/bin +/home/`r config$remote$user`/bin +``` + +However, in our case we do have an existing `python3` available so we see + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/compat/linux/x86_64/usr/bin/python3 +``` + +We need a different Python than the system provided one though, so let us load +a module to access it. diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/module-load-python.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/module-load-python.Rmd new file mode 100644 index 00000000..81d3be34 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/module-load-python.Rmd @@ -0,0 +1,8 @@ +```bash +`r config$remote$prompt` module load `r config$remote$modules$python` +`r config$remote$prompt` which python3 +``` + +```output +/cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin/python3 +``` \ No newline at end of file diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/python-executable-dir.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/python-executable-dir.Rmd new file mode 100644 index 00000000..13031bf9 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/python-executable-dir.Rmd @@ -0,0 +1,19 @@ +```bash +`r config$remote$prompt` ls /cvmfs/pilot.eessi-hpc.org/2020.12/software/x86_64/amd/zen2/software/Python/3.x.y-GCCcore-x.y.z/bin +``` + +```output +2to3 nosetests-3.8 python rst2s5.py +2to3-3.8 pasteurize python3 rst2xetex.py +chardetect pbr python3.8 rst2xml.py +cygdb pip python3.8-config rstpep2html.py +cython pip3 python3-config runxlrd.py +cythonize pip3.8 rst2html4.py sphinx-apidoc +easy_install pybabel rst2html5.py sphinx-autogen +easy_install-3.8 __pycache__ rst2html.py sphinx-build +futurize pydoc3 rst2latex.py sphinx-quickstart +idle3 pydoc3.8 rst2man.py tabulate +idle3.8 pygmentize rst2odt_prepstyles.py virtualenv +netaddr pytest rst2odt.py wheel +nosetests py.test rst2pseudoxml.py +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/software-dependencies.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/software-dependencies.Rmd new file mode 100644 index 00000000..01234436 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/modules/software-dependencies.Rmd @@ -0,0 +1,79 @@ +To demonstrate, let's use `module list`. `module list` shows all loaded +software modules. + +```bash +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 4) GMP/6.2.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 5) libffi/3.3-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 6) Python/3.x.y-GCCcore-x.y.z +``` + +```bash +`r config$remote$prompt` module load GROMACS +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z + 4) GMP/6.2.0-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z + 5) libffi/3.3-GCCcore-x.y.z 18) gompi/2020a + 6) Python/3.x.y-GCCcore-x.y.z 19) FFTW/3.3.8-gompi-2020a + 7) GCC/x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a + 8) numactl/2.0.13-GCCcore-x.y.z 21) foss/2020a + 9) libxml2/2.9.10-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... + 10) libpciaccess/0.16-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... + 11) hwloc/2.2.0-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.8... + 12) libevent/2.1.11-GCCcore-x.y.z 25) GROMACS/2020.1-foss-2020a-Python-3... + 13) UCX/1.8.0-GCCcore-x.y.z +``` + +So in this case, loading the `GROMACS` module (a bioinformatics software +package), also loaded `GMP/6.2.0-GCCcore-x.y.z` and +`SciPy-bundle/2020.03-foss-2020a-Python-3.x.y` as well. Let's try unloading the +`GROMACS` package. + +```bash +`r config$remote$prompt` module unload GROMACS +`r config$remote$prompt` module list +``` + +```output +Currently Loaded Modules: + 1) GCCcore/x.y.z 13) UCX/1.8.0-GCCcore-x.y.z + 2) Tcl/8.6.10-GCCcore-x.y.z 14) libfabric/1.11.0-GCCcore-x.y.z + 3) SQLite/3.31.1-GCCcore-x.y.z 15) PMIx/3.1.5-GCCcore-x.y.z + 4) GMP/6.2.0-GCCcore-x.y.z 16) OpenMPI/4.0.3-GCC-x.y.z + 5) libffi/3.3-GCCcore-x.y.z 17) OpenBLAS/0.3.9-GCC-x.y.z + 6) Python/3.x.y-GCCcore-x.y.z 18) gompi/2020a + 7) GCC/x.y.z 19) FFTW/3.3.8-gompi-2020a + 8) numactl/2.0.13-GCCcore-x.y.z 20) ScaLAPACK/2.1.0-gompi-2020a + 9) libxml2/2.9.10-GCCcore-x.y.z 21) foss/2020a + 10) libpciaccess/0.16-GCCcore-x.y.z 22) pybind11/2.4.3-GCCcore-x.y.z-Pytho... + 11) hwloc/2.2.0-GCCcore-x.y.z 23) SciPy-bundle/2020.03-foss-2020a-Py... + 12) libevent/2.1.11-GCCcore-x.y.z 24) networkx/2.4-foss-2020a-Python-3.x.y +``` + +So using `module unload` "un-loads" a module, and depending on how a site is +configured it may also unload all of the dependencies (in our case it does +not). If we wanted to unload everything at once, we could run `module purge` +(unloads everything). + +```bash +`r config$remote$prompt` module purge +`r config$remote$prompt` module list +``` + +```output +No modules loaded +``` + +Note that `module purge` is informative. It will also let us know if a default +set of "sticky" packages cannot be unloaded (and how to actually unload these +if we truly so desired). diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/eight-tasks.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/eight-tasks.Rmd new file mode 100644 index 00000000..dcbe379a --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/eight-tasks.Rmd @@ -0,0 +1,58 @@ +```bash +`r config$remote$prompt` nano parallel-job.sh +`r config$remote$prompt` cat parallel-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` parallel-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 8 + +# Load the computing environment we need +# (mpi4py and numpy are in SciPy-bundle) +module load `r config$remote$modules$python` +module load SciPy-bundle + +# Execute the task +mpiexec amdahl +``` + +Then submit your job. Note that the submission command has not really changed +from how we submitted the serial job: all the parallel settings are in the +batch file rather than the command line. + +```bash +`r config$remote$prompt` `r config$sched$submit$name` parallel-job.sh +``` + +As before, use the status commands to check when your job runs. + +```bash +`r config$remote$prompt` ls -t +``` + +```output +slurm-347271.out parallel-job.sh slurm-347178.out slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt +``` + +```bash +`r config$remote$prompt` cat slurm-347178.out +``` + +```output +which should take 7.688 seconds with 0.850 parallel proportion of the workload. + + Hello, World! I am process 4 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 0 of 8 on `r config$remote$node`. I will do all the serial 'work' for 4.500 seconds. + Hello, World! I am process 2 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 1 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 3 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 5 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 6 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 7 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + Hello, World! I am process 0 of 8 on `r config$remote$node`. I will do parallel 'work' for 3.188 seconds. + +Total execution time (according to rank 0): 7.697 seconds +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/four-tasks.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/four-tasks.Rmd new file mode 100644 index 00000000..2e885505 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/four-tasks.Rmd @@ -0,0 +1,56 @@ +```bash +`r config$remote$prompt` cp serial-job.sh parallel-job.sh +`r config$remote$prompt` nano parallel-job.sh +`r config$remote$prompt` cat parallel-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` parallel-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 4 + +# Load the computing environment we need +# (mpi4py and numpy are in SciPy-bundle) +module load `r config$remote$modules$python` +module load SciPy-bundle + +# Execute the task +mpiexec amdahl +``` + +Then submit your job. Note that the submission command has not really changed +from how we submitted the serial job: all the parallel settings are in the +batch file rather than the command line. + +```bash +`r config$remote$prompt` `r config$sched$submit$name` parallel-job.sh +``` + +As before, use the status commands to check when your job runs. + +```bash +`r config$remote$prompt` ls -t +``` + +```output +slurm-347178.out parallel-job.sh slurm-347087.out serial-job.sh amdahl README.md LICENSE.txt +``` + +```bash +`r config$remote$prompt` cat slurm-347178.out +``` + +```output +Doing 30.000 seconds of 'work' on 4 processors, +which should take 10.875 seconds with 0.850 parallel proportion of the workload. + + Hello, World! I am process 0 of 4 on `r config$remote$node`. I will do all the serial 'work' for 4.500 seconds. + Hello, World! I am process 2 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 1 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 3 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + Hello, World! I am process 0 of 4 on `r config$remote$node`. I will do parallel 'work' for 6.375 seconds. + +Total execution time (according to rank 0): 10.888 seconds +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/one-task.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/one-task.Rmd new file mode 100644 index 00000000..ceefdfdb --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/parallel/one-task.Rmd @@ -0,0 +1,22 @@ +```bash +`r config$remote$prompt` nano serial-job.sh +`r config$remote$prompt` cat serial-job.sh +``` + +```bash +`r config$remote$shebang` +`r config$sched$comment` `r config$sched$flag$name` solo-job +`r config$sched$comment` `r config$sched$flag$queue` `r config$sched$queue$testing` +`r config$sched$comment` -N 1 +`r config$sched$comment` -n 1 + +# Load the computing environment we need +module load `r config$remote$modules$python` + +# Execute the task +amdahl +``` + +```bash +`r config$remote$prompt` `r config$sched$submit$name` serial-job.sh +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/resources/account-history.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/resources/account-history.Rmd new file mode 100644 index 00000000..5e8a53f2 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/resources/account-history.Rmd @@ -0,0 +1,13 @@ +```output + JobID JobName Partition Account AllocCPUS State ExitCode +------------ ---------- ---------- ---------- ---------- ---------- -------- +7 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 +7.batch batch def-spons+ 1 COMPLETED 0:0 +7.extern extern def-spons+ 1 COMPLETED 0:0 +8 file.sh cpubase_b+ def-spons+ 1 COMPLETED 0:0 +8.batch batch def-spons+ 1 COMPLETED 0:0 +8.extern extern def-spons+ 1 COMPLETED 0:0 +9 example-j+ cpubase_b+ def-spons+ 1 COMPLETED 0:0 +9.batch batch def-spons+ 1 COMPLETED 0:0 +9.extern extern def-spons+ 1 COMPLETED 0:0 +``` \ No newline at end of file diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-script.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-script.Rmd new file mode 100644 index 00000000..5f76fb2c --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-script.Rmd @@ -0,0 +1,3 @@ +```output +Submitted batch job 7 +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-status.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-status.Rmd new file mode 100644 index 00000000..9011f5b4 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/basic-job-status.Rmd @@ -0,0 +1,8 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 9 cpubase_b example- user01 R 0:05 1 node1 +``` + +We can see all the details of our job, most importantly that it is in the `R` +or `RUNNING` state. Sometimes our jobs might need to wait in a queue +(`PENDING`) or have an error (`E`). diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/job-with-name-status.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/job-with-name-status.Rmd new file mode 100644 index 00000000..815f3b9d --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/job-with-name-status.Rmd @@ -0,0 +1,4 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 10 cpubase_b hello-wo user01 R 0:02 1 node1 +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/option-flags-list.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/option-flags-list.Rmd new file mode 100644 index 00000000..5e80b164 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/option-flags-list.Rmd @@ -0,0 +1,15 @@ +* `--ntasks=` or `-n `: How many CPU cores does your job need, + in total? + +* `--time ` or `-t `: + How much real-world time (walltime) will your job take to run? The `` + part can be omitted. + +* `--mem=`: How much memory on a node does your job need in + megabytes? You can also specify gigabytes using by adding a little "g" + afterwards (example: `--mem=5g`) + +* `--nodes=` or `-N `: How many separate machines does your job + need to run on? Note that if you set `ntasks` to a number greater than what + one machine can offer, {{ site.sched.name }} will set this value + automatically. diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/print-sched-variables.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/print-sched-variables.Rmd new file mode 100644 index 00000000..21fec883 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/print-sched-variables.Rmd @@ -0,0 +1,32 @@ +::: challenge + +## Job environment variables + +When {{ site.sched.name }} runs a job, it sets a number of environment +variables for the job. One of these will let us check what directory our job +script was submitted from. The `SLURM_SUBMIT_DIR` variable is set to the +directory from which our job was submitted. Using the `SLURM_SUBMIT_DIR` +variable, modify your job so that it prints out the location from which the +job was submitted. + +:::: solution + +## Solution + +```bash +{{ site.remote.prompt }} nano example-job.sh +{{ site.remote.prompt }} cat example-job.sh +``` + +```bash +`r site$remote$bash_shebang` +#SBATCH -t 00:00:30 + +echo -n "This script is running on " +hostname + +echo "This job was launched in the following directory:" +echo ${SLURM_SUBMIT_DIR} +``` +:::: +::: diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-job.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-job.Rmd new file mode 100644 index 00000000..7dde2ada --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-job.Rmd @@ -0,0 +1,3 @@ +```bash +`r config$remote$prompt` cat slurm-12.out +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-output.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-output.Rmd new file mode 100644 index 00000000..106d9ecc --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/runtime-exceeded-output.Rmd @@ -0,0 +1,5 @@ +```output +This script is running on ... +slurmstepd: error: *** JOB 12 ON node1 CANCELLED AT 2021-02-19T13:55:57 +DUE TO TIME LIMIT *** +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-begin.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-begin.Rmd new file mode 100644 index 00000000..3cfb8c95 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-begin.Rmd @@ -0,0 +1,6 @@ +```output +Submitted batch job 13 + +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) + 13 cpubase_b long_job user01 R 0:02 1 node1 +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-cancel.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-cancel.Rmd new file mode 100644 index 00000000..ccb9cd24 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-job-cancel.Rmd @@ -0,0 +1,3 @@ +```output +JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) +``` diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-multiple-jobs.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-multiple-jobs.Rmd new file mode 100644 index 00000000..5ee30f8d --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/terminate-multiple-jobs.Rmd @@ -0,0 +1,29 @@ +::: challenge +## Cancelling multiple jobs + +We can also cancel all of our jobs at once using the `-u` option. This will +delete all jobs for a specific user (in this case, yourself). Note that you +can only delete your own jobs. + +Try submitting multiple jobs and then cancelling them all. + +:::: solution + +## Solution + +First, submit a trio of jobs: + +```bash +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +`r config$remote$prompt` `r config$sched$submit$name` `r config$sched$submit$options` example-job.sh +``` + +Then, cancel them all: + +```bash +`r config$remote$prompt` `r config$sched$del` -u `r config$remote$user` +``` + +:::: +::: diff --git a/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/using-nodes-interactively.Rmd b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/using-nodes-interactively.Rmd new file mode 100644 index 00000000..0a305412 --- /dev/null +++ b/episodes/files/snippets/HPCC_MagicCastle_slurm/scheduler/using-nodes-interactively.Rmd @@ -0,0 +1,66 @@ +The `r config$sched$interactive` command runs a single command on the cluster +and then exits. Let's demonstrate this by running the `hostname` command with +`r config$sched$interactive`. (We can cancel an `r config$sched$interactive` +job with `Ctrl-c`.) + +```bash +`r config$remote.prompt` `r config$sched$interactive` hostname +``` + +```output +{{ config$remote.node` +``` + +`r config$sched$interactive` accepts all of the same options as +`r config$sched$submit.name`. However, instead of specifying these in a script, +these options are specified on the command-line when starting a job. To submit +a job that uses 2 CPUs for instance, we could use the following command: + +```bash +`r config$remote.prompt` `r config$sched$interactive` -n 2 echo "This job will use 2 CPUs." +``` + +```output +This job will use 2 CPUs. +This job will use 2 CPUs. +``` + +Typically, the resulting shell environment will be the same as that for +`r config$sched$submit.name`. + +### Interactive jobs + +Sometimes, you will need a lot of resources for interactive use. Perhaps it's +our first time running an analysis or we are attempting to debug something that +went wrong with a previous job. Fortunately, {{ config$sched$name` makes it +easy to start an interactive job with `r config$sched$interactive`: + +```bash +`r config$remote.prompt` `r config$sched$interactive` --pty bash +``` + +You should be presented with a bash prompt. Note that the prompt will likely +change to reflect your new location, in this case the compute node we are +logged on. You can also verify this with `hostname`. + +::: callout + +## Creating remote graphics + +To see graphical output inside your jobs, you need to use X11 forwarding. To +connect with this feature enabled, use the `-Y` option when you login with +the `ssh` command, e.g., `r paste("ssh -Y ", config$remote$user, "@", config$remote$login, sep="")`. + +To demonstrate what happens when you create a graphics window on the remote +node, use the `xeyes` command. A relatively adorable pair of eyes should pop +up (press `Ctrl-C` to stop). If you are using a Mac, you must have installed +XQuartz (and restarted your computer) for this to work. + +If your cluster has the +[slurm-spank-x11](https://github.com/hautreux/slurm-spank-x11) plugin +installed, you can ensure X11 forwarding within interactive jobs by using the +`--x11` option for `r config$sched$interactive` with the command +`r paste(config$sched$interactive, " --x11", " --pty bash", sep="")`. +::: + +When you are done with the interactive job, type `exit` to quit your session. diff --git a/episodes/files/snippets/README.md b/episodes/files/snippets/README.md new file mode 100644 index 00000000..56b65740 --- /dev/null +++ b/episodes/files/snippets/README.md @@ -0,0 +1,32 @@ +# HPC Carpentry Snippets Library + +This directory contains snippets of code and output that are specific +to a particular site. For example, when the lesson shows the status +of the cluster and its nodes, it is preferable to show *your* cluster +and *your* nodes. If you replace the contents of the relevant snippet, +the website gets built with your cluster details, instead of generic +values (or, more precisely, values taken from ComputeCanada). + +The snippets have been named so that the lessons use them in roughly +alphabetical order, while still reflecting something of the contents +of each file. So, if you're reading (or teaching) a lesson and notice +something amiss about half-way through, look to the files about +half-way through the directory. If it's the first or last snippet, +you're in particularly good luck. + +This alphabetical ordering was not always the case. To reduce the +headache of keeping forks, branches, and derivative works up-to-date, +we have included a utility to rename snippets from the older scheme. +To use it, run the following command, rebuild and test your site, and +commit the changes. + +```bash +$ ./rename-snippets.sh +``` + +If the naming seems counter-intuitive, please feel free to make +changes locally, and file an issue of submit a pull request to fix it +upstream. None of this is set in stone, and improvements are always +welcome. + + diff --git a/episodes/lesson_config.yaml b/episodes/lesson_config.yaml new file mode 100644 index 00000000..af0b8166 --- /dev/null +++ b/episodes/lesson_config.yaml @@ -0,0 +1,4 @@ +--- +main_config: files/snippets/HPCC_MagicCastle_slurm/_config_options.yml +snippets: HPCC_MagicCastle_slurm +... diff --git a/episodes/load_config.R b/episodes/load_config.R new file mode 100644 index 00000000..f6506897 --- /dev/null +++ b/episodes/load_config.R @@ -0,0 +1,46 @@ +## R script to chain-load lesson configuration YAML files. +## Top-level configuration is `/episodes/lesson_config.yml` + +library(yaml) + +## Load primary configuration +config <- yaml.load_file("lesson_config.yaml") + +## If "main_config" key exists, load the second configuration and merge +### print(paste("Loading ", config$main_config)) +if (!is.null(config$main_config) && file.exists(config$main_config)) { + override_config <- yaml.load_file(config$main_config) + config <- modifyList(config, override_config) +} + +snippets <- paste("files/snippets/", config$snippets, sep="") + +# Extract main and fallback paths from config +main_snippets <- config$main_snippets +fallback_snippets <- config$fallback_snippets + +# Function to choose the correct document path (or return NULL if neither exists) +choose_doc <- function(child_file) { + # Get the current document name (without extension) + current_doc <- tools::file_path_sans_ext(knitr::current_input(dir = TRUE)) + + # Build paths for the child document inside subdirectories + doc_paths <- list( + main = file.path(current_doc, main_snippets, child_file), + fallback = file.path(current_doc, fallback_snippets, child_file) + ) + print(doc_paths) + ### print(getwd()) + + # Return the valid path, or NULL if neither exists + if (file.exists(doc_paths$main)) { + print(paste("Returning", doc_paths$main)) + return(doc_paths$main) + } else if (file.exists(doc_paths$fallback)) { + print(paste("Returning", doc_paths$fallback)) + return(doc_paths$fallback) + } else { + print("Returning NULL") + return(NULL) # Return NULL if neither path exists + } +} diff --git a/index.md b/index.md index 02b64977..39ed73cc 100644 --- a/index.md +++ b/index.md @@ -2,8 +2,6 @@ site: sandpaper::sandpaper_site --- -{% include gh\_variables.html %} - This workshop is an introduction to using high-performance computing systems effectively. We can't cover every case or give an exhaustive course on parallel programming in just two days' teaching time. Instead, this workshop is intended @@ -19,7 +17,6 @@ participants to go through [shell-novice](https://swcarpentry.github.io/shell-novice/), if new to the command line (also known as terminal or shell). - :::::::::::::::::::::::::::::::::::::::::::::::::: By the end of this workshop, students will know how to: @@ -35,10 +32,8 @@ By the end of this workshop, students will know how to: ## Getting Started -To get started, follow the directions in the "[Setup]( -{{ page.root }}/setup.html)" tab to download data to your computer and follow -any installation instructions. - +To get started, please follow the "[Setup Instructions](../learners/setup.md) +to ensure you have a terminal and SSH application. :::::::::::::::::::::::::::::::::::::::::::::::::: @@ -52,9 +47,4 @@ welcome. If you are teaching this lesson in a workshop, please see the [Instructor notes](instructors/instructor-notes.md). - :::::::::::::::::::::::::::::::::::::::::::::::::: - - - - diff --git a/instructors/instructor-notes.md b/instructors/instructor-notes.Rmd similarity index 100% rename from instructors/instructor-notes.md rename to instructors/instructor-notes.Rmd diff --git a/renv.lock b/renv.lock new file mode 100644 index 00000000..d90ed3be --- /dev/null +++ b/renv.lock @@ -0,0 +1,988 @@ +{ + "R": { + "Version": "4.4.2", + "Repositories": [ + { + "Name": "CRAN", + "URL": "https://mirrors.nics.utk.edu/cran" + } + ] + }, + "Packages": { + "R6": { + "Package": "R6", + "Version": "2.6.1", + "Source": "Repository", + "Title": "Encapsulated Classes with Reference Semantics", + "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Creates classes with reference semantics, similar to R's built-in reference classes. Compared to reference classes, R6 classes are simpler and lighter-weight, and they are not built on S4 classes so they do not require the methods package. These classes allow public and private members, and they support inheritance, even when the classes are defined in different packages.", + "License": "MIT + file LICENSE", + "URL": "https://r6.r-lib.org, https://github.com/r-lib/R6", + "BugReports": "https://github.com/r-lib/R6/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Suggests": [ + "lobstr", + "testthat (>= 3.0.0)" + ], + "Config/Needs/website": "tidyverse/tidytemplate, ggplot2, microbenchmark, scales", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "base64enc": { + "Package": "base64enc", + "Version": "0.1-3", + "Source": "Repository", + "Title": "Tools for base64 encoding", + "Author": "Simon Urbanek ", + "Maintainer": "Simon Urbanek ", + "Depends": [ + "R (>= 2.9.0)" + ], + "Enhances": [ + "png" + ], + "Description": "This package provides tools for handling base64 encoding. It is more flexible than the orphaned base64 package.", + "License": "GPL-2 | GPL-3", + "URL": "http://www.rforge.net/base64enc", + "NeedsCompilation": "yes", + "Repository": "CRAN" + }, + "bslib": { + "Package": "bslib", + "Version": "0.9.0", + "Source": "Repository", + "Title": "Custom 'Bootstrap' 'Sass' Themes for 'shiny' and 'rmarkdown'", + "Authors@R": "c( person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Garrick\", \"Aden-Buie\", , \"garrick@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-7111-0077\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Javi\", \"Aguilar\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap colorpicker library\"), person(\"Thomas\", \"Park\", role = c(\"ctb\", \"cph\"), comment = \"Bootswatch library\"), person(, \"PayPal\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap accessibility plugin\") )", + "Description": "Simplifies custom 'CSS' styling of both 'shiny' and 'rmarkdown' via 'Bootstrap' 'Sass'. Supports 'Bootstrap' 3, 4 and 5 as well as their various 'Bootswatch' themes. An interactive widget is also provided for previewing themes in real time.", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/bslib/, https://github.com/rstudio/bslib", + "BugReports": "https://github.com/rstudio/bslib/issues", + "Depends": [ + "R (>= 2.10)" + ], + "Imports": [ + "base64enc", + "cachem", + "fastmap (>= 1.1.1)", + "grDevices", + "htmltools (>= 0.5.8)", + "jquerylib (>= 0.1.3)", + "jsonlite", + "lifecycle", + "memoise (>= 2.0.1)", + "mime", + "rlang", + "sass (>= 0.4.9)" + ], + "Suggests": [ + "bsicons", + "curl", + "fontawesome", + "future", + "ggplot2", + "knitr", + "magrittr", + "rappdirs", + "rmarkdown (>= 2.7)", + "shiny (> 1.8.1)", + "testthat", + "thematic", + "tools", + "utils", + "withr", + "yaml" + ], + "Config/Needs/deploy": "BH, chiflights22, colourpicker, commonmark, cpp11, cpsievert/chiflights22, cpsievert/histoslider, dplyr, DT, ggplot2, ggridges, gt, hexbin, histoslider, htmlwidgets, lattice, leaflet, lubridate, markdown, modelr, plotly, reactable, reshape2, rprojroot, rsconnect, rstudio/shiny, scales, styler, tibble", + "Config/Needs/routine": "chromote, desc, renv", + "Config/Needs/website": "brio, crosstalk, dplyr, DT, ggplot2, glue, htmlwidgets, leaflet, lorem, palmerpenguins, plotly, purrr, rprojroot, rstudio/htmltools, scales, stringr, tidyr, webshot2", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "zzzz-bs-sass, fonts, zzz-precompile, theme-*, rmd-*", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "Collate": "'accordion.R' 'breakpoints.R' 'bs-current-theme.R' 'bs-dependencies.R' 'bs-global.R' 'bs-remove.R' 'bs-theme-layers.R' 'bs-theme-preset-bootswatch.R' 'bs-theme-preset-brand.R' 'bs-theme-preset-builtin.R' 'bs-theme-preset.R' 'utils.R' 'bs-theme-preview.R' 'bs-theme-update.R' 'bs-theme.R' 'bslib-package.R' 'buttons.R' 'card.R' 'deprecated.R' 'files.R' 'fill.R' 'imports.R' 'input-dark-mode.R' 'input-switch.R' 'layout.R' 'nav-items.R' 'nav-update.R' 'navbar_options.R' 'navs-legacy.R' 'navs.R' 'onLoad.R' 'page.R' 'popover.R' 'precompiled.R' 'print.R' 'shiny-devmode.R' 'sidebar.R' 'staticimports.R' 'tooltip.R' 'utils-deps.R' 'utils-shiny.R' 'utils-tags.R' 'value-box.R' 'version-default.R' 'versions.R'", + "NeedsCompilation": "no", + "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], Garrick Aden-Buie [aut] (), Posit Software, PBC [cph, fnd], Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Javi Aguilar [ctb, cph] (Bootstrap colorpicker library), Thomas Park [ctb, cph] (Bootswatch library), PayPal [ctb, cph] (Bootstrap accessibility plugin)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "cachem": { + "Package": "cachem", + "Version": "1.1.0", + "Source": "Repository", + "Title": "Cache R Objects with Automatic Pruning", + "Description": "Key-value stores with automatic pruning. Caches can limit either their total size or the age of the oldest object (or both), automatically pruning objects to maintain the constraints.", + "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", c(\"aut\", \"cre\")), person(family = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")))", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "ByteCompile": "true", + "URL": "https://cachem.r-lib.org/, https://github.com/r-lib/cachem", + "Imports": [ + "rlang", + "fastmap (>= 1.2.0)" + ], + "Suggests": [ + "testthat" + ], + "RoxygenNote": "7.2.3", + "Config/Needs/routine": "lobstr", + "Config/Needs/website": "pkgdown", + "NeedsCompilation": "yes", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "cli": { + "Package": "cli", + "Version": "3.6.4", + "Source": "Repository", + "Title": "Helpers for Developing Command Line Interfaces", + "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"gabor@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Kirill\", \"Müller\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", , \"salim-b@pm.me\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A suite of tools to build attractive command line interfaces ('CLIs'), from semantic elements: headings, lists, alerts, paragraphs, etc. Supports custom themes via a 'CSS'-like language. It also contains a number of lower level 'CLI' elements: rules, boxes, trees, and 'Unicode' symbols with 'ASCII' alternatives. It support ANSI colors and text styles as well.", + "License": "MIT + file LICENSE", + "URL": "https://cli.r-lib.org, https://github.com/r-lib/cli", + "BugReports": "https://github.com/r-lib/cli/issues", + "Depends": [ + "R (>= 3.4)" + ], + "Imports": [ + "utils" + ], + "Suggests": [ + "callr", + "covr", + "crayon", + "digest", + "glue (>= 1.6.0)", + "grDevices", + "htmltools", + "htmlwidgets", + "knitr", + "methods", + "processx", + "ps (>= 1.3.4.9000)", + "rlang (>= 1.0.2.9003)", + "rmarkdown", + "rprojroot", + "rstudioapi", + "testthat (>= 3.2.0)", + "tibble", + "whoami", + "withr" + ], + "Config/Needs/website": "r-lib/asciicast, bench, brio, cpp11, decor, desc, fansi, prettyunits, sessioninfo, tidyverse/tidytemplate, usethis, vctrs", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "yes", + "Author": "Gábor Csárdi [aut, cre], Hadley Wickham [ctb], Kirill Müller [ctb], Salim Brüggemann [ctb] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" + }, + "digest": { + "Package": "digest", + "Version": "0.6.37", + "Source": "Repository", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Antoine\", \"Lucas\", role=\"ctb\"), person(\"Jarek\", \"Tuszynski\", role=\"ctb\"), person(\"Henrik\", \"Bengtsson\", role=\"ctb\", comment = c(ORCID = \"0000-0002-7579-5165\")), person(\"Simon\", \"Urbanek\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2297-1732\")), person(\"Mario\", \"Frasca\", role=\"ctb\"), person(\"Bryan\", \"Lewis\", role=\"ctb\"), person(\"Murray\", \"Stokely\", role=\"ctb\"), person(\"Hannes\", \"Muehleisen\", role=\"ctb\"), person(\"Duncan\", \"Murdoch\", role=\"ctb\"), person(\"Jim\", \"Hester\", role=\"ctb\"), person(\"Wush\", \"Wu\", role=\"ctb\", comment = c(ORCID = \"0000-0001-5180-0567\")), person(\"Qiang\", \"Kou\", role=\"ctb\", comment = c(ORCID = \"0000-0001-6786-5453\")), person(\"Thierry\", \"Onkelinx\", role=\"ctb\", comment = c(ORCID = \"0000-0001-8804-4216\")), person(\"Michel\", \"Lang\", role=\"ctb\", comment = c(ORCID = \"0000-0001-9754-0393\")), person(\"Viliam\", \"Simko\", role=\"ctb\"), person(\"Kurt\", \"Hornik\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4198-9911\")), person(\"Radford\", \"Neal\", role=\"ctb\", comment = c(ORCID = \"0000-0002-2473-3407\")), person(\"Kendon\", \"Bell\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9093-8312\")), person(\"Matthew\", \"de Queljoe\", role=\"ctb\"), person(\"Dmitry\", \"Selivanov\", role=\"ctb\"), person(\"Ion\", \"Suruceanu\", role=\"ctb\"), person(\"Bill\", \"Denney\", role=\"ctb\"), person(\"Dirk\", \"Schumacher\", role=\"ctb\"), person(\"András\", \"Svraka\", role=\"ctb\"), person(\"Sergey\", \"Fedorov\", role=\"ctb\"), person(\"Will\", \"Landau\", role=\"ctb\", comment = c(ORCID = \"0000-0003-1878-3253\")), person(\"Floris\", \"Vanderhaeghe\", role=\"ctb\", comment = c(ORCID = \"0000-0002-6378-6229\")), person(\"Kevin\", \"Tappe\", role=\"ctb\"), person(\"Harris\", \"McGehee\", role=\"ctb\"), person(\"Tim\", \"Mastny\", role=\"ctb\"), person(\"Aaron\", \"Peikert\", role=\"ctb\", comment = c(ORCID = \"0000-0001-7813-818X\")), person(\"Mark\", \"van der Loo\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9807-4686\")), person(\"Chris\", \"Muir\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2555-3878\")), person(\"Moritz\", \"Beller\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4852-0526\")), person(\"Sebastian\", \"Campbell\", role=\"ctb\"), person(\"Winston\", \"Chang\", role=\"ctb\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Dean\", \"Attali\", role=\"ctb\", comment = c(ORCID = \"0000-0002-5645-3493\")), person(\"Michael\", \"Chirico\", role=\"ctb\", comment = c(ORCID = \"0000-0003-0787-087X\")), person(\"Kevin\", \"Ushey\", role=\"ctb\"))", + "Date": "2024-08-19", + "Title": "Create Compact Hash Digests of R Objects", + "Description": "Implementation of a function 'digest()' for the creation of hash digests of arbitrary R objects (using the 'md5', 'sha-1', 'sha-256', 'crc32', 'xxhash', 'murmurhash', 'spookyhash', 'blake3', 'crc32c', 'xxh3_64', and 'xxh3_128' algorithms) permitting easy comparison of R language objects, as well as functions such as'hmac()' to create hash-based message authentication code. Please note that this package is not meant to be deployed for cryptographic purposes for which more comprehensive (and widely tested) libraries such as 'OpenSSL' should be used.", + "URL": "https://github.com/eddelbuettel/digest, https://dirk.eddelbuettel.com/code/digest.html", + "BugReports": "https://github.com/eddelbuettel/digest/issues", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "utils" + ], + "License": "GPL (>= 2)", + "Suggests": [ + "tinytest", + "simplermarkdown" + ], + "VignetteBuilder": "simplermarkdown", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Dirk Eddelbuettel [aut, cre] (), Antoine Lucas [ctb], Jarek Tuszynski [ctb], Henrik Bengtsson [ctb] (), Simon Urbanek [ctb] (), Mario Frasca [ctb], Bryan Lewis [ctb], Murray Stokely [ctb], Hannes Muehleisen [ctb], Duncan Murdoch [ctb], Jim Hester [ctb], Wush Wu [ctb] (), Qiang Kou [ctb] (), Thierry Onkelinx [ctb] (), Michel Lang [ctb] (), Viliam Simko [ctb], Kurt Hornik [ctb] (), Radford Neal [ctb] (), Kendon Bell [ctb] (), Matthew de Queljoe [ctb], Dmitry Selivanov [ctb], Ion Suruceanu [ctb], Bill Denney [ctb], Dirk Schumacher [ctb], András Svraka [ctb], Sergey Fedorov [ctb], Will Landau [ctb] (), Floris Vanderhaeghe [ctb] (), Kevin Tappe [ctb], Harris McGehee [ctb], Tim Mastny [ctb], Aaron Peikert [ctb] (), Mark van der Loo [ctb] (), Chris Muir [ctb] (), Moritz Beller [ctb] (), Sebastian Campbell [ctb], Winston Chang [ctb] (), Dean Attali [ctb] (), Michael Chirico [ctb] (), Kevin Ushey [ctb]", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, + "evaluate": { + "Package": "evaluate", + "Version": "1.0.3", + "Source": "Repository", + "Type": "Package", + "Title": "Parsing and Evaluation Tools that Provide More Details than the Default", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Yihui\", \"Xie\", role = \"aut\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Michael\", \"Lawrence\", role = \"ctb\"), person(\"Thomas\", \"Kluyver\", role = \"ctb\"), person(\"Jeroen\", \"Ooms\", role = \"ctb\"), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Adam\", \"Ryczkowski\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Michel\", \"Lang\", role = \"ctb\"), person(\"Karolis\", \"Koncevičius\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Parsing and evaluation tools that make it easy to recreate the command line behaviour of R.", + "License": "MIT + file LICENSE", + "URL": "https://evaluate.r-lib.org/, https://github.com/r-lib/evaluate", + "BugReports": "https://github.com/r-lib/evaluate/issues", + "Depends": [ + "R (>= 3.6.0)" + ], + "Suggests": [ + "callr", + "covr", + "ggplot2 (>= 3.3.6)", + "lattice", + "methods", + "pkgload", + "rlang", + "knitr", + "testthat (>= 3.0.0)", + "withr" + ], + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut, cre], Yihui Xie [aut] (), Michael Lawrence [ctb], Thomas Kluyver [ctb], Jeroen Ooms [ctb], Barret Schloerke [ctb], Adam Ryczkowski [ctb], Hiroaki Yutani [ctb], Michel Lang [ctb], Karolis Koncevičius [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" + }, + "fastmap": { + "Package": "fastmap", + "Version": "1.2.0", + "Source": "Repository", + "Title": "Fast Data Structures", + "Authors@R": "c( person(\"Winston\", \"Chang\", email = \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(given = \"Tessil\", role = \"cph\", comment = \"hopscotch_map library\") )", + "Description": "Fast implementation of data structures, including a key-value store, stack, and queue. Environments are commonly used as key-value stores in R, but every time a new key is used, it is added to R's global symbol table, causing a small amount of memory leakage. This can be problematic in cases where many different keys are used. Fastmap avoids this memory leak issue by implementing the map using data structures in C++.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.3", + "Suggests": [ + "testthat (>= 2.1.1)" + ], + "URL": "https://r-lib.github.io/fastmap/, https://github.com/r-lib/fastmap", + "BugReports": "https://github.com/r-lib/fastmap/issues", + "NeedsCompilation": "yes", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd], Tessil [cph] (hopscotch_map library)", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "fontawesome": { + "Package": "fontawesome", + "Version": "0.5.3", + "Source": "Repository", + "Type": "Package", + "Title": "Easily Work with 'Font Awesome' Icons", + "Description": "Easily and flexibly insert 'Font Awesome' icons into 'R Markdown' documents and 'Shiny' apps. These icons can be inserted into HTML content through inline 'SVG' tags or 'i' tags. There is also a utility function for exporting 'Font Awesome' icons as 'PNG' images for those situations where raster graphics are needed.", + "Authors@R": "c( person(\"Richard\", \"Iannone\", , \"rich@posit.co\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"ctb\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome font\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "License": "MIT + file LICENSE", + "URL": "https://github.com/rstudio/fontawesome, https://rstudio.github.io/fontawesome/", + "BugReports": "https://github.com/rstudio/fontawesome/issues", + "Encoding": "UTF-8", + "ByteCompile": "true", + "RoxygenNote": "7.3.2", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "rlang (>= 1.0.6)", + "htmltools (>= 0.5.1.1)" + ], + "Suggests": [ + "covr", + "dplyr (>= 1.0.8)", + "gt (>= 0.9.0)", + "knitr (>= 1.31)", + "testthat (>= 3.0.0)", + "rsvg" + ], + "Config/testthat/edition": "3", + "NeedsCompilation": "no", + "Author": "Richard Iannone [aut, cre] (), Christophe Dervieux [ctb] (), Winston Chang [ctb], Dave Gandy [ctb, cph] (Font-Awesome font), Posit Software, PBC [cph, fnd]", + "Maintainer": "Richard Iannone ", + "Repository": "CRAN" + }, + "fs": { + "Package": "fs", + "Version": "1.6.5", + "Source": "Repository", + "Title": "Cross-Platform File System Operations Based on 'libuv'", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"libuv project contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Joyent, Inc. and other Node contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A cross-platform interface to file system operations, built on top of the 'libuv' C library.", + "License": "MIT + file LICENSE", + "URL": "https://fs.r-lib.org, https://github.com/r-lib/fs", + "BugReports": "https://github.com/r-lib/fs/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "methods" + ], + "Suggests": [ + "covr", + "crayon", + "knitr", + "pillar (>= 1.0.0)", + "rmarkdown", + "spelling", + "testthat (>= 3.0.0)", + "tibble (>= 1.1.0)", + "vctrs (>= 0.3.0)", + "withr" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Copyright": "file COPYRIGHTS", + "Encoding": "UTF-8", + "Language": "en-US", + "RoxygenNote": "7.2.3", + "SystemRequirements": "GNU make", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut], Hadley Wickham [aut], Gábor Csárdi [aut, cre], libuv project contributors [cph] (libuv library), Joyent, Inc. and other Node contributors [cph] (libuv library), Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" + }, + "glue": { + "Package": "glue", + "Version": "1.8.0", + "Source": "Repository", + "Title": "Interpreted String Literals", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "An implementation of interpreted string literals, inspired by Python's Literal String Interpolation and Docstrings and Julia's Triple-Quoted String Literals .", + "License": "MIT + file LICENSE", + "URL": "https://glue.tidyverse.org/, https://github.com/tidyverse/glue", + "BugReports": "https://github.com/tidyverse/glue/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "methods" + ], + "Suggests": [ + "crayon", + "DBI (>= 1.2.0)", + "dplyr", + "knitr", + "magrittr", + "rlang", + "rmarkdown", + "RSQLite", + "testthat (>= 3.2.0)", + "vctrs (>= 0.3.0)", + "waldo (>= 0.5.3)", + "withr" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/Needs/website": "bench, forcats, ggbeeswarm, ggplot2, R.utils, rprintf, tidyr, tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut] (), Jennifer Bryan [aut, cre] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Jennifer Bryan ", + "Repository": "CRAN" + }, + "highr": { + "Package": "highr", + "Version": "0.11", + "Source": "Repository", + "Type": "Package", + "Title": "Syntax Highlighting for R Source Code", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Yixuan\", \"Qiu\", role = \"aut\"), person(\"Christopher\", \"Gandrud\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\") )", + "Description": "Provides syntax highlighting for R source code. Currently it supports LaTeX and HTML output. Source code of other languages is supported via Andre Simon's highlight package ().", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "xfun (>= 0.18)" + ], + "Suggests": [ + "knitr", + "markdown", + "testit" + ], + "License": "GPL", + "URL": "https://github.com/yihui/highr", + "BugReports": "https://github.com/yihui/highr/issues", + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre] (), Yixuan Qiu [aut], Christopher Gandrud [ctb], Qiang Li [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "htmltools": { + "Package": "htmltools", + "Version": "0.5.8.1", + "Source": "Repository", + "Type": "Package", + "Title": "Tools for HTML", + "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Barret\", \"Schloerke\", , \"barret@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Yihui\", \"Xie\", , \"yihui@posit.co\", role = \"aut\"), person(\"Jeff\", \"Allen\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Tools for HTML generation and output.", + "License": "GPL (>= 2)", + "URL": "https://github.com/rstudio/htmltools, https://rstudio.github.io/htmltools/", + "BugReports": "https://github.com/rstudio/htmltools/issues", + "Depends": [ + "R (>= 2.14.1)" + ], + "Imports": [ + "base64enc", + "digest", + "fastmap (>= 1.1.0)", + "grDevices", + "rlang (>= 1.0.0)", + "utils" + ], + "Suggests": [ + "Cairo", + "markdown", + "ragg", + "shiny", + "testthat", + "withr" + ], + "Enhances": [ + "knitr" + ], + "Config/Needs/check": "knitr", + "Config/Needs/website": "rstudio/quillt, bench", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "Collate": "'colors.R' 'fill.R' 'html_dependency.R' 'html_escape.R' 'html_print.R' 'htmltools-package.R' 'images.R' 'known_tags.R' 'selector.R' 'staticimports.R' 'tag_query.R' 'utils.R' 'tags.R' 'template.R'", + "NeedsCompilation": "yes", + "Author": "Joe Cheng [aut], Carson Sievert [aut, cre] (), Barret Schloerke [aut] (), Winston Chang [aut] (), Yihui Xie [aut], Jeff Allen [aut], Posit Software, PBC [cph, fnd]", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "jquerylib": { + "Package": "jquerylib", + "Version": "0.1.4", + "Source": "Repository", + "Title": "Obtain 'jQuery' as an HTML Dependency Object", + "Authors@R": "c( person(\"Carson\", \"Sievert\", role = c(\"aut\", \"cre\"), email = \"carson@rstudio.com\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", role = \"aut\", email = \"joe@rstudio.com\"), person(family = \"RStudio\", role = \"cph\"), person(family = \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library and jQuery UI library\"), person(family = \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt\") )", + "Description": "Obtain any major version of 'jQuery' () and use it in any webpage generated by 'htmltools' (e.g. 'shiny', 'htmlwidgets', and 'rmarkdown'). Most R users don't need to use this package directly, but other R packages (e.g. 'shiny', 'rmarkdown', etc.) depend on this package to avoid bundling redundant copies of 'jQuery'.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "Config/testthat/edition": "3", + "RoxygenNote": "7.0.2", + "Imports": [ + "htmltools" + ], + "Suggests": [ + "testthat" + ], + "NeedsCompilation": "no", + "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], RStudio [cph], jQuery Foundation [cph] (jQuery library and jQuery UI library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "jsonlite": { + "Package": "jsonlite", + "Version": "1.9.1", + "Source": "Repository", + "Title": "A Simple and Robust JSON Parser and Generator for R", + "License": "MIT + file LICENSE", + "Depends": [ + "methods" + ], + "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Duncan\", \"Temple Lang\", role = \"ctb\"), person(\"Lloyd\", \"Hilaiel\", role = \"cph\", comment=\"author of bundled libyajl\"))", + "URL": "https://jeroen.r-universe.dev/jsonlite https://arxiv.org/abs/1403.2805", + "BugReports": "https://github.com/jeroen/jsonlite/issues", + "Maintainer": "Jeroen Ooms ", + "VignetteBuilder": "knitr, R.rsp", + "Description": "A reasonably fast JSON parser and generator, optimized for statistical data and the web. Offers simple, flexible tools for working with JSON in R, and is particularly powerful for building pipelines and interacting with a web API. The implementation is based on the mapping described in the vignette (Ooms, 2014). In addition to converting JSON data from/to R objects, 'jsonlite' contains functions to stream, validate, and prettify JSON data. The unit tests included with the package verify that all edge cases are encoded and decoded consistently for use with dynamic data in systems and applications.", + "Suggests": [ + "httr", + "vctrs", + "testthat", + "knitr", + "rmarkdown", + "R.rsp", + "sf" + ], + "RoxygenNote": "7.2.3", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Jeroen Ooms [aut, cre] (), Duncan Temple Lang [ctb], Lloyd Hilaiel [cph] (author of bundled libyajl)", + "Repository": "CRAN" + }, + "knitr": { + "Package": "knitr", + "Version": "1.49", + "Source": "Repository", + "Type": "Package", + "Title": "A General-Purpose Package for Dynamic Report Generation in R", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Abhraneel\", \"Sarma\", role = \"ctb\"), person(\"Adam\", \"Vogt\", role = \"ctb\"), person(\"Alastair\", \"Andrew\", role = \"ctb\"), person(\"Alex\", \"Zvoleff\", role = \"ctb\"), person(\"Amar\", \"Al-Zubaidi\", role = \"ctb\"), person(\"Andre\", \"Simon\", role = \"ctb\", comment = \"the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de\"), person(\"Aron\", \"Atkins\", role = \"ctb\"), person(\"Aaron\", \"Wolen\", role = \"ctb\"), person(\"Ashley\", \"Manton\", role = \"ctb\"), person(\"Atsushi\", \"Yasumoto\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8335-495X\")), person(\"Ben\", \"Baumer\", role = \"ctb\"), person(\"Brian\", \"Diggs\", role = \"ctb\"), person(\"Brian\", \"Zhang\", role = \"ctb\"), person(\"Bulat\", \"Yapparov\", role = \"ctb\"), person(\"Cassio\", \"Pereira\", role = \"ctb\"), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person(\"David\", \"Hall\", role = \"ctb\"), person(\"David\", \"Hugh-Jones\", role = \"ctb\"), person(\"David\", \"Robinson\", role = \"ctb\"), person(\"Doug\", \"Hemken\", role = \"ctb\"), person(\"Duncan\", \"Murdoch\", role = \"ctb\"), person(\"Elio\", \"Campitelli\", role = \"ctb\"), person(\"Ellis\", \"Hughes\", role = \"ctb\"), person(\"Emily\", \"Riederer\", role = \"ctb\"), person(\"Fabian\", \"Hirschmann\", role = \"ctb\"), person(\"Fitch\", \"Simeon\", role = \"ctb\"), person(\"Forest\", \"Fang\", role = \"ctb\"), person(c(\"Frank\", \"E\", \"Harrell\", \"Jr\"), role = \"ctb\", comment = \"the Sweavel package at inst/misc/Sweavel.sty\"), person(\"Garrick\", \"Aden-Buie\", role = \"ctb\"), person(\"Gregoire\", \"Detrez\", role = \"ctb\"), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Hao\", \"Zhu\", role = \"ctb\"), person(\"Heewon\", \"Jeon\", role = \"ctb\"), person(\"Henrik\", \"Bengtsson\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Ian\", \"Lyttle\", role = \"ctb\"), person(\"Hodges\", \"Daniel\", role = \"ctb\"), person(\"Jacob\", \"Bien\", role = \"ctb\"), person(\"Jake\", \"Burkhead\", role = \"ctb\"), person(\"James\", \"Manton\", role = \"ctb\"), person(\"Jared\", \"Lander\", role = \"ctb\"), person(\"Jason\", \"Punyon\", role = \"ctb\"), person(\"Javier\", \"Luraschi\", role = \"ctb\"), person(\"Jeff\", \"Arnold\", role = \"ctb\"), person(\"Jenny\", \"Bryan\", role = \"ctb\"), person(\"Jeremy\", \"Ashkenas\", role = c(\"ctb\", \"cph\"), comment = \"the CSS file at inst/misc/docco-classic.css\"), person(\"Jeremy\", \"Stephens\", role = \"ctb\"), person(\"Jim\", \"Hester\", role = \"ctb\"), person(\"Joe\", \"Cheng\", role = \"ctb\"), person(\"Johannes\", \"Ranke\", role = \"ctb\"), person(\"John\", \"Honaker\", role = \"ctb\"), person(\"John\", \"Muschelli\", role = \"ctb\"), person(\"Jonathan\", \"Keane\", role = \"ctb\"), person(\"JJ\", \"Allaire\", role = \"ctb\"), person(\"Johan\", \"Toloe\", role = \"ctb\"), person(\"Jonathan\", \"Sidi\", role = \"ctb\"), person(\"Joseph\", \"Larmarange\", role = \"ctb\"), person(\"Julien\", \"Barnier\", role = \"ctb\"), person(\"Kaiyin\", \"Zhong\", role = \"ctb\"), person(\"Kamil\", \"Slowikowski\", role = \"ctb\"), person(\"Karl\", \"Forner\", role = \"ctb\"), person(c(\"Kevin\", \"K.\"), \"Smith\", role = \"ctb\"), person(\"Kirill\", \"Mueller\", role = \"ctb\"), person(\"Kohske\", \"Takahashi\", role = \"ctb\"), person(\"Lorenz\", \"Walthert\", role = \"ctb\"), person(\"Lucas\", \"Gallindo\", role = \"ctb\"), person(\"Marius\", \"Hofert\", role = \"ctb\"), person(\"Martin\", \"Modrák\", role = \"ctb\"), person(\"Michael\", \"Chirico\", role = \"ctb\"), person(\"Michael\", \"Friendly\", role = \"ctb\"), person(\"Michal\", \"Bojanowski\", role = \"ctb\"), person(\"Michel\", \"Kuhlmann\", role = \"ctb\"), person(\"Miller\", \"Patrick\", role = \"ctb\"), person(\"Nacho\", \"Caballero\", role = \"ctb\"), person(\"Nick\", \"Salkowski\", role = \"ctb\"), person(\"Niels Richard\", \"Hansen\", role = \"ctb\"), person(\"Noam\", \"Ross\", role = \"ctb\"), person(\"Obada\", \"Mahdi\", role = \"ctb\"), person(\"Pavel N.\", \"Krivitsky\", role = \"ctb\", comment=c(ORCID = \"0000-0002-9101-3362\")), person(\"Pedro\", \"Faria\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\"), person(\"Ramnath\", \"Vaidyanathan\", role = \"ctb\"), person(\"Richard\", \"Cotton\", role = \"ctb\"), person(\"Robert\", \"Krzyzanowski\", role = \"ctb\"), person(\"Rodrigo\", \"Copetti\", role = \"ctb\"), person(\"Romain\", \"Francois\", role = \"ctb\"), person(\"Ruaridh\", \"Williamson\", role = \"ctb\"), person(\"Sagiru\", \"Mati\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1413-3974\")), person(\"Scott\", \"Kostyshak\", role = \"ctb\"), person(\"Sebastian\", \"Meyer\", role = \"ctb\"), person(\"Sietse\", \"Brouwer\", role = \"ctb\"), person(c(\"Simon\", \"de\"), \"Bernard\", role = \"ctb\"), person(\"Sylvain\", \"Rousseau\", role = \"ctb\"), person(\"Taiyun\", \"Wei\", role = \"ctb\"), person(\"Thibaut\", \"Assus\", role = \"ctb\"), person(\"Thibaut\", \"Lamadon\", role = \"ctb\"), person(\"Thomas\", \"Leeper\", role = \"ctb\"), person(\"Tim\", \"Mastny\", role = \"ctb\"), person(\"Tom\", \"Torsney-Weir\", role = \"ctb\"), person(\"Trevor\", \"Davis\", role = \"ctb\"), person(\"Viktoras\", \"Veitas\", role = \"ctb\"), person(\"Weicheng\", \"Zhu\", role = \"ctb\"), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Zachary\", \"Foster\", role = \"ctb\"), person(\"Zhian N.\", \"Kamvar\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1458-7108\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Provides a general-purpose tool for dynamic report generation in R using Literate Programming techniques.", + "Depends": [ + "R (>= 3.6.0)" + ], + "Imports": [ + "evaluate (>= 0.15)", + "highr (>= 0.11)", + "methods", + "tools", + "xfun (>= 0.48)", + "yaml (>= 2.1.19)" + ], + "Suggests": [ + "bslib", + "codetools", + "DBI (>= 0.4-1)", + "digest", + "formatR", + "gifski", + "gridSVG", + "htmlwidgets (>= 0.7)", + "jpeg", + "JuliaCall (>= 0.11.1)", + "magick", + "litedown", + "markdown (>= 1.3)", + "png", + "ragg", + "reticulate (>= 1.4)", + "rgl (>= 0.95.1201)", + "rlang", + "rmarkdown", + "sass", + "showtext", + "styler (>= 1.2.0)", + "targets (>= 0.6.0)", + "testit", + "tibble", + "tikzDevice (>= 0.10)", + "tinytex (>= 0.46)", + "webshot", + "rstudioapi", + "svglite" + ], + "License": "GPL", + "URL": "https://yihui.org/knitr/", + "BugReports": "https://github.com/yihui/knitr/issues", + "Encoding": "UTF-8", + "VignetteBuilder": "litedown, knitr", + "SystemRequirements": "Package vignettes based on R Markdown v2 or reStructuredText require Pandoc (http://pandoc.org). The function rst2pdf() requires rst2pdf (https://github.com/rst2pdf/rst2pdf).", + "Collate": "'block.R' 'cache.R' 'utils.R' 'citation.R' 'hooks-html.R' 'plot.R' 'defaults.R' 'concordance.R' 'engine.R' 'highlight.R' 'themes.R' 'header.R' 'hooks-asciidoc.R' 'hooks-chunk.R' 'hooks-extra.R' 'hooks-latex.R' 'hooks-md.R' 'hooks-rst.R' 'hooks-textile.R' 'hooks.R' 'output.R' 'package.R' 'pandoc.R' 'params.R' 'parser.R' 'pattern.R' 'rocco.R' 'spin.R' 'table.R' 'template.R' 'utils-conversion.R' 'utils-rd2html.R' 'utils-string.R' 'utils-sweave.R' 'utils-upload.R' 'utils-vignettes.R' 'zzz.R'", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre] (), Abhraneel Sarma [ctb], Adam Vogt [ctb], Alastair Andrew [ctb], Alex Zvoleff [ctb], Amar Al-Zubaidi [ctb], Andre Simon [ctb] (the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de), Aron Atkins [ctb], Aaron Wolen [ctb], Ashley Manton [ctb], Atsushi Yasumoto [ctb] (), Ben Baumer [ctb], Brian Diggs [ctb], Brian Zhang [ctb], Bulat Yapparov [ctb], Cassio Pereira [ctb], Christophe Dervieux [ctb], David Hall [ctb], David Hugh-Jones [ctb], David Robinson [ctb], Doug Hemken [ctb], Duncan Murdoch [ctb], Elio Campitelli [ctb], Ellis Hughes [ctb], Emily Riederer [ctb], Fabian Hirschmann [ctb], Fitch Simeon [ctb], Forest Fang [ctb], Frank E Harrell Jr [ctb] (the Sweavel package at inst/misc/Sweavel.sty), Garrick Aden-Buie [ctb], Gregoire Detrez [ctb], Hadley Wickham [ctb], Hao Zhu [ctb], Heewon Jeon [ctb], Henrik Bengtsson [ctb], Hiroaki Yutani [ctb], Ian Lyttle [ctb], Hodges Daniel [ctb], Jacob Bien [ctb], Jake Burkhead [ctb], James Manton [ctb], Jared Lander [ctb], Jason Punyon [ctb], Javier Luraschi [ctb], Jeff Arnold [ctb], Jenny Bryan [ctb], Jeremy Ashkenas [ctb, cph] (the CSS file at inst/misc/docco-classic.css), Jeremy Stephens [ctb], Jim Hester [ctb], Joe Cheng [ctb], Johannes Ranke [ctb], John Honaker [ctb], John Muschelli [ctb], Jonathan Keane [ctb], JJ Allaire [ctb], Johan Toloe [ctb], Jonathan Sidi [ctb], Joseph Larmarange [ctb], Julien Barnier [ctb], Kaiyin Zhong [ctb], Kamil Slowikowski [ctb], Karl Forner [ctb], Kevin K. Smith [ctb], Kirill Mueller [ctb], Kohske Takahashi [ctb], Lorenz Walthert [ctb], Lucas Gallindo [ctb], Marius Hofert [ctb], Martin Modrák [ctb], Michael Chirico [ctb], Michael Friendly [ctb], Michal Bojanowski [ctb], Michel Kuhlmann [ctb], Miller Patrick [ctb], Nacho Caballero [ctb], Nick Salkowski [ctb], Niels Richard Hansen [ctb], Noam Ross [ctb], Obada Mahdi [ctb], Pavel N. Krivitsky [ctb] (), Pedro Faria [ctb], Qiang Li [ctb], Ramnath Vaidyanathan [ctb], Richard Cotton [ctb], Robert Krzyzanowski [ctb], Rodrigo Copetti [ctb], Romain Francois [ctb], Ruaridh Williamson [ctb], Sagiru Mati [ctb] (), Scott Kostyshak [ctb], Sebastian Meyer [ctb], Sietse Brouwer [ctb], Simon de Bernard [ctb], Sylvain Rousseau [ctb], Taiyun Wei [ctb], Thibaut Assus [ctb], Thibaut Lamadon [ctb], Thomas Leeper [ctb], Tim Mastny [ctb], Tom Torsney-Weir [ctb], Trevor Davis [ctb], Viktoras Veitas [ctb], Weicheng Zhu [ctb], Wush Wu [ctb], Zachary Foster [ctb], Zhian N. Kamvar [ctb] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "lifecycle": { + "Package": "lifecycle", + "Version": "1.0.4", + "Source": "Repository", + "Title": "Manage the Life Cycle of your Package Functions", + "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Manage the life cycle of your exported functions with shared conventions, documentation badges, and user-friendly deprecation warnings.", + "License": "MIT + file LICENSE", + "URL": "https://lifecycle.r-lib.org/, https://github.com/r-lib/lifecycle", + "BugReports": "https://github.com/r-lib/lifecycle/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "cli (>= 3.4.0)", + "glue", + "rlang (>= 1.1.0)" + ], + "Suggests": [ + "covr", + "crayon", + "knitr", + "lintr", + "rmarkdown", + "testthat (>= 3.0.1)", + "tibble", + "tidyverse", + "tools", + "vctrs", + "withr" + ], + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate, usethis", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.1", + "NeedsCompilation": "no", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" + }, + "memoise": { + "Package": "memoise", + "Version": "2.0.1", + "Source": "Repository", + "Title": "'Memoisation' of Functions", + "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = \"aut\", email = \"hadley@rstudio.com\"), person(given = \"Jim\", family = \"Hester\", role = \"aut\"), person(given = \"Winston\", family = \"Chang\", role = c(\"aut\", \"cre\"), email = \"winston@rstudio.com\"), person(given = \"Kirill\", family = \"Müller\", role = \"aut\", email = \"krlmlr+r@mailbox.org\"), person(given = \"Daniel\", family = \"Cook\", role = \"aut\", email = \"danielecook@gmail.com\"), person(given = \"Mark\", family = \"Edmondson\", role = \"ctb\", email = \"r@sunholo.com\"))", + "Description": "Cache the results of a function so that when you call it again with the same arguments it returns the previously computed value.", + "License": "MIT + file LICENSE", + "URL": "https://memoise.r-lib.org, https://github.com/r-lib/memoise", + "BugReports": "https://github.com/r-lib/memoise/issues", + "Imports": [ + "rlang (>= 0.4.10)", + "cachem" + ], + "Suggests": [ + "digest", + "aws.s3", + "covr", + "googleAuthR", + "googleCloudStorageR", + "httr", + "testthat" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.1.2", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut], Jim Hester [aut], Winston Chang [aut, cre], Kirill Müller [aut], Daniel Cook [aut], Mark Edmondson [ctb]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "mime": { + "Package": "mime", + "Version": "0.12", + "Source": "Repository", + "Type": "Package", + "Title": "Map Filenames to MIME Types", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Jeffrey\", \"Horner\", role = \"ctb\"), person(\"Beilei\", \"Bian\", role = \"ctb\") )", + "Description": "Guesses the MIME type from a filename extension using the data derived from /etc/mime.types in UNIX-type systems.", + "Imports": [ + "tools" + ], + "License": "GPL", + "URL": "https://github.com/yihui/mime", + "BugReports": "https://github.com/yihui/mime/issues", + "RoxygenNote": "7.1.1", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Yihui Xie [aut, cre] (), Jeffrey Horner [ctb], Beilei Bian [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "rappdirs": { + "Package": "rappdirs", + "Version": "0.3.3", + "Source": "Repository", + "Type": "Package", + "Title": "Application Directories: Determine Where to Save Data, Caches, and Logs", + "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = c(\"trl\", \"cre\", \"cph\"), email = \"hadley@rstudio.com\"), person(given = \"RStudio\", role = \"cph\"), person(given = \"Sridhar\", family = \"Ratnakumar\", role = \"aut\"), person(given = \"Trent\", family = \"Mick\", role = \"aut\"), person(given = \"ActiveState\", role = \"cph\", comment = \"R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs\"), person(given = \"Eddy\", family = \"Petrisor\", role = \"ctb\"), person(given = \"Trevor\", family = \"Davis\", role = c(\"trl\", \"aut\")), person(given = \"Gabor\", family = \"Csardi\", role = \"ctb\"), person(given = \"Gregory\", family = \"Jefferis\", role = \"ctb\"))", + "Description": "An easy way to determine which directories on the users computer you should use to save data, caches and logs. A port of Python's 'Appdirs' () to R.", + "License": "MIT + file LICENSE", + "URL": "https://rappdirs.r-lib.org, https://github.com/r-lib/rappdirs", + "BugReports": "https://github.com/r-lib/rappdirs/issues", + "Depends": [ + "R (>= 3.2)" + ], + "Suggests": [ + "roxygen2", + "testthat (>= 3.0.0)", + "covr", + "withr" + ], + "Copyright": "Original python appdirs module copyright (c) 2010 ActiveState Software Inc. R port copyright Hadley Wickham, RStudio. See file LICENSE for details.", + "Encoding": "UTF-8", + "RoxygenNote": "7.1.1", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Hadley Wickham [trl, cre, cph], RStudio [cph], Sridhar Ratnakumar [aut], Trent Mick [aut], ActiveState [cph] (R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs), Eddy Petrisor [ctb], Trevor Davis [trl, aut], Gabor Csardi [ctb], Gregory Jefferis [ctb]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" + }, + "renv": { + "Package": "renv", + "Version": "1.1.2", + "Source": "Repository", + "Type": "Package", + "Title": "Project Environments", + "Authors@R": "c( person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Hadley\", \"Wickham\", role = c(\"aut\"), email = \"hadley@rstudio.com\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A dependency management toolkit for R. Using 'renv', you can create and manage project-local R libraries, save the state of these libraries to a 'lockfile', and later restore your library as required. Together, these tools can help make your projects more isolated, portable, and reproducible.", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/renv/, https://github.com/rstudio/renv", + "BugReports": "https://github.com/rstudio/renv/issues", + "Imports": [ + "utils" + ], + "Suggests": [ + "BiocManager", + "cli", + "compiler", + "covr", + "cpp11", + "devtools", + "gitcreds", + "jsonlite", + "jsonvalidate", + "knitr", + "miniUI", + "modules", + "packrat", + "pak", + "R6", + "remotes", + "reticulate", + "rmarkdown", + "rstudioapi", + "shiny", + "testthat", + "uuid", + "waldo", + "yaml", + "webfakes" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "bioconductor,python,install,restore,snapshot,retrieve,remotes", + "NeedsCompilation": "no", + "Author": "Kevin Ushey [aut, cre] (), Hadley Wickham [aut] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Kevin Ushey ", + "Repository": "CRAN" + }, + "rlang": { + "Package": "rlang", + "Version": "1.1.5", + "Source": "Repository", + "Title": "Functions for Base Types and Core R and 'Tidyverse' Features", + "Description": "A toolbox for working with base types, core R features like the condition system, and core 'Tidyverse' features like tidy evaluation.", + "Authors@R": "c( person(\"Lionel\", \"Henry\", ,\"lionel@posit.co\", c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", ,\"hadley@posit.co\", \"aut\"), person(given = \"mikefc\", email = \"mikefc@coolbutuseless.com\", role = \"cph\", comment = \"Hash implementation based on Mike's xxhashlite\"), person(given = \"Yann\", family = \"Collet\", role = \"cph\", comment = \"Author of the embedded xxHash library\"), person(given = \"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", + "License": "MIT + file LICENSE", + "ByteCompile": "true", + "Biarch": "true", + "Depends": [ + "R (>= 3.5.0)" + ], + "Imports": [ + "utils" + ], + "Suggests": [ + "cli (>= 3.1.0)", + "covr", + "crayon", + "fs", + "glue", + "knitr", + "magrittr", + "methods", + "pillar", + "rmarkdown", + "stats", + "testthat (>= 3.0.0)", + "tibble", + "usethis", + "vctrs (>= 0.2.3)", + "withr" + ], + "Enhances": [ + "winch" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "URL": "https://rlang.r-lib.org, https://github.com/r-lib/rlang", + "BugReports": "https://github.com/r-lib/rlang/issues", + "Config/testthat/edition": "3", + "Config/Needs/website": "dplyr, tidyverse/tidytemplate", + "NeedsCompilation": "yes", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], mikefc [cph] (Hash implementation based on Mike's xxhashlite), Yann Collet [cph] (Author of the embedded xxHash library), Posit, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" + }, + "rmarkdown": { + "Package": "rmarkdown", + "Version": "2.29", + "Source": "Repository", + "Type": "Package", + "Title": "Dynamic Documents for R", + "Authors@R": "c( person(\"JJ\", \"Allaire\", , \"jj@posit.co\", role = \"aut\"), person(\"Yihui\", \"Xie\", , \"xie@yihui.name\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Jonathan\", \"McPherson\", , \"jonathan@posit.co\", role = \"aut\"), person(\"Javier\", \"Luraschi\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevin@posit.co\", role = \"aut\"), person(\"Aron\", \"Atkins\", , \"aron@posit.co\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Andrew\", \"Dunning\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0464-5036\")), person(\"Atsushi\", \"Yasumoto\", role = c(\"ctb\", \"cph\"), comment = c(ORCID = \"0000-0002-8335-495X\", cph = \"Number sections Lua filter\")), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Carson\", \"Sievert\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Devon\", \"Ryan\", , \"dpryan79@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Frederik\", \"Aust\", , \"frederik.aust@uni-koeln.de\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4900-788X\")), person(\"Jeff\", \"Allen\", , \"jeff@posit.co\", role = \"ctb\"), person(\"JooYoung\", \"Seo\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4064-6012\")), person(\"Malcolm\", \"Barrett\", role = \"ctb\"), person(\"Rob\", \"Hyndman\", , \"Rob.Hyndman@monash.edu\", role = \"ctb\"), person(\"Romain\", \"Lesur\", role = \"ctb\"), person(\"Roy\", \"Storey\", role = \"ctb\"), person(\"Ruben\", \"Arslan\", , \"ruben.arslan@uni-goettingen.de\", role = \"ctb\"), person(\"Sergio\", \"Oller\", role = \"ctb\"), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"jQuery UI contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt\"), person(\"Mark\", \"Otto\", role = \"ctb\", comment = \"Bootstrap library\"), person(\"Jacob\", \"Thornton\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Alexander\", \"Farkas\", role = c(\"ctb\", \"cph\"), comment = \"html5shiv library\"), person(\"Scott\", \"Jehl\", role = c(\"ctb\", \"cph\"), comment = \"Respond.js library\"), person(\"Ivan\", \"Sagalaev\", role = c(\"ctb\", \"cph\"), comment = \"highlight.js library\"), person(\"Greg\", \"Franko\", role = c(\"ctb\", \"cph\"), comment = \"tocify library\"), person(\"John\", \"MacFarlane\", role = c(\"ctb\", \"cph\"), comment = \"Pandoc templates\"), person(, \"Google, Inc.\", role = c(\"ctb\", \"cph\"), comment = \"ioslides library\"), person(\"Dave\", \"Raggett\", role = \"ctb\", comment = \"slidy library\"), person(, \"W3C\", role = \"cph\", comment = \"slidy library\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome\"), person(\"Ben\", \"Sperry\", role = \"ctb\", comment = \"Ionicons\"), person(, \"Drifty\", role = \"cph\", comment = \"Ionicons\"), person(\"Aidan\", \"Lister\", role = c(\"ctb\", \"cph\"), comment = \"jQuery StickyTabs\"), person(\"Benct Philip\", \"Jonsson\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\"), person(\"Albert\", \"Krewinkel\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\") )", + "Description": "Convert R Markdown documents into a variety of formats.", + "License": "GPL-3", + "URL": "https://github.com/rstudio/rmarkdown, https://pkgs.rstudio.com/rmarkdown/", + "BugReports": "https://github.com/rstudio/rmarkdown/issues", + "Depends": [ + "R (>= 3.0)" + ], + "Imports": [ + "bslib (>= 0.2.5.1)", + "evaluate (>= 0.13)", + "fontawesome (>= 0.5.0)", + "htmltools (>= 0.5.1)", + "jquerylib", + "jsonlite", + "knitr (>= 1.43)", + "methods", + "tinytex (>= 0.31)", + "tools", + "utils", + "xfun (>= 0.36)", + "yaml (>= 2.1.19)" + ], + "Suggests": [ + "digest", + "dygraphs", + "fs", + "rsconnect", + "downlit (>= 0.4.0)", + "katex (>= 1.4.0)", + "sass (>= 0.4.0)", + "shiny (>= 1.6.0)", + "testthat (>= 3.0.3)", + "tibble", + "vctrs", + "cleanrmd", + "withr (>= 2.4.2)", + "xml2" + ], + "VignetteBuilder": "knitr", + "Config/Needs/website": "rstudio/quillt, pkgdown", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "SystemRequirements": "pandoc (>= 1.14) - http://pandoc.org", + "NeedsCompilation": "no", + "Author": "JJ Allaire [aut], Yihui Xie [aut, cre] (), Christophe Dervieux [aut] (), Jonathan McPherson [aut], Javier Luraschi [aut], Kevin Ushey [aut], Aron Atkins [aut], Hadley Wickham [aut], Joe Cheng [aut], Winston Chang [aut], Richard Iannone [aut] (), Andrew Dunning [ctb] (), Atsushi Yasumoto [ctb, cph] (, Number sections Lua filter), Barret Schloerke [ctb], Carson Sievert [ctb] (), Devon Ryan [ctb] (), Frederik Aust [ctb] (), Jeff Allen [ctb], JooYoung Seo [ctb] (), Malcolm Barrett [ctb], Rob Hyndman [ctb], Romain Lesur [ctb], Roy Storey [ctb], Ruben Arslan [ctb], Sergio Oller [ctb], Posit Software, PBC [cph, fnd], jQuery UI contributors [ctb, cph] (jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt), Mark Otto [ctb] (Bootstrap library), Jacob Thornton [ctb] (Bootstrap library), Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Alexander Farkas [ctb, cph] (html5shiv library), Scott Jehl [ctb, cph] (Respond.js library), Ivan Sagalaev [ctb, cph] (highlight.js library), Greg Franko [ctb, cph] (tocify library), John MacFarlane [ctb, cph] (Pandoc templates), Google, Inc. [ctb, cph] (ioslides library), Dave Raggett [ctb] (slidy library), W3C [cph] (slidy library), Dave Gandy [ctb, cph] (Font-Awesome), Ben Sperry [ctb] (Ionicons), Drifty [cph] (Ionicons), Aidan Lister [ctb, cph] (jQuery StickyTabs), Benct Philip Jonsson [ctb, cph] (pagebreak Lua filter), Albert Krewinkel [ctb, cph] (pagebreak Lua filter)", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "sass": { + "Package": "sass", + "Version": "0.4.9", + "Source": "Repository", + "Type": "Package", + "Title": "Syntactically Awesome Style Sheets ('Sass')", + "Description": "An 'SCSS' compiler, powered by the 'LibSass' library. With this, R developers can use variables, inheritance, and functions to generate dynamic style sheets. The package uses the 'Sass CSS' extension language, which is stable, powerful, and CSS compatible.", + "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@rstudio.com\", \"aut\"), person(\"Timothy\", \"Mastny\", , \"tim.mastny@gmail.com\", \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Barret\", \"Schloerke\", , \"barret@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Carson\", \"Sievert\", , \"carson@rstudio.com\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Christophe\", \"Dervieux\", , \"cderv@rstudio.com\", c(\"ctb\"), comment = c(ORCID = \"0000-0003-4474-2498\")), person(family = \"RStudio\", role = c(\"cph\", \"fnd\")), person(family = \"Sass Open Source Foundation\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Greter\", \"Marcel\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Mifsud\", \"Michael\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Hampton\", \"Catlin\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Natalie\", \"Weizenbaum\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Chris\", \"Eppstein\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Adams\", \"Joseph\", role = c(\"ctb\", \"cph\"), comment = \"json.cpp\"), person(\"Trifunovic\", \"Nemanja\", role = c(\"ctb\", \"cph\"), comment = \"utf8.h\") )", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/sass/, https://github.com/rstudio/sass", + "BugReports": "https://github.com/rstudio/sass/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "SystemRequirements": "GNU make", + "Imports": [ + "fs (>= 1.2.4)", + "rlang (>= 0.4.10)", + "htmltools (>= 0.5.1)", + "R6", + "rappdirs" + ], + "Suggests": [ + "testthat", + "knitr", + "rmarkdown", + "withr", + "shiny", + "curl" + ], + "VignetteBuilder": "knitr", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Joe Cheng [aut], Timothy Mastny [aut], Richard Iannone [aut] (), Barret Schloerke [aut] (), Carson Sievert [aut, cre] (), Christophe Dervieux [ctb] (), RStudio [cph, fnd], Sass Open Source Foundation [ctb, cph] (LibSass library), Greter Marcel [ctb, cph] (LibSass library), Mifsud Michael [ctb, cph] (LibSass library), Hampton Catlin [ctb, cph] (LibSass library), Natalie Weizenbaum [ctb, cph] (LibSass library), Chris Eppstein [ctb, cph] (LibSass library), Adams Joseph [ctb, cph] (json.cpp), Trifunovic Nemanja [ctb, cph] (utf8.h)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "tinytex": { + "Package": "tinytex", + "Version": "0.56", + "Source": "Repository", + "Type": "Package", + "Title": "Helper Functions to Install and Maintain TeX Live, and Compile LaTeX Documents", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Devon\", \"Ryan\", role = \"ctb\", email = \"dpryan79@gmail.com\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Ethan\", \"Heinzen\", role = \"ctb\"), person(\"Fernando\", \"Cagua\", role = \"ctb\"), person() )", + "Description": "Helper functions to install and maintain the 'LaTeX' distribution named 'TinyTeX' (), a lightweight, cross-platform, portable, and easy-to-maintain version of 'TeX Live'. This package also contains helper functions to compile 'LaTeX' documents, and install missing 'LaTeX' packages automatically.", + "Imports": [ + "xfun (>= 0.48)" + ], + "Suggests": [ + "testit", + "rstudioapi" + ], + "License": "MIT + file LICENSE", + "URL": "https://github.com/rstudio/tinytex", + "BugReports": "https://github.com/rstudio/tinytex/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre, cph] (), Posit Software, PBC [cph, fnd], Christophe Dervieux [ctb] (), Devon Ryan [ctb] (), Ethan Heinzen [ctb], Fernando Cagua [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "xfun": { + "Package": "xfun", + "Version": "0.51", + "Source": "Repository", + "Type": "Package", + "Title": "Supporting Functions for Packages Maintained by 'Yihui Xie'", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\", URL = \"https://yihui.org\")), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Daijiang\", \"Li\", role = \"ctb\"), person(\"Xianying\", \"Tan\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", role = \"ctb\", email = \"salim-b@pm.me\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person() )", + "Description": "Miscellaneous functions commonly used in other packages maintained by 'Yihui Xie'.", + "Depends": [ + "R (>= 3.2.0)" + ], + "Imports": [ + "grDevices", + "stats", + "tools" + ], + "Suggests": [ + "testit", + "parallel", + "codetools", + "methods", + "rstudioapi", + "tinytex (>= 0.30)", + "mime", + "litedown (>= 0.4)", + "commonmark", + "knitr (>= 1.47)", + "remotes", + "pak", + "rhub", + "renv", + "curl", + "xml2", + "jsonlite", + "magick", + "yaml", + "qs", + "rmarkdown" + ], + "License": "MIT + file LICENSE", + "URL": "https://github.com/yihui/xfun", + "BugReports": "https://github.com/yihui/xfun/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "VignetteBuilder": "litedown", + "NeedsCompilation": "yes", + "Author": "Yihui Xie [aut, cre, cph] (, https://yihui.org), Wush Wu [ctb], Daijiang Li [ctb], Xianying Tan [ctb], Salim Brüggemann [ctb] (), Christophe Dervieux [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "yaml": { + "Package": "yaml", + "Version": "2.3.10", + "Source": "Repository", + "Type": "Package", + "Title": "Methods to Convert R Data to YAML and Back", + "Date": "2024-07-22", + "Suggests": [ + "RUnit" + ], + "Author": "Shawn P Garbett [aut], Jeremy Stephens [aut, cre], Kirill Simonov [aut], Yihui Xie [ctb], Zhuoer Dong [ctb], Hadley Wickham [ctb], Jeffrey Horner [ctb], reikoch [ctb], Will Beasley [ctb], Brendan O'Connor [ctb], Gregory R. Warnes [ctb], Michael Quinn [ctb], Zhian N. Kamvar [ctb], Charlie Gao [ctb]", + "Maintainer": "Shawn Garbett ", + "License": "BSD_3_clause + file LICENSE", + "Description": "Implements the 'libyaml' 'YAML' 1.1 parser and emitter () for R.", + "URL": "https://github.com/vubiostat/r-yaml/", + "BugReports": "https://github.com/vubiostat/r-yaml/issues", + "NeedsCompilation": "yes", + "Repository": "CRAN" + } + } +} diff --git a/renv/.gitignore b/renv/.gitignore new file mode 100644 index 00000000..0ec0cbba --- /dev/null +++ b/renv/.gitignore @@ -0,0 +1,7 @@ +library/ +local/ +cellar/ +lock/ +python/ +sandbox/ +staging/ diff --git a/renv/activate.R b/renv/activate.R new file mode 100644 index 00000000..0862e8b1 --- /dev/null +++ b/renv/activate.R @@ -0,0 +1,1313 @@ + +local({ + + # the requested version of renv + version <- "1.0.8" + attr(version, "sha") <- NULL + + # the project directory + project <- Sys.getenv("RENV_PROJECT") + if (!nzchar(project)) + project <- getwd() + + # use start-up diagnostics if enabled + diagnostics <- Sys.getenv("RENV_STARTUP_DIAGNOSTICS", unset = "FALSE") + if (diagnostics) { + start <- Sys.time() + profile <- tempfile("renv-startup-", fileext = ".Rprof") + utils::Rprof(profile) + on.exit({ + utils::Rprof(NULL) + elapsed <- signif(difftime(Sys.time(), start, units = "auto"), digits = 2L) + writeLines(sprintf("- renv took %s to run the autoloader.", format(elapsed))) + writeLines(sprintf("- Profile: %s", profile)) + print(utils::summaryRprof(profile)) + }, add = TRUE) + } + + # figure out whether the autoloader is enabled + enabled <- local({ + + # first, check config option + override <- getOption("renv.config.autoloader.enabled") + if (!is.null(override)) + return(override) + + # if we're being run in a context where R_LIBS is already set, + # don't load -- presumably we're being run as a sub-process and + # the parent process has already set up library paths for us + rcmd <- Sys.getenv("R_CMD", unset = NA) + rlibs <- Sys.getenv("R_LIBS", unset = NA) + if (!is.na(rlibs) && !is.na(rcmd)) + return(FALSE) + + # next, check environment variables + # prefer using the configuration one in the future + envvars <- c( + "RENV_CONFIG_AUTOLOADER_ENABLED", + "RENV_AUTOLOADER_ENABLED", + "RENV_ACTIVATE_PROJECT" + ) + + for (envvar in envvars) { + envval <- Sys.getenv(envvar, unset = NA) + if (!is.na(envval)) + return(tolower(envval) %in% c("true", "t", "1")) + } + + # enable by default + TRUE + + }) + + # bail if we're not enabled + if (!enabled) { + + # if we're not enabled, we might still need to manually load + # the user profile here + profile <- Sys.getenv("R_PROFILE_USER", unset = "~/.Rprofile") + if (file.exists(profile)) { + cfg <- Sys.getenv("RENV_CONFIG_USER_PROFILE", unset = "TRUE") + if (tolower(cfg) %in% c("true", "t", "1")) + sys.source(profile, envir = globalenv()) + } + + return(FALSE) + + } + + # avoid recursion + if (identical(getOption("renv.autoloader.running"), TRUE)) { + warning("ignoring recursive attempt to run renv autoloader") + return(invisible(TRUE)) + } + + # signal that we're loading renv during R startup + options(renv.autoloader.running = TRUE) + on.exit(options(renv.autoloader.running = NULL), add = TRUE) + + # signal that we've consented to use renv + options(renv.consent = TRUE) + + # load the 'utils' package eagerly -- this ensures that renv shims, which + # mask 'utils' packages, will come first on the search path + library(utils, lib.loc = .Library) + + # unload renv if it's already been loaded + if ("renv" %in% loadedNamespaces()) + unloadNamespace("renv") + + # load bootstrap tools + ansify <- function(text) { + if (renv_ansify_enabled()) + renv_ansify_enhanced(text) + else + renv_ansify_default(text) + } + + renv_ansify_enabled <- function() { + + override <- Sys.getenv("RENV_ANSIFY_ENABLED", unset = NA) + if (!is.na(override)) + return(as.logical(override)) + + pane <- Sys.getenv("RSTUDIO_CHILD_PROCESS_PANE", unset = NA) + if (identical(pane, "build")) + return(FALSE) + + testthat <- Sys.getenv("TESTTHAT", unset = "false") + if (tolower(testthat) %in% "true") + return(FALSE) + + iderun <- Sys.getenv("R_CLI_HAS_HYPERLINK_IDE_RUN", unset = "false") + if (tolower(iderun) %in% "false") + return(FALSE) + + TRUE + + } + + renv_ansify_default <- function(text) { + text + } + + renv_ansify_enhanced <- function(text) { + + # R help links + pattern <- "`\\?(renv::(?:[^`])+)`" + replacement <- "`\033]8;;x-r-help:\\1\a?\\1\033]8;;\a`" + text <- gsub(pattern, replacement, text, perl = TRUE) + + # runnable code + pattern <- "`(renv::(?:[^`])+)`" + replacement <- "`\033]8;;x-r-run:\\1\a\\1\033]8;;\a`" + text <- gsub(pattern, replacement, text, perl = TRUE) + + # return ansified text + text + + } + + renv_ansify_init <- function() { + + envir <- renv_envir_self() + if (renv_ansify_enabled()) + assign("ansify", renv_ansify_enhanced, envir = envir) + else + assign("ansify", renv_ansify_default, envir = envir) + + } + + `%||%` <- function(x, y) { + if (is.null(x)) y else x + } + + catf <- function(fmt, ..., appendLF = TRUE) { + + quiet <- getOption("renv.bootstrap.quiet", default = FALSE) + if (quiet) + return(invisible()) + + msg <- sprintf(fmt, ...) + cat(msg, file = stdout(), sep = if (appendLF) "\n" else "") + + invisible(msg) + + } + + header <- function(label, + ..., + prefix = "#", + suffix = "-", + n = min(getOption("width"), 78)) + { + label <- sprintf(label, ...) + n <- max(n - nchar(label) - nchar(prefix) - 2L, 8L) + if (n <= 0) + return(paste(prefix, label)) + + tail <- paste(rep.int(suffix, n), collapse = "") + paste0(prefix, " ", label, " ", tail) + + } + + heredoc <- function(text, leave = 0) { + + # remove leading, trailing whitespace + trimmed <- gsub("^\\s*\\n|\\n\\s*$", "", text) + + # split into lines + lines <- strsplit(trimmed, "\n", fixed = TRUE)[[1L]] + + # compute common indent + indent <- regexpr("[^[:space:]]", lines) + common <- min(setdiff(indent, -1L)) - leave + text <- paste(substring(lines, common), collapse = "\n") + + # substitute in ANSI links for executable renv code + ansify(text) + + } + + bootstrap <- function(version, library) { + + friendly <- renv_bootstrap_version_friendly(version) + section <- header(sprintf("Bootstrapping renv %s", friendly)) + catf(section) + + # attempt to download renv + catf("- Downloading renv ... ", appendLF = FALSE) + withCallingHandlers( + tarball <- renv_bootstrap_download(version), + error = function(err) { + catf("FAILED") + stop("failed to download:\n", conditionMessage(err)) + } + ) + catf("OK") + on.exit(unlink(tarball), add = TRUE) + + # now attempt to install + catf("- Installing renv ... ", appendLF = FALSE) + withCallingHandlers( + status <- renv_bootstrap_install(version, tarball, library), + error = function(err) { + catf("FAILED") + stop("failed to install:\n", conditionMessage(err)) + } + ) + catf("OK") + + # add empty line to break up bootstrapping from normal output + catf("") + + return(invisible()) + } + + renv_bootstrap_tests_running <- function() { + getOption("renv.tests.running", default = FALSE) + } + + renv_bootstrap_repos <- function() { + + # get CRAN repository + cran <- getOption("renv.repos.cran", "https://cloud.r-project.org") + + # check for repos override + repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA) + if (!is.na(repos)) { + + # check for RSPM; if set, use a fallback repository for renv + rspm <- Sys.getenv("RSPM", unset = NA) + if (identical(rspm, repos)) + repos <- c(RSPM = rspm, CRAN = cran) + + return(repos) + + } + + # check for lockfile repositories + repos <- tryCatch(renv_bootstrap_repos_lockfile(), error = identity) + if (!inherits(repos, "error") && length(repos)) + return(repos) + + # retrieve current repos + repos <- getOption("repos") + + # ensure @CRAN@ entries are resolved + repos[repos == "@CRAN@"] <- cran + + # add in renv.bootstrap.repos if set + default <- c(FALLBACK = "https://cloud.r-project.org") + extra <- getOption("renv.bootstrap.repos", default = default) + repos <- c(repos, extra) + + # remove duplicates that might've snuck in + dupes <- duplicated(repos) | duplicated(names(repos)) + repos[!dupes] + + } + + renv_bootstrap_repos_lockfile <- function() { + + lockpath <- Sys.getenv("RENV_PATHS_LOCKFILE", unset = "renv.lock") + if (!file.exists(lockpath)) + return(NULL) + + lockfile <- tryCatch(renv_json_read(lockpath), error = identity) + if (inherits(lockfile, "error")) { + warning(lockfile) + return(NULL) + } + + repos <- lockfile$R$Repositories + if (length(repos) == 0) + return(NULL) + + keys <- vapply(repos, `[[`, "Name", FUN.VALUE = character(1)) + vals <- vapply(repos, `[[`, "URL", FUN.VALUE = character(1)) + names(vals) <- keys + + return(vals) + + } + + renv_bootstrap_download <- function(version) { + + sha <- attr(version, "sha", exact = TRUE) + + methods <- if (!is.null(sha)) { + + # attempting to bootstrap a development version of renv + c( + function() renv_bootstrap_download_tarball(sha), + function() renv_bootstrap_download_github(sha) + ) + + } else { + + # attempting to bootstrap a release version of renv + c( + function() renv_bootstrap_download_tarball(version), + function() renv_bootstrap_download_cran_latest(version), + function() renv_bootstrap_download_cran_archive(version) + ) + + } + + for (method in methods) { + path <- tryCatch(method(), error = identity) + if (is.character(path) && file.exists(path)) + return(path) + } + + stop("All download methods failed") + + } + + renv_bootstrap_download_impl <- function(url, destfile) { + + mode <- "wb" + + # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715 + fixup <- + Sys.info()[["sysname"]] == "Windows" && + substring(url, 1L, 5L) == "file:" + + if (fixup) + mode <- "w+b" + + args <- list( + url = url, + destfile = destfile, + mode = mode, + quiet = TRUE + ) + + if ("headers" %in% names(formals(utils::download.file))) { + headers <- renv_bootstrap_download_custom_headers(url) + if (length(headers) && is.character(headers)) + args$headers <- headers + } + + do.call(utils::download.file, args) + + } + + renv_bootstrap_download_custom_headers <- function(url) { + + headers <- getOption("renv.download.headers") + if (is.null(headers)) + return(character()) + + if (!is.function(headers)) + stopf("'renv.download.headers' is not a function") + + headers <- headers(url) + if (length(headers) == 0L) + return(character()) + + if (is.list(headers)) + headers <- unlist(headers, recursive = FALSE, use.names = TRUE) + + ok <- + is.character(headers) && + is.character(names(headers)) && + all(nzchar(names(headers))) + + if (!ok) + stop("invocation of 'renv.download.headers' did not return a named character vector") + + headers + + } + + renv_bootstrap_download_cran_latest <- function(version) { + + spec <- renv_bootstrap_download_cran_latest_find(version) + type <- spec$type + repos <- spec$repos + + baseurl <- utils::contrib.url(repos = repos, type = type) + ext <- if (identical(type, "source")) + ".tar.gz" + else if (Sys.info()[["sysname"]] == "Windows") + ".zip" + else + ".tgz" + name <- sprintf("renv_%s%s", version, ext) + url <- paste(baseurl, name, sep = "/") + + destfile <- file.path(tempdir(), name) + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (inherits(status, "condition")) + return(FALSE) + + # report success and return + destfile + + } + + renv_bootstrap_download_cran_latest_find <- function(version) { + + # check whether binaries are supported on this system + binary <- + getOption("renv.bootstrap.binary", default = TRUE) && + !identical(.Platform$pkgType, "source") && + !identical(getOption("pkgType"), "source") && + Sys.info()[["sysname"]] %in% c("Darwin", "Windows") + + types <- c(if (binary) "binary", "source") + + # iterate over types + repositories + for (type in types) { + for (repos in renv_bootstrap_repos()) { + + # build arguments for utils::available.packages() call + args <- list(type = type, repos = repos) + + # add custom headers if available -- note that + # utils::available.packages() will pass this to download.file() + if ("headers" %in% names(formals(utils::download.file))) { + headers <- renv_bootstrap_download_custom_headers(repos) + if (length(headers) && is.character(headers)) + args$headers <- headers + } + + # retrieve package database + db <- tryCatch( + as.data.frame( + do.call(utils::available.packages, args), + stringsAsFactors = FALSE + ), + error = identity + ) + + if (inherits(db, "error")) + next + + # check for compatible entry + entry <- db[db$Package %in% "renv" & db$Version %in% version, ] + if (nrow(entry) == 0) + next + + # found it; return spec to caller + spec <- list(entry = entry, type = type, repos = repos) + return(spec) + + } + } + + # if we got here, we failed to find renv + fmt <- "renv %s is not available from your declared package repositories" + stop(sprintf(fmt, version)) + + } + + renv_bootstrap_download_cran_archive <- function(version) { + + name <- sprintf("renv_%s.tar.gz", version) + repos <- renv_bootstrap_repos() + urls <- file.path(repos, "src/contrib/Archive/renv", name) + destfile <- file.path(tempdir(), name) + + for (url in urls) { + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (identical(status, 0L)) + return(destfile) + + } + + return(FALSE) + + } + + renv_bootstrap_download_tarball <- function(version) { + + # if the user has provided the path to a tarball via + # an environment variable, then use it + tarball <- Sys.getenv("RENV_BOOTSTRAP_TARBALL", unset = NA) + if (is.na(tarball)) + return() + + # allow directories + if (dir.exists(tarball)) { + name <- sprintf("renv_%s.tar.gz", version) + tarball <- file.path(tarball, name) + } + + # bail if it doesn't exist + if (!file.exists(tarball)) { + + # let the user know we weren't able to honour their request + fmt <- "- RENV_BOOTSTRAP_TARBALL is set (%s) but does not exist." + msg <- sprintf(fmt, tarball) + warning(msg) + + # bail + return() + + } + + catf("- Using local tarball '%s'.", tarball) + tarball + + } + + renv_bootstrap_github_token <- function() { + for (envvar in c("GITHUB_TOKEN", "GITHUB_PAT", "GH_TOKEN")) { + envval <- Sys.getenv(envvar, unset = NA) + if (!is.na(envval)) + return(envval) + } + } + + renv_bootstrap_download_github <- function(version) { + + enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE") + if (!identical(enabled, "TRUE")) + return(FALSE) + + # prepare download options + token <- renv_bootstrap_github_token() + if (is.null(token)) + token <- "" + + if (nzchar(Sys.which("curl")) && nzchar(token)) { + fmt <- "--location --fail --header \"Authorization: token %s\"" + extra <- sprintf(fmt, token) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "curl", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } else if (nzchar(Sys.which("wget")) && nzchar(token)) { + fmt <- "--header=\"Authorization: token %s\"" + extra <- sprintf(fmt, token) + saved <- options("download.file.method", "download.file.extra") + options(download.file.method = "wget", download.file.extra = extra) + on.exit(do.call(base::options, saved), add = TRUE) + } + + url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version) + name <- sprintf("renv_%s.tar.gz", version) + destfile <- file.path(tempdir(), name) + + status <- tryCatch( + renv_bootstrap_download_impl(url, destfile), + condition = identity + ) + + if (!identical(status, 0L)) + return(FALSE) + + renv_bootstrap_download_augment(destfile) + + return(destfile) + + } + + # Add Sha to DESCRIPTION. This is stop gap until #890, after which we + # can use renv::install() to fully capture metadata. + renv_bootstrap_download_augment <- function(destfile) { + sha <- renv_bootstrap_git_extract_sha1_tar(destfile) + if (is.null(sha)) { + return() + } + + # Untar + tempdir <- tempfile("renv-github-") + on.exit(unlink(tempdir, recursive = TRUE), add = TRUE) + untar(destfile, exdir = tempdir) + pkgdir <- dir(tempdir, full.names = TRUE)[[1]] + + # Modify description + desc_path <- file.path(pkgdir, "DESCRIPTION") + desc_lines <- readLines(desc_path) + remotes_fields <- c( + "RemoteType: github", + "RemoteHost: api.github.com", + "RemoteRepo: renv", + "RemoteUsername: rstudio", + "RemotePkgRef: rstudio/renv", + paste("RemoteRef: ", sha), + paste("RemoteSha: ", sha) + ) + writeLines(c(desc_lines[desc_lines != ""], remotes_fields), con = desc_path) + + # Re-tar + local({ + old <- setwd(tempdir) + on.exit(setwd(old), add = TRUE) + + tar(destfile, compression = "gzip") + }) + invisible() + } + + # Extract the commit hash from a git archive. Git archives include the SHA1 + # hash as the comment field of the tarball pax extended header + # (see https://www.kernel.org/pub/software/scm/git/docs/git-archive.html) + # For GitHub archives this should be the first header after the default one + # (512 byte) header. + renv_bootstrap_git_extract_sha1_tar <- function(bundle) { + + # open the bundle for reading + # We use gzcon for everything because (from ?gzcon) + # > Reading from a connection which does not supply a 'gzip' magic + # > header is equivalent to reading from the original connection + conn <- gzcon(file(bundle, open = "rb", raw = TRUE)) + on.exit(close(conn)) + + # The default pax header is 512 bytes long and the first pax extended header + # with the comment should be 51 bytes long + # `52 comment=` (11 chars) + 40 byte SHA1 hash + len <- 0x200 + 0x33 + res <- rawToChar(readBin(conn, "raw", n = len)[0x201:len]) + + if (grepl("^52 comment=", res)) { + sub("52 comment=", "", res) + } else { + NULL + } + } + + renv_bootstrap_install <- function(version, tarball, library) { + + # attempt to install it into project library + dir.create(library, showWarnings = FALSE, recursive = TRUE) + output <- renv_bootstrap_install_impl(library, tarball) + + # check for successful install + status <- attr(output, "status") + if (is.null(status) || identical(status, 0L)) + return(status) + + # an error occurred; report it + header <- "installation of renv failed" + lines <- paste(rep.int("=", nchar(header)), collapse = "") + text <- paste(c(header, lines, output), collapse = "\n") + stop(text) + + } + + renv_bootstrap_install_impl <- function(library, tarball) { + + # invoke using system2 so we can capture and report output + bin <- R.home("bin") + exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R" + R <- file.path(bin, exe) + + args <- c( + "--vanilla", "CMD", "INSTALL", "--no-multiarch", + "-l", shQuote(path.expand(library)), + shQuote(path.expand(tarball)) + ) + + system2(R, args, stdout = TRUE, stderr = TRUE) + + } + + renv_bootstrap_platform_prefix <- function() { + + # construct version prefix + version <- paste(R.version$major, R.version$minor, sep = ".") + prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-") + + # include SVN revision for development versions of R + # (to avoid sharing platform-specific artefacts with released versions of R) + devel <- + identical(R.version[["status"]], "Under development (unstable)") || + identical(R.version[["nickname"]], "Unsuffered Consequences") + + if (devel) + prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r") + + # build list of path components + components <- c(prefix, R.version$platform) + + # include prefix if provided by user + prefix <- renv_bootstrap_platform_prefix_impl() + if (!is.na(prefix) && nzchar(prefix)) + components <- c(prefix, components) + + # build prefix + paste(components, collapse = "/") + + } + + renv_bootstrap_platform_prefix_impl <- function() { + + # if an explicit prefix has been supplied, use it + prefix <- Sys.getenv("RENV_PATHS_PREFIX", unset = NA) + if (!is.na(prefix)) + return(prefix) + + # if the user has requested an automatic prefix, generate it + auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA) + if (is.na(auto) && getRversion() >= "4.4.0") + auto <- "TRUE" + + if (auto %in% c("TRUE", "True", "true", "1")) + return(renv_bootstrap_platform_prefix_auto()) + + # empty string on failure + "" + + } + + renv_bootstrap_platform_prefix_auto <- function() { + + prefix <- tryCatch(renv_bootstrap_platform_os(), error = identity) + if (inherits(prefix, "error") || prefix %in% "unknown") { + + msg <- paste( + "failed to infer current operating system", + "please file a bug report at https://github.com/rstudio/renv/issues", + sep = "; " + ) + + warning(msg) + + } + + prefix + + } + + renv_bootstrap_platform_os <- function() { + + sysinfo <- Sys.info() + sysname <- sysinfo[["sysname"]] + + # handle Windows + macOS up front + if (sysname == "Windows") + return("windows") + else if (sysname == "Darwin") + return("macos") + + # check for os-release files + for (file in c("/etc/os-release", "/usr/lib/os-release")) + if (file.exists(file)) + return(renv_bootstrap_platform_os_via_os_release(file, sysinfo)) + + # check for redhat-release files + if (file.exists("/etc/redhat-release")) + return(renv_bootstrap_platform_os_via_redhat_release()) + + "unknown" + + } + + renv_bootstrap_platform_os_via_os_release <- function(file, sysinfo) { + + # read /etc/os-release + release <- utils::read.table( + file = file, + sep = "=", + quote = c("\"", "'"), + col.names = c("Key", "Value"), + comment.char = "#", + stringsAsFactors = FALSE + ) + + vars <- as.list(release$Value) + names(vars) <- release$Key + + # get os name + os <- tolower(sysinfo[["sysname"]]) + + # read id + id <- "unknown" + for (field in c("ID", "ID_LIKE")) { + if (field %in% names(vars) && nzchar(vars[[field]])) { + id <- vars[[field]] + break + } + } + + # read version + version <- "unknown" + for (field in c("UBUNTU_CODENAME", "VERSION_CODENAME", "VERSION_ID", "BUILD_ID")) { + if (field %in% names(vars) && nzchar(vars[[field]])) { + version <- vars[[field]] + break + } + } + + # join together + paste(c(os, id, version), collapse = "-") + + } + + renv_bootstrap_platform_os_via_redhat_release <- function() { + + # read /etc/redhat-release + contents <- readLines("/etc/redhat-release", warn = FALSE) + + # infer id + id <- if (grepl("centos", contents, ignore.case = TRUE)) + "centos" + else if (grepl("redhat", contents, ignore.case = TRUE)) + "redhat" + else + "unknown" + + # try to find a version component (very hacky) + version <- "unknown" + + parts <- strsplit(contents, "[[:space:]]")[[1L]] + for (part in parts) { + + nv <- tryCatch(numeric_version(part), error = identity) + if (inherits(nv, "error")) + next + + version <- nv[1, 1] + break + + } + + paste(c("linux", id, version), collapse = "-") + + } + + renv_bootstrap_library_root_name <- function(project) { + + # use project name as-is if requested + asis <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT_ASIS", unset = "FALSE") + if (asis) + return(basename(project)) + + # otherwise, disambiguate based on project's path + id <- substring(renv_bootstrap_hash_text(project), 1L, 8L) + paste(basename(project), id, sep = "-") + + } + + renv_bootstrap_library_root <- function(project) { + + prefix <- renv_bootstrap_profile_prefix() + + path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA) + if (!is.na(path)) + return(paste(c(path, prefix), collapse = "/")) + + path <- renv_bootstrap_library_root_impl(project) + if (!is.null(path)) { + name <- renv_bootstrap_library_root_name(project) + return(paste(c(path, prefix, name), collapse = "/")) + } + + renv_bootstrap_paths_renv("library", project = project) + + } + + renv_bootstrap_library_root_impl <- function(project) { + + root <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA) + if (!is.na(root)) + return(root) + + type <- renv_bootstrap_project_type(project) + if (identical(type, "package")) { + userdir <- renv_bootstrap_user_dir() + return(file.path(userdir, "library")) + } + + } + + renv_bootstrap_validate_version <- function(version, description = NULL) { + + # resolve description file + # + # avoid passing lib.loc to `packageDescription()` below, since R will + # use the loaded version of the package by default anyhow. note that + # this function should only be called after 'renv' is loaded + # https://github.com/rstudio/renv/issues/1625 + description <- description %||% packageDescription("renv") + + # check whether requested version 'version' matches loaded version of renv + sha <- attr(version, "sha", exact = TRUE) + valid <- if (!is.null(sha)) + renv_bootstrap_validate_version_dev(sha, description) + else + renv_bootstrap_validate_version_release(version, description) + + if (valid) + return(TRUE) + + # the loaded version of renv doesn't match the requested version; + # give the user instructions on how to proceed + dev <- identical(description[["RemoteType"]], "github") + remote <- if (dev) + paste("rstudio/renv", description[["RemoteSha"]], sep = "@") + else + paste("renv", description[["Version"]], sep = "@") + + # display both loaded version + sha if available + friendly <- renv_bootstrap_version_friendly( + version = description[["Version"]], + sha = if (dev) description[["RemoteSha"]] + ) + + fmt <- heredoc(" + renv %1$s was loaded from project library, but this project is configured to use renv %2$s. + - Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile. + - Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library. + ") + catf(fmt, friendly, renv_bootstrap_version_friendly(version), remote) + + FALSE + + } + + renv_bootstrap_validate_version_dev <- function(version, description) { + + expected <- description[["RemoteSha"]] + if (!is.character(expected)) + return(FALSE) + + pattern <- sprintf("^\\Q%s\\E", version) + grepl(pattern, expected, perl = TRUE) + + } + + renv_bootstrap_validate_version_release <- function(version, description) { + expected <- description[["Version"]] + is.character(expected) && identical(expected, version) + } + + renv_bootstrap_hash_text <- function(text) { + + hashfile <- tempfile("renv-hash-") + on.exit(unlink(hashfile), add = TRUE) + + writeLines(text, con = hashfile) + tools::md5sum(hashfile) + + } + + renv_bootstrap_load <- function(project, libpath, version) { + + # try to load renv from the project library + if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) + return(FALSE) + + # warn if the version of renv loaded does not match + renv_bootstrap_validate_version(version) + + # execute renv load hooks, if any + hooks <- getHook("renv::autoload") + for (hook in hooks) + if (is.function(hook)) + tryCatch(hook(), error = warnify) + + # load the project + renv::load(project) + + TRUE + + } + + renv_bootstrap_profile_load <- function(project) { + + # if RENV_PROFILE is already set, just use that + profile <- Sys.getenv("RENV_PROFILE", unset = NA) + if (!is.na(profile) && nzchar(profile)) + return(profile) + + # check for a profile file (nothing to do if it doesn't exist) + path <- renv_bootstrap_paths_renv("profile", profile = FALSE, project = project) + if (!file.exists(path)) + return(NULL) + + # read the profile, and set it if it exists + contents <- readLines(path, warn = FALSE) + if (length(contents) == 0L) + return(NULL) + + # set RENV_PROFILE + profile <- contents[[1L]] + if (!profile %in% c("", "default")) + Sys.setenv(RENV_PROFILE = profile) + + profile + + } + + renv_bootstrap_profile_prefix <- function() { + profile <- renv_bootstrap_profile_get() + if (!is.null(profile)) + return(file.path("profiles", profile, "renv")) + } + + renv_bootstrap_profile_get <- function() { + profile <- Sys.getenv("RENV_PROFILE", unset = "") + renv_bootstrap_profile_normalize(profile) + } + + renv_bootstrap_profile_set <- function(profile) { + profile <- renv_bootstrap_profile_normalize(profile) + if (is.null(profile)) + Sys.unsetenv("RENV_PROFILE") + else + Sys.setenv(RENV_PROFILE = profile) + } + + renv_bootstrap_profile_normalize <- function(profile) { + + if (is.null(profile) || profile %in% c("", "default")) + return(NULL) + + profile + + } + + renv_bootstrap_path_absolute <- function(path) { + + substr(path, 1L, 1L) %in% c("~", "/", "\\") || ( + substr(path, 1L, 1L) %in% c(letters, LETTERS) && + substr(path, 2L, 3L) %in% c(":/", ":\\") + ) + + } + + renv_bootstrap_paths_renv <- function(..., profile = TRUE, project = NULL) { + renv <- Sys.getenv("RENV_PATHS_RENV", unset = "renv") + root <- if (renv_bootstrap_path_absolute(renv)) NULL else project + prefix <- if (profile) renv_bootstrap_profile_prefix() + components <- c(root, renv, prefix, ...) + paste(components, collapse = "/") + } + + renv_bootstrap_project_type <- function(path) { + + descpath <- file.path(path, "DESCRIPTION") + if (!file.exists(descpath)) + return("unknown") + + desc <- tryCatch( + read.dcf(descpath, all = TRUE), + error = identity + ) + + if (inherits(desc, "error")) + return("unknown") + + type <- desc$Type + if (!is.null(type)) + return(tolower(type)) + + package <- desc$Package + if (!is.null(package)) + return("package") + + "unknown" + + } + + renv_bootstrap_user_dir <- function() { + dir <- renv_bootstrap_user_dir_impl() + path.expand(chartr("\\", "/", dir)) + } + + renv_bootstrap_user_dir_impl <- function() { + + # use local override if set + override <- getOption("renv.userdir.override") + if (!is.null(override)) + return(override) + + # use R_user_dir if available + tools <- asNamespace("tools") + if (is.function(tools$R_user_dir)) + return(tools$R_user_dir("renv", "cache")) + + # try using our own backfill for older versions of R + envvars <- c("R_USER_CACHE_DIR", "XDG_CACHE_HOME") + for (envvar in envvars) { + root <- Sys.getenv(envvar, unset = NA) + if (!is.na(root)) + return(file.path(root, "R/renv")) + } + + # use platform-specific default fallbacks + if (Sys.info()[["sysname"]] == "Windows") + file.path(Sys.getenv("LOCALAPPDATA"), "R/cache/R/renv") + else if (Sys.info()[["sysname"]] == "Darwin") + "~/Library/Caches/org.R-project.R/R/renv" + else + "~/.cache/R/renv" + + } + + renv_bootstrap_version_friendly <- function(version, shafmt = NULL, sha = NULL) { + sha <- sha %||% attr(version, "sha", exact = TRUE) + parts <- c(version, sprintf(shafmt %||% " [sha: %s]", substring(sha, 1L, 7L))) + paste(parts, collapse = "") + } + + renv_bootstrap_exec <- function(project, libpath, version) { + if (!renv_bootstrap_load(project, libpath, version)) + renv_bootstrap_run(project, libpath, version) + } + + renv_bootstrap_run <- function(project, libpath, version) { + + # perform bootstrap + bootstrap(version, libpath) + + # exit early if we're just testing bootstrap + if (!is.na(Sys.getenv("RENV_BOOTSTRAP_INSTALL_ONLY", unset = NA))) + return(TRUE) + + # try again to load + if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { + return(renv::load(project = project)) + } + + # failed to download or load renv; warn the user + msg <- c( + "Failed to find an renv installation: the project will not be loaded.", + "Use `renv::activate()` to re-initialize the project." + ) + + warning(paste(msg, collapse = "\n"), call. = FALSE) + + } + + renv_json_read <- function(file = NULL, text = NULL) { + + jlerr <- NULL + + # if jsonlite is loaded, use that instead + if ("jsonlite" %in% loadedNamespaces()) { + + json <- tryCatch(renv_json_read_jsonlite(file, text), error = identity) + if (!inherits(json, "error")) + return(json) + + jlerr <- json + + } + + # otherwise, fall back to the default JSON reader + json <- tryCatch(renv_json_read_default(file, text), error = identity) + if (!inherits(json, "error")) + return(json) + + # report an error + if (!is.null(jlerr)) + stop(jlerr) + else + stop(json) + + } + + renv_json_read_jsonlite <- function(file = NULL, text = NULL) { + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") + jsonlite::fromJSON(txt = text, simplifyVector = FALSE) + } + + renv_json_read_patterns <- function() { + + list( + + # objects + list("{", "\t\n\tobject(\t\n\t"), + list("}", "\t\n\t)\t\n\t"), + + # arrays + list("[", "\t\n\tarray(\t\n\t"), + list("]", "\n\t\n)\n\t\n"), + + # maps + list(":", "\t\n\t=\t\n\t") + + ) + + } + + renv_json_read_envir <- function() { + + envir <- new.env(parent = emptyenv()) + + envir[["+"]] <- `+` + envir[["-"]] <- `-` + + envir[["object"]] <- function(...) { + result <- list(...) + names(result) <- as.character(names(result)) + result + } + + envir[["array"]] <- list + + envir[["true"]] <- TRUE + envir[["false"]] <- FALSE + envir[["null"]] <- NULL + + envir + + } + + renv_json_read_remap <- function(object, patterns) { + + # repair names if necessary + if (!is.null(names(object))) { + + nms <- names(object) + for (pattern in patterns) + nms <- gsub(pattern[[2L]], pattern[[1L]], nms, fixed = TRUE) + names(object) <- nms + + } + + # repair strings if necessary + if (is.character(object)) { + for (pattern in patterns) + object <- gsub(pattern[[2L]], pattern[[1L]], object, fixed = TRUE) + } + + # recurse for other objects + if (is.recursive(object)) + for (i in seq_along(object)) + object[i] <- list(renv_json_read_remap(object[[i]], patterns)) + + # return remapped object + object + + } + + renv_json_read_default <- function(file = NULL, text = NULL) { + + # read json text + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") + + # convert into something the R parser will understand + patterns <- renv_json_read_patterns() + transformed <- text + for (pattern in patterns) + transformed <- gsub(pattern[[1L]], pattern[[2L]], transformed, fixed = TRUE) + + # parse it + rfile <- tempfile("renv-json-", fileext = ".R") + on.exit(unlink(rfile), add = TRUE) + writeLines(transformed, con = rfile) + json <- parse(rfile, keep.source = FALSE, srcfile = NULL)[[1L]] + + # evaluate in safe environment + result <- eval(json, envir = renv_json_read_envir()) + + # fix up strings if necessary + renv_json_read_remap(result, patterns) + + } + + + # load the renv profile, if any + renv_bootstrap_profile_load(project) + + # construct path to library root + root <- renv_bootstrap_library_root(project) + + # construct library prefix for platform + prefix <- renv_bootstrap_platform_prefix() + + # construct full libpath + libpath <- file.path(root, prefix) + + # run bootstrap code + renv_bootstrap_exec(project, libpath, version) + + invisible() + +}) diff --git a/renv/profiles/lesson-requirements/renv.lock b/renv/profiles/lesson-requirements/renv.lock new file mode 100644 index 00000000..57824b77 --- /dev/null +++ b/renv/profiles/lesson-requirements/renv.lock @@ -0,0 +1,984 @@ +{ + "R": { + "Version": "4.4.2", + "Repositories": [ + { + "Name": "carpentries", + "URL": "https://carpentries.r-universe.dev" + }, + { + "Name": "carpentries_archive", + "URL": "https://carpentries.github.io/drat" + }, + { + "Name": "CRAN", + "URL": "https://cran.rstudio.com" + } + ] + }, + "Packages": { + "R6": { + "Package": "R6", + "Version": "2.5.1", + "Source": "Repository", + "Title": "Encapsulated Classes with Reference Semantics", + "Authors@R": "person(\"Winston\", \"Chang\", role = c(\"aut\", \"cre\"), email = \"winston@stdout.org\")", + "Description": "Creates classes with reference semantics, similar to R's built-in reference classes. Compared to reference classes, R6 classes are simpler and lighter-weight, and they are not built on S4 classes so they do not require the methods package. These classes allow public and private members, and they support inheritance, even when the classes are defined in different packages.", + "Depends": [ + "R (>= 3.0)" + ], + "Suggests": [ + "testthat", + "pryr" + ], + "License": "MIT + file LICENSE", + "URL": "https://r6.r-lib.org, https://github.com/r-lib/R6/", + "BugReports": "https://github.com/r-lib/R6/issues", + "RoxygenNote": "7.1.1", + "NeedsCompilation": "no", + "Author": "Winston Chang [aut, cre]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "base64enc": { + "Package": "base64enc", + "Version": "0.1-3", + "Source": "Repository", + "Title": "Tools for base64 encoding", + "Author": "Simon Urbanek ", + "Maintainer": "Simon Urbanek ", + "Depends": [ + "R (>= 2.9.0)" + ], + "Enhances": [ + "png" + ], + "Description": "This package provides tools for handling base64 encoding. It is more flexible than the orphaned base64 package.", + "License": "GPL-2 | GPL-3", + "URL": "http://www.rforge.net/base64enc", + "NeedsCompilation": "yes", + "Repository": "CRAN" + }, + "bslib": { + "Package": "bslib", + "Version": "0.8.0", + "Source": "Repository", + "Title": "Custom 'Bootstrap' 'Sass' Themes for 'shiny' and 'rmarkdown'", + "Authors@R": "c( person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Garrick\", \"Aden-Buie\", , \"garrick@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-7111-0077\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Javi\", \"Aguilar\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap colorpicker library\"), person(\"Thomas\", \"Park\", role = c(\"ctb\", \"cph\"), comment = \"Bootswatch library\"), person(, \"PayPal\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap accessibility plugin\") )", + "Description": "Simplifies custom 'CSS' styling of both 'shiny' and 'rmarkdown' via 'Bootstrap' 'Sass'. Supports 'Bootstrap' 3, 4 and 5 as well as their various 'Bootswatch' themes. An interactive widget is also provided for previewing themes in real time.", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/bslib/, https://github.com/rstudio/bslib", + "BugReports": "https://github.com/rstudio/bslib/issues", + "Depends": [ + "R (>= 2.10)" + ], + "Imports": [ + "base64enc", + "cachem", + "fastmap (>= 1.1.1)", + "grDevices", + "htmltools (>= 0.5.8)", + "jquerylib (>= 0.1.3)", + "jsonlite", + "lifecycle", + "memoise (>= 2.0.1)", + "mime", + "rlang", + "sass (>= 0.4.9)" + ], + "Suggests": [ + "bsicons", + "curl", + "fontawesome", + "future", + "ggplot2", + "knitr", + "magrittr", + "rappdirs", + "rmarkdown (>= 2.7)", + "shiny (> 1.8.1)", + "testthat", + "thematic", + "withr" + ], + "Config/Needs/deploy": "BH, chiflights22, colourpicker, commonmark, cpp11, cpsievert/chiflights22, cpsievert/histoslider, dplyr, DT, ggplot2, ggridges, gt, hexbin, histoslider, htmlwidgets, lattice, leaflet, lubridate, modelr, plotly, reactable, reshape2, rprojroot, rsconnect, rstudio/shiny, scales, styler, tibble", + "Config/Needs/routine": "chromote, desc, renv", + "Config/Needs/website": "brio, crosstalk, dplyr, DT, ggplot2, glue, htmlwidgets, leaflet, lorem, palmerpenguins, plotly, purrr, rprojroot, rstudio/htmltools, scales, stringr, tidyr, webshot2", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "zzzz-bs-sass, fonts, zzz-precompile, theme-*, rmd-*", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "Collate": "'accordion.R' 'breakpoints.R' 'bs-current-theme.R' 'bs-dependencies.R' 'bs-global.R' 'bs-remove.R' 'bs-theme-layers.R' 'bs-theme-preset-bootswatch.R' 'bs-theme-preset-builtin.R' 'bs-theme-preset.R' 'utils.R' 'bs-theme-preview.R' 'bs-theme-update.R' 'bs-theme.R' 'bslib-package.R' 'buttons.R' 'card.R' 'deprecated.R' 'files.R' 'fill.R' 'imports.R' 'input-dark-mode.R' 'input-switch.R' 'layout.R' 'nav-items.R' 'nav-update.R' 'navs-legacy.R' 'navs.R' 'onLoad.R' 'page.R' 'popover.R' 'precompiled.R' 'print.R' 'shiny-devmode.R' 'sidebar.R' 'staticimports.R' 'tooltip.R' 'utils-deps.R' 'utils-shiny.R' 'utils-tags.R' 'value-box.R' 'version-default.R' 'versions.R'", + "NeedsCompilation": "no", + "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], Garrick Aden-Buie [aut] (), Posit Software, PBC [cph, fnd], Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Javi Aguilar [ctb, cph] (Bootstrap colorpicker library), Thomas Park [ctb, cph] (Bootswatch library), PayPal [ctb, cph] (Bootstrap accessibility plugin)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "cachem": { + "Package": "cachem", + "Version": "1.1.0", + "Source": "Repository", + "Title": "Cache R Objects with Automatic Pruning", + "Description": "Key-value stores with automatic pruning. Caches can limit either their total size or the age of the oldest object (or both), automatically pruning objects to maintain the constraints.", + "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", c(\"aut\", \"cre\")), person(family = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")))", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "ByteCompile": "true", + "URL": "https://cachem.r-lib.org/, https://github.com/r-lib/cachem", + "Imports": [ + "rlang", + "fastmap (>= 1.2.0)" + ], + "Suggests": [ + "testthat" + ], + "RoxygenNote": "7.2.3", + "Config/Needs/routine": "lobstr", + "Config/Needs/website": "pkgdown", + "NeedsCompilation": "yes", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "cli": { + "Package": "cli", + "Version": "3.6.3", + "Source": "Repository", + "Title": "Helpers for Developing Command Line Interfaces", + "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Kirill\", \"Müller\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", , \"salim-b@pm.me\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A suite of tools to build attractive command line interfaces ('CLIs'), from semantic elements: headings, lists, alerts, paragraphs, etc. Supports custom themes via a 'CSS'-like language. It also contains a number of lower level 'CLI' elements: rules, boxes, trees, and 'Unicode' symbols with 'ASCII' alternatives. It support ANSI colors and text styles as well.", + "License": "MIT + file LICENSE", + "URL": "https://cli.r-lib.org, https://github.com/r-lib/cli", + "BugReports": "https://github.com/r-lib/cli/issues", + "Depends": [ + "R (>= 3.4)" + ], + "Imports": [ + "utils" + ], + "Suggests": [ + "callr", + "covr", + "crayon", + "digest", + "glue (>= 1.6.0)", + "grDevices", + "htmltools", + "htmlwidgets", + "knitr", + "methods", + "mockery", + "processx", + "ps (>= 1.3.4.9000)", + "rlang (>= 1.0.2.9003)", + "rmarkdown", + "rprojroot", + "rstudioapi", + "testthat", + "tibble", + "whoami", + "withr" + ], + "Config/Needs/website": "r-lib/asciicast, bench, brio, cpp11, decor, desc, fansi, prettyunits, sessioninfo, tidyverse/tidytemplate, usethis, vctrs", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.3", + "NeedsCompilation": "yes", + "Author": "Gábor Csárdi [aut, cre], Hadley Wickham [ctb], Kirill Müller [ctb], Salim Brüggemann [ctb] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" + }, + "digest": { + "Package": "digest", + "Version": "0.6.37", + "Source": "Repository", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Antoine\", \"Lucas\", role=\"ctb\"), person(\"Jarek\", \"Tuszynski\", role=\"ctb\"), person(\"Henrik\", \"Bengtsson\", role=\"ctb\", comment = c(ORCID = \"0000-0002-7579-5165\")), person(\"Simon\", \"Urbanek\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2297-1732\")), person(\"Mario\", \"Frasca\", role=\"ctb\"), person(\"Bryan\", \"Lewis\", role=\"ctb\"), person(\"Murray\", \"Stokely\", role=\"ctb\"), person(\"Hannes\", \"Muehleisen\", role=\"ctb\"), person(\"Duncan\", \"Murdoch\", role=\"ctb\"), person(\"Jim\", \"Hester\", role=\"ctb\"), person(\"Wush\", \"Wu\", role=\"ctb\", comment = c(ORCID = \"0000-0001-5180-0567\")), person(\"Qiang\", \"Kou\", role=\"ctb\", comment = c(ORCID = \"0000-0001-6786-5453\")), person(\"Thierry\", \"Onkelinx\", role=\"ctb\", comment = c(ORCID = \"0000-0001-8804-4216\")), person(\"Michel\", \"Lang\", role=\"ctb\", comment = c(ORCID = \"0000-0001-9754-0393\")), person(\"Viliam\", \"Simko\", role=\"ctb\"), person(\"Kurt\", \"Hornik\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4198-9911\")), person(\"Radford\", \"Neal\", role=\"ctb\", comment = c(ORCID = \"0000-0002-2473-3407\")), person(\"Kendon\", \"Bell\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9093-8312\")), person(\"Matthew\", \"de Queljoe\", role=\"ctb\"), person(\"Dmitry\", \"Selivanov\", role=\"ctb\"), person(\"Ion\", \"Suruceanu\", role=\"ctb\"), person(\"Bill\", \"Denney\", role=\"ctb\"), person(\"Dirk\", \"Schumacher\", role=\"ctb\"), person(\"András\", \"Svraka\", role=\"ctb\"), person(\"Sergey\", \"Fedorov\", role=\"ctb\"), person(\"Will\", \"Landau\", role=\"ctb\", comment = c(ORCID = \"0000-0003-1878-3253\")), person(\"Floris\", \"Vanderhaeghe\", role=\"ctb\", comment = c(ORCID = \"0000-0002-6378-6229\")), person(\"Kevin\", \"Tappe\", role=\"ctb\"), person(\"Harris\", \"McGehee\", role=\"ctb\"), person(\"Tim\", \"Mastny\", role=\"ctb\"), person(\"Aaron\", \"Peikert\", role=\"ctb\", comment = c(ORCID = \"0000-0001-7813-818X\")), person(\"Mark\", \"van der Loo\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9807-4686\")), person(\"Chris\", \"Muir\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2555-3878\")), person(\"Moritz\", \"Beller\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4852-0526\")), person(\"Sebastian\", \"Campbell\", role=\"ctb\"), person(\"Winston\", \"Chang\", role=\"ctb\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Dean\", \"Attali\", role=\"ctb\", comment = c(ORCID = \"0000-0002-5645-3493\")), person(\"Michael\", \"Chirico\", role=\"ctb\", comment = c(ORCID = \"0000-0003-0787-087X\")), person(\"Kevin\", \"Ushey\", role=\"ctb\"))", + "Date": "2024-08-19", + "Title": "Create Compact Hash Digests of R Objects", + "Description": "Implementation of a function 'digest()' for the creation of hash digests of arbitrary R objects (using the 'md5', 'sha-1', 'sha-256', 'crc32', 'xxhash', 'murmurhash', 'spookyhash', 'blake3', 'crc32c', 'xxh3_64', and 'xxh3_128' algorithms) permitting easy comparison of R language objects, as well as functions such as'hmac()' to create hash-based message authentication code. Please note that this package is not meant to be deployed for cryptographic purposes for which more comprehensive (and widely tested) libraries such as 'OpenSSL' should be used.", + "URL": "https://github.com/eddelbuettel/digest, https://dirk.eddelbuettel.com/code/digest.html", + "BugReports": "https://github.com/eddelbuettel/digest/issues", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "utils" + ], + "License": "GPL (>= 2)", + "Suggests": [ + "tinytest", + "simplermarkdown" + ], + "VignetteBuilder": "simplermarkdown", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Dirk Eddelbuettel [aut, cre] (), Antoine Lucas [ctb], Jarek Tuszynski [ctb], Henrik Bengtsson [ctb] (), Simon Urbanek [ctb] (), Mario Frasca [ctb], Bryan Lewis [ctb], Murray Stokely [ctb], Hannes Muehleisen [ctb], Duncan Murdoch [ctb], Jim Hester [ctb], Wush Wu [ctb] (), Qiang Kou [ctb] (), Thierry Onkelinx [ctb] (), Michel Lang [ctb] (), Viliam Simko [ctb], Kurt Hornik [ctb] (), Radford Neal [ctb] (), Kendon Bell [ctb] (), Matthew de Queljoe [ctb], Dmitry Selivanov [ctb], Ion Suruceanu [ctb], Bill Denney [ctb], Dirk Schumacher [ctb], András Svraka [ctb], Sergey Fedorov [ctb], Will Landau [ctb] (), Floris Vanderhaeghe [ctb] (), Kevin Tappe [ctb], Harris McGehee [ctb], Tim Mastny [ctb], Aaron Peikert [ctb] (), Mark van der Loo [ctb] (), Chris Muir [ctb] (), Moritz Beller [ctb] (), Sebastian Campbell [ctb], Winston Chang [ctb] (), Dean Attali [ctb] (), Michael Chirico [ctb] (), Kevin Ushey [ctb]", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, + "evaluate": { + "Package": "evaluate", + "Version": "1.0.0", + "Source": "Repository", + "Type": "Package", + "Title": "Parsing and Evaluation Tools that Provide More Details than the Default", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Yihui\", \"Xie\", role = \"aut\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Michael\", \"Lawrence\", role = \"ctb\"), person(\"Thomas\", \"Kluyver\", role = \"ctb\"), person(\"Jeroen\", \"Ooms\", role = \"ctb\"), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Adam\", \"Ryczkowski\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Michel\", \"Lang\", role = \"ctb\"), person(\"Karolis\", \"Koncevičius\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Parsing and evaluation tools that make it easy to recreate the command line behaviour of R.", + "License": "MIT + file LICENSE", + "URL": "https://evaluate.r-lib.org/, https://github.com/r-lib/evaluate", + "BugReports": "https://github.com/r-lib/evaluate/issues", + "Depends": [ + "R (>= 3.6.0)" + ], + "Suggests": [ + "covr", + "ggplot2 (>= 3.3.6)", + "lattice", + "methods", + "rlang", + "testthat (>= 3.0.0)", + "withr" + ], + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut, cre], Yihui Xie [aut] (), Michael Lawrence [ctb], Thomas Kluyver [ctb], Jeroen Ooms [ctb], Barret Schloerke [ctb], Adam Ryczkowski [ctb], Hiroaki Yutani [ctb], Michel Lang [ctb], Karolis Koncevičius [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" + }, + "fastmap": { + "Package": "fastmap", + "Version": "1.2.0", + "Source": "Repository", + "Title": "Fast Data Structures", + "Authors@R": "c( person(\"Winston\", \"Chang\", email = \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(given = \"Tessil\", role = \"cph\", comment = \"hopscotch_map library\") )", + "Description": "Fast implementation of data structures, including a key-value store, stack, and queue. Environments are commonly used as key-value stores in R, but every time a new key is used, it is added to R's global symbol table, causing a small amount of memory leakage. This can be problematic in cases where many different keys are used. Fastmap avoids this memory leak issue by implementing the map using data structures in C++.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.3", + "Suggests": [ + "testthat (>= 2.1.1)" + ], + "URL": "https://r-lib.github.io/fastmap/, https://github.com/r-lib/fastmap", + "BugReports": "https://github.com/r-lib/fastmap/issues", + "NeedsCompilation": "yes", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd], Tessil [cph] (hopscotch_map library)", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "fontawesome": { + "Package": "fontawesome", + "Version": "0.5.2", + "Source": "Repository", + "Type": "Package", + "Title": "Easily Work with 'Font Awesome' Icons", + "Description": "Easily and flexibly insert 'Font Awesome' icons into 'R Markdown' documents and 'Shiny' apps. These icons can be inserted into HTML content through inline 'SVG' tags or 'i' tags. There is also a utility function for exporting 'Font Awesome' icons as 'PNG' images for those situations where raster graphics are needed.", + "Authors@R": "c( person(\"Richard\", \"Iannone\", , \"rich@posit.co\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"ctb\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome font\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "License": "MIT + file LICENSE", + "URL": "https://github.com/rstudio/fontawesome, https://rstudio.github.io/fontawesome/", + "BugReports": "https://github.com/rstudio/fontawesome/issues", + "Encoding": "UTF-8", + "ByteCompile": "true", + "RoxygenNote": "7.2.3", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "rlang (>= 1.0.6)", + "htmltools (>= 0.5.1.1)" + ], + "Suggests": [ + "covr", + "dplyr (>= 1.0.8)", + "knitr (>= 1.31)", + "testthat (>= 3.0.0)", + "rsvg" + ], + "Config/testthat/edition": "3", + "NeedsCompilation": "no", + "Author": "Richard Iannone [aut, cre] (), Christophe Dervieux [ctb] (), Winston Chang [ctb], Dave Gandy [ctb, cph] (Font-Awesome font), Posit Software, PBC [cph, fnd]", + "Maintainer": "Richard Iannone ", + "Repository": "CRAN" + }, + "fs": { + "Package": "fs", + "Version": "1.6.4", + "Source": "Repository", + "Title": "Cross-Platform File System Operations Based on 'libuv'", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"libuv project contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Joyent, Inc. and other Node contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A cross-platform interface to file system operations, built on top of the 'libuv' C library.", + "License": "MIT + file LICENSE", + "URL": "https://fs.r-lib.org, https://github.com/r-lib/fs", + "BugReports": "https://github.com/r-lib/fs/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "methods" + ], + "Suggests": [ + "covr", + "crayon", + "knitr", + "pillar (>= 1.0.0)", + "rmarkdown", + "spelling", + "testthat (>= 3.0.0)", + "tibble (>= 1.1.0)", + "vctrs (>= 0.3.0)", + "withr" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Copyright": "file COPYRIGHTS", + "Encoding": "UTF-8", + "Language": "en-US", + "RoxygenNote": "7.2.3", + "SystemRequirements": "GNU make", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut], Hadley Wickham [aut], Gábor Csárdi [aut, cre], libuv project contributors [cph] (libuv library), Joyent, Inc. and other Node contributors [cph] (libuv library), Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" + }, + "glue": { + "Package": "glue", + "Version": "1.7.0", + "Source": "Repository", + "Title": "Interpreted String Literals", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "An implementation of interpreted string literals, inspired by Python's Literal String Interpolation and Docstrings and Julia's Triple-Quoted String Literals .", + "License": "MIT + file LICENSE", + "URL": "https://glue.tidyverse.org/, https://github.com/tidyverse/glue", + "BugReports": "https://github.com/tidyverse/glue/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "methods" + ], + "Suggests": [ + "crayon", + "DBI (>= 1.2.0)", + "dplyr", + "knitr", + "magrittr", + "rlang", + "rmarkdown", + "RSQLite", + "testthat (>= 3.2.0)", + "vctrs (>= 0.3.0)", + "waldo (>= 0.3.0)", + "withr" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/Needs/website": "bench, forcats, ggbeeswarm, ggplot2, R.utils, rprintf, tidyr, tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.3.9000", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut] (), Jennifer Bryan [aut, cre] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Jennifer Bryan ", + "Repository": "CRAN" + }, + "highr": { + "Package": "highr", + "Version": "0.11", + "Source": "Repository", + "Type": "Package", + "Title": "Syntax Highlighting for R Source Code", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Yixuan\", \"Qiu\", role = \"aut\"), person(\"Christopher\", \"Gandrud\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\") )", + "Description": "Provides syntax highlighting for R source code. Currently it supports LaTeX and HTML output. Source code of other languages is supported via Andre Simon's highlight package ().", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "xfun (>= 0.18)" + ], + "Suggests": [ + "knitr", + "markdown", + "testit" + ], + "License": "GPL", + "URL": "https://github.com/yihui/highr", + "BugReports": "https://github.com/yihui/highr/issues", + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre] (), Yixuan Qiu [aut], Christopher Gandrud [ctb], Qiang Li [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "htmltools": { + "Package": "htmltools", + "Version": "0.5.8.1", + "Source": "Repository", + "Type": "Package", + "Title": "Tools for HTML", + "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Barret\", \"Schloerke\", , \"barret@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Yihui\", \"Xie\", , \"yihui@posit.co\", role = \"aut\"), person(\"Jeff\", \"Allen\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Tools for HTML generation and output.", + "License": "GPL (>= 2)", + "URL": "https://github.com/rstudio/htmltools, https://rstudio.github.io/htmltools/", + "BugReports": "https://github.com/rstudio/htmltools/issues", + "Depends": [ + "R (>= 2.14.1)" + ], + "Imports": [ + "base64enc", + "digest", + "fastmap (>= 1.1.0)", + "grDevices", + "rlang (>= 1.0.0)", + "utils" + ], + "Suggests": [ + "Cairo", + "markdown", + "ragg", + "shiny", + "testthat", + "withr" + ], + "Enhances": [ + "knitr" + ], + "Config/Needs/check": "knitr", + "Config/Needs/website": "rstudio/quillt, bench", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "Collate": "'colors.R' 'fill.R' 'html_dependency.R' 'html_escape.R' 'html_print.R' 'htmltools-package.R' 'images.R' 'known_tags.R' 'selector.R' 'staticimports.R' 'tag_query.R' 'utils.R' 'tags.R' 'template.R'", + "NeedsCompilation": "yes", + "Author": "Joe Cheng [aut], Carson Sievert [aut, cre] (), Barret Schloerke [aut] (), Winston Chang [aut] (), Yihui Xie [aut], Jeff Allen [aut], Posit Software, PBC [cph, fnd]", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "jquerylib": { + "Package": "jquerylib", + "Version": "0.1.4", + "Source": "Repository", + "Title": "Obtain 'jQuery' as an HTML Dependency Object", + "Authors@R": "c( person(\"Carson\", \"Sievert\", role = c(\"aut\", \"cre\"), email = \"carson@rstudio.com\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", role = \"aut\", email = \"joe@rstudio.com\"), person(family = \"RStudio\", role = \"cph\"), person(family = \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library and jQuery UI library\"), person(family = \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt\") )", + "Description": "Obtain any major version of 'jQuery' () and use it in any webpage generated by 'htmltools' (e.g. 'shiny', 'htmlwidgets', and 'rmarkdown'). Most R users don't need to use this package directly, but other R packages (e.g. 'shiny', 'rmarkdown', etc.) depend on this package to avoid bundling redundant copies of 'jQuery'.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "Config/testthat/edition": "3", + "RoxygenNote": "7.0.2", + "Imports": [ + "htmltools" + ], + "Suggests": [ + "testthat" + ], + "NeedsCompilation": "no", + "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], RStudio [cph], jQuery Foundation [cph] (jQuery library and jQuery UI library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "jsonlite": { + "Package": "jsonlite", + "Version": "1.8.9", + "Source": "Repository", + "Title": "A Simple and Robust JSON Parser and Generator for R", + "License": "MIT + file LICENSE", + "Depends": [ + "methods" + ], + "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Duncan\", \"Temple Lang\", role = \"ctb\"), person(\"Lloyd\", \"Hilaiel\", role = \"cph\", comment=\"author of bundled libyajl\"))", + "URL": "https://jeroen.r-universe.dev/jsonlite https://arxiv.org/abs/1403.2805", + "BugReports": "https://github.com/jeroen/jsonlite/issues", + "Maintainer": "Jeroen Ooms ", + "VignetteBuilder": "knitr, R.rsp", + "Description": "A reasonably fast JSON parser and generator, optimized for statistical data and the web. Offers simple, flexible tools for working with JSON in R, and is particularly powerful for building pipelines and interacting with a web API. The implementation is based on the mapping described in the vignette (Ooms, 2014). In addition to converting JSON data from/to R objects, 'jsonlite' contains functions to stream, validate, and prettify JSON data. The unit tests included with the package verify that all edge cases are encoded and decoded consistently for use with dynamic data in systems and applications.", + "Suggests": [ + "httr", + "vctrs", + "testthat", + "knitr", + "rmarkdown", + "R.rsp", + "sf" + ], + "RoxygenNote": "7.2.3", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Jeroen Ooms [aut, cre] (), Duncan Temple Lang [ctb], Lloyd Hilaiel [cph] (author of bundled libyajl)", + "Repository": "CRAN" + }, + "knitr": { + "Package": "knitr", + "Version": "1.48", + "Source": "Repository", + "Type": "Package", + "Title": "A General-Purpose Package for Dynamic Report Generation in R", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Abhraneel\", \"Sarma\", role = \"ctb\"), person(\"Adam\", \"Vogt\", role = \"ctb\"), person(\"Alastair\", \"Andrew\", role = \"ctb\"), person(\"Alex\", \"Zvoleff\", role = \"ctb\"), person(\"Amar\", \"Al-Zubaidi\", role = \"ctb\"), person(\"Andre\", \"Simon\", role = \"ctb\", comment = \"the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de\"), person(\"Aron\", \"Atkins\", role = \"ctb\"), person(\"Aaron\", \"Wolen\", role = \"ctb\"), person(\"Ashley\", \"Manton\", role = \"ctb\"), person(\"Atsushi\", \"Yasumoto\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8335-495X\")), person(\"Ben\", \"Baumer\", role = \"ctb\"), person(\"Brian\", \"Diggs\", role = \"ctb\"), person(\"Brian\", \"Zhang\", role = \"ctb\"), person(\"Bulat\", \"Yapparov\", role = \"ctb\"), person(\"Cassio\", \"Pereira\", role = \"ctb\"), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person(\"David\", \"Hall\", role = \"ctb\"), person(\"David\", \"Hugh-Jones\", role = \"ctb\"), person(\"David\", \"Robinson\", role = \"ctb\"), person(\"Doug\", \"Hemken\", role = \"ctb\"), person(\"Duncan\", \"Murdoch\", role = \"ctb\"), person(\"Elio\", \"Campitelli\", role = \"ctb\"), person(\"Ellis\", \"Hughes\", role = \"ctb\"), person(\"Emily\", \"Riederer\", role = \"ctb\"), person(\"Fabian\", \"Hirschmann\", role = \"ctb\"), person(\"Fitch\", \"Simeon\", role = \"ctb\"), person(\"Forest\", \"Fang\", role = \"ctb\"), person(c(\"Frank\", \"E\", \"Harrell\", \"Jr\"), role = \"ctb\", comment = \"the Sweavel package at inst/misc/Sweavel.sty\"), person(\"Garrick\", \"Aden-Buie\", role = \"ctb\"), person(\"Gregoire\", \"Detrez\", role = \"ctb\"), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Hao\", \"Zhu\", role = \"ctb\"), person(\"Heewon\", \"Jeon\", role = \"ctb\"), person(\"Henrik\", \"Bengtsson\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Ian\", \"Lyttle\", role = \"ctb\"), person(\"Hodges\", \"Daniel\", role = \"ctb\"), person(\"Jacob\", \"Bien\", role = \"ctb\"), person(\"Jake\", \"Burkhead\", role = \"ctb\"), person(\"James\", \"Manton\", role = \"ctb\"), person(\"Jared\", \"Lander\", role = \"ctb\"), person(\"Jason\", \"Punyon\", role = \"ctb\"), person(\"Javier\", \"Luraschi\", role = \"ctb\"), person(\"Jeff\", \"Arnold\", role = \"ctb\"), person(\"Jenny\", \"Bryan\", role = \"ctb\"), person(\"Jeremy\", \"Ashkenas\", role = c(\"ctb\", \"cph\"), comment = \"the CSS file at inst/misc/docco-classic.css\"), person(\"Jeremy\", \"Stephens\", role = \"ctb\"), person(\"Jim\", \"Hester\", role = \"ctb\"), person(\"Joe\", \"Cheng\", role = \"ctb\"), person(\"Johannes\", \"Ranke\", role = \"ctb\"), person(\"John\", \"Honaker\", role = \"ctb\"), person(\"John\", \"Muschelli\", role = \"ctb\"), person(\"Jonathan\", \"Keane\", role = \"ctb\"), person(\"JJ\", \"Allaire\", role = \"ctb\"), person(\"Johan\", \"Toloe\", role = \"ctb\"), person(\"Jonathan\", \"Sidi\", role = \"ctb\"), person(\"Joseph\", \"Larmarange\", role = \"ctb\"), person(\"Julien\", \"Barnier\", role = \"ctb\"), person(\"Kaiyin\", \"Zhong\", role = \"ctb\"), person(\"Kamil\", \"Slowikowski\", role = \"ctb\"), person(\"Karl\", \"Forner\", role = \"ctb\"), person(c(\"Kevin\", \"K.\"), \"Smith\", role = \"ctb\"), person(\"Kirill\", \"Mueller\", role = \"ctb\"), person(\"Kohske\", \"Takahashi\", role = \"ctb\"), person(\"Lorenz\", \"Walthert\", role = \"ctb\"), person(\"Lucas\", \"Gallindo\", role = \"ctb\"), person(\"Marius\", \"Hofert\", role = \"ctb\"), person(\"Martin\", \"Modrák\", role = \"ctb\"), person(\"Michael\", \"Chirico\", role = \"ctb\"), person(\"Michael\", \"Friendly\", role = \"ctb\"), person(\"Michal\", \"Bojanowski\", role = \"ctb\"), person(\"Michel\", \"Kuhlmann\", role = \"ctb\"), person(\"Miller\", \"Patrick\", role = \"ctb\"), person(\"Nacho\", \"Caballero\", role = \"ctb\"), person(\"Nick\", \"Salkowski\", role = \"ctb\"), person(\"Niels Richard\", \"Hansen\", role = \"ctb\"), person(\"Noam\", \"Ross\", role = \"ctb\"), person(\"Obada\", \"Mahdi\", role = \"ctb\"), person(\"Pavel N.\", \"Krivitsky\", role = \"ctb\", comment=c(ORCID = \"0000-0002-9101-3362\")), person(\"Pedro\", \"Faria\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\"), person(\"Ramnath\", \"Vaidyanathan\", role = \"ctb\"), person(\"Richard\", \"Cotton\", role = \"ctb\"), person(\"Robert\", \"Krzyzanowski\", role = \"ctb\"), person(\"Rodrigo\", \"Copetti\", role = \"ctb\"), person(\"Romain\", \"Francois\", role = \"ctb\"), person(\"Ruaridh\", \"Williamson\", role = \"ctb\"), person(\"Sagiru\", \"Mati\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1413-3974\")), person(\"Scott\", \"Kostyshak\", role = \"ctb\"), person(\"Sebastian\", \"Meyer\", role = \"ctb\"), person(\"Sietse\", \"Brouwer\", role = \"ctb\"), person(c(\"Simon\", \"de\"), \"Bernard\", role = \"ctb\"), person(\"Sylvain\", \"Rousseau\", role = \"ctb\"), person(\"Taiyun\", \"Wei\", role = \"ctb\"), person(\"Thibaut\", \"Assus\", role = \"ctb\"), person(\"Thibaut\", \"Lamadon\", role = \"ctb\"), person(\"Thomas\", \"Leeper\", role = \"ctb\"), person(\"Tim\", \"Mastny\", role = \"ctb\"), person(\"Tom\", \"Torsney-Weir\", role = \"ctb\"), person(\"Trevor\", \"Davis\", role = \"ctb\"), person(\"Viktoras\", \"Veitas\", role = \"ctb\"), person(\"Weicheng\", \"Zhu\", role = \"ctb\"), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Zachary\", \"Foster\", role = \"ctb\"), person(\"Zhian N.\", \"Kamvar\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1458-7108\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Provides a general-purpose tool for dynamic report generation in R using Literate Programming techniques.", + "Depends": [ + "R (>= 3.3.0)" + ], + "Imports": [ + "evaluate (>= 0.15)", + "highr (>= 0.11)", + "methods", + "tools", + "xfun (>= 0.44)", + "yaml (>= 2.1.19)" + ], + "Suggests": [ + "bslib", + "codetools", + "DBI (>= 0.4-1)", + "digest", + "formatR", + "gifski", + "gridSVG", + "htmlwidgets (>= 0.7)", + "jpeg", + "JuliaCall (>= 0.11.1)", + "magick", + "markdown (>= 1.3)", + "png", + "ragg", + "reticulate (>= 1.4)", + "rgl (>= 0.95.1201)", + "rlang", + "rmarkdown", + "sass", + "showtext", + "styler (>= 1.2.0)", + "targets (>= 0.6.0)", + "testit", + "tibble", + "tikzDevice (>= 0.10)", + "tinytex (>= 0.46)", + "webshot", + "rstudioapi", + "svglite" + ], + "License": "GPL", + "URL": "https://yihui.org/knitr/", + "BugReports": "https://github.com/yihui/knitr/issues", + "Encoding": "UTF-8", + "VignetteBuilder": "knitr", + "SystemRequirements": "Package vignettes based on R Markdown v2 or reStructuredText require Pandoc (http://pandoc.org). The function rst2pdf() requires rst2pdf (https://github.com/rst2pdf/rst2pdf).", + "Collate": "'block.R' 'cache.R' 'utils.R' 'citation.R' 'hooks-html.R' 'plot.R' 'defaults.R' 'concordance.R' 'engine.R' 'highlight.R' 'themes.R' 'header.R' 'hooks-asciidoc.R' 'hooks-chunk.R' 'hooks-extra.R' 'hooks-latex.R' 'hooks-md.R' 'hooks-rst.R' 'hooks-textile.R' 'hooks.R' 'output.R' 'package.R' 'pandoc.R' 'params.R' 'parser.R' 'pattern.R' 'rocco.R' 'spin.R' 'table.R' 'template.R' 'utils-conversion.R' 'utils-rd2html.R' 'utils-string.R' 'utils-sweave.R' 'utils-upload.R' 'utils-vignettes.R' 'zzz.R'", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre] (), Abhraneel Sarma [ctb], Adam Vogt [ctb], Alastair Andrew [ctb], Alex Zvoleff [ctb], Amar Al-Zubaidi [ctb], Andre Simon [ctb] (the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de), Aron Atkins [ctb], Aaron Wolen [ctb], Ashley Manton [ctb], Atsushi Yasumoto [ctb] (), Ben Baumer [ctb], Brian Diggs [ctb], Brian Zhang [ctb], Bulat Yapparov [ctb], Cassio Pereira [ctb], Christophe Dervieux [ctb], David Hall [ctb], David Hugh-Jones [ctb], David Robinson [ctb], Doug Hemken [ctb], Duncan Murdoch [ctb], Elio Campitelli [ctb], Ellis Hughes [ctb], Emily Riederer [ctb], Fabian Hirschmann [ctb], Fitch Simeon [ctb], Forest Fang [ctb], Frank E Harrell Jr [ctb] (the Sweavel package at inst/misc/Sweavel.sty), Garrick Aden-Buie [ctb], Gregoire Detrez [ctb], Hadley Wickham [ctb], Hao Zhu [ctb], Heewon Jeon [ctb], Henrik Bengtsson [ctb], Hiroaki Yutani [ctb], Ian Lyttle [ctb], Hodges Daniel [ctb], Jacob Bien [ctb], Jake Burkhead [ctb], James Manton [ctb], Jared Lander [ctb], Jason Punyon [ctb], Javier Luraschi [ctb], Jeff Arnold [ctb], Jenny Bryan [ctb], Jeremy Ashkenas [ctb, cph] (the CSS file at inst/misc/docco-classic.css), Jeremy Stephens [ctb], Jim Hester [ctb], Joe Cheng [ctb], Johannes Ranke [ctb], John Honaker [ctb], John Muschelli [ctb], Jonathan Keane [ctb], JJ Allaire [ctb], Johan Toloe [ctb], Jonathan Sidi [ctb], Joseph Larmarange [ctb], Julien Barnier [ctb], Kaiyin Zhong [ctb], Kamil Slowikowski [ctb], Karl Forner [ctb], Kevin K. Smith [ctb], Kirill Mueller [ctb], Kohske Takahashi [ctb], Lorenz Walthert [ctb], Lucas Gallindo [ctb], Marius Hofert [ctb], Martin Modrák [ctb], Michael Chirico [ctb], Michael Friendly [ctb], Michal Bojanowski [ctb], Michel Kuhlmann [ctb], Miller Patrick [ctb], Nacho Caballero [ctb], Nick Salkowski [ctb], Niels Richard Hansen [ctb], Noam Ross [ctb], Obada Mahdi [ctb], Pavel N. Krivitsky [ctb] (), Pedro Faria [ctb], Qiang Li [ctb], Ramnath Vaidyanathan [ctb], Richard Cotton [ctb], Robert Krzyzanowski [ctb], Rodrigo Copetti [ctb], Romain Francois [ctb], Ruaridh Williamson [ctb], Sagiru Mati [ctb] (), Scott Kostyshak [ctb], Sebastian Meyer [ctb], Sietse Brouwer [ctb], Simon de Bernard [ctb], Sylvain Rousseau [ctb], Taiyun Wei [ctb], Thibaut Assus [ctb], Thibaut Lamadon [ctb], Thomas Leeper [ctb], Tim Mastny [ctb], Tom Torsney-Weir [ctb], Trevor Davis [ctb], Viktoras Veitas [ctb], Weicheng Zhu [ctb], Wush Wu [ctb], Zachary Foster [ctb], Zhian N. Kamvar [ctb] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "lifecycle": { + "Package": "lifecycle", + "Version": "1.0.4", + "Source": "Repository", + "Title": "Manage the Life Cycle of your Package Functions", + "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Manage the life cycle of your exported functions with shared conventions, documentation badges, and user-friendly deprecation warnings.", + "License": "MIT + file LICENSE", + "URL": "https://lifecycle.r-lib.org/, https://github.com/r-lib/lifecycle", + "BugReports": "https://github.com/r-lib/lifecycle/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "cli (>= 3.4.0)", + "glue", + "rlang (>= 1.1.0)" + ], + "Suggests": [ + "covr", + "crayon", + "knitr", + "lintr", + "rmarkdown", + "testthat (>= 3.0.1)", + "tibble", + "tidyverse", + "tools", + "vctrs", + "withr" + ], + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate, usethis", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.1", + "NeedsCompilation": "no", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" + }, + "memoise": { + "Package": "memoise", + "Version": "2.0.1", + "Source": "Repository", + "Title": "'Memoisation' of Functions", + "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = \"aut\", email = \"hadley@rstudio.com\"), person(given = \"Jim\", family = \"Hester\", role = \"aut\"), person(given = \"Winston\", family = \"Chang\", role = c(\"aut\", \"cre\"), email = \"winston@rstudio.com\"), person(given = \"Kirill\", family = \"Müller\", role = \"aut\", email = \"krlmlr+r@mailbox.org\"), person(given = \"Daniel\", family = \"Cook\", role = \"aut\", email = \"danielecook@gmail.com\"), person(given = \"Mark\", family = \"Edmondson\", role = \"ctb\", email = \"r@sunholo.com\"))", + "Description": "Cache the results of a function so that when you call it again with the same arguments it returns the previously computed value.", + "License": "MIT + file LICENSE", + "URL": "https://memoise.r-lib.org, https://github.com/r-lib/memoise", + "BugReports": "https://github.com/r-lib/memoise/issues", + "Imports": [ + "rlang (>= 0.4.10)", + "cachem" + ], + "Suggests": [ + "digest", + "aws.s3", + "covr", + "googleAuthR", + "googleCloudStorageR", + "httr", + "testthat" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.1.2", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut], Jim Hester [aut], Winston Chang [aut, cre], Kirill Müller [aut], Daniel Cook [aut], Mark Edmondson [ctb]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" + }, + "mime": { + "Package": "mime", + "Version": "0.12", + "Source": "Repository", + "Type": "Package", + "Title": "Map Filenames to MIME Types", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Jeffrey\", \"Horner\", role = \"ctb\"), person(\"Beilei\", \"Bian\", role = \"ctb\") )", + "Description": "Guesses the MIME type from a filename extension using the data derived from /etc/mime.types in UNIX-type systems.", + "Imports": [ + "tools" + ], + "License": "GPL", + "URL": "https://github.com/yihui/mime", + "BugReports": "https://github.com/yihui/mime/issues", + "RoxygenNote": "7.1.1", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Yihui Xie [aut, cre] (), Jeffrey Horner [ctb], Beilei Bian [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "rappdirs": { + "Package": "rappdirs", + "Version": "0.3.3", + "Source": "Repository", + "Type": "Package", + "Title": "Application Directories: Determine Where to Save Data, Caches, and Logs", + "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = c(\"trl\", \"cre\", \"cph\"), email = \"hadley@rstudio.com\"), person(given = \"RStudio\", role = \"cph\"), person(given = \"Sridhar\", family = \"Ratnakumar\", role = \"aut\"), person(given = \"Trent\", family = \"Mick\", role = \"aut\"), person(given = \"ActiveState\", role = \"cph\", comment = \"R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs\"), person(given = \"Eddy\", family = \"Petrisor\", role = \"ctb\"), person(given = \"Trevor\", family = \"Davis\", role = c(\"trl\", \"aut\")), person(given = \"Gabor\", family = \"Csardi\", role = \"ctb\"), person(given = \"Gregory\", family = \"Jefferis\", role = \"ctb\"))", + "Description": "An easy way to determine which directories on the users computer you should use to save data, caches and logs. A port of Python's 'Appdirs' () to R.", + "License": "MIT + file LICENSE", + "URL": "https://rappdirs.r-lib.org, https://github.com/r-lib/rappdirs", + "BugReports": "https://github.com/r-lib/rappdirs/issues", + "Depends": [ + "R (>= 3.2)" + ], + "Suggests": [ + "roxygen2", + "testthat (>= 3.0.0)", + "covr", + "withr" + ], + "Copyright": "Original python appdirs module copyright (c) 2010 ActiveState Software Inc. R port copyright Hadley Wickham, RStudio. See file LICENSE for details.", + "Encoding": "UTF-8", + "RoxygenNote": "7.1.1", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Hadley Wickham [trl, cre, cph], RStudio [cph], Sridhar Ratnakumar [aut], Trent Mick [aut], ActiveState [cph] (R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs), Eddy Petrisor [ctb], Trevor Davis [trl, aut], Gabor Csardi [ctb], Gregory Jefferis [ctb]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" + }, + "renv": { + "Package": "renv", + "Version": "1.0.8", + "Source": "Repository", + "Type": "Package", + "Title": "Project Environments", + "Authors@R": "c( person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Hadley\", \"Wickham\", role = c(\"aut\"), email = \"hadley@rstudio.com\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A dependency management toolkit for R. Using 'renv', you can create and manage project-local R libraries, save the state of these libraries to a 'lockfile', and later restore your library as required. Together, these tools can help make your projects more isolated, portable, and reproducible.", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/renv/, https://github.com/rstudio/renv", + "BugReports": "https://github.com/rstudio/renv/issues", + "Imports": [ + "utils" + ], + "Suggests": [ + "BiocManager", + "cli", + "covr", + "cpp11", + "devtools", + "gitcreds", + "jsonlite", + "jsonvalidate", + "knitr", + "miniUI", + "packrat", + "pak", + "R6", + "remotes", + "reticulate", + "rmarkdown", + "rstudioapi", + "shiny", + "testthat", + "uuid", + "waldo", + "yaml", + "webfakes" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "bioconductor,python,install,restore,snapshot,retrieve,remotes", + "NeedsCompilation": "no", + "Author": "Kevin Ushey [aut, cre] (), Hadley Wickham [aut] (), Posit Software, PBC [cph, fnd]", + "Maintainer": "Kevin Ushey ", + "Repository": "CRAN" + }, + "rlang": { + "Package": "rlang", + "Version": "1.1.4", + "Source": "Repository", + "Title": "Functions for Base Types and Core R and 'Tidyverse' Features", + "Description": "A toolbox for working with base types, core R features like the condition system, and core 'Tidyverse' features like tidy evaluation.", + "Authors@R": "c( person(\"Lionel\", \"Henry\", ,\"lionel@posit.co\", c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", ,\"hadley@posit.co\", \"aut\"), person(given = \"mikefc\", email = \"mikefc@coolbutuseless.com\", role = \"cph\", comment = \"Hash implementation based on Mike's xxhashlite\"), person(given = \"Yann\", family = \"Collet\", role = \"cph\", comment = \"Author of the embedded xxHash library\"), person(given = \"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", + "License": "MIT + file LICENSE", + "ByteCompile": "true", + "Biarch": "true", + "Depends": [ + "R (>= 3.5.0)" + ], + "Imports": [ + "utils" + ], + "Suggests": [ + "cli (>= 3.1.0)", + "covr", + "crayon", + "fs", + "glue", + "knitr", + "magrittr", + "methods", + "pillar", + "rmarkdown", + "stats", + "testthat (>= 3.0.0)", + "tibble", + "usethis", + "vctrs (>= 0.2.3)", + "withr" + ], + "Enhances": [ + "winch" + ], + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "URL": "https://rlang.r-lib.org, https://github.com/r-lib/rlang", + "BugReports": "https://github.com/r-lib/rlang/issues", + "Config/testthat/edition": "3", + "Config/Needs/website": "dplyr, tidyverse/tidytemplate", + "NeedsCompilation": "yes", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], mikefc [cph] (Hash implementation based on Mike's xxhashlite), Yann Collet [cph] (Author of the embedded xxHash library), Posit, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" + }, + "rmarkdown": { + "Package": "rmarkdown", + "Version": "2.28", + "Source": "Repository", + "Type": "Package", + "Title": "Dynamic Documents for R", + "Authors@R": "c( person(\"JJ\", \"Allaire\", , \"jj@posit.co\", role = \"aut\"), person(\"Yihui\", \"Xie\", , \"xie@yihui.name\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Jonathan\", \"McPherson\", , \"jonathan@posit.co\", role = \"aut\"), person(\"Javier\", \"Luraschi\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevin@posit.co\", role = \"aut\"), person(\"Aron\", \"Atkins\", , \"aron@posit.co\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Andrew\", \"Dunning\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0464-5036\")), person(\"Atsushi\", \"Yasumoto\", role = c(\"ctb\", \"cph\"), comment = c(ORCID = \"0000-0002-8335-495X\", cph = \"Number sections Lua filter\")), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Carson\", \"Sievert\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Devon\", \"Ryan\", , \"dpryan79@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Frederik\", \"Aust\", , \"frederik.aust@uni-koeln.de\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4900-788X\")), person(\"Jeff\", \"Allen\", , \"jeff@posit.co\", role = \"ctb\"), person(\"JooYoung\", \"Seo\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4064-6012\")), person(\"Malcolm\", \"Barrett\", role = \"ctb\"), person(\"Rob\", \"Hyndman\", , \"Rob.Hyndman@monash.edu\", role = \"ctb\"), person(\"Romain\", \"Lesur\", role = \"ctb\"), person(\"Roy\", \"Storey\", role = \"ctb\"), person(\"Ruben\", \"Arslan\", , \"ruben.arslan@uni-goettingen.de\", role = \"ctb\"), person(\"Sergio\", \"Oller\", role = \"ctb\"), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"jQuery UI contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt\"), person(\"Mark\", \"Otto\", role = \"ctb\", comment = \"Bootstrap library\"), person(\"Jacob\", \"Thornton\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Alexander\", \"Farkas\", role = c(\"ctb\", \"cph\"), comment = \"html5shiv library\"), person(\"Scott\", \"Jehl\", role = c(\"ctb\", \"cph\"), comment = \"Respond.js library\"), person(\"Ivan\", \"Sagalaev\", role = c(\"ctb\", \"cph\"), comment = \"highlight.js library\"), person(\"Greg\", \"Franko\", role = c(\"ctb\", \"cph\"), comment = \"tocify library\"), person(\"John\", \"MacFarlane\", role = c(\"ctb\", \"cph\"), comment = \"Pandoc templates\"), person(, \"Google, Inc.\", role = c(\"ctb\", \"cph\"), comment = \"ioslides library\"), person(\"Dave\", \"Raggett\", role = \"ctb\", comment = \"slidy library\"), person(, \"W3C\", role = \"cph\", comment = \"slidy library\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome\"), person(\"Ben\", \"Sperry\", role = \"ctb\", comment = \"Ionicons\"), person(, \"Drifty\", role = \"cph\", comment = \"Ionicons\"), person(\"Aidan\", \"Lister\", role = c(\"ctb\", \"cph\"), comment = \"jQuery StickyTabs\"), person(\"Benct Philip\", \"Jonsson\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\"), person(\"Albert\", \"Krewinkel\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\") )", + "Maintainer": "Yihui Xie ", + "Description": "Convert R Markdown documents into a variety of formats.", + "License": "GPL-3", + "URL": "https://github.com/rstudio/rmarkdown, https://pkgs.rstudio.com/rmarkdown/", + "BugReports": "https://github.com/rstudio/rmarkdown/issues", + "Depends": [ + "R (>= 3.0)" + ], + "Imports": [ + "bslib (>= 0.2.5.1)", + "evaluate (>= 0.13)", + "fontawesome (>= 0.5.0)", + "htmltools (>= 0.5.1)", + "jquerylib", + "jsonlite", + "knitr (>= 1.43)", + "methods", + "tinytex (>= 0.31)", + "tools", + "utils", + "xfun (>= 0.36)", + "yaml (>= 2.1.19)" + ], + "Suggests": [ + "digest", + "dygraphs", + "fs", + "rsconnect", + "downlit (>= 0.4.0)", + "katex (>= 1.4.0)", + "sass (>= 0.4.0)", + "shiny (>= 1.6.0)", + "testthat (>= 3.0.3)", + "tibble", + "vctrs", + "cleanrmd", + "withr (>= 2.4.2)", + "xml2" + ], + "VignetteBuilder": "knitr", + "Config/Needs/website": "rstudio/quillt, pkgdown", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "SystemRequirements": "pandoc (>= 1.14) - http://pandoc.org", + "NeedsCompilation": "no", + "Author": "JJ Allaire [aut], Yihui Xie [aut, cre] (), Christophe Dervieux [aut] (), Jonathan McPherson [aut], Javier Luraschi [aut], Kevin Ushey [aut], Aron Atkins [aut], Hadley Wickham [aut], Joe Cheng [aut], Winston Chang [aut], Richard Iannone [aut] (), Andrew Dunning [ctb] (), Atsushi Yasumoto [ctb, cph] (, Number sections Lua filter), Barret Schloerke [ctb], Carson Sievert [ctb] (), Devon Ryan [ctb] (), Frederik Aust [ctb] (), Jeff Allen [ctb], JooYoung Seo [ctb] (), Malcolm Barrett [ctb], Rob Hyndman [ctb], Romain Lesur [ctb], Roy Storey [ctb], Ruben Arslan [ctb], Sergio Oller [ctb], Posit Software, PBC [cph, fnd], jQuery UI contributors [ctb, cph] (jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt), Mark Otto [ctb] (Bootstrap library), Jacob Thornton [ctb] (Bootstrap library), Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Alexander Farkas [ctb, cph] (html5shiv library), Scott Jehl [ctb, cph] (Respond.js library), Ivan Sagalaev [ctb, cph] (highlight.js library), Greg Franko [ctb, cph] (tocify library), John MacFarlane [ctb, cph] (Pandoc templates), Google, Inc. [ctb, cph] (ioslides library), Dave Raggett [ctb] (slidy library), W3C [cph] (slidy library), Dave Gandy [ctb, cph] (Font-Awesome), Ben Sperry [ctb] (Ionicons), Drifty [cph] (Ionicons), Aidan Lister [ctb, cph] (jQuery StickyTabs), Benct Philip Jonsson [ctb, cph] (pagebreak Lua filter), Albert Krewinkel [ctb, cph] (pagebreak Lua filter)", + "Repository": "CRAN" + }, + "sass": { + "Package": "sass", + "Version": "0.4.9", + "Source": "Repository", + "Type": "Package", + "Title": "Syntactically Awesome Style Sheets ('Sass')", + "Description": "An 'SCSS' compiler, powered by the 'LibSass' library. With this, R developers can use variables, inheritance, and functions to generate dynamic style sheets. The package uses the 'Sass CSS' extension language, which is stable, powerful, and CSS compatible.", + "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@rstudio.com\", \"aut\"), person(\"Timothy\", \"Mastny\", , \"tim.mastny@gmail.com\", \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Barret\", \"Schloerke\", , \"barret@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Carson\", \"Sievert\", , \"carson@rstudio.com\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Christophe\", \"Dervieux\", , \"cderv@rstudio.com\", c(\"ctb\"), comment = c(ORCID = \"0000-0003-4474-2498\")), person(family = \"RStudio\", role = c(\"cph\", \"fnd\")), person(family = \"Sass Open Source Foundation\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Greter\", \"Marcel\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Mifsud\", \"Michael\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Hampton\", \"Catlin\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Natalie\", \"Weizenbaum\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Chris\", \"Eppstein\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Adams\", \"Joseph\", role = c(\"ctb\", \"cph\"), comment = \"json.cpp\"), person(\"Trifunovic\", \"Nemanja\", role = c(\"ctb\", \"cph\"), comment = \"utf8.h\") )", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/sass/, https://github.com/rstudio/sass", + "BugReports": "https://github.com/rstudio/sass/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "SystemRequirements": "GNU make", + "Imports": [ + "fs (>= 1.2.4)", + "rlang (>= 0.4.10)", + "htmltools (>= 0.5.1)", + "R6", + "rappdirs" + ], + "Suggests": [ + "testthat", + "knitr", + "rmarkdown", + "withr", + "shiny", + "curl" + ], + "VignetteBuilder": "knitr", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Joe Cheng [aut], Timothy Mastny [aut], Richard Iannone [aut] (), Barret Schloerke [aut] (), Carson Sievert [aut, cre] (), Christophe Dervieux [ctb] (), RStudio [cph, fnd], Sass Open Source Foundation [ctb, cph] (LibSass library), Greter Marcel [ctb, cph] (LibSass library), Mifsud Michael [ctb, cph] (LibSass library), Hampton Catlin [ctb, cph] (LibSass library), Natalie Weizenbaum [ctb, cph] (LibSass library), Chris Eppstein [ctb, cph] (LibSass library), Adams Joseph [ctb, cph] (json.cpp), Trifunovic Nemanja [ctb, cph] (utf8.h)", + "Maintainer": "Carson Sievert ", + "Repository": "CRAN" + }, + "tinytex": { + "Package": "tinytex", + "Version": "0.53", + "Source": "Repository", + "Type": "Package", + "Title": "Helper Functions to Install and Maintain TeX Live, and Compile LaTeX Documents", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Devon\", \"Ryan\", role = \"ctb\", email = \"dpryan79@gmail.com\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Ethan\", \"Heinzen\", role = \"ctb\"), person(\"Fernando\", \"Cagua\", role = \"ctb\"), person() )", + "Description": "Helper functions to install and maintain the 'LaTeX' distribution named 'TinyTeX' (), a lightweight, cross-platform, portable, and easy-to-maintain version of 'TeX Live'. This package also contains helper functions to compile 'LaTeX' documents, and install missing 'LaTeX' packages automatically.", + "Imports": [ + "xfun (>= 0.29)" + ], + "Suggests": [ + "testit", + "rstudioapi" + ], + "License": "MIT + file LICENSE", + "URL": "https://github.com/rstudio/tinytex", + "BugReports": "https://github.com/rstudio/tinytex/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Yihui Xie [aut, cre, cph] (), Posit Software, PBC [cph, fnd], Christophe Dervieux [ctb] (), Devon Ryan [ctb] (), Ethan Heinzen [ctb], Fernando Cagua [ctb]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "xfun": { + "Package": "xfun", + "Version": "0.47", + "Source": "Repository", + "Type": "Package", + "Title": "Supporting Functions for Packages Maintained by 'Yihui Xie'", + "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Daijiang\", \"Li\", role = \"ctb\"), person(\"Xianying\", \"Tan\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", role = \"ctb\", email = \"salim-b@pm.me\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person() )", + "Description": "Miscellaneous functions commonly used in other packages maintained by 'Yihui Xie'.", + "Depends": [ + "R (>= 3.2.0)" + ], + "Imports": [ + "grDevices", + "stats", + "tools" + ], + "Suggests": [ + "testit", + "parallel", + "codetools", + "methods", + "rstudioapi", + "tinytex (>= 0.30)", + "mime", + "markdown (>= 1.5)", + "knitr (>= 1.47)", + "htmltools", + "remotes", + "pak", + "rhub", + "renv", + "curl", + "xml2", + "jsonlite", + "magick", + "yaml", + "qs", + "rmarkdown" + ], + "License": "MIT + file LICENSE", + "URL": "https://github.com/yihui/xfun", + "BugReports": "https://github.com/yihui/xfun/issues", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "VignetteBuilder": "knitr", + "NeedsCompilation": "yes", + "Author": "Yihui Xie [aut, cre, cph] (), Wush Wu [ctb], Daijiang Li [ctb], Xianying Tan [ctb], Salim Brüggemann [ctb] (), Christophe Dervieux [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Yihui Xie ", + "Repository": "CRAN" + }, + "yaml": { + "Package": "yaml", + "Version": "2.3.10", + "Source": "Repository", + "Type": "Package", + "Title": "Methods to Convert R Data to YAML and Back", + "Date": "2024-07-22", + "Suggests": [ + "RUnit" + ], + "Author": "Shawn P Garbett [aut], Jeremy Stephens [aut, cre], Kirill Simonov [aut], Yihui Xie [ctb], Zhuoer Dong [ctb], Hadley Wickham [ctb], Jeffrey Horner [ctb], reikoch [ctb], Will Beasley [ctb], Brendan O'Connor [ctb], Gregory R. Warnes [ctb], Michael Quinn [ctb], Zhian N. Kamvar [ctb], Charlie Gao [ctb]", + "Maintainer": "Shawn Garbett ", + "License": "BSD_3_clause + file LICENSE", + "Description": "Implements the 'libyaml' 'YAML' 1.1 parser and emitter () for R.", + "URL": "https://github.com/vubiostat/r-yaml/", + "BugReports": "https://github.com/vubiostat/r-yaml/issues", + "NeedsCompilation": "yes", + "Repository": "CRAN" + } + } +} diff --git a/renv/profiles/lesson-requirements/renv/.gitignore b/renv/profiles/lesson-requirements/renv/.gitignore new file mode 100644 index 00000000..0ec0cbba --- /dev/null +++ b/renv/profiles/lesson-requirements/renv/.gitignore @@ -0,0 +1,7 @@ +library/ +local/ +cellar/ +lock/ +python/ +sandbox/ +staging/ diff --git a/renv/profiles/lesson-requirements/renv/settings.json b/renv/profiles/lesson-requirements/renv/settings.json new file mode 100644 index 00000000..ffdbb320 --- /dev/null +++ b/renv/profiles/lesson-requirements/renv/settings.json @@ -0,0 +1,19 @@ +{ + "bioconductor.version": null, + "external.libraries": [], + "ignored.packages": [], + "package.dependency.fields": [ + "Imports", + "Depends", + "LinkingTo" + ], + "ppm.enabled": null, + "ppm.ignored.urls": [], + "r.version": null, + "snapshot.type": "implicit", + "use.cache": true, + "vcs.ignore.cellar": true, + "vcs.ignore.library": true, + "vcs.ignore.local": true, + "vcs.manage.ignores": true +} diff --git a/renv/settings.json b/renv/settings.json new file mode 100644 index 00000000..ffdbb320 --- /dev/null +++ b/renv/settings.json @@ -0,0 +1,19 @@ +{ + "bioconductor.version": null, + "external.libraries": [], + "ignored.packages": [], + "package.dependency.fields": [ + "Imports", + "Depends", + "LinkingTo" + ], + "ppm.enabled": null, + "ppm.ignored.urls": [], + "r.version": null, + "snapshot.type": "implicit", + "use.cache": true, + "vcs.ignore.cellar": true, + "vcs.ignore.library": true, + "vcs.ignore.local": true, + "vcs.manage.ignores": true +} diff --git a/site/README.md b/site/README.md index 42997e3d..0a00291c 100644 --- a/site/README.md +++ b/site/README.md @@ -1,2 +1,2 @@ This directory contains rendered lesson materials. Please do not edit files -here. +here.