diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..c292c4b08fd3efcb9494815910d67afd4ebe6b49 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,33 @@ +image: singularityware/singularity:gitlab-2.6 + +build: + script: + - /bin/bash .gitlabci/build.sh Singularity + + # step 1. build the container! + # You can add any other sregistry push commands here, and specify a client + # (and make sure your define the encrypted environment credentials in gitlab + # to push to your storage locations of choice + + - mkdir -p build && cp *.simg build + - mkdir -p build && cp Singularity* build + + # Step 2. Take a look at "artifacts" below and add the paths you want added + # You can also add the entire build folder. You can also upload to storage + # clients defined by sregistry, here are some examples + # https://singularityhub.github.io/sregistry-cli/clients + # Environment variables must be defined in CI encrypted secrets/settings + # https://code.stanford.edu/help/ci/variables/README#variables). + #- /bin/bash build.sh --uri collection/container --cli google-storage Singularity + #- /bin/bash build.sh --uri collection/container --cli google-drive Singularity + #- /bin/bash build.sh --uri collection/container --cli globus Singularity + #- /bin/bash build.sh --uri collection/container --cli registry Singularity + + # This is where you can save job artifacts + # https://docs.gitlab.com/ee/user/project/pipelines/job_artifacts.html + # You can specify the path to containers or the build folder to save. + # Don't forget to save your recipes too! + artifacts: + paths: + - build/Singularity.simg + - build/Singularity diff --git a/.gitlabci/build.sh b/.gitlabci/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd76f40e3b814ff957c6430977446855567bf15f --- /dev/null +++ b/.gitlabci/build.sh @@ -0,0 +1,175 @@ +#!/bin/bash + +# build.sh will build a Singularity container. It's not overly complicated. +# +# USAGE: build.sh --uri collection-name/container-name --cli registry Singularity +# build.sh --uri collection-name/container-name --cli registry +# build.sh Singularity + +# Copyright (C) 2017-2018 Vanessa Sochat. + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +# License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +set -o errexit +set -o nounset + +function usage() { + + echo "USAGE: build [recipe] [options]" + echo "" + echo "OPTIONS: + + Image Format + --uri -u if uploading, a uri to give to sregistry + --cli -c the sregistry client to use (if uploading) + --help -h show this help and exit + " +} + +# --- Option processing -------------------------------------------------------- + +uri="" +cli="" +tag="" + +while true; do + case ${1:-} in + -h|--help|help) + usage + exit 0 + ;; + -u|-uri) + shift + uri="${1:-}" + shift + ;; + -t|--tag) + shift + tag="${1:-}" + shift + ;; + -c|--cli) + shift + cli="${1:-}" + shift + ;; + \?) printf "illegal option: -%s\n" "${1:-}" >&2 + usage + exit 1 + ;; + -*) + printf "illegal option: -%s\n" "${1:-}" >&2 + usage + exit 1 + ;; + *) + break; + ;; + esac +done + +################################################################################ +### Recipe File ################################################################ +################################################################################ + + +if [ $# == 0 ] ; then + recipe="Singularity" +else + recipe=$1 +fi + +echo "" +echo "Image Recipe: ${recipe}" + + +################################################################################ +### Storage Client ############################################################# +################################################################################ + +is_valid_client () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +# Test if client is valid + +clients=("google-storage" "registry" "globus" "dropbox" "google-drive") + +if [ "${cli}" != "" ]; then + is_valid_client "${cli}" "${clients[@]}" + if [ $? -ne 0 ]; then + echo "${cli} is not a valid choice! Choose from ${clients[@]}"; + exit 1 + fi + echo "Storage Client: ${cli}" +else + echo "Storage Client: none" +fi + + +################################################################################ +### Build! ##################################################################### +################################################################################ + +# Continue if the image recipe is found + +if [ -f "$recipe" ]; then + + imagefile="${recipe}.simg" + + echo "Creating $imagefile using $recipe..." + singularity build $imagefile $recipe + + # If the image is successfully built, test it and upload (examples) + + if [ -f "${imagefile}" ]; then + + # Example testing using run (you could also use test command) + + echo "Testing the image... Marco!" + singularity exec $imagefile echo "Polo!" + + # Example sregistry commands to push to endpoints + + if [ "${cli}" != "" ]; then + + # If the uri isn't provided, he gets a robot name + if [ "${uri}" == "" ]; then + uri=$(python -c "from sregistry.logger.namer import RobotNamer; bot=RobotNamer(); print(bot.generate())") + fi + + # If a tag is provided, add to uri + if [ "${tag}" != "" ]; then + uri="${uri}:${tag}" + fi + + echo "Pushing ${uri} to ${cli}://" + echo "SREGISTRY_CLIENT=${cli} sregistry push --name ${uri} ${imagefile}" + SREGISTRY_CLIENT="${cli}" sregistry push --name "${uri}" "${imagefile}" + + else + echo "Skipping upload. Image $imagefile is finished!" + fi + + fi + +else + + echo "Singularity recipe ${recipe} not found!" + exit 1 + +fi diff --git a/.gitlabci/setup.sh b/.gitlabci/setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..44d2ef301543f2bfd1af041bdc21f0a400949843 --- /dev/null +++ b/.gitlabci/setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +apt-get update && apt-get install -y wget git \ + build-essential \ + squashfs-tools \ + libtool \ + autotools-dev \ + libarchive-dev \ + automake \ + autoconf \ + uuid-dev \ + libssl-dev + + +sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers + +# Check Python + +echo "Python Version:" +python --version +pip install sregistry[all] +sregistry version + +echo "sregistry Version:" + +# Install Singularity + +cd /tmp && \ + git clone -b vault/release-2.5 https://www.github.com/sylabs/singularity.git + cd singularity && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local && \ + make && make install diff --git a/.gitlabci/sregistry-gitlab.png b/.gitlabci/sregistry-gitlab.png new file mode 100644 index 0000000000000000000000000000000000000000..a14e917a20fb32011329af6bc6abd04012acd0d6 Binary files /dev/null and b/.gitlabci/sregistry-gitlab.png differ diff --git a/.gitlabci/sregistry-gitlab.xcf b/.gitlabci/sregistry-gitlab.xcf new file mode 100644 index 0000000000000000000000000000000000000000..f2742162bfc602f92d3d9c5cd573f29e334e042d Binary files /dev/null and b/.gitlabci/sregistry-gitlab.xcf differ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9743abd38a2228666c9d76377f3aaf26682bd80a --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, Vanessa Sochat +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 200b90b4812742cb61f1975fc0098c311ba4ea1c..e71a94c9072d12660e9a8772c700f5ddcc35872c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,107 @@ -# NEXW_2019_General-Singularity-Image +# Singularity Builder GitLab -Recipe to build a general-use Singularity image for bioinformatics and cellular genomics. \ No newline at end of file + + +This is a simple example of how you can achieve: + + - version control of your recipes + - versioning to include image hash *and* commit id + - build of associated container and + - push to a storage endpoint (or GitLab artifact) + +for a reproducible build workflow. + +**Why should this be managed via GitLab?** + +GitLab, by way of easy integration with continuous integration, is an easy way +to have a workflow set up where multiple people can collaborate on a container recipe, +the recipe can be tested (with whatever testing you need), discussed in pull requests, +and then finally pushed to be a GitLab artifact, to your storage of choice +or to Singularity Registry. + +**Why should I use this instead of a service?** + +You could use a remote builder, but if you do the build in a continuous integration +service you get complete control over it. This means everything from the version of +Singularity to use, to the tests that you run for your container. You have a lot more +freedom in the rate of building, and organization of your repository, because it's you +that writes the configuration. Although the default would work for most, you can +edit the build, setup, and circle configuration file in the +[.gitlabci](.gitlabci) folder to fit your needs. + +## Quick Start + +Add your Singularity recipes to this repository, and edit the build commands in +the [build.sh](.gitlabci/build.sh) file. This is where you can specify endpoints +(Singularity Registry, Dropbox, Google Storage, AWS) along with container names +(the uri) and tag. You can build as many recipes as you like, just add another line! + +```yaml + # recipe relative to repository base + - /bin/bash .gitlabci/build.sh Singularity + - /bin/bash .gitlabci/build.sh --uri collection/container --tag tacos --cli google-storage Singularity + - /bin/bash .gitlabci/build.sh --uri collection/container --cli google-drive Singularity + - /bin/bash .gitlabci/build.sh --uri collection/container --cli globus Singularity + - /bin/bash .gitlabci/build.sh --uri collection/container --cli registry Singularity +``` + +For each client that you use, required environment variables (e.g., credentials to push, +or interact with the API) must be defined in the (encrypted) Travis environment. To +know what variables to define, along with usage for the various clients, see +the [client specific pages](https://singularityhub.github.io/sregistry-cli/clients) + +## Detailed Started + +### 0. Fork this repository + +You can clone and tweak, but it's easiest likely to get started with our example +files and edit them as you need. + +### 1. Get to Know GitLab + +We will be working with [GitLab](https://www.gitlab.com) repositories, which have +a built-in Continuous Integration service. Note that if you aren't using GitLab.com +or need to configure or enable runners, you can do this by going to your project page, +and clicking on the Settings --> CI/CD and selecting your "Runners." Typically a runner +is some external service, or the GitLab shared runners. I used GitLab.com so the +shared runners were already active for me. + +### 2. Add your Recipe(s) + +For the example here, we have a single recipe named "Singularity" that is provided +as an input argument to the [build script](.gitlabci/build.sh). You could add another +recipe, and then of course call the build to happen more than once. +The build script will name the image based on the recipe, and you of course +can change this. Just write the path to it (relative to the repository base) in +your [.gitlab-ci.yml](.gitlab-ci.yml). + + +### 3. Configure Singularity + +The basic steps to [setup](.gitlabci/setup.sh) the build are the following: + + - Install Singularity, we use the release 2.6 branch as it was the last to not be written in GoLang. You could of course change the lines in [setup.sh](.gitlabci/setup.sh) to use a specific tagged release, an older version, or development version. + - Install the sregistry client, if needed. The [sregistry client](https://singularityhub.github.io/sregistry-cli) allows you to issue a command like "sregistry push ..." to upload a finished image to one of your cloud / storage endpoints. By default, the push won't happen, and you will just build an image using the CI. + +### 4. Configure the Build + +The basic steps for the [build](.gitlabci/build.sh) are the following: + + - Running build.sh with no inputs will default to a recipe called "Singularity" in the base of the repository. You can provide an argument to point to a different recipe path, always relative to the base of your repository. + - If you want to define a particular unique resource identifier for a finished container (to be uploaded to your storage endpoint) you can do that with `--uri collection/container`. If you don't define one, a robot name will be generated. + - You can add `--uri` to specify a custom name, and this can include the tag, OR you can specify `--tag` to go along with a name without one. It depends on which is easier for you. + - If you add `--cli` then this is telling the build script that you have defined the [needed environment variables](https://circleci.com/docs/2.0/env-vars/) for your [client of choice](https://singularityhub.github.io/sregistry-cli/clients) and you want successful builds to be pushed to your storage endpoint. See [here](https://singularityhub.github.io/sregistry-cli/clients) for a list of current client endpoints, or roll your own! + +See the [.gitlab-ci.yml](.gitlab-ci.yml) for examples of this build.sh command (commented out). If there is some cloud service that you'd like that is not provided, please [open an issue](https://www.github.com/singularityhub/sregistry-cli/issues). + +### 5. Pull your Container + +If you want to pull from the artifacts folder in GitLab, check out the +[Singularity Registry Client](https://www.github.com/singularityhub/sregistry-cli), and +specifically the [GitLab client](https://singularityhub.github.io/client-gitlab). The client +will help you to pull a container stored as an artifact given that you know the job identifier, +and if you change the job name (default is build) that will need to be provided too! +See the [GitLab client](https://singularityhub.github.io/client-gitlab) pages for more information +on how to do this. If you don't want to use GitLab artifacts for deployment, this is okay too! +Just remember that if you are using Singularity Registry Client (or GitLab for that matter) +to deploy a container elsewhere, you likely need to configure environment variables to help authenticate you. Instructions can be found [here](https://code.stanford.edu/help/ci/variables/README#variables). diff --git a/Singularity b/Singularity new file mode 100644 index 0000000000000000000000000000000000000000..acecc191e482458e8e156ff9e7646db481b98e2a --- /dev/null +++ b/Singularity @@ -0,0 +1,5 @@ +Bootstrap: docker +From: ubuntu:16.04 + +%runscript + exec echo "Polo $@!"