├── .buildkite ├── duniverse.yml ├── obi-batch-install.sh ├── obi-binary.sh ├── obi-ci-install.sh ├── obi-gather-results.sh ├── obi-index.sh ├── obi-list-all-pkgs.sh ├── opam-admin.sh ├── opam-admin.yml ├── opam-bulk.yml ├── opam-containers.yml ├── opam-index.yml ├── opam-matrix.sh └── opam-matrix.yml ├── .duniverse ├── dune.sxp └── opam.sxp ├── .gitignore ├── .travis.yml ├── CHANGES.md ├── CONTRIBUTING.md ├── LICENSE.md ├── METADATA.md ├── Makefile ├── README.md ├── dune-project ├── obi-dune.opam ├── obi-gitlab.opam ├── obi.opam ├── opam-ci.opam ├── src ├── lib │ ├── dune │ ├── obi.ml │ ├── obi.mli │ ├── ocaml_version_sexp.ml │ └── opamJsonActions.ml ├── obi-dune │ ├── README.md │ ├── dune │ ├── obi_dune.ml │ └── scripts │ │ └── obi-ci-install.sh ├── obi-gitlab │ ├── dune │ └── gitlab.ml └── opam-ci │ ├── dune │ ├── main.ml │ ├── repos.ml │ └── user.ml └── test └── opamJson ├── dune ├── owl-fail.json └── test.ml /.buildkite/duniverse.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: 'Clone Duniverse' 3 | command: 4 | - opam pin add -ny jbuilder --dev 5 | - opam pin add -ny dune https://github.com/ocaml/dune.git 6 | - opam pin add -ny duniverse https://github.com/avsm/duniverse.git 7 | - opam --yes depext -uiy dune 8 | - opam --yes depext -iy duniverse 9 | - git clone git://github.com/ocaml/opam /home/opam/opam 10 | - cd /home/opam/opam 11 | - opam exec -- duniverse opam opam-devel dune-release utop bun odoc opam-depext -vv && opam exec -- duniverse lock -vv && cat .duniverse*.sxp 12 | agents: 13 | docker: "true" 14 | arch: "amd64" 15 | plugins: 16 | docker#v1.1.1: 17 | image: "ocaml/opam2" 18 | always_pull: true 19 | -------------------------------------------------------------------------------- /.buildkite/obi-batch-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # Either build an artefact directly or spawn more if there are a lot of packages 3 | 4 | hub=$1 5 | tag=$2 6 | arch=$3 7 | pkg=$4 8 | 9 | docker pull $hub:$tag 10 | docker run --rm $hub:$tag opam list -s --all-versions --installable $pkg | sort > toinstall.txt 11 | docker run --rm $hub:$tag opam list -s --all-versions $pkg | sort > allpossible.txt 12 | ls -la *.txt 13 | echo toinstall: 14 | cat toinstall.txt 15 | echo allpossible: 16 | cat allpossible.txt 17 | echo diffing: 18 | comm -23 allpossible.txt toinstall.txt > uninstallable.txt 19 | 20 | echo --- Checking uninstallable 21 | wc -l uninstallable.txt 22 | echo Uninstallable packages in this switch are: 23 | cat uninstallable.txt 24 | 25 | echo --- Checking installable 26 | echo To install: 27 | cat toinstall.txt 28 | 29 | numlines=`wc -l < toinstall.txt` 30 | 31 | # workaround: bap takes forever to build in serial 32 | if [[ $pkg == bap-* ]]; then 33 | echo --- Special case for bap 34 | tospawn=`tail -n +2 toinstall.txt` 35 | tobuild=`head -1 toinstall.txt` 36 | elif [ $numlines -gt 4 ]; then 37 | echo --- Splitting packages 38 | tospawn=`tail -n +5 toinstall.txt` 39 | tobuild=`head -4 toinstall.txt` 40 | else 41 | tobuild=`cat toinstall.txt` 42 | tospawn= 43 | fi 44 | 45 | if [ ! -z "$tospawn" ]; then 46 | echo --- Spawning jobs for excess packages 47 | echo steps: > tobuild.yml 48 | for i in $tospawn; do 49 | cat <> tobuild.yml 50 | - label: ":right-facing_fist:" 51 | agents: 52 | docker: "true" 53 | arch: "$arch" 54 | os: "linux" 55 | timeout_in_minutes: 60 56 | retry: 57 | automatic: true 58 | command: 59 | - mkdir -p $tag/results 60 | - docker pull $hub:$tag 61 | - rm -rf output && mkdir -p output 62 | - docker run --privileged --rm -v \`pwd\`/output:/mnt -v opam2-archive:/home/opam/.opam/download-cache $hub:$tag obi-ci-install $i 63 | - cp output/$i.txt $tag/results/$i.txt 64 | - cp output/$i.json $tag/results/$i.json 65 | - buildkite-agent artifact upload $tag/results/$i.txt 66 | - buildkite-agent artifact upload $tag/results/$i.json 67 | - rm -rf output 68 | EOL 69 | done 70 | cat tobuild.yml 71 | buildkite-agent pipeline upload tobuild.yml 72 | fi 73 | 74 | echo "--- Building packages" 75 | for i in $tobuild; do 76 | echo --- Building $i 77 | mkdir -p $tag/results 78 | docker pull $hub:$tag 79 | rm -rf output && mkdir -p output 80 | docker run --privileged --rm -v `pwd`/output:/mnt -v opam2-archive:/home/opam/.opam/download-cache $hub:$tag obi-ci-install $i 81 | cp output/$i.txt $tag/results/$i.txt 82 | cp output/$i.json $tag/results/$i.json 83 | buildkite-agent artifact upload $tag/results/$i.txt 84 | buildkite-agent artifact upload $tag/results/$i.json 85 | rm -rf output 86 | done 87 | 88 | echo "--- Recording uninstallable results" 89 | for i in `cat uninstallable.txt`; do 90 | echo --- Building $i 91 | mkdir -p $tag/results 92 | rm -rf output && mkdir -p output 93 | docker run --privileged --rm -v `pwd`/output:/mnt -v opam2-archive:/home/opam/.opam/download-cache $hub:$tag obi-ci-install $i 94 | cp output/$i.txt $tag/results/$i.txt 95 | cp output/$i.json $tag/results/$i.json 96 | buildkite-agent artifact upload $tag/results/$i.txt 97 | buildkite-agent artifact upload $tag/results/$i.json 98 | rm -rf output 99 | done 100 | -------------------------------------------------------------------------------- /.buildkite/obi-binary.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | hub=${1:-ocaml/opam2-staging} 4 | tag=${2:-obi-buildkite} 5 | arches="amd64 arm64 ppc64le" 6 | 7 | dockerfile_for_arch() { 8 | arch=$1 9 | DFILE="FROM ocaml/opam2 10 | RUN sudo apt-get update && sudo apt-get -y install m4 pkg-config 11 | RUN opam switch 4.06 12 | COPY . /home/opam/src/ 13 | RUN sudo chown -R opam /home/opam/src 14 | RUN git -C /home/opam/opam-repository pull origin master && opam update 15 | RUN opam pin add -n --dev dockerfile-opam 16 | RUN opam pin add -n --dev ocaml-version 17 | RUN opam install -y -j10 --deps-only /home/opam/src 18 | RUN cd /home/opam/src && opam exec -- jbuilder build 19 | RUN sudo cp /home/opam/src/_build/install/default/bin/obi-buildkite /usr/bin/obi-buildkite 20 | RUN sudo chmod a+x /usr/bin/obi-buildkite 21 | " 22 | echo -n " - \"echo " 23 | printf '%q' "$DFILE" | sed -e 's/^\$/$$/g' 24 | echo "> Dockerfile\"" 25 | } 26 | 27 | build_one_arch() { 28 | arch=$1 29 | cat < $1.txt 2>&1 14 | exitcode=$? 15 | endtime=`date +%s` 16 | echo $exitcode $starttime $endtime >> $1.txt 17 | sudo cp $1.json /mnt/$1.json 18 | sudo cp $1.txt /mnt/$1.txt 19 | -------------------------------------------------------------------------------- /.buildkite/obi-gather-results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | tag=$1 4 | arch=$2 5 | ov=$3 6 | distro=$4 7 | rev=$5 8 | 9 | rm -rf $tag 10 | buildkite-agent artifact download "$tag/results/*" . 11 | echo $arch > $tag/arch 12 | echo $ov > $tag/ov 13 | echo $distro > $tag/distro 14 | echo $rev > $tag/rev 15 | tar -jcvf results-$tag.tar.bz2 $tag 16 | buildkite-agent artifact upload results-$tag.tar.bz2 17 | if [ -d obi-logs ]; then 18 | git -C obi-logs clean -dxf 19 | git -C obi-logs fetch git@github.com:ocaml/obi-logs 20 | git -C obi-logs reset --hard origin/builds 21 | else 22 | git clone -b builds --depth=1 git@github.com:ocaml/obi-logs 23 | fi 24 | docker pull ocaml/opam2-staging:obi-buildkite 25 | docker run -it -v `pwd`/$tag:/home/opam/$tag -v `pwd`/obi-logs:/home/opam/obi-logs ocaml/opam2-staging:obi-buildkite obi-buildkite process -vv -i /home/opam/$tag -o /home/opam/obi-logs 26 | ssh-add -D && ssh-add ~/.ssh/id_rsa.bulk && ssh-add -l 27 | cd obi-logs && find . -type f && git add . && git pull --commit && git commit -m "update $tag" && git push -u origin builds 28 | rm -rf obi-logs 29 | -------------------------------------------------------------------------------- /.buildkite/obi-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | echo --- Cloning opam repository 4 | cd /home/opam 5 | git -C opam-repository pull origin master 6 | opam update 7 | echo --- Cloning obi-logs 8 | rm -rf obi-logs 9 | git clone -b builds --depth=1 git://github.com/ocaml/obi-logs 10 | echo --- Generating index 11 | obi-buildkite index -i obi-logs -r opam-repository -vv > index.sxp 12 | echo --- Uploading result 13 | buildkite-agent artifact upload index.sxp 14 | cd obi-logs && buildkite-agent artifact upload maintainers.sxp && buildkite-agent artifact upload tags.sxp 15 | 16 | ssh-add -D && ssh-add ~/.ssh/id_rsa.bulk && ssh-add -l 17 | cd obi-logs && find . -type f && git add . && git pull --commit && git commit -m "update $tag" && git push -u origin builds 18 | rm -rf obi-logs 19 | -------------------------------------------------------------------------------- /.buildkite/obi-list-all-pkgs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | hub=$1 3 | tag=$2 4 | rev=$3 5 | buildkite-agent artifact download "$tag/*" . 6 | echo --- Building bulk image 7 | docker build --no-cache --rm --pull -t $hub:$tag -f $tag/Dockerfile.$rev . 8 | echo --- Pushing bulk image to Hub 9 | docker push $hub:$tag 10 | echo --- Listing packages to buikd 11 | mkdir -p $tag 12 | docker run $hub:$tag opam list -a -s | sort -R > $tag/pkgs.txt 13 | buildkite-agent artifact upload $tag/pkgs.txt 14 | echo --- Generating and spawning build jobs 15 | cat $tag/pkgs.txt | xargs -n 1 -I __NAME__ sh -c "sed -e 's/__PKG__/__NAME__/g' < $tag/template.yml > $tag/build-__NAME__.yml" 16 | echo steps: > all.yml 17 | (for i in `cat $tag/pkgs.txt`; do 18 | cat $tag/build-$i.yml; 19 | done) | grep -v ^steps >> all.yml 20 | buildkite-agent pipeline upload all.yml 21 | -------------------------------------------------------------------------------- /.buildkite/opam-admin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | rev=$1 4 | srev=`echo $rev | cut -c1-6` 5 | echo 'steps:' 6 | 7 | cmd() { 8 | cmd=$1 9 | cat <&1 | tee $cmd.txt 19 | - buildkite-agent artifact upload $cmd.txt 20 | plugins: 21 | docker#v1.1.1: 22 | image: "ocaml/opam2" 23 | always_pull: true 24 | env: 25 | OPAMCOLOR: "never" 26 | EOL 27 | } 28 | 29 | cmd "lint" 30 | cmd "cache" 31 | cmd "check" 32 | 33 | cat < rev' 6 | - '.buildkite/opam-admin.sh `cat rev` | buildkite-agent pipeline upload' 7 | -------------------------------------------------------------------------------- /.buildkite/opam-bulk.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: 'Opam Repo Rev' 3 | command: 4 | - 'cd /home/opam && obi-buildkite bulk --arch=$ARCH --opam-repo-rev=$OPAM_REPO_REV -vv' 5 | - 'cd /home/opam/_results && buildkite-agent artifact upload "bulk-*/*"' 6 | - 'cat /home/opam/_results/bulk.yml | buildkite-agent pipeline upload' 7 | agents: 8 | docker: "true" 9 | os: "linux" 10 | pusher: "true" 11 | plugins: 12 | docker#v1.1.1: 13 | image: "ocaml/opam2-staging:obi-buildkite" 14 | workdir: /src 15 | always_pull: true 16 | environment: 17 | - DISTRO=${DISTRO?} 18 | - OCAML_VERSION=${OCAML_VERSION?} 19 | - OPAM_REPO_REV=${OPAM_REPO_REV?} 20 | - ARCH=${ARCH?} 21 | docker-login#v1.0.0: 22 | username: avsm 23 | -------------------------------------------------------------------------------- /.buildkite/opam-containers.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: 'Build' 3 | command: 4 | - 'obi-buildkite gen -vv -o /home/opam/_results' 5 | - 'cd /home/opam/_results && buildkite-agent artifact upload "phase*-*/*"' 6 | - 'cd /home/opam/_results && buildkite-agent artifact upload "README.md"' 7 | - 'cat /home/opam/_results/phase1.yml | buildkite-agent pipeline upload' 8 | concurrency: 1 9 | concurrency_group: "opam/build" 10 | agents: 11 | docker: "true" 12 | os: "linux" 13 | plugins: 14 | docker#v1.1.1: 15 | image: "ocaml/opam2-staging:obi-buildkite" 16 | always_pull: true 17 | - wait 18 | - command: 19 | - 'rm -rf infrastructure.wiki/' 20 | - 'git clone git@github.com:ocaml/infrastructure.wiki' 21 | - 'buildkite-agent artifact download "README.md" .' 22 | - 'mv README.md infrastructure.wiki/Containers.md' 23 | - 'git config --global user.email "bactrian@ocaml.org"' 24 | - 'git config --global user.name "Bactrian the Build Bot"' 25 | - '(cd infrastructure.wiki && git add . && git commit -m containers -a && git push) || true' 26 | label: ':rocket: README' 27 | concurrency: 1 28 | concurrency_group: "opam/push" 29 | agents: 30 | githubpusher: true 31 | -------------------------------------------------------------------------------- /.buildkite/opam-index.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: 'Build Index' 3 | command: 4 | - ./.buildkite/obi-index.sh 5 | agents: 6 | docker: "true" 7 | os: "linux" 8 | plugins: 9 | docker#v1.1.1: 10 | image: "ocaml/opam2-staging:obi-buildkite" 11 | always_pull: true 12 | - wait 13 | - label: 'Push Index' 14 | command: 15 | - ssh-add -D && ssh-add ~/.ssh/id_rsa.bulk && ssh-add -l 16 | - git config --global user.email 'bactrian@ocaml.org' && git config --global user.name 'Bactrian the Build Bot' 17 | - rm -rf index 18 | - git clone git@github.com:ocaml/obi-logs index --reference . 19 | - git -C index checkout --orphan index 20 | - git -C index reset 21 | - git -C index clean -dxf 22 | - buildkite-agent artifact download 'index.sxp' index 23 | - git -C index add . 24 | - git -C index commit -m 'BuildKite Update' || true 25 | - git -C index push origin index -f 26 | - rm -rf index 27 | retry: 28 | automatic: 29 | limit: 3 30 | agents: 31 | githubpusher: "true" 32 | - label: 'Push Maintainer and Tag Cache' 33 | command: 34 | - ssh-add -D && ssh-add ~/.ssh/id_rsa.bulk && ssh-add -l 35 | - git config --global user.email 'bactrian@ocaml.org' && git config --global user.name 'Bactrian the Build Bot' 36 | - rm -rf obi-logs 37 | - git clone git@github.com:ocaml/obi-logs -b builds 38 | - buildkite-agent artifact download 'maintainers.sxp' obi-logs 39 | - buildkite-agent artifact download 'tags.sxp' obi-logs 40 | - git -C obi-logs add tags.sxp maintainers.sxp 41 | - git -C obi-logs commit -m 'BuildKite Cache Update' || true 42 | - git -C obi-logs push origin builds 43 | - rm -rf obi-logs 44 | retry: 45 | automatic: 46 | limit: 3 47 | agents: 48 | githubpusher: "true" 49 | -------------------------------------------------------------------------------- /.buildkite/opam-matrix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | rev=$1 4 | srev=`echo $rev | cut -c1-6` 5 | echo 'steps:' 6 | 7 | build() { 8 | distro=$1 9 | ov=$2 10 | arch=$3 11 | cat < rev' 8 | - '.buildkite/opam-matrix.sh `cat rev` | buildkite-agent pipeline upload' 9 | -------------------------------------------------------------------------------- /.duniverse/dune.sxp: -------------------------------------------------------------------------------- 1 | ((repos 2 | (((dir ezjsonm) (upstream https://github.com/mirage/ezjsonm.git) 3 | (ref 0.6.0)) 4 | ((dir sexplib) (upstream https://github.com/janestreet/sexplib.git) 5 | (ref v0.11.0)) 6 | ((dir stdio) (upstream https://github.com/janestreet/stdio.git) 7 | (ref v0.11.0)) 8 | ((dir ocamlgraph) (upstream git://github.com/dune-universe/ocamlgraph.git) 9 | (ref duniverse-v1.8.8)) 10 | ((dir sexplib0) (upstream https://github.com/janestreet/sexplib0.git) 11 | (ref v0.11.0)) 12 | ((dir uutf) (upstream git://github.com/dune-universe/uutf.git) 13 | (ref duniverse-v1.0.1)) 14 | ((dir ocamlfind) (upstream git://github.com/dune-universe/lib-findlib.git) 15 | (ref duniverse-1.8.0)) 16 | ((dir topkg) (upstream git://github.com/dune-universe/topkg.git) 17 | (ref duniverse-v0.9.1)) 18 | ((dir rresult) (upstream git://github.com/dune-universe/rresult.git) 19 | (ref duniverse-v0.5.0)) 20 | ((dir uri) (upstream https://github.com/mirage/ocaml-uri.git) 21 | (ref v1.9.7)) 22 | ((dir csv) (upstream https://github.com/Chris00/ocaml-csv.git) (ref 2.1)) 23 | ((dir opam-core) (upstream git://github.com/dune-universe/opam.git) 24 | (ref duniverse-2.0.0-rc4)) 25 | ((dir base) (upstream https://github.com/janestreet/base.git) 26 | (ref v0.11.1)) 27 | ((dir cstruct) (upstream https://github.com/mirage/ocaml-cstruct.git) 28 | (ref v3.2.1)) 29 | ((dir ptime) (upstream git://github.com/dune-universe/ptime.git) 30 | (ref duniverse-v0.8.4)) 31 | ((dir ppx_sexp_conv) 32 | (upstream https://github.com/janestreet/ppx_sexp_conv.git) (ref v0.11.2)) 33 | ((dir ppxlib) (upstream https://github.com/ocaml-ppx/ppxlib.git) 34 | (ref 0.3.0)) 35 | ((dir dockerfile) (upstream https://github.com/avsm/ocaml-dockerfile.git) 36 | (ref v5.0.0)) 37 | ((dir yaml) (upstream https://github.com/avsm/ocaml-yaml.git) 38 | (ref v0.2.1)) 39 | ((dir fmt) (upstream git://github.com/dune-universe/fmt.git) 40 | (ref duniverse-v0.8.5)) 41 | ((dir bos) (upstream git://github.com/dune-universe/bos.git) 42 | (ref duniverse-v0.2.0)) 43 | ((dir ocaml-migrate-parsetree) 44 | (upstream https://github.com/ocaml-ppx/ocaml-migrate-parsetree.git) 45 | (ref v1.0.11)) 46 | ((dir ppx_derivers) 47 | (upstream https://github.com/ocaml-ppx/ppx_derivers.git) (ref 1.0)) 48 | ((dir ocaml-compiler-libs) 49 | (upstream https://github.com/janestreet/ocaml-compiler-libs.git) 50 | (ref v0.11.0)) 51 | ((dir stringext) (upstream https://github.com/rgrinberg/stringext.git) 52 | (ref 1.5.0)) 53 | ((dir jsonm) (upstream git://github.com/dune-universe/jsonm.git) 54 | (ref duniverse-v1.0.1)) 55 | ((dir result) (upstream https://github.com/janestreet/result.git) 56 | (ref 1.3)) 57 | ((dir cppo) (upstream https://github.com/mjambon/cppo.git) (ref v1.6.4)) 58 | ((dir configurator) 59 | (upstream https://github.com/janestreet/configurator.git) (ref v0.11.0)) 60 | ((dir cmdliner) (upstream git://github.com/dune-universe/cmdliner.git) 61 | (ref duniverse-v1.0.2)) 62 | ((dir hex) (upstream https://github.com/mirage/ocaml-hex.git) 63 | (ref v1.2.0)) 64 | ((dir parsexp) (upstream https://github.com/janestreet/parsexp.git) 65 | (ref v0.11.0)) 66 | ((dir logs) (upstream git://github.com/dune-universe/logs.git) 67 | (ref duniverse-v0.6.2)) 68 | ((dir re) (upstream https://github.com/ocaml/ocaml-re.git) (ref 1.7.3)) 69 | ((dir astring) (upstream git://github.com/dune-universe/astring.git) 70 | (ref duniverse-v0.8.3)) 71 | ((dir fpath) (upstream git://github.com/dune-universe/fpath.git) 72 | (ref duniverse-v0.7.2)) 73 | ((dir ocaml-version) (upstream https://github.com/avsm/ocaml-version.git) 74 | (ref v0.3.0))))) -------------------------------------------------------------------------------- /.duniverse/opam.sxp: -------------------------------------------------------------------------------- 1 | ((roots 2 | (((name obi)) ((name obi-buildkite)) ((name obi-gitlab)) ((name opam-ci)))) 3 | (excludes 4 | (((name obi)) ((name obi-buildkite)) ((name obi-gitlab)) ((name opam-ci)))) 5 | (pins ()) 6 | (pkgs 7 | (((package ((name ocamlfind) (version (1.8.0)))) 8 | (dev_repo (Duniverse_fork lib-findlib)) (tag (1.8.0))) 9 | ((package ((name result) (version (1.3)))) 10 | (dev_repo (Github (janestreet result))) (tag (1.3))) 11 | ((package ((name ocaml-migrate-parsetree) (version (1.0.11)))) 12 | (dev_repo (Github (ocaml-ppx ocaml-migrate-parsetree))) (tag (v1.0.11))) 13 | ((package ((name sexplib0) (version (v0.11.0)))) 14 | (dev_repo (Github (janestreet sexplib0))) (tag (v0.11.0))) 15 | ((package ((name cppo) (version (1.6.4)))) 16 | (dev_repo (Github (mjambon cppo))) (tag (v1.6.4))) 17 | ((package ((name base) (version (v0.11.1)))) 18 | (dev_repo (Github (janestreet base))) (tag (v0.11.1))) 19 | ((package ((name ppx_derivers) (version (1.0)))) 20 | (dev_repo (Github (ocaml-ppx ppx_derivers))) (tag (1.0))) 21 | ((package ((name ocaml-compiler-libs) (version (v0.11.0)))) 22 | (dev_repo (Github (janestreet ocaml-compiler-libs))) (tag (v0.11.0))) 23 | ((package ((name stdio) (version (v0.11.0)))) 24 | (dev_repo (Github (janestreet stdio))) (tag (v0.11.0))) 25 | ((package ((name ppxlib) (version (0.3.0)))) 26 | (dev_repo (Github (ocaml-ppx ppxlib))) (tag (0.3.0))) 27 | ((package ((name topkg) (version (0.9.1)))) 28 | (dev_repo (Duniverse_fork topkg)) (tag (v0.9.1))) 29 | ((package ((name ppx_sexp_conv) (version (v0.11.2)))) 30 | (dev_repo (Github (janestreet ppx_sexp_conv))) (tag (v0.11.2))) 31 | ((package ((name re) (version (1.7.3)))) 32 | (dev_repo (Github (ocaml ocaml-re))) (tag (1.7.3))) 33 | ((package ((name parsexp) (version (v0.11.0)))) 34 | (dev_repo (Github (janestreet parsexp))) (tag (v0.11.0))) 35 | ((package ((name configurator) (version (v0.11.0)))) 36 | (dev_repo (Github (janestreet configurator))) (tag (v0.11.0))) 37 | ((package ((name sexplib) (version (v0.11.0)))) 38 | (dev_repo (Github (janestreet sexplib))) (tag (v0.11.0))) 39 | ((package ((name cmdliner) (version (1.0.2)))) 40 | (dev_repo (Duniverse_fork cmdliner)) (tag (v1.0.2))) 41 | ((package ((name fmt) (version (0.8.5)))) (dev_repo (Duniverse_fork fmt)) 42 | (tag (v0.8.5))) 43 | ((package ((name uutf) (version (1.0.1)))) 44 | (dev_repo (Duniverse_fork uutf)) (tag (v1.0.1))) 45 | ((package ((name cstruct) (version (3.2.1)))) 46 | (dev_repo (Github (mirage ocaml-cstruct))) (tag (v3.2.1))) 47 | ((package ((name astring) (version (0.8.3)))) 48 | (dev_repo (Duniverse_fork astring)) (tag (v0.8.3))) 49 | ((package ((name logs) (version (0.6.2)))) 50 | (dev_repo (Duniverse_fork logs)) (tag (v0.6.2))) 51 | ((package ((name stringext) (version (1.5.0)))) 52 | (dev_repo (Github (rgrinberg stringext))) (tag (1.5.0))) 53 | ((package ((name ptime) (version (0.8.4)))) 54 | (dev_repo (Duniverse_fork ptime)) (tag (v0.8.4))) 55 | ((package ((name rresult) (version (0.5.0)))) 56 | (dev_repo (Duniverse_fork rresult)) (tag (v0.5.0))) 57 | ((package ((name uri) (version (1.9.7)))) 58 | (dev_repo (Github (mirage ocaml-uri))) (tag (v1.9.7))) 59 | ((package ((name integers) (version (0.2.2)))) 60 | (dev_repo (Github (ocamllabs ocaml-integers))) (tag (v0.2.2)) 61 | (is_dune false)) 62 | ((package ((name jsonm) (version (1.0.1)))) 63 | (dev_repo (Duniverse_fork jsonm)) (tag (v1.0.1))) 64 | ((package ((name hex) (version (1.2.0)))) 65 | (dev_repo (Github (mirage ocaml-hex))) (tag (v1.2.0))) 66 | ((package ((name ocamlgraph) (version (1.8.8)))) 67 | (dev_repo (Duniverse_fork ocamlgraph)) (tag (v1.8.8))) 68 | ((package ((name ctypes) (version (0.14.0)))) 69 | (dev_repo (Github (ocamllabs ocaml-ctypes))) (tag (0.14.0)) 70 | (is_dune false)) 71 | ((package ((name ezjsonm) (version (0.6.0)))) 72 | (dev_repo (Github (mirage ezjsonm))) (tag (0.6.0))) 73 | ((package ((name opam-core) (version (2.0.0)))) 74 | (dev_repo (Duniverse_fork opam)) (tag (2.0.0-rc4))) 75 | ((package ((name csv) (version (2.1)))) 76 | (dev_repo (Github (Chris00 ocaml-csv))) (tag (2.1))) 77 | ((package ((name fpath) (version (0.7.2)))) 78 | (dev_repo (Duniverse_fork fpath)) (tag (v0.7.2))) 79 | ((package ((name dockerfile) (version (5.0.0)))) 80 | (dev_repo (Github (avsm ocaml-dockerfile))) (tag (v5.0.0))) 81 | ((package ((name ocaml-version) (version (0.3.0)))) 82 | (dev_repo (Github (avsm ocaml-version))) (tag (v0.3.0))) 83 | ((package ((name bos) (version (0.2.0)))) (dev_repo (Duniverse_fork bos)) 84 | (tag (v0.2.0))) 85 | ((package ((name dockerfile-opam) (version (5.0.0)))) 86 | (dev_repo (Github (avsm ocaml-dockerfile))) (tag (v5.0.0))) 87 | ((package ((name dockerfile-cmd) (version (5.0.0)))) 88 | (dev_repo (Github (avsm ocaml-dockerfile))) (tag (v5.0.0))) 89 | ((package ((name yaml) (version (0.2.1)))) 90 | (dev_repo (Github (avsm ocaml-yaml))) (tag (v0.2.1))) 91 | ((package ((name textwrap) (version (0.2)))) 92 | (dev_repo (Github (superbobry ocaml-textwrap))) (tag (0.2)) 93 | (is_dune false)))) 94 | (remotes ()) (branch master) (opam_switch ocaml-system)) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | .merlin 3 | *.install 4 | .DS_Store 5 | .*.swp 6 | _opam 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | install: wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-docker.sh 3 | script: bash -ex .travis-docker.sh 4 | services: 5 | - docker 6 | env: 7 | global: 8 | - PINS="obi:. opam-ci:. obi-buildkite:." 9 | - DISTRO="debian-stable" 10 | matrix: 11 | - PACKAGE="obi" OCAML_VERSION="4.05.0" 12 | - PACKAGE="obi" OCAML_VERSION="4.06.0" 13 | - PACKAGE="opam-ci" OCAML_VERSION="4.05.0" 14 | - PACKAGE="opam-ci" OCAML_VERSION="4.06.0" 15 | - PACKAGE="obi-buildkite" OCAML_VERSION="4.05.0" 16 | - PACKAGE="obi-buildkite" OCAML_VERSION="4.06.0" 17 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ## dev 2 | 3 | * Port build system to Dune. 4 | * Add a `obi-gitlab` tool to coordinate CI via GitLab. 5 | 6 | ## v1.0.0 7 | 8 | * Initial public release of BuildKite Obi and the `opam-ci` tool. 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | OCaml and opam are community-orientated open source projects, and 2 | the build infrastructure around it is designed to be openly accessible, 3 | within the limits of security required by our infrastruture. 4 | Accordingly, this repository publishes the code that powers the 5 | bulk build instructure, and the results of the runs are available 6 | in the repository under a liberal CC0 license. 7 | 8 | It was originally developed at OCaml Labs and is now maintained by 9 | OCaml Labs, Tarides as well as developers from the OCaml and opam 10 | communities. 11 | 12 | Contributions to Obi are welcome and should be submitted via GitHub 13 | pull requests. Obi code is distributed under the ISC license and 14 | contributors are required to sign their work in order to certify that 15 | they have the right to submit it under this license. See the following 16 | section for more details. 17 | 18 | Signing contributions 19 | --------------------- 20 | 21 | We require that you sign your contributions. Your signature certifies 22 | that you wrote the patch or otherwise have the right to pass it on as 23 | an open-source patch. The rules are pretty simple: if you can certify 24 | the below (from [developercertificate.org][dco]): 25 | 26 | ``` 27 | Developer Certificate of Origin 28 | Version 1.1 29 | 30 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 31 | 1 Letterman Drive 32 | Suite D4700 33 | San Francisco, CA, 94129 34 | 35 | Everyone is permitted to copy and distribute verbatim copies of this 36 | license document, but changing it is not allowed. 37 | 38 | 39 | Developer's Certificate of Origin 1.1 40 | 41 | By making a contribution to this project, I certify that: 42 | 43 | (a) The contribution was created in whole or in part by me and I 44 | have the right to submit it under the open source license 45 | indicated in the file; or 46 | 47 | (b) The contribution is based upon previous work that, to the best 48 | of my knowledge, is covered under an appropriate open source 49 | license and I have the right under that license to submit that 50 | work with modifications, whether created in whole or in part 51 | by me, under the same open source license (unless I am 52 | permitted to submit under a different license), as indicated 53 | in the file; or 54 | 55 | (c) The contribution was provided directly to me by some other 56 | person who certified (a), (b) or (c) and I have not modified 57 | it. 58 | 59 | (d) I understand and agree that this project and the contribution 60 | are public and that a record of the contribution (including all 61 | personal information I submit with it, including my sign-off) is 62 | maintained indefinitely and may be redistributed consistent with 63 | this project or the open source license(s) involved. 64 | ``` 65 | 66 | Then you just add a line to every git commit message: 67 | 68 | ``` 69 | Signed-off-by: Joe Smith 70 | ``` 71 | 72 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 73 | 74 | If you set your `user.name` and `user.email` git configs, you can sign 75 | your commit automatically with `git commit -s`. 76 | 77 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 Anil Madhavapeddy and other authors 2 | 3 | Permission to use, copy, modify, and distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /METADATA.md: -------------------------------------------------------------------------------- 1 | This lists the changes in the Obi metadata, for when we want to parse old logs 2 | in the future. 3 | 4 | ## 3 5 | 6 | - No changes yet. 7 | 8 | ## 2 9 | 10 | - Remove `Obi.Index.start_time` and `end_time` and replace with `duration` 11 | that represents the interval only. 12 | - Add various `omit_nil` annotations to lists that can be empty in the sexp. 13 | 14 | ## 1 15 | 16 | - Initial version of the metadata sexp file. 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build clean test 2 | 3 | build: 4 | dune build --profile=release 5 | 6 | test: 7 | dune runtest 8 | 9 | install: 10 | dune install 11 | 12 | uninstall: 13 | dune uninstall 14 | 15 | clean: 16 | dune clean 17 | 18 | doc: 19 | dune build @doc 20 | 21 | publish-doc: doc 22 | rm -rf .gh-pages 23 | git clone `git config --get remote.origin.url` .gh-pages --reference . 24 | git -C .gh-pages checkout --orphan gh-pages 25 | git -C .gh-pages reset 26 | git -C .gh-pages clean -dxf 27 | cp -r _build/default/_doc/_html/* .gh-pages/ 28 | git -C .gh-pages add . 29 | git -C .gh-pages commit -m "Update Pages" 30 | git -C .gh-pages push origin gh-pages -f 31 | rm -rf .gh-pages 32 | 33 | test-all: 34 | sh ./.docker-run.sh 35 | 36 | REPO=../../mirage/opam-repository 37 | PACKAGES=$(REPO)/packages 38 | # until we have https://github.com/ocaml/opam-publish/issues/38 39 | pkg-%: 40 | topkg opam pkg -n $* 41 | mkdir -p $(PACKAGES)/$* 42 | cp -r _build/$*.* $(PACKAGES)/$*/ 43 | cd $(PACKAGES) && git add $* 44 | 45 | PKGS=$(basename $(wildcard *.opam)) 46 | opam-pkg: 47 | $(MAKE) $(PKGS:%=pkg-%) 48 | 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## obi -- interfaces to the OCaml Build Infrastructure 2 | 3 | This repository contains the scripts, libraries and command-line tools to 4 | access the opam2 bulk build infrastructure that checks on the health of the 5 | [opam](https://opam.ocaml.org) package manager. 6 | 7 |

8 | 9 |

10 | 11 | The main services and repositories associates with this infrastructure are: 12 | 13 | - **Documentation:** 14 | - 15 | - is rebuilt automatically with the latest information 16 | - 17 | - **GitHub:** Git repositories 18 | - : for the source code 19 | - : for the build logs 20 | - **Docker Hub:** container images 21 | - : opam2 and OCaml compiler images 22 | - : intermediate container images for bulk builds 23 | - **Coordination:** 24 | - : the coordination Hub (account required until [buildkite#137](https://github.com/buildkite/feedback/issues/137) is resolved) 25 | - **Users:** 26 | - The results database is also used by 27 | 28 | ## Getting Started 29 | 30 | The main tool you will want to try out is `opam-ci`, which provides 31 | CLI access to build results. You can try this out by: 32 | 33 | opam update 34 | opam ci --help 35 | opam ci status --help 36 | opam ci logs --help 37 | 38 | See the [online documentation](https://github.com/ocaml/infrastructure/wiki/Using-the-opam-ci-tool) for more information. 39 | 40 | ## Further Information 41 | 42 | - [CHANGES.md](CHANGES.md) is the repository changelog for source code and 43 | Buildkite scripts. 44 | - [METADATA.md](METADATA.md) contains a changelog for the Obi sexp format 45 | that is published on . 46 | 47 | While we are assembling the documentation, please contact @avsm for more information. 48 | -------------------------------------------------------------------------------- /dune-project: -------------------------------------------------------------------------------- 1 | (lang dune 1.0) 2 | (name obi) 3 | -------------------------------------------------------------------------------- /obi-dune.opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | maintainer: "Anil Madhavapeddy " 3 | authors: "Anil Madhavapeddy " 4 | license: "ISC" 5 | tags: ["org:mirage" "org:ocamllabs"] 6 | homepage: "https://github.com/ocaml/obi" 7 | doc: "https://ocaml.github.io/obi/" 8 | bug-reports: "https://github.com/ocaml/obi/issues" 9 | depends: [ 10 | "ocaml" {>= "4.05.0"} 11 | "dune" {build} 12 | "bos" 13 | "fmt" 14 | "logs" 15 | "obi" 16 | "cmdliner" 17 | ] 18 | build: [ 19 | ["dune" "subst"] {pinned} 20 | ["dune" "build" "-p" name "-j" jobs] 21 | ] 22 | dev-repo: "git+https://github.com/ocaml/obi.git" 23 | synopsis: "Generate Dune files for opam bulk builds" 24 | -------------------------------------------------------------------------------- /obi-gitlab.opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | maintainer: "Anil Madhavapeddy " 3 | authors: "Anil Madhavapeddy " 4 | license: "ISC" 5 | tags: ["org:mirage" "org:ocamllabs"] 6 | homepage: "https://github.com/ocaml/obi" 7 | doc: "https://ocaml.github.io/obi/" 8 | bug-reports: "https://github.com/ocaml/obi/issues" 9 | depends: [ 10 | "ocaml" {>= "4.05.0"} 11 | "obi" 12 | "dune" {build} 13 | "dockerfile-opam" {>= "5.0.0"} 14 | "dockerfile-cmd" {>= "5.0.0"} 15 | "ezjsonm" 16 | "uri" 17 | "ptime" 18 | "yaml" {>= "0.2.0"} 19 | ] 20 | build: [ 21 | ["dune" "subst"] {pinned} 22 | ["dune" "build" "-p" name "-j" jobs] 23 | ] 24 | dev-repo: "git+https://github.com/ocaml/obi.git" 25 | synopsis: "Interface the OCaml Build Infrastructure with GitLab" 26 | -------------------------------------------------------------------------------- /obi.opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | maintainer: "Anil Madhavapeddy " 3 | authors: "Anil Madhavapeddy " 4 | license: "ISC" 5 | tags: ["org:mirage" "org:ocamllabs"] 6 | homepage: "https://github.com/ocaml/obi" 7 | doc: "https://ocaml.github.io/obi/" 8 | bug-reports: "https://github.com/ocaml/obi/issues" 9 | depends: [ 10 | "ocaml" {>= "4.05.0"} 11 | "dune" {build} 12 | "dockerfile-opam" {>= "6.0.0"} 13 | "dockerfile-cmd" {>= "6.0.0"} 14 | "ocaml-version" {>= "1.0.0"} 15 | "yaml" {>= "0.2.0"} 16 | "uri" 17 | "ezjsonm" 18 | "opam-core" 19 | "ptime" 20 | "textwrap" 21 | ] 22 | build: [ 23 | ["dune" "subst"] {pinned} 24 | ["dune" "build" "-p" name "-j" jobs] 25 | ] 26 | dev-repo: "git+https://github.com/ocaml/obi.git" 27 | synopsis: "Interface to OCaml Build Infrastructure" 28 | description: """ 29 | This package contains the OCaml libraries to access the opam2 bulk build 30 | infrastructure that checks on the health of the [opam](https://opam.ocaml.org) 31 | package manager. 32 | 33 | The main services and repositories associates with this infrastructure are: 34 | 35 | - **Documentation:** 36 | - 37 | - is rebuilt automatically with the latest information 38 | - **GitHub:** Git repositories 39 | - : for the source code 40 | - **Docker Hub:** container images 41 | - : opam2 and OCaml compiler images 42 | - : intermediate container images for bulk builds 43 | """ 44 | -------------------------------------------------------------------------------- /opam-ci.opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | maintainer: "Anil Madhavapeddy " 3 | authors: "Anil Madhavapeddy " 4 | license: "ISC" 5 | tags: "org:ocamllabs" 6 | homepage: "https://github.com/ocaml/obi" 7 | doc: "https://ocaml.github.io/obi/" 8 | bug-reports: "https://github.com/ocaml/obi/issues" 9 | depends: [ 10 | "ocaml" {>= "4.05.0"} 11 | "dune" {build} 12 | "obi" 13 | "bos" 14 | "cmdliner" 15 | "fmt" 16 | "logs" 17 | "rresult" 18 | "textwrap" 19 | ] 20 | flags: plugin 21 | build: [ 22 | ["dune" "subst"] {pinned} 23 | ["dune" "build" "-p" name "-j" jobs] 24 | ] 25 | dev-repo: "git+https://github.com/ocaml/obi.git" 26 | synopsis: "CLI interface to the OCaml Build Infrastructure" 27 | -------------------------------------------------------------------------------- /src/lib/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name obi) 3 | (public_name obi) 4 | (libraries ocaml-version sexplib dockerfile-opam ezjsonm opam-core) 5 | (preprocess (per_module ((pps ppx_sexp_conv) obi)))) 6 | -------------------------------------------------------------------------------- /src/lib/obi.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | open Sexplib.Conv 18 | 19 | module Ocaml_version = struct 20 | include Ocaml_version 21 | include Ocaml_version_sexp 22 | end 23 | 24 | module Builds = struct 25 | type build_result = 26 | { code: [`Signaled of int | `Exited of int] 27 | ; actions: string 28 | ; start_time: float 29 | ; end_time: float } 30 | [@@deriving sexp] 31 | 32 | type pkg = {name: string; versions: (string * build_result) list} 33 | [@@deriving sexp] 34 | 35 | type params = 36 | { arch: Ocaml_version.arch 37 | ; distro: Dockerfile_distro.t 38 | ; ov: Ocaml_version.t } 39 | [@@deriving sexp] 40 | 41 | type batch = {rev: string; params: params; pkgs: pkg list} [@@deriving sexp] 42 | end 43 | 44 | module Index = struct 45 | type maintainers = (string * string list) list [@@deriving sexp] 46 | 47 | type tags = (string * string list) list [@@deriving sexp] 48 | 49 | type deps = (string * string * [`Fail | `Ok | `Skipped]) list 50 | [@@deriving sexp] 51 | 52 | type result = 53 | [ `Ok 54 | | `Fail of int * deps 55 | | `Depfail of deps 56 | | `Uninstallable of string list[@sexp.omit_nil] 57 | | `No_sources of string list[@sexp.omit_nil] 58 | | `Solver_failure ] 59 | [@@deriving sexp] 60 | 61 | let pp_result ppf result = 62 | let open Format in 63 | match result with 64 | | `Ok -> pp_print_string ppf "ok" 65 | | `Fail (exit, _deps) -> 66 | pp_print_string ppf "exit code " ; 67 | pp_print_int ppf exit 68 | | `Depfail _deps -> pp_print_string ppf "dependency failed" 69 | | `Uninstallable _sl -> 70 | pp_print_string ppf "uninstallable due to constraints" 71 | | `No_sources _sl -> pp_print_string ppf "unable to fetch package sources" 72 | | `Solver_failure -> pp_print_string ppf "internal opam solver failure" 73 | 74 | type params = 75 | { arch: Ocaml_version.arch 76 | ; distro: Dockerfile_distro.t 77 | ; ov: Ocaml_version.t } 78 | [@@deriving sexp] 79 | 80 | type metadata = 81 | { rev: string 82 | ; params: params 83 | ; build_result: result 84 | ; duration: float 85 | ; log: string list [@sexp.omit_nil] } 86 | [@@deriving sexp] 87 | 88 | type pkg = 89 | { name: string 90 | ; mutable maintainers: string list 91 | ; mutable tags: string list [@sexp.omit_nil] 92 | ; mutable versions: (string * metadata list) list } 93 | [@@deriving sexp] 94 | 95 | type t = {version: int [@default 0]; packages: pkg list} [@@deriving sexp] 96 | 97 | let current_version = 2 98 | end 99 | 100 | module VersionCompare = OpamVersionCompare 101 | module OpamJsonActions = OpamJsonActions 102 | -------------------------------------------------------------------------------- /src/lib/obi.mli: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | (** Data structures for accessing results of opam2 package build results *) 18 | 19 | (** Index of all the opam2 builds. The [Index] module contains types for the 20 | opam2 bulk build results. *) 21 | module Index : sig 22 | (** [deps] is a list of package dependencies, where each entry is a tuple of 23 | [name, version, status] *) 24 | type deps = (string * string * [`Fail | `Ok | `Skipped]) list 25 | [@@deriving sexp] 26 | 27 | (** [result] represents the exit result of an [opam] invocation. The raw exit 28 | codes are parsed to look for special opam exit codes, which indicate 29 | errors such as the internal solver failing to find a result. *) 30 | type result = 31 | [ `Ok 32 | | `Fail of int * deps 33 | | `Depfail of deps 34 | | `Uninstallable of string list 35 | | `No_sources of string list 36 | | `Solver_failure ] 37 | [@@deriving sexp] 38 | 39 | val pp_result : Format.formatter -> result -> unit 40 | (** [pp_result ppf result] will print a human-readable result to formatter 41 | [ppf] *) 42 | 43 | (** [params] represents some of the build parameters that opam packages are 44 | tested against. These include the CPU architecture, OS distribution and 45 | OCaml compiler version. *) 46 | type params = 47 | { arch: Ocaml_version.arch (** CPU architecture *) 48 | ; distro: Dockerfile_distro.t (** Operating system distribution *) 49 | ; ov: Ocaml_version.t (** OCaml compiler version *) } 50 | [@@deriving sexp] 51 | 52 | (** [metadata] contains the results and parameters for a single build run of 53 | a package and version. *) 54 | type metadata = 55 | { rev: string 56 | (** opam-repository git revision hash that this run was built against *) 57 | ; params: params (** build {!params} for this run *) 58 | ; build_result: result 59 | (** result of the execution of this set of parameters *) 60 | ; duration: float 61 | (** wall clock time of the duration of the build in Unix seconds *) 62 | ; log: string list 63 | (** list of log lines. This is blank unless there is an error *) } 64 | [@@deriving sexp] 65 | 66 | (** [pkg] collects all the metadata for all versions of a given opam package. *) 67 | type pkg = 68 | { name: string (** opam package name *) 69 | ; mutable maintainers: string list 70 | (** list of maintainers associated with this package. As a heuristic, 71 | all of the maintainers listed for all versions are bundled 72 | together here for simplicity. *) 73 | ; mutable tags: string list 74 | (** list of tags associated with this package. As a heuristic, all of 75 | the tags listed of all versions are bundled together here for 76 | simplicity. *) 77 | ; mutable versions: (string * metadata list) list 78 | (** list of a tuple of opam package version and the list of build 79 | results. There is a list of results since there are usually 80 | multiple different runs for every version, to test it against 81 | different compiler and CPU/OS combinations. *) } 82 | [@@deriving sexp] 83 | 84 | (** [t] contains a version number of the metadata system, which is set to 85 | {!current_version} for the latest builds. Older clients can parse the 86 | sexpression to determine if they need to upgrade for a newer format. The 87 | [CHANGES.md] file in the Obi source code repository tracks metadata 88 | version number bumps. *) 89 | type t = {version: int; packages: pkg list} [@@deriving sexp] 90 | 91 | val current_version : int 92 | (** [current_version] is a monotonically increasing number for the version of 93 | Obi in use. *) 94 | 95 | (** [maintainers] is an association list of the package name (including 96 | version string) to the list of maintainers registered for that package. 97 | As a heuristic, all of the maintainers listed for all versions are 98 | bundled here for simplicity. *) 99 | type maintainers = (string * string list) list [@@deriving sexp] 100 | 101 | (** [tags] is an association list of the package name (including version 102 | string) to the list of tags registered for that package. As a heuristic, 103 | all of the tags listed for all versions are bundled here for simplicity. *) 104 | type tags = (string * string list) list [@@deriving sexp] 105 | end 106 | 107 | (** Results of individual builds for a particular parameters. This is not 108 | intended for direct use by users, as the results are collected and 109 | aggregated into the {!Index} module which is easier to consume by tools. *) 110 | module Builds : sig 111 | (** [build_result] records the results of a single job build. *) 112 | type build_result = 113 | { code: [`Signaled of int | `Exited of int] 114 | ; actions: string 115 | ; start_time: float 116 | ; end_time: float } 117 | [@@deriving sexp] 118 | 119 | (** [pkg] records the results of the runs for every version of a package. *) 120 | type pkg = {name: string; versions: (string * build_result) list} 121 | [@@deriving sexp] 122 | 123 | (** [params] is the configuration for a run. *) 124 | type params = 125 | { arch: Ocaml_version.arch 126 | ; distro: Dockerfile_distro.t 127 | ; ov: Ocaml_version.t } 128 | [@@deriving sexp] 129 | 130 | (** [batch] represents the results of a bulk build with one configuration 131 | against a single opam-repository revision. *) 132 | type batch = 133 | { rev: string (** opam-repository git revision *) 134 | ; params: params 135 | ; pkgs: pkg list } 136 | [@@deriving sexp] 137 | end 138 | 139 | module VersionCompare : sig 140 | val compare : string -> string -> int 141 | end 142 | 143 | module OpamJsonActions : sig 144 | val installs : Ezjsonm.value -> Index.deps 145 | end 146 | -------------------------------------------------------------------------------- /src/lib/ocaml_version_sexp.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | open Ocaml_version 18 | 19 | let sexp_of_t t = Sexplib.Sexp.Atom (to_string t) 20 | 21 | let t_of_sexp t = 22 | match t with 23 | | Sexplib.Sexp.Atom t -> of_string_exn t 24 | | _ -> failwith "invalid input for Ocaml_version.t_of_sexp" 25 | 26 | let sexp_of_arch t = Sexplib.Sexp.Atom (string_of_arch t) 27 | 28 | let arch_of_sexp t = 29 | match t with 30 | | Sexplib.Sexp.Atom t -> ( 31 | match arch_of_string t with 32 | | Ok a -> a 33 | | Error (`Msg m) -> 34 | failwith ("invalid input for Ocaml_version.arch_of_sexp: " ^ m) ) 35 | | _ -> failwith "invalid input for Ocaml_version.arch_of_sexp" 36 | -------------------------------------------------------------------------------- /src/lib/opamJsonActions.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | let installs j = 18 | let open Ezjsonm in 19 | try 20 | get_dict j |> List.assoc "results" 21 | |> get_list (fun v -> 22 | try 23 | let d = get_dict v in 24 | let action = List.assoc "action" d in 25 | let build = get_dict action |> List.assoc "build" |> get_dict in 26 | let name = List.assoc "name" build |> get_string in 27 | let version = List.assoc "version" build |> get_string in 28 | let result = List.assoc "result" d in 29 | let is_error = 30 | try get_dict result |> List.mem_assoc "process-error" 31 | with _ -> false 32 | in 33 | let is_ok = try get_string result = "OK" with _ -> false in 34 | let is_skipped = 35 | try get_dict result |> List.mem_assoc "aborted" with _ -> false 36 | in 37 | let status = 38 | if is_error then `Fail 39 | else if is_ok then `Ok 40 | else if is_skipped then `Skipped 41 | else raise (Invalid_argument "Unable to parse JSON") 42 | in 43 | Some (name, version, status) 44 | with Not_found -> None ) 45 | |> List.fold_left (fun a -> function None -> a | Some x -> x :: a) [] 46 | with Not_found -> [] 47 | -------------------------------------------------------------------------------- /src/obi-dune/README.md: -------------------------------------------------------------------------------- 1 | This is a WIP to generate bulk build runes via Dune. 2 | -------------------------------------------------------------------------------- /src/obi-dune/dune: -------------------------------------------------------------------------------- 1 | (executables 2 | (names obi_dune) 3 | (public_names obi-dune) 4 | (libraries obi bos cmdliner fmt fmt.cli logs.fmt fmt.tty logs.cli) 5 | (package obi-dune) 6 | ) 7 | 8 | (rule 9 | (targets scripts.b64) 10 | (deps scripts/obi-ci-install.sh) 11 | (action (with-stdout-to %{targets} (run base64 %{deps})))) 12 | 13 | (rule 14 | (targets scripts.ml) 15 | (action (write-file scripts.ml "let obi_ci_install = \"%{read:scripts.b64}\""))) 16 | -------------------------------------------------------------------------------- /src/obi-dune/obi_dune.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | module OV = Ocaml_version 18 | open Astring 19 | open Bos 20 | open Rresult 21 | open R.Infix 22 | 23 | module Rules = struct 24 | 25 | let base ~rev = 26 | let script = String.cuts ~sep:"\n" Scripts.obi_ci_install |> String.concat ~sep:"" in 27 | Fmt.strf 28 | {| 29 | (alias (name bulk) (deps results.tar.bz2)) 30 | (rule (targets custom) (mode fallback) (action (write-file custom "LABEL obi.no-custom=true\n"))) 31 | (rule (targets rev) (mode fallback) (action (write-file rev %S))) 32 | (rule (targets Dockerfile) (action (with-stdout-to %%{targets} ( 33 | progn 34 | (echo "FROM ocaml/opam2:%%{read-lines:ov}\n") 35 | (echo "WORKDIR /home/opam/opam-repository\n") 36 | (echo "RUN git pull origin master\n") 37 | (echo "RUN git checkout %%{read-lines:rev}\n") 38 | (echo "RUN opam update -uy\n") 39 | (echo "RUN opam switch %%{read-lines:ov}\n") 40 | (echo "%%{read:custom}") 41 | (echo "RUN echo '%s' | base64 -d > /tmp/obi-ci-install\n") 42 | (echo "RUN sudo mv /tmp/obi-ci-install /usr/bin/obi-ci-install\n") 43 | (echo "RUN sudo chmod a+x /usr/bin/obi-ci-install\n"))))) 44 | (rule (targets arch) (action (with-stdout-to arch (run uname -m)))) 45 | (rule (targets image-name) 46 | (action (write-file image-name "obi-%%{read-lines:distro}_ocaml-%%{read-lines:ov}_%%{read-lines:arch}_%%{read-lines:rev}"))) 47 | (rule (targets Dockerfile.log) (deps Dockerfile) 48 | (action (with-outputs-to Dockerfile.log (run docker build --no-cache --pull -t %%{read:image-name} --rm --force-rm .)))) |} rev script 49 | 50 | let build_one p : string = 51 | Fmt.strf 52 | {| 53 | (rule (targets %s.json) (deps %s.tar) (action (with-stdout-to %%{targets} (run tar -xOf %%{deps} %s.json)))) 54 | (rule (targets %s.txt) (deps %s.tar) (action (with-stdout-to %%{targets} (run tar -xOf %%{deps} %s.txt)))) 55 | 56 | (rule (targets %s.tar) (deps Dockerfile.log) 57 | (action (with-stdout-to %%{targets} 58 | (run docker run --privileged --rm -v opam2-archive:/home/opam/.opam/download-cache %%{read:image-name} obi-ci-install %s)))) |} 59 | p p p p p p p p 60 | 61 | let collect packages : string = 62 | let pkg ext x = Fmt.strf "%s.%s" x ext in 63 | let json_deps = String.concat ~sep:" " (List.map (pkg "json") packages) in 64 | let txt_deps = String.concat ~sep:" " (List.map (pkg "txt") packages) in 65 | Fmt.strf 66 | {| 67 | (rule (targets results.tar.bz2) (deps %s %s arch ov rev distro) (action (run tar -jcf %%{targets} %%{deps}))) 68 | |} json_deps txt_deps 69 | 70 | let gen ~rev packages = 71 | [ base ~rev 72 | ; collect packages ] 73 | @ List.map build_one packages 74 | |> String.concat ~sep:"\n" 75 | end 76 | 77 | let packages ~rev () = 78 | OS.Cmd.( 79 | run_out 80 | Cmd.( 81 | v "docker" % "run" % "--rm" % "ocaml/opam2:alpine" % "sh" % "-c" 82 | % Fmt.strf 83 | "git -C /home/opam/opam-repository pull -q origin master && git \ 84 | checkout -q %s && opam update >/dev/null && opam list -s -a" 85 | rev) 86 | |> to_lines) 87 | 88 | let opam_repo_rev rev = 89 | match rev with 90 | | Some rev -> Ok rev 91 | | None -> begin 92 | OS.Cmd.(run_out Cmd.(v "git" % "ls-remote" % "https://github.com/ocaml/opam-repository.git" % "refs/heads/master") |> to_string) >>= fun l -> 93 | match String.cuts ~sep:"\t" l with 94 | | hd::_ -> Logs.debug (fun l -> l "opam repo rev %s" hd); Ok hd 95 | | _ -> Error (`Msg "unable to get remote head of opam-repository") 96 | end 97 | 98 | let gen_bulk_rules rev () = 99 | opam_repo_rev rev >>= fun rev -> 100 | Logs.app (fun l -> l "Using opam repo rev %s" rev); 101 | OS.File.write Fpath.(v "rev") rev >>= fun () -> 102 | Logs.app (fun l -> l "Generating package list...") ; 103 | packages ~rev () 104 | >>= fun packages -> 105 | Logs.app (fun l -> l "... %d packages found." (List.length packages)); 106 | Rules.gen ~rev packages 107 | |> fun dune -> 108 | OS.File.exists Fpath.(v "dune") >>= function 109 | | true -> R.error_msg "`dune` file exists, please delete it and rerun" 110 | | false -> OS.File.write Fpath.(v "dune") dune 111 | 112 | open Cmdliner 113 | 114 | let setup_logs = 115 | let setup_log style_renderer level = 116 | Fmt_tty.setup_std_outputs ?style_renderer (); 117 | Logs.set_level level; 118 | Logs.set_reporter (Logs_fmt.reporter ()) in 119 | let global_option_section = "COMMON OPTIONS" in 120 | Term.(const setup_log 121 | $ Fmt_cli.style_renderer ~docs:global_option_section () 122 | $ Logs_cli.level ~docs:global_option_section ()) 123 | 124 | let opam_repo_rev_t = 125 | let doc = "opam repo git rev" in 126 | let open Arg in 127 | value & opt (some string) None 128 | & info ["opam-repo-rev"] ~docv:"OPAM_REPO_REV" ~doc 129 | 130 | let opam_cmd = 131 | let doc = "generate bulk build dune rules for a particular opam repo rev" in 132 | let exits = Term.default_exits in 133 | let man = [`S Manpage.s_description; `P "TODO"] in 134 | ( Term.(term_result (const gen_bulk_rules $ opam_repo_rev_t $ setup_logs)) 135 | , Term.info "rules" ~doc ~sdocs:Manpage.s_common_options ~exits ~man ) 136 | 137 | let default_cmd = 138 | let doc = "TODO" in 139 | let sdocs = Manpage.s_common_options in 140 | ( Term.(ret (const (fun _ -> `Help (`Pager, None)) $ pure ())) 141 | , Term.info "obi-dune" ~version:"%%VERSION%%" ~doc ~sdocs ) 142 | 143 | let cmds = [opam_cmd] 144 | 145 | let () = Term.(exit @@ eval_choice default_cmd cmds) 146 | -------------------------------------------------------------------------------- /src/obi-dune/scripts/obi-ci-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export OPAMERRLOGLEN=0 4 | export OPAMYES=1 5 | export OPAMJSON=$1.json 6 | 7 | starttime=`date +%s` 8 | opam depext -iy -j 2 $1 > $1.txt 2>&1 9 | exitcode=$? 10 | endtime=`date +%s` 11 | echo $exitcode $starttime $endtime >> $1.txt 12 | tar -cf - $1.txt $1.json 13 | -------------------------------------------------------------------------------- /src/obi-gitlab/dune: -------------------------------------------------------------------------------- 1 | (executables 2 | (names gitlab) 3 | (public_names obi-gitlab) 4 | (libraries obi dockerfile-opam dockerfile-cmd uri ptime yaml ezjsonm) 5 | (preprocess (per_module ((pps ppx_sexp_conv) import))) 6 | (flags -w -9) 7 | (package obi-gitlab) 8 | ) 9 | -------------------------------------------------------------------------------- /src/obi-gitlab/gitlab.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | module L = Dockerfile_linux 18 | module D = Dockerfile_distro 19 | module C = Dockerfile_cmd 20 | module G = Dockerfile_gen 21 | module O = Dockerfile_opam 22 | module OV = Ocaml_version 23 | open Rresult 24 | open R.Infix 25 | 26 | type copts = {staging_hub_id: string; prod_hub_id: string; results_dir: Fpath.t} 27 | 28 | let copts staging_hub_id prod_hub_id results_dir = 29 | {staging_hub_id; prod_hub_id; results_dir} 30 | 31 | let arches = OV.arches 32 | 33 | type build_t = {ov: Ocaml_version.t; distro: D.t} 34 | 35 | let docs {prod_hub_id; _} = 36 | let distros ds = 37 | List.map 38 | (fun distro -> 39 | let name = D.human_readable_string_of_distro distro in 40 | let tag = D.tag_of_distro distro in 41 | let arches = 42 | String.concat " " 43 | (List.map OV.string_of_arch 44 | (D.distro_arches OV.Releases.latest distro)) 45 | in 46 | Fmt.strf "| %s | `%s` | %s | `docker run %s:%s`" name tag arches 47 | prod_hub_id tag ) 48 | ds 49 | |> String.concat "\n" 50 | in 51 | let latest_distros = distros D.latest_distros in 52 | let active_distros = distros (D.active_distros `X86_64) in 53 | let dev_versions_of_ocaml = 54 | String.concat " " (List.map OV.to_string OV.Releases.dev) 55 | in 56 | let intro = 57 | Fmt.strf 58 | {|# OCaml Container Infrastructure 59 | 60 | This repository contains a set of [Docker](http://docker.com) container definitions 61 | for various combination of [OCaml](https://ocaml.org) and the 62 | [opam](https://opam.ocaml.org) package manager. The containers come preinstalled with 63 | an opam2 environment, and are particularly suitable for use with continuous integration 64 | systems such as [Travis CI](https://travis-ci.org). All the containers are hosted 65 | on the [Docker Hub ocaml/opam2](http://hub.docker.com/r/ocaml/opam2) repository. 66 | 67 | Using it is as simple as: 68 | 69 | ``` 70 | docker pull %s 71 | docker run -it %s bash 72 | ``` 73 | 74 | This will get you an interactive development environment (including on [Docker for Mac](https://www.docker.com/docker-mac)). You can grab a specific OS distribution and test out external dependencies as well: 75 | 76 | ``` 77 | docker run %s:ubuntu opam depext -i cohttp-lwt-unix tls 78 | ``` 79 | 80 | There are a number of different variants available that are regularly rebuilt on the ocaml.org infrastructure and pushed to the [Docker Hub](http://hub.docker.com/r/ocaml/opam2). Each of the containers contains the Dockerfile used to build it in `/Dockerfile` within the container image. 81 | 82 | 83 | Using The Defaults 84 | ================== 85 | 86 | The `%s` Docker remote has a default `latest` tag that provides the %s Linux distribution with the latest release of the OCaml compiler (%s). 87 | The [opam-depext](https://github.com/ocaml/opam-depext) plugin can be used to install external system libraries in a distro-portable way. 88 | 89 | The default user is `opam` in the `/home/opam` directory, with a copy of the [opam-repository](https://github.com/ocaml/opam-repository) 90 | checked out in `/home/opam/opam-repository`. You can supply your own source code by volume mounting it anywhere in the container, 91 | but bear in mind that it should be owned by the `opam` user (uid `1000` in all distributions). 92 | 93 | 94 | Selecting a Specific Compiler 95 | ============================= 96 | 97 | The default container comes with the latest compiler activated, but also a number of other switches for older revisions of OCaml. You can 98 | switch to these to test compatibility in CI by iterating through older revisions. 99 | 100 | For example: 101 | 102 | ``` 103 | $ docker run %s opam switch 104 | switch compiler description 105 | 4.02 ocaml-base-compiler.4.02.3 4.02 106 | 4.03 ocaml-base-compiler.4.03.0 4.03 107 | 4.04 ocaml-base-compiler.4.04.2 4.04 108 | 4.05 ocaml-base-compiler.4.05.0 4.05 109 | -> 4.06 ocaml-base-compiler.4.06.1 4.06 110 | ``` 111 | 112 | Note that the name of the switch drops the minor patch release (e.g. `4.06` _vs_ `4.06.1`), since you should always be using the latest patch revision of the compiler. 113 | 114 | Accessing Compiler Variants 115 | =========================== 116 | 117 | Modern versions of OCaml also feature a number of variants, such as the experimental flambda inliner or [AFL fuzzing](http://lcamtuf.coredump.cx/afl/) support. These are also conveniently available using the `` tag. For example: 118 | 119 | ``` 120 | $ docker run %s:4.06 opam switch 121 | switch compiler description 122 | -> 4.06 ocaml-base-compiler.4.06.1 4.06 123 | 4.06+afl ocaml-variants.4.06.1+afl 4.06+afl 124 | 4.06+default-unsafe-string ocaml-variants.4.06.1+default-unsafe-string 4.06+default-unsafe-string 125 | 4.06+flambda ocaml-variants.4.06.1+flambda 4.06+flambda 126 | 4.06+force-safe-string ocaml-variants.4.06.1+force-safe-string 4.06+force-safe-string 127 | ``` 128 | 129 | In this case, the `4.06` container has the latest patch release (4.06.1) activated by default, but the other variant compilers are available easily via `opam switch` without having to compile them yourself. Using this more specific tag also helps you pin the version of OCaml that your CI system will be testing with, as the default `latest` tag will be regularly upgraded to keep up with upstream OCaml releases. 130 | 131 | 132 | Selecting Linux Distributions 133 | ============================= 134 | 135 | There are also tags available to select other Linux distributions, which is useful to validate and test the behaviour of your package in CI. 136 | 137 | Distribution | Tag | Architectures | Command 138 | ------------ | --- | ------------- | ------- 139 | %s 140 | 141 | The tags above are for the latest version of the distribution, and are upgraded to the latest upstream stable releases. You can also select a specific version number in the tag name to obtain a particular OS release. However, these will eventually time out once they are out of their support window, so try to use the version-free aliases described earlier unless you really know that you want a specific release. When a specific release does time out, the container will be replaced by one that always displays an error message requesting you to upgrade your CI script. 142 | 143 | 144 | Distribution | Tag | Architectures | Command 145 | ------------ | --- | ------------- | ------- 146 | %s 147 | 148 | 149 | Multi-architecture Containers 150 | ============================= 151 | 152 | The observant reader will notice that the distributions listed above have more than one architecture. We are building an increasing number of packages on non-x86 containers, starting with ARM64 and soon to include PPC64. 153 | 154 | Using the multiarch images is simple, as the correct one will be selected depending on your host architecture. The images are built using [docker manifest](https://docs.docker.com/edge/engine/reference/commandline/manifest/). 155 | 156 | Development Versions of the Compiler 157 | ==================================== 158 | 159 | You can also access development versions of the OCaml compiler (currently %s) that have not yet been released. These are rebuilt around once a day, so you may lag a few commits behind the main master branch. You can reference it via the version `v4.08` tag as with other compilers, but please do bear in mind that this is a development version and so subject to more breakage than a stable release. 160 | 161 | ``` 162 | $ docker run -it ocaml/opam2:v4.08 switch 163 | switch compiler description 164 | -> 4.08 ocaml-variants.4.08.0+trunk 4.08 165 | 4.08+trunk+afl ocaml-variants.4.08.0+trunk+afl 4.08+trunk+afl 166 | 4.08+trunk+flambda ocaml-variants.4.08.0+trunk+flambda 4.08+trunk+flambda 167 | $ docker run -it ocaml/opam2:v4.08 ocaml --version 168 | The OCaml toplevel, version 4.08.0+dev7-2018-11-10 169 | ``` 170 | 171 | Just the Binaries Please 172 | ======================== 173 | 174 | All of the OCaml containers here are built over a smaller container that just installs the `opam` binary first, without having an OCaml compiler installed. Sometimes for advanced uses, you may want to initialise your own opam environment. In this case, you can access the base container directly via the `-opam` tag (e.g. `debian-9-opam`). Bear in mind that this base image will be deleted in the future when the distribution goes out of support, so please only use these low-level opam containers if one of the options listed above isn't sufficient for your usecase. 175 | 176 | There are a large number of distribution and OCaml version combinations that are regularly built that are not mentioned here. For the advanced user who needs a specific combination, the full current list can be found on the [Docker Hub](http://hub.docker.com/r/ocaml/opam2). However, please try to use the shorter aliases rather than these explicit versions if you can, since then your builds will not error as the upstream versions advance. 177 | 178 | Package Sandboxing 179 | ================== 180 | 181 | The Docker containers install opam2's [Bubblewrap](https://github.com/projectatomic/bubblewrap) tool that is used for sandboxing builds. However, due to the way that Linux sandboxing works, this may not work with all Docker installations since unprivileged containers cannot create new Linux namespaces on some installations. Thus, sandboxing is disabled by default in the containers that have opam initialised. 182 | 183 | If you can run containers with `docker run --privileged`, then you can enable opam sandboxing within the container by running `opam-sandbox-enable` within the container. This will ensure that every package is restricted to only writing within `~/.opam` and is the recommended way of doing testing. 184 | 185 | Questions and Feedback 186 | ====================== 187 | 188 | We are constantly improving and maintaining this infrastructure, so please get in touch with Anil Madhavapeddy `` if you have any questions or requests for improvement. Note that until opam 2.0 is released, this infrastructure is considered to be in a beta stage and subject to change. 189 | 190 | This is all possible thanks to generous infrastructure contributions from [Packet.net](https://www.packet.net), [IBM](http://ibm.com), [Azure](https://azure.microsoft.com/en-gb/) and [Rackspace](http://rackspace.com), as well as a dedicated machine cluster funded by [Jane Street](http://janestreet.com). The Docker Hub also provides a huge amount of storage space for our containers. We use hundreds of build agents running on [GitLab](http://gitlab.com) in order to regularly generate the large volume of updates that this infrastructure needs, including the multiarch builds. 191 | 192 | |} 193 | prod_hub_id prod_hub_id prod_hub_id prod_hub_id 194 | (D.human_readable_string_of_distro D.master_distro) 195 | OV.(to_string Releases.latest) 196 | prod_hub_id prod_hub_id latest_distros active_distros 197 | dev_versions_of_ocaml 198 | in 199 | intro 200 | 201 | let assoc_hashtbl l = 202 | let h = Hashtbl.create 7 in 203 | List.iter 204 | (fun (k, v) -> 205 | match Hashtbl.find h k with 206 | | _, a -> Hashtbl.replace h k (None, v :: a) 207 | | exception Not_found -> Hashtbl.add h k (None, [v]) ) 208 | l ; 209 | h 210 | 211 | let docker_build_and_push_cmds ~distro ~arch ~tag prefix = 212 | let file = Fmt.strf "%s-%s/Dockerfile.%s" prefix arch distro in 213 | `A 214 | [ `String "docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD" 215 | ; `String (Fmt.strf "echo >> %s" file) 216 | ; `String (Fmt.strf "echo ADD %s /Dockerfile >> %s" file file) 217 | ; `String (Fmt.strf "docker build --no-cache --force-rm --rm --pull -t %s -f %s ." tag file) 218 | ; `String (Fmt.strf "docker push %s" tag) 219 | ; `String (Fmt.strf "docker rmi %s" tag) ] 220 | 221 | let gen_multiarch ~staging_hub_id ~prod_hub_id h suffix name = 222 | Hashtbl.fold 223 | (fun f (tag, arches) acc -> 224 | let tags = 225 | List.map 226 | (fun arch -> 227 | Fmt.strf "%s:%s%s-linux-%s" staging_hub_id f suffix 228 | (OV.string_of_arch arch) ) 229 | arches 230 | in 231 | let l = String.concat " " tags in 232 | let pulls = 233 | List.map (fun t -> `String (Fmt.strf "docker pull %s" t)) tags 234 | in 235 | let tag = 236 | match tag with None -> Fmt.strf "%s%s" f suffix | Some t -> t 237 | in 238 | let annotates = 239 | List.map2 240 | (fun atag arch -> 241 | let flags = 242 | match arch with 243 | | `Aarch32 -> "--arch arm --variant v7" 244 | | `I386 -> "--arch 386" 245 | | `Aarch64 -> "--arch arm64 --variant v8" 246 | | `X86_64 -> "--arch amd64" 247 | | `Ppc64le -> "--arch ppc64le" 248 | in 249 | `String 250 | (Fmt.strf "docker manifest annotate %s:%s %s %s" prod_hub_id tag 251 | atag flags) ) 252 | tags arches 253 | in 254 | let script = 255 | `A 256 | ( [`String "docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD"] 257 | @ pulls 258 | @ [ `String 259 | (Fmt.strf "docker manifest push -p %s:%s || true" prod_hub_id tag) 260 | ; `String 261 | (Fmt.strf "docker manifest create %s:%s %s" prod_hub_id tag l) 262 | ] 263 | @ annotates 264 | @ [ `String (Fmt.strf "docker manifest inspect %s:%s" prod_hub_id tag) 265 | ; `String 266 | (Fmt.strf "docker manifest push -p %s:%s" prod_hub_id tag) ] ) 267 | in 268 | let cmds : Yaml.value = 269 | `O 270 | [ ("stage", `String (Fmt.strf "%s-multiarch" name)) 271 | ; ("retry", `String "2") 272 | ; ("except", `A [`String "pushes"]) 273 | ; ("tags", `A [`String "shell"; `String "amd64"]) 274 | ; ("script", script) ] 275 | in 276 | let jobname = match tag.[0] with '0' .. '9' -> "v" ^ tag | _ -> tag in 277 | (jobname, cmds) :: acc ) 278 | h [] 279 | 280 | let gen_ocaml ({staging_hub_id; prod_hub_id; results_dir; _} as opts) () = 281 | ignore (Bos.OS.Dir.create ~path:true results_dir) ; 282 | let ocaml_dockerfiles = 283 | List.map 284 | (fun arch -> 285 | let prefix = Fmt.strf "ocaml-%s" (OV.string_of_arch arch) in 286 | let results_dir = Fpath.(results_dir / prefix) in 287 | ignore (Bos.OS.Dir.create ~path:true results_dir) ; 288 | let all_compilers = 289 | D.active_distros arch 290 | |> List.map (O.all_ocaml_compilers prod_hub_id arch) 291 | in 292 | let each_compiler = 293 | D.active_tier1_distros arch 294 | |> List.map (O.separate_ocaml_compilers prod_hub_id arch) 295 | |> List.flatten 296 | in 297 | let dfiles = all_compilers @ each_compiler in 298 | ignore (G.generate_dockerfiles ~crunch:true results_dir dfiles) ; 299 | List.map (fun (f, _) -> (f, arch)) dfiles ) 300 | arches 301 | |> List.flatten 302 | in 303 | let ocaml_arch_builds = 304 | List.map 305 | (fun (f, arch) -> 306 | let arch = OV.string_of_arch arch in 307 | let label = Fmt.strf "%s-linux-%s" f arch in 308 | let tag = Fmt.strf "%s:%s" staging_hub_id label in 309 | let cmds = 310 | `O 311 | [ ("stage", `String "ocaml-builds") 312 | ; ("retry", `String "2") 313 | ; ("except", `A [`String "pushes"]) 314 | ; ("tags", `A [`String "shell"; `String (match arch with "i386" -> "amd64" | "arm32v7" -> "arm64" | _ -> arch) ]) 315 | ; ( "script" 316 | , docker_build_and_push_cmds ~distro:f ~arch ~tag "ocaml" ) ] 317 | in 318 | (label, cmds) ) 319 | ocaml_dockerfiles 320 | in 321 | let ocaml_dockerfiles_by_arch = assoc_hashtbl ocaml_dockerfiles in 322 | let ocaml_multiarch_dockerfiles = 323 | gen_multiarch ~staging_hub_id ~prod_hub_id ocaml_dockerfiles_by_arch "" 324 | "ocaml" 325 | in 326 | (* Generate aliases for OCaml releases and distros *) 327 | let distro_alias_multiarch = 328 | let distro_aliases = Hashtbl.create 7 in 329 | List.iter 330 | (fun ldistro -> 331 | let distro = D.resolve_alias ldistro in 332 | let f = D.tag_of_distro distro in 333 | let tag = D.tag_of_distro ldistro in 334 | let arches = 335 | try snd (Hashtbl.find ocaml_dockerfiles_by_arch f) 336 | with Not_found -> [] 337 | in 338 | Hashtbl.add distro_aliases f (Some tag, arches) ; 339 | (* Add an alias for "latest" for Debian Stable too *) 340 | if ldistro = `Debian `Stable then 341 | Hashtbl.add distro_aliases f (Some "latest", arches) ) 342 | D.latest_distros ; 343 | gen_multiarch ~staging_hub_id ~prod_hub_id distro_aliases "" "alias" 344 | in 345 | let ocaml_alias_multiarch = 346 | let ocaml_aliases = Hashtbl.create 7 in 347 | List.iter 348 | (fun ov -> 349 | let ov = OV.with_patch ov None in 350 | let distro = D.resolve_alias (`Debian `Stable) in 351 | let f = 352 | Fmt.strf "%s-ocaml-%s" (D.tag_of_distro distro) (OV.to_string ov) 353 | in 354 | let arches = 355 | try snd (Hashtbl.find ocaml_dockerfiles_by_arch f) 356 | with Not_found -> [] 357 | in 358 | let tag = OV.to_string ov in 359 | Hashtbl.add ocaml_aliases f (Some tag, arches) ) 360 | OV.Releases.recent_with_dev ; 361 | gen_multiarch ~staging_hub_id ~prod_hub_id ocaml_aliases "" "alias" 362 | in 363 | let yml = 364 | `O 365 | ( ocaml_arch_builds @ ocaml_multiarch_dockerfiles @ distro_alias_multiarch 366 | @ ocaml_alias_multiarch ) 367 | |> Yaml.to_string_exn ~len:256000 368 | in 369 | Bos.OS.File.write Fpath.(results_dir / "ocaml-builds.yml") yml 370 | >>= fun () -> Bos.OS.File.write Fpath.(results_dir / "README.md") (docs opts) 371 | 372 | let gen_opam ({staging_hub_id; prod_hub_id; results_dir; _} as opts) () = 373 | ignore (Bos.OS.Dir.create ~path:true results_dir) ; 374 | let opam_dockerfiles = 375 | List.map 376 | (fun arch -> 377 | let prefix = Fmt.strf "opam-%s" (OV.string_of_arch arch) in 378 | let results_dir = Fpath.(results_dir / prefix) in 379 | ignore (Bos.OS.Dir.create ~path:true results_dir) ; 380 | let distros = 381 | List.filter 382 | (D.distro_supported_on arch OV.Releases.latest) 383 | (D.active_distros arch) 384 | in 385 | let open Dockerfile in 386 | let dfiles = List.map (fun distro -> O.gen_opam2_distro ~arch distro) distros in 387 | ignore (G.generate_dockerfiles ~crunch:true results_dir dfiles) ; 388 | List.map (fun (f, _) -> (f, arch)) dfiles ) 389 | arches 390 | |> List.flatten 391 | in 392 | let opam_arch_builds = 393 | List.map 394 | (fun (f, arch) -> 395 | let arch = OV.string_of_arch arch in 396 | let tag = Fmt.strf "%s:%s-opam-linux-%s" staging_hub_id f arch in 397 | let label = Fmt.strf "%s-opam-linux-%s" f arch in 398 | let cmds = 399 | `O 400 | [ ("stage", `String "opam-builds") 401 | ; ("retry", `String "2") 402 | ; ("except", `A [`String "pushes"]) 403 | ; ("tags", `A [`String "shell"; `String (match arch with "i386" -> "amd64" | "arm32v7" -> "arm64" | _ -> arch)]) 404 | ; ("script", docker_build_and_push_cmds ~distro:f ~arch ~tag "opam") 405 | ] 406 | in 407 | (label, cmds) ) 408 | opam_dockerfiles 409 | in 410 | let opam_dockerfiles_by_arch = assoc_hashtbl opam_dockerfiles in 411 | let opam_multiarch_dockerfiles = 412 | gen_multiarch ~staging_hub_id ~prod_hub_id opam_dockerfiles_by_arch "-opam" 413 | "opam" 414 | in 415 | let yml = 416 | `O (opam_arch_builds @ opam_multiarch_dockerfiles) 417 | |> Yaml.to_string_exn ~len:256000 418 | in 419 | Bos.OS.File.write Fpath.(results_dir / "opam-builds.yml") yml 420 | >>= fun () -> Bos.OS.File.write Fpath.(results_dir / "README.md") (docs opts) 421 | 422 | let pkg_version pkg = 423 | let open Astring in 424 | match String.cut ~sep:"." pkg with 425 | | None -> Error (`Msg "invalid pkg") 426 | | Some (pkg, ver) -> Ok (pkg, ver) 427 | 428 | open Cmdliner 429 | 430 | let setup_logs = C.setup_logs () 431 | 432 | let fpath = Arg.conv ~docv:"PATH" (Fpath.of_string, Fpath.pp) 433 | 434 | let arch = 435 | let doc = "CPU architecture to perform build on" in 436 | let term = 437 | Arg.enum [("i386", `I386); ("amd64", `X86_64); ("arm64", `Aarch64); ("ppc64le", `Ppc64le)] 438 | in 439 | Arg.(value & opt term `X86_64 & info ["arch"] ~docv:"ARCH" ~doc) 440 | 441 | let opam_repo_rev = 442 | let doc = "opam repo git rev" in 443 | let open Arg in 444 | required 445 | & opt (some string) None 446 | & info ["opam-repo-rev"] ~docv:"OPAM_REPO_REV" ~doc 447 | 448 | let copts_t = 449 | let docs = Manpage.s_common_options in 450 | let staging_hub_id = 451 | let doc = "Docker Hub user/repo to push to for staging builds" in 452 | let open Arg in 453 | value 454 | & opt string "ocaml/opam2-staging" 455 | & info ["staging-hub-id"] ~docv:"STAGING_HUB_ID" ~doc ~docs 456 | in 457 | let prod_hub_id = 458 | let doc = 459 | "Docker Hub user/repo to push to for production multiarch builds" 460 | in 461 | let open Arg in 462 | value & opt string "ocaml/opam2" 463 | & info ["prod-hub-id"] ~docv:"PROD_HUB_ID" ~doc ~docs 464 | in 465 | let results_dir = 466 | let doc = "Directory in which to store bulk build results" in 467 | let open Arg in 468 | value 469 | & opt fpath (Fpath.v "_results") 470 | & info ["o"; "results-dir"] ~docv:"RESULTS_DIR" ~doc ~docs 471 | in 472 | Term.(const copts $ staging_hub_id $ prod_hub_id $ results_dir) 473 | 474 | let buildv ov distro = 475 | Ocaml_version.of_string_exn ov 476 | |> fun ov -> 477 | let distro = 478 | match D.distro_of_tag distro with 479 | | None -> failwith "unknown distro" 480 | | Some distro -> distro 481 | in 482 | {ov; distro} 483 | 484 | let build_t = 485 | let ocaml_version = 486 | let doc = "ocaml version to build" in 487 | let env = Arg.env_var "OCAML_VERSION" ~doc in 488 | let open Arg in 489 | value & opt string "4.06.1" 490 | & info ["ocaml-version"] ~docv:"OCAML_VERSION" ~env ~doc 491 | in 492 | let distro = 493 | let doc = "distro to build" in 494 | let env = Arg.env_var "DISTRO" ~doc in 495 | let open Arg in 496 | value & opt string "debian-9" & info ["distro"] ~env ~docv:"DISTRO" ~doc 497 | in 498 | Term.(const buildv $ ocaml_version $ distro) 499 | 500 | let opam_cmd = 501 | let doc = "generate, build and push base opam container images" in 502 | let exits = Term.default_exits in 503 | let man = 504 | [ `S Manpage.s_description 505 | ; `P "Generate and build base $(b,opam) container images." ] 506 | in 507 | ( Term.(term_result (const gen_opam $ copts_t $ setup_logs)) 508 | , Term.info "opam" ~doc ~sdocs:Manpage.s_common_options ~exits ~man ) 509 | 510 | let ocaml_cmd = 511 | let doc = "generate, build and push base ocaml container images" in 512 | let exits = Term.default_exits in 513 | let man = 514 | [ `S Manpage.s_description 515 | ; `P "Generate and build base $(b,ocaml) container images." ] 516 | in 517 | ( Term.(term_result (const gen_ocaml $ copts_t $ setup_logs)) 518 | , Term.info "ocaml" ~doc ~sdocs:Manpage.s_common_options ~exits ~man ) 519 | 520 | let default_cmd = 521 | let doc = "build and push opam and OCaml multiarch container images" in 522 | let sdocs = Manpage.s_common_options in 523 | ( Term.(ret (const (fun _ -> `Help (`Pager, None)) $ pure ())) 524 | , Term.info "obi-git" ~version:"%%VERSION%%" ~doc ~sdocs ) 525 | 526 | let cmds = [opam_cmd; ocaml_cmd] 527 | 528 | let () = Term.(exit @@ eval_choice default_cmd cmds) 529 | -------------------------------------------------------------------------------- /src/opam-ci/dune: -------------------------------------------------------------------------------- 1 | (executables 2 | (names main) 3 | (public_names opam-ci) 4 | (libraries obi bos cmdliner fmt.cli logs.fmt fmt.tty logs.cli rresult textwrap) 5 | (flags -w -9) 6 | (package opam-ci)) 7 | -------------------------------------------------------------------------------- /src/opam-ci/main.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | open User 18 | open Cmdliner 19 | 20 | let setup_logs = 21 | let setup_log level = 22 | Fmt_tty.setup_std_outputs ~style_renderer:`Ansi_tty () ; 23 | Logs.set_level level ; 24 | Logs.set_reporter (Logs_fmt.reporter ()) 25 | in 26 | let global_option_section = "COMMON OPTIONS" in 27 | Term.(const setup_log $ Logs_cli.level ~docs:global_option_section ()) 28 | 29 | let copts_t = 30 | let docs = Manpage.s_common_options in 31 | let refresh = 32 | let doc = 33 | "How to refresh status logs. $(docv) defaults to $(i,poll) which pulls \ 34 | from GitHub every hour. $(i,network) will force the metadata to be \ 35 | pulled, and $(i,local) will only use the cached version locally." 36 | in 37 | let open Arg in 38 | let term = 39 | Arg.enum [("local", `Local); ("poll", `Poll); ("network", `Network)] 40 | in 41 | value & opt term `Poll & info ["refresh"] ~docv:"REFRESH_LOGS" ~doc ~docs 42 | in 43 | Term.(const copts $ refresh) 44 | 45 | let filter_t = 46 | let maintainer = 47 | let doc = 48 | "Match packages via a case-insensitive substring check on the \ 49 | $(b,maintainer) and $(b,tags) fields in the opam metadata. $(docv) can \ 50 | be repeated multiple times to include more filters." 51 | in 52 | let open Arg in 53 | value & opt_all string [] 54 | & info ["maintainer"; "m"] ~docv:"MAINTAINER" ~doc 55 | in 56 | let filters_t = 57 | let doc = 58 | "Filter the list of packages displayed. $(docv) defaults to \ 59 | $(i,failures) to show all packages with errors. $(i,recent) will show \ 60 | results for the latest version of each package. $(i,variants) will \ 61 | list packages that regress with a compiler variant such as safe-string \ 62 | or flambda. $(i,variants:fl), $(i,variants:rc) and $(i,variants:ss) \ 63 | will show failures for just the flambda, release-candidate or \ 64 | safe-string tests respectively. $(i,lagging) will show the packages \ 65 | whose latest release is uninstallable against the latest version of \ 66 | the compiler. $(i,orphaned) will list all the packages that do not \ 67 | have an active maintainer, and so are looking for community help to \ 68 | take care of them. $(i,all) will show all known results for all \ 69 | versions including successes." 70 | in 71 | let open Arg in 72 | let term = 73 | Arg.enum 74 | [ ("all", `All) 75 | ; ("failures", `Failures) 76 | ; ("recent", `Recent) 77 | ; ("lagging", `Lagging) 78 | ; ("orphaned", `Orphaned) 79 | ; ("variants:ss", `Variants `SS) 80 | ; ("variants:fl", `Variants `Flambda) 81 | ; ("variants:rc", `Variants `RC) ] 82 | in 83 | value & opt term `Failures & info ["filter"; "f"] ~docv:"FILTERS" ~doc 84 | in 85 | Term.(const filters $ maintainer $ filters_t) 86 | 87 | let param_t = 88 | let distro = 89 | let doc = "List only logs relating to this distribution" in 90 | let open Arg in 91 | let term = 92 | conv ~docv:"DISTRO" 93 | ( (fun d -> 94 | match D.distro_of_tag d with 95 | | Some r -> Ok r 96 | | None -> Error (`Msg ("unknown distribution " ^ d)) ) 97 | , fun ppf d -> Fmt.pf ppf "%s" (D.tag_of_distro d) ) 98 | in 99 | value & opt (some term) None & info ["distro"] ~docv:"DISTRO" ~doc 100 | in 101 | let ov = 102 | let doc = "List only logs relating to this OCaml compiler version" in 103 | let open Arg in 104 | let term = conv ~docv:"COMPILER" (OV.of_string, OV.pp) in 105 | value & opt (some term) None & info ["compiler"] ~doc 106 | in 107 | let arch = 108 | let doc = "List only logs relating to this CPU architecture" in 109 | let open Arg in 110 | let term = 111 | enum [("amd64", `X86_64); ("arm64", `Aarch64); ("ppc64le", `Ppc64le)] 112 | in 113 | value & opt (some term) None & info ["arch"] ~doc 114 | in 115 | Term.(const params $ distro $ ov $ arch) 116 | 117 | let status_cmd = 118 | let doc = "summary of builds across compilers, OS and CPUs" in 119 | let exits = Term.default_exits in 120 | let packages_t = 121 | let doc = 122 | "optional list of package names (without versions) to filter the status \ 123 | list by. If omitted, all packages matching the criteria are shown. You \ 124 | can also use $(i,-m) to filter by maintainer or tag or $(i,-f) by \ 125 | other filter criteria (see below)." 126 | in 127 | Arg.(value & pos_all string [] & info [] ~docv:"PACKAGES" ~doc) 128 | in 129 | let man = 130 | [ `S Manpage.s_description 131 | ; `P 132 | "The status view shows a panel of icons that represent different \ 133 | combinations of ways to build opam packages. From left to right, \ 134 | these are:" 135 | ; `I 136 | ( "$(b,Compiler)" 137 | , "The circled numbers represent OCaml compiler versions (a circled 6 \ 138 | is OCaml 4.06, a circled 7 is 4.07, and so on). " ) 139 | ; `I 140 | ( "$(b,Distro)" 141 | , "The square letters indicate different OS distributions. $(i,D) is \ 142 | Debian, $(i,F) is Fedora, $(i,A) is Alpine, $(i,U) is Ubuntu and \ 143 | $(i,S) is OpenSUSE." ) 144 | ; `I 145 | ( "$(b,CPU Architecture)" 146 | , "The small circled letters represent different CPU architectures. \ 147 | $(i,x) represents x86_64, $(i,a) is arm64 and $(i,p) is \ 148 | PowerPC64LE." ) 149 | ; `P 150 | "Some compiler variants are also tested to track down specific \ 151 | problems, shown by the icons to the far right of the display." 152 | ; `I 153 | ( "$(b,safe-string)" 154 | , "The $(i,ss) icon is for 'safe-string' failures, which would happen \ 155 | in OCaml 4.06 due to the switch to immutable strings." ) 156 | ; `I 157 | ( "$(b,flambda)" 158 | , "The $(i,fl) icon is for packages that fail to compile with the \ 159 | flambda variant of the compiler." ) 160 | ; `I 161 | ( "$(b,release-candidate)" 162 | , "The $(i,flag) icon is for packages that fail to compile with the \ 163 | latest release candidate of OCaml; this is useful to figure out \ 164 | how much of the ecosystem works with a soon-to-be-released \ 165 | compiler." ) 166 | ; `P 167 | "The colours indicate the result of the build: $(i,white) indicates \ 168 | the package was not built due to constraints, $(i,green) is a \ 169 | successful build, $(i, yellow) indicates the build was skipped due \ 170 | to a dependency failure, $(i, red) is a direct build failure of that \ 171 | package, and $(i,magenta) and $(i,blue) indicate package metadata \ 172 | errors such as a failure of the solver to find a solution or the \ 173 | package sources being unavailable." ] 174 | in 175 | ( (let open Term in 176 | term_result 177 | (const show_status $ copts_t $ filter_t $ packages_t $ setup_logs)) 178 | , Term.info "status" ~doc ~exits ~man ) 179 | 180 | let logs_cmd = 181 | let doc = "display detailed failure logs for opam packages" in 182 | let exits = Term.default_exits in 183 | let pkg_t = Arg.(required & pos 0 (some string) None & info [] ~doc) in 184 | let man = 185 | [ `S Manpage.s_description 186 | ; `P 187 | "The logs view will show the build details available for the various \ 188 | configuration combinations. To start, you can try to show the \ 189 | results for a single package:" 190 | ; `P "opam-ci logs cdrom" 191 | ; `P 192 | "If just one failure is found, then the build logs are shown for that \ 193 | failure. If there is more than one failure, the output will give you \ 194 | a more precise command line to enter to select just one of the \ 195 | failures. For example:" 196 | ; `P 197 | "opam-ci logs cdrom.0.9.1 --compiler=4.06 --arch=amd64 \ 198 | --distro=alpine-3.7" 199 | ; `P 200 | "There will often be repeat failures, so just pick one of them and \ 201 | hopefully many of the other errors will be related." ] 202 | in 203 | ( (let open Term in 204 | term_result (const show_logs $ pkg_t $ copts_t $ param_t $ setup_logs)) 205 | , Term.info "logs" ~doc ~exits ~man ) 206 | 207 | let default_cmd = 208 | let doc = "opam 2.0 package manager continuous integration interface" in 209 | let sdocs = Manpage.s_common_options in 210 | let man = 211 | [ `S Manpage.s_description 212 | ; `P 213 | "The $(tname) tool provides an interface to the opam 2.0 continuous\n \ 214 | integration cluster, which regularly rebuilds the package repository \ 215 | across\n \ 216 | a variety of OCaml compiler versions, operating system distributions \ 217 | and \n \ 218 | CPU architectures. These builds are done regularly in remote \ 219 | infrastructure and\n \ 220 | the results are pushed to a metadata repository where they are \ 221 | fetched by this\n \ 222 | command to give you a summary of the status of your own packages." 223 | ; `P 224 | "The $(i,opam-ci status) command shows a dashboard of the build results\n \ 225 | across this matrix. Packages can be filtered by maintainer \ 226 | substrings or tag names in the\n \ 227 | opam package description, so you see only those relevant to you." 228 | ; `P 229 | "The $(i,opam-ci logs) command will show you the build errors\n \ 230 | so you can fix them. It also generates a Dockerfile of the precise \ 231 | build to reproduce the environment locally for you." 232 | ; `P 233 | "To get started, try these commands with the maintainer argument \ 234 | replaced with your own information or tags:" 235 | ; `P "# show all the failing MirageOS packages" 236 | ; `Noblank 237 | ; `P "opam-ci status -m org:mirage | less -R" 238 | ; `P "# show all the packages maintained by anil@recoil.org" 239 | ; `Noblank 240 | ; `P "opam-ci status -m anil@recoil.org --filter=all | less -R" 241 | ; `P 242 | "# show all the packages failing on the latest RC of the OCaml compiler" 243 | ; `Noblank 244 | ; `P "opam-ci status --filter=variants:rc | less -R" 245 | ; `P "# display all failure logs for the mirage-xen package" 246 | ; `Noblank 247 | ; `P "opam-ci logs mirage-xen" 248 | ; `P 249 | "See the $(b,--help) pages for the subcommands below for more \ 250 | detailed information." ] 251 | in 252 | ( Term.(ret (const (fun _ -> `Help (`Pager, None)) $ pure ())) 253 | , Term.info "opam-ci" ~version:"%%VERSION%%" ~doc ~sdocs ~man ) 254 | 255 | let cmds = [status_cmd; logs_cmd] 256 | 257 | let () = Term.(exit @@ eval_choice default_cmd cmds) 258 | -------------------------------------------------------------------------------- /src/opam-ci/repos.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | open Bos 18 | open Rresult 19 | 20 | let home_dir () = OS.Env.(value "HOME" path ~absent:(Fpath.v ".")) 21 | 22 | let obi_dir () = 23 | let absent = Fpath.(home_dir () / ".obi") in 24 | OS.Env.(value ~log:Logs.Debug "OBI_HOME" path ~absent) 25 | 26 | let remote_logs_repo () = 27 | let absent = "https://github.com/ocaml/obi-logs.git" in 28 | OS.Env.(value ~log:Logs.Debug "OBI_LOGS_REPO" string ~absent) 29 | 30 | let logs_polling_interval = 3600. 31 | 32 | let run_git_in_repo ~repo args = 33 | OS.Cmd.(run Cmd.(v "git" % "-C" % p repo %% of_list args)) 34 | 35 | let run_git args = OS.Cmd.(run Cmd.(v "git" %% of_list args)) 36 | 37 | let rec init ?(retry= false) ?(refresh= `Poll) () = 38 | let d = obi_dir () in 39 | Logs.info (fun l -> l "Initialising in %a" Fpath.pp d) ; 40 | OS.Dir.create ~path:true d 41 | >>= fun _ -> 42 | let local_logs_repo = Fpath.(d / "obi-logs") in 43 | let local_logs_mtime = Fpath.(d / "last-pulled") in 44 | OS.Dir.exists local_logs_repo 45 | >>= fun repo_exists -> 46 | ( if repo_exists then ( 47 | let refresh = 48 | match (retry, refresh) with 49 | | _, `Local -> `Local 50 | | _, `Network -> `Network 51 | | true, `Poll -> `Network 52 | | false, `Poll -> 53 | let poll = 54 | OS.Path.stat local_logs_mtime 55 | >>= fun stats -> 56 | let curtime = Unix.gettimeofday () in 57 | let mtime = stats.Unix.st_mtime in 58 | if curtime -. mtime > logs_polling_interval then ( 59 | Logs.debug (fun l -> 60 | l "Obi logs repo polling time exceeded; pulling from network" 61 | ) ; 62 | Ok `Network ) 63 | else ( 64 | Logs.debug (fun l -> 65 | l 66 | "Obi logs repo has been pulled recently so not pulling \ 67 | from network" ) ; 68 | Ok `Local ) 69 | in 70 | match poll with 71 | | Ok r -> r 72 | | Error (`Msg m) -> 73 | Logs.debug (fun l -> l "Forcing network poll due to: %s" m) ; 74 | `Network 75 | in 76 | match refresh with 77 | | `Network -> 78 | Logs.debug (fun l -> l "Fetching latest Obi logs") ; 79 | run_git_in_repo ~repo:local_logs_repo 80 | ["fetch"; "-q"; remote_logs_repo (); "index"] 81 | >>= fun () -> 82 | run_git_in_repo ~repo:local_logs_repo 83 | ["remote"; "set-url"; "origin"; remote_logs_repo ()] 84 | >>= fun () -> 85 | run_git_in_repo ~repo:local_logs_repo 86 | ["reset"; "-q"; "--hard"; "origin/index"] 87 | >>= fun () -> OS.File.write local_logs_mtime "" 88 | | `Local -> 89 | Logs.debug (fun l -> l "Using existing Obi logs") ; 90 | Ok () ) 91 | else 92 | match refresh with 93 | | `Network | `Poll -> 94 | Logs.debug (fun l -> l "Cloning fresh Obi logs") ; 95 | run_git 96 | [ "clone" 97 | ; "-q" 98 | ; "--depth=1" 99 | ; "-b" 100 | ; "index" 101 | ; remote_logs_repo () 102 | ; Cmd.p local_logs_repo ] 103 | | `Local -> 104 | Logs.err (fun l -> 105 | l 106 | "Must use --refresh=poll or --refresh=network to fetch initial \ 107 | logs" ) ; 108 | Error 109 | (`Msg 110 | "Must use --refresh=poll or --refresh=network to fetch initial logs") 111 | ) 112 | >>= fun () -> 113 | (* TODO store multiple versions based on date? *) 114 | let state = Fpath.(local_logs_repo / "index.sxp") in 115 | Logs.debug (fun l -> l "Parsing state file %a" Fpath.pp state) ; 116 | OS.File.read state 117 | >>= fun s -> 118 | try 119 | let t = Obi.Index.t_of_sexp (Sexplib.Sexp.of_string s) in 120 | let pkgs = t.Obi.Index.packages in 121 | Logs.debug (fun l -> l "Found metadata for %d packages" (List.length pkgs)) ; 122 | Ok pkgs 123 | with _exn -> 124 | try 125 | Printexc.record_backtrace true ; 126 | (* Manually parse the sexpression to get the version string *) 127 | let version = 128 | let open Sexplib.Sexp in 129 | let s = of_string s in 130 | match s with 131 | | List (List [Atom "version"; Atom v] :: _) -> int_of_string v 132 | | _ -> raise (Failure "unable to find version string in index.sxp") 133 | in 134 | let err = 135 | Fmt.strf 136 | "Your opam-ci client is out of date.\n\ 137 | It is using Obi metadata version %d but the upstream logs have \ 138 | version %d (higher indicates a newer version).\n\ 139 | Please run `opam update -u` to get the latest version of opam-ci \ 140 | that is compatible with the latest log format.\n\ 141 | If that does not work, you can try pinning to the development \ 142 | version of opam-ci via `opam pin add opam-ci --dev`.\n\ 143 | If that also does not help, then please report an issue at \ 144 | https://github.com/ocaml/obi/issues" 145 | Obi.Index.current_version version 146 | in 147 | if retry then Error (`Msg err) else init ~retry:true ~refresh () 148 | with exn -> 149 | let err = 150 | Fmt.strf 151 | "We have encountered a total failure to parse the upstream metadata.\n\ 152 | Please run `opam update -u` to get the latest version of opam-ci \ 153 | that is compatible with the latest log format.\n\ 154 | If that does not work, try pinning to the development \ 155 | version of opam-ci via `opam pin add opam-ci --dev`.\n\ 156 | Please report this issue with at \ 157 | https://github.com/ocaml/obi/issues with this backtrace:\n\ 158 | %s\n\n \ 159 | %s\n" 160 | (Printexc.to_string exn) 161 | (Printexc.get_backtrace ()) 162 | in 163 | if retry then Error (`Msg err) else init ~retry:true ~refresh () 164 | -------------------------------------------------------------------------------- /src/opam-ci/user.ml: -------------------------------------------------------------------------------- 1 | (* Copyright (c) 2018 Anil Madhavapeddy 2 | * 3 | * Permission to use, copy, modify, and distribute this software for any 4 | * purpose with or without fee is hereby granted, provided that the above 5 | * copyright notice and this permission notice appear in all copies. 6 | * 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | * 15 | *) 16 | 17 | open Rresult 18 | open Astring 19 | module OV = Ocaml_version 20 | module D = Dockerfile_distro 21 | 22 | module U = struct 23 | let c3 = "③ " 24 | 25 | let c4 = "④ " 26 | 27 | let c5 = "⑤ " 28 | 29 | let c6 = "⑥ " 30 | 31 | let c7 = "⑦ " 32 | 33 | let c8 = "⑧ " 34 | 35 | let tick = "✓ " 36 | 37 | let cross = "✘ " 38 | 39 | let debian = "🄳 " 40 | 41 | let fedora = "🄵 " 42 | 43 | let ubuntu = "🅄 " 44 | 45 | let opensuse = "🅂 " 46 | 47 | let alpine = "🄰 " 48 | 49 | let flambda = "fl " 50 | 51 | let ss = "∬ " 52 | 53 | let release = "⚐ " 54 | 55 | let amd64 = "ⓧ " 56 | 57 | let arm64 = "ⓐ " 58 | 59 | let ppc64 = "ⓟ " 60 | 61 | let arm32 = "ⓐ " (* TODO fix these unicode characters *) 62 | end 63 | 64 | module A = struct 65 | open Obi.Index 66 | 67 | let ovs = 68 | List.map OV.of_string_exn ["4.03"; "4.04"; "4.05"; "4.06"; "4.07"; "4.08"] 69 | 70 | let ov_stable = OV.of_string_exn "4.06" 71 | 72 | let ov_rc = OV.of_string_exn "4.07" 73 | 74 | let ov_stable_uss = OV.of_string_exn "4.06+default-unsafe-string" 75 | 76 | let ov_stable_fl = OV.of_string_exn "4.06+flambda" 77 | 78 | let base_distro = `Debian `V9 79 | 80 | let other_distros = 81 | [`Alpine `V3_7; `Ubuntu `V18_04; `Fedora `V28; `OpenSUSE `V42_3] 82 | 83 | let distros = base_distro :: other_distros 84 | 85 | let find ?(distro= base_distro) ?(ov= ov_stable) ?(arch= `X86_64) 86 | (m: metadata list) = 87 | List.find_opt 88 | (fun m -> 89 | m.params.distro = distro && m.params.ov = ov && m.params.arch = arch ) 90 | m 91 | 92 | (* Latest stable *) 93 | let stable ms = List.filter (fun m -> m.params.distro = base_distro) ms 94 | 95 | type res = [`Ok | `Fail | `Uninstallable | `Unknown] 96 | 97 | let classify m = 98 | match m with 99 | | None -> `Unknown 100 | | Some {build_result= `Ok} -> `Ok 101 | | Some {build_result= `Fail _} -> `Fail 102 | | Some {build_result= `Depfail _} -> `Depfail 103 | | Some {build_result= `Uninstallable _} -> `Uninstallable 104 | | Some {build_result= `Solver_failure} -> `Solver_failure 105 | | Some {build_result= `No_sources _} -> `No_sources 106 | 107 | let latest_version versions = 108 | List.sort (fun a b -> Obi.VersionCompare.compare (fst b) (fst a)) versions 109 | |> List.hd 110 | 111 | let test_two_versions a b m = 112 | let ss = find ~ov:a m in 113 | let ss' = classify ss in 114 | let uss = find ~ov:b m in 115 | let uss' = classify uss in 116 | match (ss', uss') with 117 | | `Fail, `Ok | `Ok, `Fail -> `Fail 118 | | `Ok, `Ok -> `Ok 119 | | `Fail, `Fail -> `Ok 120 | | `Depfail, _ | _, `Depfail -> `Depfail 121 | | `Uninstallable, _ | _, `Uninstallable -> `Uninstallable 122 | | `Solver_failure, _ | _, `Solver_failure -> `Solver_failure 123 | | `No_sources, _ | _, `No_sources -> `No_sources 124 | | `Unknown, _ | _, `Unknown -> `Unknown 125 | 126 | let test_safe_string m = test_two_versions ov_stable ov_stable_uss m 127 | 128 | let test_flambda m = test_two_versions ov_stable ov_stable_fl m 129 | 130 | let test_ocaml406to7 m = test_two_versions ov_stable ov_rc m 131 | 132 | let is_fail = function `Fail | `No_sources -> true | _ -> false 133 | 134 | (* There are any failures *) 135 | let has_fails m = List.exists (fun m -> classify (Some m) |> is_fail) m 136 | 137 | let has_variant_fails ty m = 138 | match ty with 139 | | `Flambda -> test_flambda m |> is_fail 140 | | `SS -> test_safe_string m |> is_fail 141 | | `RC -> test_ocaml406to7 m |> is_fail 142 | 143 | let any_variant_fails m = 144 | (* TODO add an enumeration of the variants rather than hardcoding here *) 145 | List.exists (fun ty -> has_variant_fails ty m) [`Flambda; `SS; `RC] 146 | 147 | let is_lagging ms = 148 | let version, m = latest_version ms in 149 | let ss = find ~ov:ov_stable m |> classify in 150 | let uss = find ~ov:ov_rc m |> classify in 151 | match (ss, uss) with 152 | | (`Ok | `Unknown | `Uninstallable), `Uninstallable -> true 153 | | _ -> false 154 | 155 | let is_unmaintained pkg = 156 | pkg.maintainers = [] 157 | || List.mem "https://github.com/ocaml/opam-repository/issues" 158 | pkg.maintainers 159 | || List.mem "contact@ocamlpro.com" pkg.maintainers 160 | end 161 | 162 | module S = struct 163 | open Obi.Index 164 | 165 | let u_of_ov = 166 | let open U in 167 | function 168 | | "4.03" -> c3 169 | | "4.04" -> c4 170 | | "4.05" -> c5 171 | | "4.06" -> c6 172 | | "4.07" -> c7 173 | | "4.08" -> c8 174 | | _ -> "?" 175 | 176 | let u_of_distro = 177 | let open U in 178 | function 179 | | `Debian _ -> debian 180 | | `Fedora _ -> fedora 181 | | `OpenSUSE _ -> opensuse 182 | | `Alpine _ -> alpine 183 | | `Ubuntu _ -> ubuntu 184 | | _ -> "?" 185 | 186 | let u_of_arch = 187 | U.(function `I386|`X86_64 -> amd64 | `Aarch64 -> arm64 | `Ppc64le -> ppc64 | `Aarch32 -> arm32 ) 188 | 189 | let render_classify ppf fn m u = 190 | match fn m |> A.classify with 191 | | `Unknown -> Fmt.(pf ppf "%a" (styled `Underline string) u) 192 | | `Ok -> Fmt.(pf ppf "%a" (styled `Green string) u) 193 | | `Uninstallable -> Fmt.(pf ppf "%a" string u) 194 | | `Solver_failure -> 195 | Fmt.(pf ppf "%a" (styled `Underline (styled `Magenta string)) u) 196 | | `No_sources -> Fmt.(pf ppf "%a" (styled `Blue string) u) 197 | | `Fail -> Fmt.(pf ppf "%a" (styled `Red (styled `Bold string)) u) 198 | | `Depfail -> Fmt.(pf ppf "%a" (styled `Yellow string) u) 199 | 200 | let compilers ppf (m: metadata list) = 201 | List.iter 202 | (fun ov -> 203 | u_of_ov (OV.to_string ov) |> render_classify ppf (A.find ~ov) m ) 204 | A.ovs 205 | 206 | let distros ppf m = 207 | List.iter 208 | (fun distro -> 209 | u_of_distro distro |> render_classify ppf (A.find ~distro) m ) 210 | A.distros 211 | 212 | let arches ppf m = 213 | List.iter 214 | (fun arch -> u_of_arch arch |> render_classify ppf (A.find ~arch) m) 215 | OV.arches 216 | 217 | let variants ppf m = 218 | let col = function 219 | | `No_sources -> `Blue 220 | | `Uninstallable -> `White 221 | | `Unknown -> `Underline 222 | | `Ok -> `Green 223 | | `Depfail -> `Yellow 224 | | `Solver_failure -> `Magenta 225 | | `Fail -> `Red 226 | in 227 | let run fn u = Fmt.(pf ppf "%a" (styled (col (fn m)) string) u) in 228 | run A.test_safe_string U.ss ; 229 | run A.test_flambda U.flambda ; 230 | run A.test_ocaml406to7 U.release 231 | end 232 | 233 | type copts = {refresh: [`Local | `Poll | `Network]} 234 | 235 | type filters = 236 | { maintainers: string list 237 | ; filters: 238 | [ `All 239 | | `Failures 240 | | `Recent 241 | | `Lagging 242 | | `Orphaned 243 | | `Variants of [`Flambda | `RC | `SS] ] } 244 | 245 | type params = {distro: D.t option; ov: OV.t option; arch: OV.arch option} 246 | 247 | let filters maintainers filters = {maintainers; filters} 248 | 249 | let copts refresh = {refresh} 250 | 251 | let params distro ov arch = {distro; ov; arch} 252 | 253 | let check_maintainer ~maintainers pkg = 254 | let open Obi.Index in 255 | match maintainers with 256 | | [] -> true 257 | | _ -> 258 | let l = List.map String.Ascii.lowercase (pkg.maintainers @ pkg.tags) in 259 | List.exists 260 | (fun sub -> List.exists (fun p -> String.find_sub ~sub p <> None) l) 261 | maintainers 262 | 263 | let render_package_version ppf (version, metadata) = 264 | Fmt.(pf ppf "%14s " version) ; 265 | S.compilers ppf metadata ; 266 | Fmt.(pf ppf " ") ; 267 | S.distros ppf metadata ; 268 | Fmt.(pf ppf " ") ; 269 | S.arches ppf metadata ; 270 | Fmt.(pf ppf " ") ; 271 | S.variants ppf metadata ; 272 | Fmt.(pf ppf "@\n%!") 273 | 274 | let render_package_logs ppf name version metadata = 275 | let open Obi.Index in 276 | let p = metadata.params in 277 | (let open Fmt in 278 | pf ppf "@\n%a %a %s %s %s (%a) (opam-repository %s):@\n" 279 | (styled `Bold (styled `Blue string)) 280 | "====>" 281 | (styled `Bold string) 282 | (name ^ "." ^ version) 283 | (OV.to_string p.ov) 284 | (D.human_readable_string_of_distro p.distro) 285 | (OV.string_of_arch p.arch) pp_result metadata.build_result 286 | (String.with_range ~len:8 metadata.rev)) ; 287 | match metadata.log with 288 | | [] -> Fmt.(pf ppf "@\n") 289 | | _logs -> 290 | let w = 291 | Wrapper.make ~initial_indent:" " ~subsequent_indent:" " 292 | ~drop_whitespace:true 100 293 | in 294 | List.iter 295 | (fun l -> List.iter (fun s -> Fmt.pf ppf "%s@\n" s) (Wrapper.wrap w l)) 296 | metadata.log 297 | 298 | let render_packages ppf name pkgs = 299 | List.iter 300 | (fun v -> 301 | Fmt.pf ppf "%35s %!" name ; 302 | render_package_version ppf v ) 303 | pkgs 304 | 305 | let render_package ppf ~filters pkg = 306 | let open Obi.Index in 307 | match filters with 308 | | `All -> render_packages ppf pkg.name pkg.versions 309 | | `Lagging -> 310 | if A.is_lagging pkg.versions then 311 | render_packages ppf pkg.name [A.latest_version pkg.versions] 312 | | `Orphaned -> 313 | if A.is_unmaintained pkg then 314 | render_packages ppf pkg.name [A.latest_version pkg.versions] 315 | | `Failures -> 316 | List.filter (fun (_, m) -> A.has_fails m) pkg.versions 317 | |> render_packages ppf pkg.name 318 | | `All_variants -> 319 | List.filter (fun (_, m) -> A.any_variant_fails m) pkg.versions 320 | |> render_packages ppf pkg.name 321 | | `Variants ty -> 322 | List.filter (fun (_, m) -> A.has_variant_fails ty m) pkg.versions 323 | |> render_packages ppf pkg.name 324 | | `Recent -> render_packages ppf pkg.name [A.latest_version pkg.versions] 325 | 326 | let render_package_details ppf pkg name version {distro; ov; arch} = 327 | let open Obi.Index in 328 | (* Determine how many results match the result set *) 329 | let ms = 330 | List.fold_left 331 | (fun acc (v, ml) -> 332 | let metas = 333 | List.filter 334 | (fun m -> 335 | let distro_ok = 336 | match distro with 337 | | None -> true 338 | | Some d when d = m.params.distro -> true 339 | | Some _ -> false 340 | in 341 | let ov_ok = 342 | match ov with 343 | | None -> true 344 | | Some ov when ov = m.params.ov -> true 345 | | Some _ -> false 346 | in 347 | let arch_ok = 348 | match arch with 349 | | None -> true 350 | | Some arch when arch = m.params.arch -> true 351 | | Some _ -> false 352 | in 353 | let is_err = 354 | match m.build_result with 355 | | `Ok -> false 356 | | `Uninstallable _ -> false 357 | | _ -> true 358 | in 359 | is_err && arch_ok && ov_ok && distro_ok ) 360 | ml 361 | in 362 | match version with 363 | | None -> (v, metas) :: acc 364 | | Some ver when ver = v -> (v, metas) :: acc 365 | | Some _ -> acc ) 366 | [] pkg.versions 367 | in 368 | let total_res = List.fold_left (fun a (_, ms) -> List.length ms + a) 0 ms in 369 | match total_res with 370 | | 0 -> 371 | let open Fmt in 372 | pf ppf "%a: No failures found with these constraints\n%!" 373 | (styled `Bold string) 374 | name 375 | | 1 -> 376 | List.iter 377 | (fun (version, m) -> 378 | List.iter (render_package_logs ppf name version) m ; 379 | List.iter 380 | (fun m -> 381 | (let open Fmt in 382 | pf ppf "@\n%a %a Dockerfile for %s %s %s:@\n" 383 | (styled `Bold (styled `Blue string)) 384 | "====>" 385 | (styled `Bold string) 386 | (name ^ "." ^ version) 387 | (OV.to_string m.params.ov) 388 | (D.human_readable_string_of_distro m.params.distro) 389 | (OV.string_of_arch m.params.arch)) ; 390 | let dfile = 391 | let open Dockerfile in 392 | Dockerfile_opam.bulk_build "ocaml/opam2" m.params.distro 393 | m.params.ov m.rev 394 | @@ run "opam depext -uiv %s.%s" name version 395 | in 396 | Fmt.pf ppf "%s@\n" (Dockerfile.string_of_t dfile) ) 397 | m ) 398 | ms 399 | | _ -> 400 | (let open Fmt in 401 | pf ppf 402 | "%a: multiple build failures found with different configuration \ 403 | parameters.@\n\ 404 | Please refine the command to select exactly one of the following:@\n" 405 | (styled `Bold string) 406 | name) ; 407 | List.iter 408 | (fun (version, ms) -> 409 | List.iter 410 | (fun m -> 411 | let open Fmt in 412 | pf ppf 413 | " opam-ci logs %s.%s --compiler=%s --arch=%s --distro=%s\n%!" 414 | name version (OV.to_string m.params.ov) 415 | (OV.string_of_arch m.params.arch) 416 | (D.tag_of_distro m.params.distro) ) 417 | ms ) 418 | ms 419 | 420 | let show_status {refresh} {maintainers; filters} packages () = 421 | let open Obi.Index in 422 | Repos.init ~refresh () 423 | >>= fun pkgs -> 424 | let ppf = Fmt.stdout in 425 | let pkgs = 426 | match packages with 427 | | [] -> pkgs 428 | | _ -> List.filter (fun pkg -> List.mem pkg.name packages) pkgs 429 | in 430 | let pkgs = List.sort (fun a b -> String.compare a.name b.name) pkgs in 431 | List.iter 432 | (fun pkg -> 433 | if check_maintainer ~maintainers pkg then render_package ppf ~filters pkg 434 | ) 435 | pkgs ; 436 | (let open Fmt in 437 | pf ppf "@\nTry '%a' to see details of build failures for each package.@\n@\n" 438 | (styled `Bold string) 439 | "opam-ci logs ") ; 440 | Ok () 441 | 442 | let show_logs pkg {refresh} params () = 443 | let open Obi.Index in 444 | let name, version = 445 | match String.cut ~sep:"." pkg with 446 | | None -> (pkg, None) (* No version string *) 447 | | Some (name, version) -> (name, Some version) 448 | in 449 | let ppf = Fmt.stdout in 450 | Repos.init ~refresh () 451 | >>= fun pkgs -> 452 | match List.find_opt (fun p -> p.name = name) pkgs with 453 | | None -> Error (`Msg (Fmt.strf "Package %s not found" pkg)) 454 | | Some pkg -> 455 | render_package_details ppf pkg name version params ; 456 | Ok () 457 | -------------------------------------------------------------------------------- /test/opamJson/dune: -------------------------------------------------------------------------------- 1 | (executables 2 | (names test) 3 | (libraries alcotest obi)) 4 | 5 | (alias 6 | (name runtest) 7 | (deps test.exe owl-fail.json) 8 | (action (run ${exe:test.exe}))) 9 | -------------------------------------------------------------------------------- /test/opamJson/owl-fail.json: -------------------------------------------------------------------------------- 1 | {"opam-version":"2.0.0~rc2 (2e8b5e77af72008601db84c22482df179852dec2)","command-line":["opam","install","--json","x.json","owl"],"command-line":["opam","install","--json","x.json","owl"],"switch":"default","request":{"action":"install","install":["owl"],"remove":[],"upgrade":[],"criteria":"-removed,-count[version-lag,request],-count[version-lag,changed],-changed"},"solution":[{"install":{"name":"owl-base","version":"0.3.8"}},{"install":{"name":"ocamlify","version":"0.0.1"}},{"install":{"name":"plplot","version":"5.11.0"}},{"install":{"name":"ocamlmod","version":"0.0.9"}},{"install":{"name":"conf-openblas","version":"0.2.0"}},{"install":{"name":"oasis","version":"0.4.11"}},{"install":{"name":"eigen","version":"0.0.5"}},{"install":{"name":"owl","version":"0.3.8"}}],"results":[{"action":{"build":{"name":"conf-openblas","version":"0.2.0"}},"result":{"process-error":{"code":"1","duration":0.316798210144043,"info":{"context":"2.0.0~rc2 | macos/x86_64 | ocaml-system.4.06.0 | file:///Users/avsm/src/git/ocaml/opam-repository","path":"~/.opam/default/.opam-switch/build/conf-openblas.0.2.0","command":"/bin/sh -exc cc $CFLAGS -I/usr/local/opt/openblas/include test.c -L/usr/local/opt/openblas/lib -lopenblas","exit-code":"1","env-file":"~/.opam/log/conf-openblas-39760-038aac.env","output-file":"~/.opam/log/conf-openblas-39760-038aac.out"},"output":["+ cc -I/usr/local/opt/openblas/include test.c -L/usr/local/opt/openblas/lib -lopenblas","test.c:1:10: fatal error: 'cblas.h' file not found","#include "," ^~~~~~~~~","1 error generated."]}},"duration":0.3405649662017822},{"action":{"build":{"name":"plplot","version":"5.11.0"}},"result":{"process-error":{"code":"1","duration":7.617889881134033,"info":{"context":"2.0.0~rc2 | macos/x86_64 | ocaml-system.4.06.0 | file:///Users/avsm/src/git/ocaml/opam-repository","path":"~/.opam/default/.opam-switch/build/plplot.5.11.0","command":"~/.opam/default/bin/ocaml setup.ml -build","exit-code":"1","env-file":"~/.opam/log/plplot-39760-ab22a1.env","output-file":"~/.opam/log/plplot-39760-ab22a1.out"},"output":["ocamlopt.opt unix.cmxa -I /Users/avsm/.opam/default/lib/ocamlbuild /Users/avsm/.opam/default/lib/ocamlbuild/ocamlbuildlib.cmxa myocamlbuild.ml /Users/avsm/.opam/default/lib/ocamlbuild/ocamlbuild.cmx -o myocamlbuild","Failure: No valid installation of plplot or plplotd found.","E: Failure(\"Command ''/Users/avsm/.opam/default/bin/ocamlbuild' src/libplplot_stubs.a src/dllplplot_stubs.so src/plplot.cma src/plplot.cmxa src/plplot.a src/plplot.cmxs -tag debug' terminated with error code 2\")"]}},"duration":10.00927495956421},{"action":{"build":{"name":"ocamlify","version":"0.0.1"}},"result":"OK","duration":3.861037015914917},{"action":{"build":{"name":"owl-base","version":"0.3.8"}},"result":"OK","duration":13.55557489395142},{"action":{"build":{"name":"ocamlmod","version":"0.0.9"}},"result":"OK","duration":6.359157085418701},{"action":{"install":{"name":"conf-openblas","version":"0.2.0"}},"result":{"aborted":[{"build":{"name":"conf-openblas","version":"0.2.0"}}]}},{"action":{"install":{"name":"plplot","version":"5.11.0"}},"result":{"aborted":[{"build":{"name":"plplot","version":"5.11.0"}}]}},{"action":{"install":{"name":"ocamlify","version":"0.0.1"}},"result":"OK","duration":10.24873995780945},{"action":{"install":{"name":"owl-base","version":"0.3.8"}},"result":"OK","duration":7.104352951049805},{"action":{"install":{"name":"ocamlmod","version":"0.0.9"}},"result":"OK","duration":5.545505046844482},{"action":{"build":{"name":"oasis","version":"0.4.11"}},"result":"OK","duration":36.26007294654846},{"action":{"install":{"name":"oasis","version":"0.4.11"}},"result":"OK","duration":6.608125925064087},{"action":{"build":{"name":"eigen","version":"0.0.5"}},"result":"OK","duration":58.22816801071167},{"action":{"install":{"name":"eigen","version":"0.0.5"}},"result":"OK","duration":5.453004121780396},{"action":{"build":{"name":"owl","version":"0.3.8"}},"result":{"aborted":[{"build":{"name":"conf-openblas","version":"0.2.0"}},{"build":{"name":"plplot","version":"5.11.0"}}]}},{"action":{"install":{"name":"owl","version":"0.3.8"}},"result":{"aborted":[{"build":{"name":"conf-openblas","version":"0.2.0"}},{"build":{"name":"plplot","version":"5.11.0"}}]}}]} -------------------------------------------------------------------------------- /test/opamJson/test.ml: -------------------------------------------------------------------------------- 1 | module O = Obi.OpamJsonActions 2 | 3 | let test_owl () = 4 | open_in "owl-fail.json" |> Ezjsonm.from_channel |> O.installs 5 | |> fun actions -> 6 | Alcotest.(check int) "found results" (List.length actions) 8 7 | 8 | let test_json = [("Owl fail", `Quick, test_owl)] 9 | 10 | let () = Alcotest.run "Test Opam JSON" [("test_json", test_json)] 11 | --------------------------------------------------------------------------------