Setting up benchmarks

.JuliaFormatter.toml

@@ -0,0 +1,7 @@
+margin = 100
+indent = 2
+whitespace_typedefs = true
+whitespace_ops_in_indices = true
+remove_extra_newlines = true
+annotate_untyped_fields_with_any = false
+normalize_line_endings = "unix"

.cirrus.yml

@@ -1,17 +1,17 @@
 freebsd_instance:
-  image: freebsd-12-0-release-amd64
+  image: freebsd-12-1-release-amd64
 task:
   name: FreeBSD
   artifacts_cache:
     folder: ~/.julia/artifacts
   env:
-    JULIA_VERSION: 1.6
-    JULIA_VERSION: nightly
+    matrix:
+      JULIA_VERSION: 1
+      JULIA_VERSION: nightly
+  allow_failures: $JULIA_VERSION == 'nightly'
   install_script:
     - sh -c "$(fetch https://raw.githubusercontent.com/ararslan/CirrusCI.jl/master/bin/install.sh -o -)"
   build_script:
     - cirrusjl build
   test_script:
     - cirrusjl test
-  coverage_script:
-    - cirrusjl coverage codecov

.github/workflows/format_pr.yml

@@ -0,0 +1,32 @@
+# https://github.com/julia-actions/julia-format/blob/master/workflows/format_pr.yml
+name: format-pr
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install JuliaFormatter and format
+        run: |
+          julia  -e 'import Pkg; Pkg.add("JuliaFormatter")'
+          julia  -e 'using JuliaFormatter; format(".")'
+      # https://github.com/marketplace/actions/create-pull-request
+      # https://github.com/peter-evans/create-pull-request#reference-example
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: ":robot: Format .jl files"
+          title: '[AUTO] JuliaFormatter.jl run'
+          branch: auto-juliaformatter-pr
+          delete-branch: true
+          labels: formatting, automated pr, no changelog
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.gitignore

@@ -3,3 +3,5 @@
 *.jl.mem
 /Manifest.toml
 /docs/build/
+benchmark/Manifest.toml
+

CITATION.cff

@@ -0,0 +1,10 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Template
+    given-names: JSO
+    orcid: https://orcid.org/0123-4567-89AB-CDEF
+title: "JSO Template"
+version: 2.0.4
+doi: 10.5281/zenodo.1234
+date-released: 2021-10-04

Jenkinsfile

@@ -0,0 +1,119 @@
+def bmarkFile = 'run_benchmarks.jl'
+def prNumber = BRANCH_NAME.tokenize("PR-")[0]
+pipeline {
+  agent any
+  environment {
+    REPO_EXISTS = fileExists "$repo"
+  }
+  options {
+    skipDefaultCheckout true
+  }
+  triggers {
+    GenericTrigger(
+     genericVariables: [
+        [
+            key: 'action', 
+            value: '$.action',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '[^(created)]', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'comment',
+            value: '$.comment.body',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'org',
+            value: '$.organization.login',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: 'JuliaSmoothOptimizers' //Optional, defaults to empty string
+        ],
+        [
+            key: 'pullrequest',
+            value: '$.issue.number',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '[^0-9]', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'repo',
+            value: '$.repository.name',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ]
+     ],
+
+     causeString: 'Triggered on $comment',
+
+     token: "JSOTemplate",
+
+     printContributedVariables: true,
+     printPostContent: true,
+
+     silentResponse: false,
+
+     regexpFilterText: '$comment $pullrequest',
+     regexpFilterExpression: '@JSOBot runbenchmarks( .*\\.jl)? ' + prNumber
+    )
+  }
+  stages {
+    stage('clone repo') {
+      when {
+        expression { REPO_EXISTS == 'false' }
+      }
+      steps {
+        sh 'git clone https://${GITHUB_AUTH}@github.com/$org/$repo.git'
+      }
+    }
+    stage('checkout on new branch') {
+      steps {
+        dir(WORKSPACE + "/$repo") {
+          sh '''
+            git clean -fd
+            git checkout main
+            git pull origin main
+            git fetch origin
+            LOCAL_BRANCH_NAME="temp_bmark"
+            git branch -D $LOCAL_BRANCH_NAME || true
+            git fetch origin pull/$pullrequest/head:$LOCAL_BRANCH_NAME
+            git checkout $LOCAL_BRANCH_NAME --
+          '''
+        }
+      }
+    }
+    stage('run benchmarks') {
+      steps {
+        script {
+          def data = env.comment.tokenize(' ')
+          if (data.size() > 2) {
+            bmarkFile = data.get(2);
+          }
+        }
+        dir(WORKSPACE + "/$repo") {
+        sh "mkdir -p $HOME/benchmarks/${org}/${repo}"
+        sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log push_benchmarks.sh $bmarkFile"
+        }   
+      }
+    }
+  }
+  post {
+    success {
+      echo "SUCCESS!"  
+    }
+    cleanup {
+      dir(WORKSPACE + "/$repo") {
+      sh 'printenv'
+      sh '''
+      git clean -fd
+      git checkout main
+      '''
+      }
+    }
+  }
+}
+

benchmark/Project.toml

@@ -0,0 +1,12 @@
+[deps]
+ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
+GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+JSOTemplate = "adf32eb0-fe7e-47bb-93e4-fe96e72f6839"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+SolverBenchmark = "581a75fa-a23a-52d0-a590-d6201de2218a"

benchmark/benchmarks.jl

@@ -0,0 +1,5 @@
+using BenchmarkTools
+
+const SUITE = BenchmarkGroup()
+
+# Write your benchmarks here:

benchmark/run_benchmarks.jl

@@ -0,0 +1,85 @@
+using Pkg
+bmark_dir = @__DIR__
+repo_name = string(split(ARGS[1], ".")[1])
+bmarkname = lowercase(repo_name)
+
+# if we are running these benchmarks from the git repository
+# we want to develop the package instead of using the release
+if isdir(joinpath(bmark_dir, "..", ".git"))
+    Pkg.develop(PackageSpec(url = joinpath(bmark_dir, "..")))
+end
+
+using DataFrames
+using GitHub
+using JLD2
+using JSON
+using PkgBenchmark
+using Plots
+
+using SolverBenchmark
+
+# NB: benchmarkpkg will run benchmarks/benchmarks.jl by default
+commit = benchmarkpkg(repo_name)  # current state of repository
+main = benchmarkpkg(repo_name, "main")
+judgement = judge(commit, main)
+
+commit_stats = bmark_results_to_dataframes(commit)
+main_stats = bmark_results_to_dataframes(main)
+judgement_stats = judgement_results_to_dataframes(judgement)
+
+export_markdown("judgement_$(bmarkname).md", judgement)
+export_markdown("main.md", main)
+export_markdown("$(bmarkname).md", commit)
+
+function profile_solvers_from_pkgbmark(stats::Dict{Symbol,DataFrame})
+    # guard against zero gctimes
+    costs = [
+        df -> df[!, :time],
+        df -> df[!, :memory],
+        df -> df[!, :gctime] .+ 1,
+        df -> df[!, :allocations],
+    ]
+    profile_solvers(stats, costs, ["time", "memory", "gctime+1", "allocations"])
+end
+
+# extract stats for each benchmark to plot profiles
+# files_dict will be part of json_dict below
+files_dict = Dict{String,Any}()
+file_num = 1
+for k ∈ keys(judgement_stats)
+    global file_num
+    k_stats = Dict{Symbol,DataFrame}(:commit => commit_stats[k], :main => main_stats[k])
+    save_stats(k_stats, "$(bmarkname)_vs_main_$(k).jld2", force = true)
+
+    k_profile = profile_solvers_from_pkgbmark(k_stats)
+    savefig("profiles_commit_vs_main_$(k).svg")
+    # read contents of svg file to add to gist
+    k_svgfile = open("profiles_commit_vs_main_$(k).svg", "r") do fd
+        readlines(fd)
+    end
+    # file_num makes sure svg files appear before md files (added below)
+    files_dict["$(file_num)_$(k).svg"] = Dict{String,Any}("content" => join(k_svgfile))
+    file_num += 1
+end
+
+for mdfile ∈ [:judgement, :main, :commit]
+    global file_num
+    files_dict["$(file_num)_$(mdfile).md"] =
+        Dict{String,Any}("content" => "$(sprint(export_markdown, eval(mdfile)))")
+    file_num += 1
+end
+
+jldopen("$(bmarkname)_vs_main_judgement.jld2", "w") do file
+    file["jstats"] = judgement_stats
+end
+
+# json description of gist
+json_dict = Dict{String,Any}(
+    "description" => "$(repo_name) repository benchmark",
+    "public" => true,
+    "files" => files_dict,
+)
+
+open("gist.json", "w") do f
+    JSON.print(f, json_dict)
+end