Skip to content

Commit 55011d2

Browse files
mikeland73mikeland73
authored
[perf] Cache nixpkgs resolution (#2576)
## Summary This caches resolutions for unlocked nixpkgs (the default) for 90 days on the user's machine. Otherwise, everytime devbox tries to resolve `github:NixOS/nixpkgs/nixpkgs-unstable` it will download a new version of nixpkgs. This is slow (40+ seconds) and usually nothing changes. ### If a user wants to update nixpkgs there are two ways of doing it: * `devbox update` without any arguments will update nixpkgs. This is existing functionality. * `devbox update nixpkgs [...pkgs]` will also update nixpkgs. I'm not convinced updating nixpkgs when using `update` without arguments is best, but I didn't want to break past functionality. It adds 40+ seconds to update operations when there is no package to update. An alternative command syntax considered was `devbox update --nixpkgs`. The benefit of doing this would be to avoid conflicts in the future. I did not use `stdenv` because it already exists and it is a different package. ### Bug fixes: * Updating a specific package (e.g. `devbox update go`) no longer updates nixpkgs ### Alternative approaches: * We could try to search the local nix database to try to find existing nixpkgs, but this seemed more trouble than it was worth. ## How was it tested? * Created a new project and logged the cache being used. * Replaced the cache file data with an old version, created a new project and observed the old version used. * Tried several permutations of `devbox update` with and without packages ## Community Contribution License All community contributions in this pull request are licensed to the project maintainers under the terms of the [Apache 2 License](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request, I represent that I have the right to license the contributions to the project maintainers under the Apache 2 License as stated in the [Community Contribution License](https://github.com/jetify-com/opensource/blob/main/CONTRIBUTING.md#community-contribution-license).
1 parent a117fcf commit 55011d2

File tree

8 files changed

+142
-80
lines changed

8 files changed

+142
-80
lines changed

‎devbox.lock‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,173 +2,174 @@
22
"lockfile_version": "1",
33
"packages": {
44
"fd@latest": {
5-
"last_modified": "2025-02-07T11:26:36Z",
6-
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#fd",
5+
"last_modified": "2025-03-11T17:52:14Z",
6+
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#fd",
77
"source": "devbox-search",
88
"version": "10.2.0",
99
"systems": {
1010
"aarch64-darwin": {
1111
"outputs": [
1212
{
1313
"name": "out",
14-
"path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0",
14+
"path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0",
1515
"default": true
1616
}
1717
],
18-
"store_path": "/nix/store/ad5m54pfn9k39v80lpyhrnsh336nqrp5-fd-10.2.0"
18+
"store_path": "/nix/store/40pdazk980kp3h26py4hjyx9rys1g14n-fd-10.2.0"
1919
},
2020
"aarch64-linux": {
2121
"outputs": [
2222
{
2323
"name": "out",
24-
"path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0",
24+
"path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0",
2525
"default": true
2626
}
2727
],
28-
"store_path": "/nix/store/vn3mny38qmf3lm809rpcvahxqwhkqb7m-fd-10.2.0"
28+
"store_path": "/nix/store/76zcwa1d33vciy4gyqvk6jl2n3g1542q-fd-10.2.0"
2929
},
3030
"x86_64-darwin": {
3131
"outputs": [
3232
{
3333
"name": "out",
34-
"path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0",
34+
"path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0",
3535
"default": true
3636
}
3737
],
38-
"store_path": "/nix/store/6kdv4iy9j3svncq0vs47wxfvvn7flcr5-fd-10.2.0"
38+
"store_path": "/nix/store/ys9qmljs0ag7j040radgg48l6pvjmv9l-fd-10.2.0"
3939
},
4040
"x86_64-linux": {
4141
"outputs": [
4242
{
4343
"name": "out",
44-
"path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0",
44+
"path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0",
4545
"default": true
4646
}
4747
],
48-
"store_path": "/nix/store/x58pg72qw2xv1vvs4pbqw63zhkdkp331-fd-10.2.0"
48+
"store_path": "/nix/store/rrdvpl7rym4ia0h7rfz1vmlcvvivj30j-fd-10.2.0"
4949
}
5050
}
5151
},
5252
"git@latest": {
53-
"last_modified": "2025-02-07T11:26:36Z",
54-
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#git",
53+
"last_modified": "2025-03-11T17:52:14Z",
54+
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#git",
5555
"source": "devbox-search",
56-
"version": "2.47.2",
56+
"version": "2.48.1",
5757
"systems": {
5858
"aarch64-darwin": {
5959
"outputs": [
6060
{
6161
"name": "out",
62-
"path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2",
62+
"path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1",
6363
"default": true
6464
},
6565
{
6666
"name": "doc",
67-
"path": "/nix/store/rh151iwgy4h8yv8kxd5facw57cyj0bav-git-2.47.2-doc"
67+
"path": "/nix/store/086knqdw7fjgzczp0i6nad95s2v6jbya-git-2.48.1-doc"
6868
}
6969
],
70-
"store_path": "/nix/store/9z3jhc0rlj3zaw8nd1zka9vli6w0q11g-git-2.47.2"
70+
"store_path": "/nix/store/b3sci30zzzlj3rzj1y89cijnd6zcwapk-git-2.48.1"
7171
},
7272
"aarch64-linux": {
7373
"outputs": [
7474
{
7575
"name": "out",
76-
"path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2",
76+
"path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1",
7777
"default": true
7878
},
7979
{
8080
"name": "debug",
81-
"path": "/nix/store/8vfpmf3vjgzl2psip76p0f9h11sb6y3p-git-2.47.2-debug"
81+
"path": "/nix/store/xqqsvzlilh843rm6knykyng81apapr33-git-2.48.1-debug"
8282
},
8383
{
8484
"name": "doc",
85-
"path": "/nix/store/c25mq3q83dvw3k5pb0qr5333g3cycylq-git-2.47.2-doc"
85+
"path": "/nix/store/485b32ys0s2dvjfisn7405ildmpqvfzk-git-2.48.1-doc"
8686
}
8787
],
88-
"store_path": "/nix/store/gx5y37qcfqdvn0h6swjd04dmqjjh3nk7-git-2.47.2"
88+
"store_path": "/nix/store/pck1dr5jxrd5b8nmfasbn13z422jhcfm-git-2.48.1"
8989
},
9090
"x86_64-darwin": {
9191
"outputs": [
9292
{
9393
"name": "out",
94-
"path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2",
94+
"path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1",
9595
"default": true
9696
},
9797
{
9898
"name": "doc",
99-
"path": "/nix/store/xmh2djjrnbpiqqgpblrcbavnqh0nv4km-git-2.47.2-doc"
99+
"path": "/nix/store/cgv7qa0ix059ma9a0qac0bywfvl3k7k2-git-2.48.1-doc"
100100
}
101101
],
102-
"store_path": "/nix/store/39xx5gx3hxigs1b5ldw5i2jr84vsn3rf-git-2.47.2"
102+
"store_path": "/nix/store/9qjzgsf9mvdp6sfd7xyzhgrahl2qhhp6-git-2.48.1"
103103
},
104104
"x86_64-linux": {
105105
"outputs": [
106106
{
107107
"name": "out",
108-
"path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2",
108+
"path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1",
109109
"default": true
110110
},
111111
{
112-
"name": "debug",
113-
"path": "/nix/store/jyz4nvcd3bci4vg2sfsmvrq0fp9mzr5a-git-2.47.2-debug"
112+
"name": "doc",
113+
"path": "/nix/store/hjczhs1dm3hzij7mx5c91rkzqvkb89av-git-2.48.1-doc"
114114
},
115115
{
116-
"name": "doc",
117-
"path": "/nix/store/lb4nipdhlwrxdavz7gdkcik6lkz3cbdm-git-2.47.2-doc"
116+
"name": "debug",
117+
"path": "/nix/store/bk8xndavdnc2qgyvc6hcc8h29lk9jzqb-git-2.48.1-debug"
118118
}
119119
],
120-
"store_path": "/nix/store/33g65w5cc9n8fr0hxj84282xmv4l7hyl-git-2.47.2"
120+
"store_path": "/nix/store/lqx2rv26sdndpa2vyy2vxsahj03km69z-git-2.48.1"
121121
}
122122
}
123123
},
124124
"github:NixOS/nixpkgs/nixpkgs-unstable": {
125-
"resolved": "github:NixOS/nixpkgs/73cf49b8ad837ade2de76f87eb53fc85ed5d4680?lastModified=1739866667&narHash=sha256-EO1ygNKZlsAC9avfcwHkKGMsmipUk1Uc0TbrEZpkn64%3D"
125+
"last_modified": "2025-04-07T13:23:10Z",
126+
"resolved": "github:NixOS/nixpkgs/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b?lastModified=1744032190&narHash=sha256-KSlfrncSkcu1YE%2BuuJ%2FPTURsSlThoGkRqiGDVdbiE%2Fk%3D"
126127
},
127128
"go@latest": {
128-
"last_modified": "2025-02-12T00:10:52Z",
129-
"resolved": "github:NixOS/nixpkgs/83a2581c81ff5b06f7c1a4e7cc736a455dfcf7b4#go_1_24",
129+
"last_modified": "2025-03-11T17:52:14Z",
130+
"resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#go",
130131
"source": "devbox-search",
131-
"version": "1.24.0",
132+
"version": "1.24.1",
132133
"systems": {
133134
"aarch64-darwin": {
134135
"outputs": [
135136
{
136137
"name": "out",
137-
"path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0",
138+
"path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1",
138139
"default": true
139140
}
140141
],
141-
"store_path": "/nix/store/qldcnifalkvyah0wnv7m4zb854yd9l88-go-1.24.0"
142+
"store_path": "/nix/store/ja4jxx60lh1qfqfl4z4p2rff56ia1c3c-go-1.24.1"
142143
},
143144
"aarch64-linux": {
144145
"outputs": [
145146
{
146147
"name": "out",
147-
"path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0",
148+
"path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1",
148149
"default": true
149150
}
150151
],
151-
"store_path": "/nix/store/rrxgml7w4pfmibjbspkdvrw8vd2vnarb-go-1.24.0"
152+
"store_path": "/nix/store/8ply43gnxk1xwichr81mpgbjcd9a1y5w-go-1.24.1"
152153
},
153154
"x86_64-darwin": {
154155
"outputs": [
155156
{
156157
"name": "out",
157-
"path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0",
158+
"path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1",
158159
"default": true
159160
}
160161
],
161-
"store_path": "/nix/store/7imv22pl4qrjwvi6jzlfb305rc2min45-go-1.24.0"
162+
"store_path": "/nix/store/87yxrfx5lh78bdz393i33cr5z23x06q4-go-1.24.1"
162163
},
163164
"x86_64-linux": {
164165
"outputs": [
165166
{
166167
"name": "out",
167-
"path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0",
168+
"path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1",
168169
"default": true
169170
}
170171
],
171-
"store_path": "/nix/store/vh5d5bj1sljdhdypy80x1ydx2jx6rv2q-go-1.24.0"
172+
"store_path": "/nix/store/cfjhl0kn7xc65466pha9fkrvigw3g72n-go-1.24.1"
172173
}
173174
}
174175
}

‎internal/devbox/update.go‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package devbox
66
import (
77
"context"
88
"fmt"
9+
"slices"
910

1011
"github.com/pkg/errors"
1112
"go.jetify.com/devbox/internal/devbox/devopt"
@@ -20,6 +21,20 @@ import (
2021
)
2122

2223
func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error {
24+
if len(opts.Pkgs) == 0 || slices.Contains(opts.Pkgs, "nixpkgs") {
25+
if err := d.lockfile.UpdateStdenv(); err != nil {
26+
return err
27+
}
28+
// if nixpkgs is the only package to update, just return here.
29+
if len(opts.Pkgs) == 1 {
30+
return nil
31+
}
32+
// Otherwise, remove nixpkgs and continue
33+
opts.Pkgs = slices.DeleteFunc(opts.Pkgs, func(pkg string) bool {
34+
return pkg == "nixpkgs"
35+
})
36+
}
37+
2338
inputs, err := d.inputsToUpdate(opts)
2439
if err != nil {
2540
return err
@@ -65,9 +80,6 @@ func (d *Devbox) Update(ctx context.Context, opts devopt.UpdateOpts) error {
6580
}
6681
}
6782

68-
if err := d.updateStdenv(); err != nil {
69-
return err
70-
}
7183
mode := update
7284
if opts.NoInstall {
7385
mode = noInstall
@@ -110,15 +122,6 @@ func (d *Devbox) inputsToUpdate(
110122
return pkgsToUpdate, nil
111123
}
112124

113-
func (d *Devbox) updateStdenv() error {
114-
err := d.lockfile.Remove(d.Stdenv().String())
115-
if err != nil {
116-
return err
117-
}
118-
d.lockfile.Stdenv() // will re-resolve the stdenv flake
119-
return nil
120-
}
121-
122125
func (d *Devbox) updateDevboxPackage(pkg *devpkg.Package) error {
123126
resolved, err := d.lockfile.FetchResolvedPackage(pkg.Raw)
124127
if err != nil {

‎internal/lock/lockfile.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,23 @@ func (f *File) Save() error {
141141
return cuecfg.WriteFile(lockFilePath(f.devboxProject.ProjectDir()), f)
142142
}
143143

144+
func (f *File) UpdateStdenv() error {
145+
if err := nix.ClearFlakeCache(f.devboxProject.Stdenv()); err != nil {
146+
return err
147+
}
148+
if err := f.Remove(f.devboxProject.Stdenv().String()); err != nil {
149+
return err
150+
}
151+
return f.Add(f.devboxProject.Stdenv().String())
152+
}
153+
154+
// TODO: We should improve a few issues with this function:
155+
// * It shared the same name as Devbox.Stdenv() which is confusing.
156+
// * Since File implements DevboxProject, IDEs really struggle to accurately find call sites.
157+
// (side note, we should remove DevboxProject interface)
158+
// * This function forces a resolution of the stdenv flake which is slow and doesn't give us a
159+
// chance to "prep" the user for some waiting.
160+
// * Should we rename to Nixpkgs() ? Stdenv feels a bit ambiguous.
144161
func (f *File) Stdenv() flake.Ref {
145162
unlocked := f.devboxProject.Stdenv()
146163
pkg, err := f.Resolve(unlocked.String())

‎internal/lock/resolve.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ func (f *File) FetchResolvedPackage(pkg string) (*Package, error) {
3939
return nil, err
4040
}
4141
return &Package{
42-
Resolved: installable.String(),
42+
Resolved: installable.String(),
43+
LastModified: time.Unix(installable.Ref.LastModified, 0).UTC().Format(time.RFC3339),
4344
}, nil
4445
}
4546

@@ -220,7 +221,25 @@ func lockFlake(ctx context.Context, ref flake.Ref) (flake.Ref, error) {
220221
return ref, nil
221222
}
222223

223-
meta, err := nix.ResolveFlake(ctx, ref)
224+
var meta nix.FlakeMetadata
225+
var err error
226+
// For nixpkgs, we cache resolutions (currently flakeCacheTTL=30 days) to avoid downloading
227+
// new nixpkgs too often which is really slow and rarely changes anything.
228+
//
229+
// Ideally we can do something similar for all packages (flake and otherwise)
230+
// Specifically, if user adds python@3.12 (or python@latest that resolves to 3.12) and that
231+
// package is already installed, we should use it instead of using 3.12 from search service
232+
// (which may have different store path). This would allow all devbox projects to share packages
233+
// if the version resolution is the same.
234+
//
235+
// That said, the logic for caching resolved versions and non-locked flake references would not
236+
// be the same.
237+
if ref.IsNixpkgs() {
238+
meta, err = nix.ResolveCachedFlake(ctx, ref)
239+
} else {
240+
meta, err = nix.ResolveFlake(ctx, ref)
241+
}
242+
224243
if err != nil {
225244
return ref, err
226245
}

‎internal/nix/flake.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,23 @@ package nix
33
import (
44
"context"
55
"encoding/json"
6+
"time"
67

78
"go.jetify.com/devbox/nix/flake"
9+
"go.jetify.com/pkg/filecache"
810
)
911

12+
const flakeCacheTTL = time.Hour * 24 * 30
13+
14+
var flakeFileCache = filecache.New[FlakeMetadata]("devbox/flakes")
15+
1016
type FlakeMetadata struct {
11-
Description string `json:"description"`
12-
Original flake.Ref `json:"original"`
13-
Resolved flake.Ref `json:"resolved"`
14-
Locked flake.Ref `json:"locked"`
15-
Path string `json:"path"`
17+
Description string `json:"description"`
18+
LastModified int64 `json:"lastModified"`
19+
Locked flake.Ref `json:"locked"`
20+
Original flake.Ref `json:"original"`
21+
Path string `json:"path"`
22+
Resolved flake.Ref `json:"resolved"`
1623
}
1724

1825
func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
@@ -28,3 +35,18 @@ func ResolveFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
2835
}
2936
return meta, nil
3037
}
38+
39+
func ResolveCachedFlake(ctx context.Context, ref flake.Ref) (FlakeMetadata, error) {
40+
return flakeFileCache.GetOrSet(ref.String(), func() (FlakeMetadata, time.Duration, error) {
41+
meta, err := ResolveFlake(ctx, ref)
42+
if err != nil {
43+
return FlakeMetadata{}, 0, err
44+
}
45+
return meta, flakeCacheTTL, nil
46+
})
47+
}
48+
49+
func ClearFlakeCache(ref flake.Ref) error {
50+
// TODO: Add unset to filecache
51+
return flakeFileCache.Set(ref.String(), FlakeMetadata{}, -1)
52+
}

0 commit comments

Comments
 (0)