[wip] Add project cache

Author: andrzej-kaczmarek

newt/cache/cache.go

@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package cache
+
+import (
+	"encoding/gob"
+	"fmt"
+	"mynewt.apache.org/newt/newt/interfaces"
+	"mynewt.apache.org/newt/newt/repo"
+	"mynewt.apache.org/newt/util"
+	"os"
+	"path/filepath"
+)
+
+type ProjectCache struct {
+	baseDir string
+}
+
+func InitCache(projDir string) *ProjectCache {
+	pc := ProjectCache{}
+	pc.baseDir = filepath.Join(projDir, ".cache")
+
+	if _, err := os.Stat(pc.baseDir); os.IsNotExist(err) {
+		if err := os.Mkdir(pc.baseDir, 0754); err != nil {
+			return nil
+		}
+	}
+
+	return &pc
+}
+
+func (pc *ProjectCache) getPackagesFile(repo *repo.Repo) string {
+	return fmt.Sprintf(filepath.Join(pc.baseDir, repo.Name()))
+}
+
+func (pc *ProjectCache) AddPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface) {
+	cacheName := pc.getPackagesFile(repo)
+	var dirList []string
+
+	for _, v := range pkgMap {
+		dirList = append(dirList, v.BasePath())
+	}
+
+	f, err := os.Create(cacheName)
+	if err != nil {
+		util.OneTimeWarning("Failed to create cache file for \"%s\"", repo.Name())
+		return
+	}
+
+	defer f.Close()
+
+	enc := gob.NewEncoder(f)
+	err = enc.Encode(dirList)
+	if err != nil {
+		util.OneTimeWarning("Failed to update cache file for \"%s\"", repo.Name())
+	}
+}
+
+func (pc *ProjectCache) GetPackagesDirs(repo *repo.Repo) []string {
+	cacheName := pc.getPackagesFile(repo)
+	var dirList []string
+
+	f, err := os.Open(cacheName)
+	if err != nil {
+		util.OneTimeWarning("Failed to open cache file for \"%s\"", repo.Name())
+		return nil
+	}
+
+	defer f.Close()
+
+	enc := gob.NewDecoder(f)
+	err = enc.Decode(&dirList)
+	if err != nil {
+		util.OneTimeWarning("Failed to read cache for \"%s\"", repo.Name())
+		return nil
+	}
+
+	return dirList
+}

newt/pkg/localpackage.go

@@ -449,8 +449,34 @@ func LocalPackageSpecialName(dirName string) bool {
 	return ok
 }
 
+func ReadPackage(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface,
+	pkgPath string) ([]string, error) {
+
+	var warnings []string
+
+	pkg, err := LoadLocalPackage(repo, pkgPath)
+	if err != nil {
+		warnings = append(warnings, err.Error())
+		return warnings, nil
+	}
+
+	if oldPkg, ok := pkgMap[pkg.Name()]; ok {
+		oldlPkg := oldPkg.(*LocalPackage)
+		warnings = append(warnings,
+			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
+				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
+				oldlPkg.BasePath(), pkg.BasePath()))
+
+		return warnings, nil
+	}
+
+	pkgMap[pkg.Name()] = pkg
+
+	return warnings, nil
+}
+
 func ReadLocalPackageRecursive(repo *repo.Repo,
-	pkgList map[string]interfaces.PackageInterface, basePath string,
+	pkgMap map[string]interfaces.PackageInterface, basePath string,
 	pkgName string, searchedMap map[string]struct{}) ([]string, error) {
 
 	var warnings []string
@@ -465,7 +491,7 @@ func ReadLocalPackageRecursive(repo *repo.Repo,
 			continue
 		}
 
-		subWarnings, err := ReadLocalPackageRecursive(repo, pkgList,
+		subWarnings, err := ReadLocalPackageRecursive(repo, pkgMap,
 			basePath, filepath.Join(pkgName, name), searchedMap)
 		warnings = append(warnings, subWarnings...)
 		if err != nil {
@@ -477,39 +503,21 @@ func ReadLocalPackageRecursive(repo *repo.Repo,
 		return warnings, nil
 	}
 
-	pkg, err := LoadLocalPackage(repo, filepath.Join(basePath, pkgName))
-	if err != nil {
-		warnings = append(warnings, err.Error())
-		return warnings, nil
-	}
+	var subWarnings []string
+	subWarnings, err = ReadPackage(repo, pkgMap, filepath.Join(basePath, pkgName))
+	warnings = append(warnings, subWarnings...)
 
-	if oldPkg, ok := pkgList[pkg.Name()]; ok {
-		oldlPkg := oldPkg.(*LocalPackage)
-		warnings = append(warnings,
-			fmt.Sprintf("Multiple packages with same pkg.name=%s "+
-				"in repo %s; path1=%s path2=%s", oldlPkg.Name(), repo.Name(),
-				oldlPkg.BasePath(), pkg.BasePath()))
-
-		return warnings, nil
-	}
-
-	pkgList[pkg.Name()] = pkg
-
-	return warnings, nil
+	return warnings, err
 }
 
-func ReadLocalPackages(repo *repo.Repo, basePath string) (
-	*map[string]interfaces.PackageInterface, []string, error) {
-
-	pkgMap := &map[string]interfaces.PackageInterface{}
-
+func ReadLocalPackages(repo *repo.Repo, pkgMap map[string]interfaces.PackageInterface, basePath string) ([]string, error) {
 	// Keep track of which directories we have traversed.  Prevent infinite
 	// loops caused by symlink cycles by not inspecting the same directory
 	// twice.
 	searchedMap := map[string]struct{}{}
 
-	warnings, err := ReadLocalPackageRecursive(repo, *pkgMap,
+	warnings, err := ReadLocalPackageRecursive(repo, pkgMap,
 		basePath, "", searchedMap)
 
-	return pkgMap, warnings, err
+	return warnings, err
 }

newt/project/project.go

@@ -21,6 +21,7 @@ package project
 
 import (
 	"fmt"
+	"mynewt.apache.org/newt/newt/cache"
 	"os"
 	"path"
 	"path/filepath"
@@ -57,6 +58,8 @@ type Project struct {
 	// Base path of the project
 	BasePath string
 
+	cache *cache.ProjectCache
+
 	packages interfaces.PackageList
 
 	// Contains all the repos that form this project.  Each repo is in one of
@@ -634,6 +637,13 @@ func (proj *Project) Init(dir string, download bool) error {
 		return err
 	}
 
+	if util.EnableProjectCache {
+		proj.cache = cache.InitCache(proj.BasePath)
+		if proj.cache == nil {
+			util.OneTimeWarning("Failed to initialize project cache")
+		}
+	}
+
 	return nil
 }
 
@@ -741,12 +751,37 @@ func (proj *Project) loadPackageList() error {
 	// packages / store them in the project package list.
 	repos := proj.Repos()
 	for name, repo := range repos {
-		list, warnings, err := pkg.ReadLocalPackages(repo, repo.Path())
-		if err == nil {
-			proj.packages[name] = list
+		pkgMap := &map[string]interfaces.PackageInterface{}
+
+		var dirList []string
+
+		if proj.cache != nil {
+			dirList = proj.cache.GetPackagesDirs(repo)
 		}
 
-		proj.warnings = append(proj.warnings, warnings...)
+		if proj.cache != nil && dirList != nil {
+			log.Debug("Using cache for packages in \"%s\"\n", repo.Name())
+			for _, pkgPath := range dirList {
+				warnings, err := pkg.ReadPackage(repo, *pkgMap, pkgPath)
+				if err == nil {
+					proj.packages[name] = pkgMap
+				}
+
+				proj.warnings = append(proj.warnings, warnings...)
+			}
+		} else {
+			log.Debug("Not using cache for packages in \"%s\"\n", repo.Name())
+			warnings, err := pkg.ReadLocalPackages(repo, *pkgMap, repo.Path())
+			if err == nil {
+				proj.packages[name] = pkgMap
+			}
+
+			if proj.cache != nil {
+				proj.cache.AddPackages(repo, *pkgMap)
+			}
+
+			proj.warnings = append(proj.warnings, warnings...)
+		}
 	}
 
 	return nil

newt/settings/settings.go

@@ -59,6 +59,7 @@ func processNewtrc(yc ycfg.YCfg) {
 
 	util.SkipNewtCompat, _ = yc.GetValBoolDflt("skip_newt_compat", nil, false)
 	util.SkipSyscfgRepoHash, _ = yc.GetValBoolDflt("skip_syscfg_repo_hash", nil, false)
+	util.EnableProjectCache, _ = yc.GetValBoolDflt("project_cache", nil, false)
 }
 
 func readNewtrc() ycfg.YCfg {

util/util.go

@@ -51,6 +51,7 @@ var ShallowCloneDepth int
 var logFile *os.File
 var SkipNewtCompat bool
 var SkipSyscfgRepoHash bool
+var EnableProjectCache bool
 
 func ParseEqualsPair(v string) (string, string, error) {
 	s := strings.Split(v, "=")