Merge pull request #71 from gvallee/container_manifest

Create a manifest for all containers that are created with SyMPI

Author: gvallee

internal/pkg/container/container.go

@@ -21,6 +21,7 @@ import (
 	"github.com/sylabs/singularity-mpi/internal/pkg/checker"
 	"github.com/sylabs/singularity-mpi/internal/pkg/implem"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sy"
+	"github.com/sylabs/singularity-mpi/internal/pkg/syexec"
 	"github.com/sylabs/singularity-mpi/internal/pkg/sys"
 	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
 )
@@ -104,10 +105,6 @@ func Create(container *Config, sysCfg *sys.Config) error {
 
 	log.Printf("- Creating image %s...", container.Path)
 
-	// We only let the mpirun command run for 10 minutes max
-	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*2*time.Minute)
-	defer cancel()
-
 	// The definition file is ready so we simple build the container using the Singularity command
 	if sysCfg.Debug {
 		err = checker.CheckDefFile(container.DefFile)
@@ -117,24 +114,28 @@ func Create(container *Config, sysCfg *sys.Config) error {
 	}
 
 	log.Printf("-> Using definition file %s", container.DefFile)
-	var stdout, stderr bytes.Buffer
-	var cmd *exec.Cmd
+
+	var cmd syexec.SyCmd
+	singularityVersion := sy.GetVersion(sysCfg)
+	cmd.ManifestData = []string{"Singularity version: " + singularityVersion}
+	cmd.ManifestDir = container.InstallDir
+	cmd.ExecDir = container.BuildDir
 	if sy.IsSudoCmd("build", sysCfg) {
-		log.Printf("-> Running %s %s %s %s %s\n", sysCfg.SudoBin, sysCfg.SingularityBin, "build", container.Path, container.DefFile)
-		cmd = exec.CommandContext(ctx, sysCfg.SudoBin, sysCfg.SingularityBin, "build", container.Path, container.DefFile)
+		cmd.BinPath = sysCfg.SudoBin
+		cmd.ManifestFileHash = []string{sysCfg.SingularityBin, container.DefFile}
+		cmd.CmdArgs = []string{sysCfg.SingularityBin, "build", container.Path, container.DefFile}
 	} else if sysCfg.Nopriv {
-		log.Printf("-> Running %s %s %s %s\n", sysCfg.SingularityBin, "build --fakeroot", container.Path, container.DefFile)
-		cmd = exec.CommandContext(ctx, sysCfg.SingularityBin, "build", "--fakeroot", container.Path, container.DefFile)
+		cmd.BinPath = sysCfg.SingularityBin
+		cmd.ManifestFileHash = []string{container.DefFile}
+		cmd.CmdArgs = []string{"build", "--fakeroot", container.Path, container.DefFile}
 	} else {
-		log.Printf("-> Running %s %s %s %s\n", sysCfg.SingularityBin, "build", container.Path, container.DefFile)
-		cmd = exec.CommandContext(ctx, sysCfg.SingularityBin, "build", container.Path, container.DefFile)
+		cmd.BinPath = sysCfg.SingularityBin
+		cmd.ManifestFileHash = []string{container.DefFile}
+		cmd.CmdArgs = []string{"build", container.Path, container.DefFile}
 	}
-	cmd.Dir = container.BuildDir
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err = cmd.Run()
-	if err != nil {
-		return fmt.Errorf("failed to execute command - stdout: %s; stderr: %s; err: %s", stdout.String(), stderr.String(), err)
+	res := cmd.Run()
+	if res.Err != nil {
+		return fmt.Errorf("failed to execute command - stdout: %s; stderr: %s; err: %s", res.Stdout, res.Stderr, res.Err)
 	}
 
 	// We make all SIF file executable to make it easier to integrate with other tools

internal/pkg/ldd/debian.go

@@ -61,7 +61,7 @@ func DebianGetDependencies(output string) []string {
 
 	lines := strings.Split(output, "\n")
 
-	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*time.Minute)
 	defer cancel()
 
 	// the package of interest is the one for the current architecture

internal/pkg/ldd/rpm.go

@@ -30,7 +30,7 @@ func RPMGetDependencies(output string) []string {
 
 	lines := strings.Split(output, "\n")
 
-	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*time.Minute)
 	defer cancel()
 
 	// the package of interest is the one for the current architecture

internal/pkg/manifest/manifest.go

@@ -18,7 +18,7 @@ func Create(filepath string, entries []string) error {
 		return fmt.Errorf("failed to create %s: %s", filepath, err)
 	}
 
-	_, err = f.WriteString(strings.Join(entries, " "))
+	_, err = f.WriteString(strings.Join(entries, "\n"))
 	if err != nil {
 		return fmt.Errorf("failed to write to %s: %s", filepath, err)
 	}

internal/pkg/sy/sy.go

@@ -244,7 +244,7 @@ func Configure(env *buildenv.Info, sysCfg *sys.Config, extraArgs []string) error
 		}
 	}
 	singularityManifestPath := filepath.Join(env.InstallDir, "install.MANIFEST")
-	err := manifest.Create(singularityManifestPath, args)
+	err := manifest.Create(singularityManifestPath, []string{strings.Join(args, " ")})
 	if err != nil {
 		return fmt.Errorf("failed to create installation manifest: %s", err)
 	}
@@ -339,3 +339,26 @@ func GetSIFArchs(imgPath string, sysCfg *sys.Config) ([]string, error) {
 
 	return getArchsFromSIFListOutput(stdout.String()), nil
 }
+
+// GetVersion returned the version of Singularity that is currently used
+func GetVersion(sysCfg *sys.Config) string {
+	if sysCfg.SingularityBin == "" {
+		// Not a fatal error, we just log the error
+		log.Printf("path to the singularity binary is undefined")
+		return ""
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), sys.CmdTimeout*time.Minute)
+	defer cancel()
+	var stdout bytes.Buffer
+	cmd := exec.CommandContext(ctx, sysCfg.SingularityBin, "version")
+	cmd.Stdout = &stdout
+	err := cmd.Run()
+	if err != nil {
+		// Not a fatal error, we just log the error
+		log.Printf("failed to execute singularity version: %s", err)
+		return ""
+	}
+
+	return stdout.String()
+}

internal/pkg/syexec/syexec.go

@@ -6,8 +6,21 @@
 package syexec
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"log"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/sylabs/singularity-mpi/internal/pkg/manifest"
+	"github.com/sylabs/singularity-mpi/internal/pkg/sys"
+	util "github.com/sylabs/singularity-mpi/internal/pkg/util/file"
 )
 
 // Result represents the result of the execution of a command
@@ -25,18 +38,118 @@ type SyCmd struct {
 	// Cmd represents the command to execute to submit the job
 	Cmd *exec.Cmd
 
+	// Timeout is the maximum time a command can run
+	Timeout time.Duration
+
 	// BinPath is the path to the binary to execute
 	BinPath string
 
 	// CmdArgs is a slice of string representing the command's arguments
 	CmdArgs []string
 
+	// ExecDir is the directory where to execute the command
+	ExecDir string
+
 	// Env is a slice of string representing the environment to be used with the command
-	Env     []string
+	Env []string
 
 	// Ctx is the context of the command to execute to submit a job
 	Ctx context.Context
 
 	// CancelFn is the function to cancel the command to submit a job
 	CancelFn context.CancelFunc
+
+	// ManifestDir is the directory where to create the manifest related to the command execution
+	ManifestDir string
+
+	// ManifestData is extra content to add to the manifest
+	ManifestData []string
+
+	// ManifestFileHash is a list of absolute path to files for which we want a hash in the manifest
+	ManifestFileHash []string
+}
+
+func getFileHash(path string) string {
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	hasher := sha256.New()
+	_, err = io.Copy(hasher, f)
+	if err != nil {
+		return ""
+	}
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func hashFiles(files []string) []string {
+	var hashData []string
+
+	for _, file := range files {
+		hash := getFileHash(file)
+		hashData = append(hashData, file+": "+hash)
+	}
+
+	return hashData
+}
+
+// Run executes a syexec command and creates the appropriate manifest (when possible)
+func (c *SyCmd) Run() Result {
+	var res Result
+
+	cmdTimeout := c.Timeout
+	if cmdTimeout == 0 {
+		cmdTimeout = sys.CmdTimeout
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout*time.Minute)
+	defer cancel()
+
+	var stderr, stdout bytes.Buffer
+	if c.Cmd == nil {
+		c.Cmd = exec.CommandContext(ctx, c.BinPath, c.CmdArgs...)
+		c.Cmd.Dir = c.ExecDir
+		c.Cmd.Stdout = &stdout
+		c.Cmd.Stderr = &stderr
+	}
+
+	log.Printf("-> Running %s %s\n", c.BinPath, strings.Join(c.CmdArgs, " "))
+	err := c.Cmd.Run()
+	res.Stderr = stderr.String()
+	res.Stdout = stdout.String()
+	if err != nil {
+		res.Err = err
+		return res
+	}
+
+	if c.ManifestDir != "" {
+		path := filepath.Join(c.ManifestDir, "exec.MANIFEST")
+		if !util.FileExists(path) {
+			currentTime := time.Now()
+			data := []string{"Command: " + c.BinPath + strings.Join(c.CmdArgs, " ") + "\n"}
+			data = append(data, "Execution path: "+c.ExecDir)
+			data = append(data, "Execution time: "+currentTime.Format("2006-01-02 15:04:05"))
+			data = append(data, c.ManifestData...)
+
+			filesToHash := []string{c.BinPath} // we always get the fingerprint of the binary we execute
+			filesToHash = append(filesToHash, c.ManifestFileHash...)
+			hashData := hashFiles(filesToHash)
+			data = append(data, hashData...)
+
+			err := manifest.Create(path, data)
+			if err != nil {
+				// This is not a fatal error, we just log it
+				log.Printf("failed to create manifest: %s", err)
+			}
+			log.Printf("-> Manifest successfully created (%s)", path)
+
+		} else {
+			log.Printf("Manifest %s already exists, skipping...", err)
+		}
+	}
+
+	return res
 }

internal/pkg/sys/sys.go

@@ -22,7 +22,7 @@ const (
 	// image containers and install MPI
 	DefaultSympiInstallDir = ".sympi"
 
-	// CmdTimetout is the maximum time we allow a command to run
+	// CmdTimeout is the maximum time we allow a command to run
 	CmdTimeout = 20
 
 	// DefaultUbuntuDistro is the default Ubuntu distribution we use