Sync dev to main

Makefile

@@ -5,15 +5,51 @@ COVERAGE_THRESHOLD ?= 50.0
 # Build del progetto
 build:
 	@echo "Building proxsave..."
-	@VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0-dev); \
+	@VERSION=$$( \
+		if git describe --tags --exact-match >/dev/null 2>&1 && [ -z "$$(git status --porcelain 2>/dev/null)" ]; then \
+			git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0-dev; \
+		else \
+			desc=$$(git describe --tags --long --dirty --always 2>/dev/null || echo dev); \
+			dirty=""; \
+			case "$$desc" in *-dirty) dirty=".dirty"; desc=$${desc%-dirty};; esac; \
+			sha_part=$${desc##*-}; \
+			sha=$${sha_part#g}; \
+			rest=$${desc%-*}; \
+			n=$${rest##*-}; \
+			tag=$${rest%-*}; \
+			if [ "$$tag" = "$$desc" ] || [ -z "$$n" ] || [ -z "$$sha" ] || [ "$$sha_part" = "$$desc" ]; then \
+				echo "0.0.0-dev.0+g$${desc}$$dirty"; \
+			else \
+				echo "$$tag-dev.$$n+g$$sha$$dirty"; \
+			fi; \
+		fi \
+	); \
 	COMMIT=$$(git rev-parse --short HEAD 2>/dev/null || echo dev); \
 	BUILD_TIME=$$(date -u +"%Y-%m-%dT%H:%M:%SZ"); \
 	go build -ldflags="-X 'main.buildTime=$$BUILD_TIME' -X 'github.com/tis24dev/proxsave/internal/version.Version=$$VERSION' -X 'github.com/tis24dev/proxsave/internal/version.Commit=$$COMMIT' -X 'github.com/tis24dev/proxsave/internal/version.Date=$$BUILD_TIME'" -o build/proxsave ./cmd/proxsave
 
 # Build ottimizzato per release
 build-release:
 	@echo "Building release..."
-	@VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0-dev); \
+	@VERSION=$$( \
+		if git describe --tags --exact-match >/dev/null 2>&1 && [ -z "$$(git status --porcelain 2>/dev/null)" ]; then \
+			git describe --tags --abbrev=0 2>/dev/null || echo 0.0.0-dev; \
+		else \
+			desc=$$(git describe --tags --long --dirty --always 2>/dev/null || echo dev); \
+			dirty=""; \
+			case "$$desc" in *-dirty) dirty=".dirty"; desc=$${desc%-dirty};; esac; \
+			sha_part=$${desc##*-}; \
+			sha=$${sha_part#g}; \
+			rest=$${desc%-*}; \
+			n=$${rest##*-}; \
+			tag=$${rest%-*}; \
+			if [ "$$tag" = "$$desc" ] || [ -z "$$n" ] || [ -z "$$sha" ] || [ "$$sha_part" = "$$desc" ]; then \
+				echo "0.0.0-dev.0+g$${desc}$$dirty"; \
+			else \
+				echo "$$tag-dev.$$n+g$$sha$$dirty"; \
+			fi; \
+		fi \
+	); \
 	COMMIT=$$(git rev-parse --short HEAD 2>/dev/null || echo dev); \
 	BUILD_TIME=$$(date -u +"%Y-%m-%dT%H:%M:%SZ"); \
 	go build -ldflags="-s -w -X 'main.buildTime=$$BUILD_TIME' -X 'github.com/tis24dev/proxsave/internal/version.Version=$$VERSION' -X 'github.com/tis24dev/proxsave/internal/version.Commit=$$COMMIT' -X 'github.com/tis24dev/proxsave/internal/version.Date=$$BUILD_TIME'" -o build/proxsave ./cmd/proxsave

cmd/prefilter-manual/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/tis24dev/proxsave/internal/backup"
+	"github.com/tis24dev/proxsave/internal/logging"
+	"github.com/tis24dev/proxsave/internal/types"
+)
+
+func parseLogLevel(raw string) types.LogLevel {
+	switch strings.ToLower(strings.TrimSpace(raw)) {
+	case "debug":
+		return types.LogLevelDebug
+	case "info", "":
+		return types.LogLevelInfo
+	case "warning", "warn":
+		return types.LogLevelWarning
+	case "error":
+		return types.LogLevelError
+	default:
+		return types.LogLevelInfo
+	}
+}
+
+func main() {
+	var (
+		root       string
+		maxSize    int64
+		levelLabel string
+	)
+
+	flag.StringVar(&root, "root", "/tmp/test_prefilter", "Root directory to run prefilter on")
+	flag.Int64Var(&maxSize, "max-size", 8*1024*1024, "Max file size (bytes) to prefilter")
+	flag.StringVar(&levelLabel, "log-level", "info", "Log level: debug|info|warn|error")
+	flag.Parse()
+
+	root = filepath.Clean(strings.TrimSpace(root))
+	if root == "" || root == "." {
+		root = string(os.PathSeparator)
+	}
+
+	logger := logging.New(parseLogLevel(levelLabel), false)
+	logger.SetOutput(os.Stdout)
+
+	cfg := backup.OptimizationConfig{
+		EnablePrefilter:           true,
+		PrefilterMaxFileSizeBytes: maxSize,
+	}
+
+	if err := backup.ApplyOptimizations(context.Background(), logger, root, cfg); err != nil {
+		logger.Error("Prefilter failed: %v", err)
+		os.Exit(1)
+	}
+}

cmd/proxsave/main.go

@@ -1057,6 +1057,7 @@
 	checkerConfig.MinDiskPrimaryGB = cfg.MinDiskPrimaryGB
 	checkerConfig.MinDiskSecondaryGB = cfg.MinDiskSecondaryGB
 	checkerConfig.MinDiskCloudGB = cfg.MinDiskCloudGB
+	checkerConfig.FsIoTimeout = time.Duration(cfg.FsIoTimeoutSeconds) * time.Second
 	checkerConfig.DryRun = dryRun
 	checkerDone := logging.DebugStart(logger, "pre-backup check config", "dry_run=%v", dryRun)
 	if err := checkerConfig.Validate(); err != nil {
@@ -1569,7 +1570,7 @@
 		}
 		fmt.Printf("\r  Remaining: %ds   ", int(remaining.Seconds()))
 
 		select {
 		case <-ticker.C:
 			continue
 		}

cmd/proxsave/runtime_helpers.go

@@ -273,13 +273,13 @@ func detectFilesystemInfo(ctx context.Context, backend storage.Storage, path str
 		return nil, nil
 	}
 
-		if !fsInfo.SupportsOwnership {
-			if backend != nil && backend.Location() == storage.LocationCloud {
-				logger.Debug("%s [%s] does not support ownership changes (cloud remote); chown/chmod already disabled", path, fsInfo.Type)
-			} else {
-				logger.Info("%s [%s] does not support ownership changes; chown/chmod will be skipped", path, fsInfo.Type)
-			}
+	if !fsInfo.SupportsOwnership {
+		if backend != nil && backend.Location() == storage.LocationCloud {
+			logger.Debug("%s [%s] does not support ownership changes (cloud remote); chown/chmod already disabled", path, fsInfo.Type)
+		} else {
+			logger.Info("%s [%s] does not support ownership changes; chown/chmod will be skipped", path, fsInfo.Type)
 		}
+	}
 
 	return fsInfo, nil
 }

docs/BACKUP_ENV_MAPPING.md

@@ -68,7 +68,6 @@ PORT_WHITELIST            = SAME
 PVE_BACKUP_INCLUDE_PATTERN = SAME
 PVE_CLUSTER_PATH          = SAME
 PVE_CONFIG_PATH           = SAME
-PXAR_STOP_ON_CAP          = SAME
 RCLONE_BANDWIDTH_LIMIT    = SAME
 RCLONE_FLAGS              = SAME
 SECONDARY_LOG_PATH        = SAME
@@ -88,12 +87,17 @@ WEBHOOK_TIMEOUT           = SAME
 ## Go-only variables (new)
 
 SYSTEM_ROOT_PREFIX = NEW (Go-only) → Override system root for collection (testing/chroot). Empty or "/" uses the real root.
+PVESH_TIMEOUT = NEW (Go-only) → Timeout (seconds) for each `pvesh` command execution (0=disabled).
+FS_IO_TIMEOUT = NEW (Go-only) → Timeout (seconds) for filesystem probes (stat/readdir/statfs) on storages (0=disabled). Helps avoid hangs on unreachable network mounts.
+NOTE: PBS restore behavior is selected interactively during `--restore` and is intentionally not configured via `backup.env`.
 BACKUP_PBS_S3_ENDPOINTS = NEW (Go-only) → Collect `s3.cfg` and S3 endpoint snapshots (PBS).
 BACKUP_PBS_NODE_CONFIG = NEW (Go-only) → Collect `node.cfg` and node snapshots (PBS).
 BACKUP_PBS_ACME_ACCOUNTS = NEW (Go-only) → Collect `acme/accounts.cfg` and ACME account snapshots (PBS).
 BACKUP_PBS_ACME_PLUGINS = NEW (Go-only) → Collect `acme/plugins.cfg` and ACME plugin snapshots (PBS).
 BACKUP_PBS_METRIC_SERVERS = NEW (Go-only) → Collect `metricserver.cfg` (PBS).
 BACKUP_PBS_TRAFFIC_CONTROL = NEW (Go-only) → Collect `traffic-control.cfg` and traffic-control snapshots (PBS).
+BACKUP_PBS_NOTIFICATIONS = NEW (Go-only) → Collect `notifications.cfg` and notification snapshots (PBS).
+BACKUP_PBS_NOTIFICATIONS_PRIV = NEW (Go-only) → Collect `notifications-priv.cfg` (PBS notification secrets/credentials).
 BACKUP_PBS_NETWORK_CONFIG = NEW (Go-only) → Collect `network.cfg` and network snapshots (PBS), independent from BACKUP_NETWORK_CONFIGS (system).
 
 ## Renamed variables / Supported aliases in Go
@@ -140,17 +144,18 @@ STORAGE_WARNING_THRESHOLD_SECONDARY = SEMANTIC CHANGE → MIN_DISK_SPACE_SECONDA
 
 AUTO_DETECT_DATASTORES    = LEGACY (Bash only, auto-detect handled internally in Go)
 BACKUP_COROSYNC_CONFIG    = LEGACY (Go always uses COROSYNC_CONFIG_PATH / cluster)
-BACKUP_SMALL_PXAR         = LEGACY (in Go, PXAR tuning is more granular via PXAR_*_*)
+BACKUP_SMALL_PXAR         = LEGACY (no equivalent in Go; PXAR metadata sampling is bounded)
 CLOUD_BACKUP_REQUIRED     = LEGACY (secondary is always optional = warning only, non-blocking)
 CLOUD_PARALLEL_UPLOAD_TIMEOUT = LEGACY (in Go, timeouts are RCLONE_TIMEOUT_*)
 ENABLE_EMOJI_LOG          = LEGACY (log formatting handled internally in Go)
 ENABLE_LOG_MANAGEMENT     = LEGACY (log management in Go via LogPath/retention)
 MAX_CLOUD_LOGS            = LEGACY (Bash only; in Go log retention follows MAX_CLOUD_BACKUPS/CloudRetentionDays)
 MAX_LOCAL_LOGS            = LEGACY (Bash only; in Go log retention follows MAX_LOCAL_BACKUPS/LocalRetentionDays)
-MAX_PXAR_SIZE             = LEGACY (in Go there are PXAR_SCAN_MAX_ROOTS / budget, not the same semantics)
+MAX_PXAR_SIZE             = LEGACY (no equivalent in Go)
 MAX_SECONDARY_LOGS        = LEGACY (Bash only; in Go log retention follows MAX_SECONDARY_BACKUPS/SecondaryRetentionDays)
 MIN_BASH_VERSION          = LEGACY (specific only to Bash script)
 MULTI_STORAGE_PARALLEL    = LEGACY (in Go there is parallel storage management, not controlled by this variable)
+PXAR_STOP_ON_CAP          = LEGACY (Go no longer uses this tuning knob)
 REMOVE_UNAUTHORIZED_FILES = LEGACY (in Go there is no hard delete flag; checks are more conservative)
 SECONDARY_BACKUP_REQUIRED = LEGACY (secondary is always optional = warning only, non-blocking)
 SKIP_CLOUD_VERIFICATION   = LEGACY (verifications always performed)

docs/CLI_REFERENCE.md

@@ -789,6 +789,9 @@ CONFIG_FILE=/etc/pbs/prod.env ./build/proxsave
 # Force dry-run mode
 DRY_RUN=true ./build/proxsave
 
+# PBS restore behavior
+# Selected interactively during `--restore` on PBS hosts (Merge vs Clean 1:1).
+
 # Set debug level
 DEBUG_LEVEL=extreme ./build/proxsave --log-level debug
 

docs/CONFIGURATION.md

@@ -6,6 +6,7 @@ Complete reference for all 200+ configuration variables in `configs/backup.env`.
 
 - [Configuration File Location](#configuration-file-location)
 - [General Settings](#general-settings)
+- [Restore (PBS)](#restore-pbs)
 - [Security Settings](#security-settings)
 - [Disk Space](#disk-space)
 - [Storage Paths](#storage-paths)
@@ -74,6 +75,25 @@ PROFILING_ENABLED=true             # true | false (profiles written under LOG_PA
 
 ---
 
+## Restore (PBS)
+
+PBS restore behavior is chosen **interactively at restore time** on PBS hosts (not via `backup.env`).
+
+You will be asked to choose a behavior:
+- **Merge (existing PBS)**: intended for restoring onto an already operational PBS; ProxSave applies supported PBS categories via `proxmox-backup-manager` without deleting existing objects that are not in the backup.
+- **Clean 1:1 (fresh PBS install)**: intended for restoring onto a new, clean PBS; ProxSave attempts to make supported PBS objects match the backup (may remove objects that exist on the system but are not in the backup).
+
+ProxSave applies supported PBS staged categories via API automatically (and may fall back to file-based staged apply only in **Clean 1:1** mode).
+
+**Current API coverage**:
+- Node + traffic control (`pbs_host`)
+- Datastores + S3 endpoints (`datastore_pbs`)
+- Remotes (`pbs_remotes`)
+- Jobs (sync/verify/prune) (`pbs_jobs`)
+- Notifications endpoints/matchers (`pbs_notifications`)
+
+---
+
 ## Security Settings
 
 ```bash
@@ -321,7 +341,29 @@ PREFILTER_MAX_FILE_SIZE_MB=8       # Skip prefilter for files >8MB
 
 - **Smart chunking**: Splits large files for parallel processing
 - **Deduplication**: Detects duplicate data blocks (reduces storage)
-- **Prefilter**: Analyzes small files before compression (optimizes algorithm selection)
+- **Prefilter**: Applies safe, semantic-preserving normalization to small text/JSON files to improve compression (e.g. removes CR from CRLF line endings and minifies JSON). It does **not** reorder, de-indent, or strip structured configuration files, and it avoids touching Proxmox/PBS structured config paths (e.g. `etc/pve/**`, `etc/proxmox-backup/**`).
+
+### Prefilter (`ENABLE_PREFILTER`) — details and risks
+
+**What it does** (on the *staged* backup tree, before compression):
+- Removes `\r` from CRLF text files (`.txt`, `.log`, `.md`, `.conf`, `.cfg`, `.ini`) to normalize line endings
+- Minifies JSON (`.json`) while keeping valid JSON semantics
+
+**What it does not do**:
+- It does **not** reorder lines, remove indentation, or otherwise rewrite whitespace/ordering-sensitive structured configs.
+- It does **not** follow symlinks (symlinks are skipped).
+- It skips Proxmox/PBS structured configuration paths where formatting/order matters, such as:
+  - `etc/pve/**`
+  - `etc/proxmox-backup/**`
+  - `etc/systemd/system/**`
+  - `etc/ssh/**`
+  - `etc/pam.d/**`
+
+**Why you might disable it** (even though it's safe):
+- If you need maximum fidelity (bit-for-bit) of text/JSON formatting as originally collected (CRLF preservation, JSON pretty-printing, etc.)
+- If you prefer the most conservative pipeline possible (forensics/compliance)
+
+**Important**: Prefilter never edits files on the host system — it only operates on the temporary staging directory that will be archived.
 
 ---
 
@@ -918,6 +960,8 @@ BACKUP_PVE_REPLICATION=true        # VM/CT replication config
 
 # PVE backup files
 BACKUP_PVE_BACKUP_FILES=true       # Include backup files from /var/lib/vz/dump
+PVESH_TIMEOUT=15                   # Timeout (seconds) for each `pvesh` call (0=disabled)
+FS_IO_TIMEOUT=30                   # Timeout (seconds) for filesystem probes on storages (stat/readdir/statfs). Helps avoid hangs on unreachable network mounts (0=disabled)
 BACKUP_SMALL_PVE_BACKUPS=false     # Include small backups only
 MAX_PVE_BACKUP_SIZE=100M           # Max size for "small" backups
 PVE_BACKUP_INCLUDE_PATTERN=        # Glob patterns to include
@@ -938,6 +982,24 @@ BACKUP_VM_CONFIGS=true             # VM/CT config files
 # PBS datastore configs
 BACKUP_DATASTORE_CONFIGS=true      # Datastore definitions
 
+# S3 endpoints (used by S3 datastores)
+BACKUP_PBS_S3_ENDPOINTS=true       # s3.cfg (S3 endpoints, used by S3 datastores)
+
+# Node/global config
+BACKUP_PBS_NODE_CONFIG=true        # node.cfg (global PBS settings)
+
+# ACME
+BACKUP_PBS_ACME_ACCOUNTS=true      # acme/accounts.cfg
+BACKUP_PBS_ACME_PLUGINS=true       # acme/plugins.cfg
+
+# Integrations
+BACKUP_PBS_METRIC_SERVERS=true     # metricserver.cfg
+BACKUP_PBS_TRAFFIC_CONTROL=true    # traffic-control.cfg
+
+# Notifications
+BACKUP_PBS_NOTIFICATIONS=true      # notifications.cfg (targets/matchers/endpoints)
+BACKUP_PBS_NOTIFICATIONS_PRIV=true # notifications-priv.cfg (secrets/credentials for endpoints)
+
 # User and permissions
 BACKUP_USER_CONFIGS=true           # PBS users and tokens
 
@@ -953,26 +1015,25 @@ BACKUP_VERIFICATION_JOBS=true      # Backup verification schedules
 # Tape backup
 BACKUP_TAPE_CONFIGS=true           # Tape library configuration
 
+# Network configuration (PBS)
+BACKUP_PBS_NETWORK_CONFIG=true     # network.cfg (PBS), independent from BACKUP_NETWORK_CONFIGS (system)
+
 # Prune schedules
 BACKUP_PRUNE_SCHEDULES=true        # Retention prune schedules
 
 # PXAR metadata scanning
 PXAR_SCAN_ENABLE=false             # Enable PXAR file metadata collection
 PXAR_SCAN_DS_CONCURRENCY=3         # Datastores scanned in parallel
-PXAR_SCAN_INTRA_CONCURRENCY=4      # Workers per datastore
-PXAR_SCAN_FANOUT_LEVEL=2           # Directory depth for fan-out
-PXAR_SCAN_MAX_ROOTS=2048           # Max worker roots per datastore
-PXAR_STOP_ON_CAP=false             # Stop enumeration at max roots
-PXAR_ENUM_READDIR_WORKERS=4        # Parallel ReadDir workers
-PXAR_ENUM_BUDGET_MS=0              # Time budget for enumeration (0=disabled)
-PXAR_FILE_INCLUDE_PATTERN=         # Include patterns (default: *.pxar, catalog.pxar*)
+PXAR_FILE_INCLUDE_PATTERN=         # Include patterns (default: *.pxar, *.pxar.*, catalog.pxar*)
 PXAR_FILE_EXCLUDE_PATTERN=         # Exclude patterns (e.g., *.tmp, *.lock)
 ```
 
 **Note (PBS snapshot behavior)**: ProxSave snapshots `PBS_CONFIG_PATH` (`/etc/proxmox-backup`) for completeness. When a PBS feature is disabled, proxsave excludes the corresponding well-known config files from that snapshot (for example, `remote.cfg` is excluded when `BACKUP_REMOTE_CONFIGS=false`) and also skips the related command outputs.
 
 **PXAR scanning**: Collects metadata from Proxmox Backup Server .pxar archives.
 
+**Note**: `PXAR_FILE_INCLUDE_PATTERN` and `PXAR_FILE_EXCLUDE_PATTERN` are also reused for file sampling in PVE datastore metadata. Leave them empty to use the built-in defaults per platform.
+
 ### Override Collection Paths
 
 ```bash

docs/RESTORE_DIAGRAMS.md

@@ -138,6 +138,8 @@ flowchart TD
     style CheckboxMenu fill:#87CEEB
 ```
 
+**Note (PBS)**: ProxSave applies supported PBS staged categories via `proxmox-backup-manager` by default. In **Clean 1:1** mode it may fall back to writing staged `*.cfg` files back to `/etc/proxmox-backup` when API apply is unavailable or fails.
+
 ---
 
 ## Service Management Flow