diff --git a/backend/migrations/20251119_000000_multi_branch_sync.sql b/backend/migrations/20251119_000000_multi_branch_sync.sql new file mode 100644 index 0000000000..0c6e6867d0 --- /dev/null +++ b/backend/migrations/20251119_000000_multi_branch_sync.sql @@ -0,0 +1,7 @@ + +ALTER TABLE package_ops ADD COLUMN instance_id TEXT NULL + REFERENCES instances(id) ON DELETE SET NULL; + +-- Create index for querying ops by instance +CREATE INDEX IF NOT EXISTS idx_package_ops_instance + ON package_ops(instance_id) WHERE instance_id IS NOT NULL; diff --git a/backend/src/BuiltinCli/Builtin.fs b/backend/src/BuiltinCli/Builtin.fs index 7ecf024798..629d9c26e3 100644 --- a/backend/src/BuiltinCli/Builtin.fs +++ b/backend/src/BuiltinCli/Builtin.fs @@ -14,6 +14,7 @@ let builtins = Libs.File.builtins Libs.Execution.builtins Libs.Output.builtins + Libs.Process.builtins Libs.Stdin.builtins Libs.Time.builtins Libs.Terminal.builtins ] diff --git a/backend/src/BuiltinCli/BuiltinCli.fsproj b/backend/src/BuiltinCli/BuiltinCli.fsproj index 323944bff9..254904ec91 100644 --- a/backend/src/BuiltinCli/BuiltinCli.fsproj +++ b/backend/src/BuiltinCli/BuiltinCli.fsproj @@ -14,6 +14,7 @@ + diff --git a/backend/src/BuiltinCli/Libs/Process.fs b/backend/src/BuiltinCli/Libs/Process.fs new file mode 100644 index 0000000000..a07385e30a --- /dev/null +++ b/backend/src/BuiltinCli/Libs/Process.fs @@ -0,0 +1,131 @@ +/// Standard libraries for process management +module BuiltinCli.Libs.Process + +open System.Threading.Tasks +open FSharp.Control.Tasks + +open Prelude +open LibExecution.RuntimeTypes + +module Dval = LibExecution.Dval +module Builtin = LibExecution.Builtin +open Builtin.Shortcuts + +let fns : List = + [ { name = fn "processSpawnBackground" 0 + typeParams = [] + parameters = + [ Param.make "args" (TList TString) "Arguments to pass to the CLI" ] + returnType = TypeReference.result TInt64 TString + description = + "Spawns the current CLI executable in the background with the given arguments. Returns the process ID (PID) on success." + fn = + (function + | _state, _, _, [ DList(_vtTODO, args) ] -> + uply { + try + let argStrings = + args + |> List.map (fun arg -> + match arg with + | DString s -> s + | _ -> Exception.raiseInternal "Expected string arguments" []) + + // Get the current executable path + let currentExe = + System.Diagnostics.Process.GetCurrentProcess().MainModule.FileName + + let psi = System.Diagnostics.ProcessStartInfo() + psi.FileName <- currentExe + psi.UseShellExecute <- false + psi.CreateNoWindow <- true + // Redirect to prevent inheriting parent's streams + psi.RedirectStandardOutput <- true + psi.RedirectStandardError <- true + psi.RedirectStandardInput <- true + + // Add arguments + for arg in argStrings do + psi.ArgumentList.Add(arg) + + let proc = System.Diagnostics.Process.Start(psi) + + if isNull proc then + return + Dval.resultError + KTInt64 + KTString + (DString "Failed to start background process") + else + return Dval.resultOk KTInt64 KTString (DInt64(int64 proc.Id)) + with ex -> + return + Dval.resultError + KTInt64 + KTString + (DString $"Error spawning process: {ex.Message}") + } + | _ -> incorrectArgs ()) + sqlSpec = NotYetImplemented + previewable = Impure + deprecated = NotDeprecated } + + + { name = fn "processIsRunning" 0 + typeParams = [] + parameters = [ Param.make "pid" TInt64 "Process ID to check" ] + returnType = TBool + description = "Checks if a process with the given PID is currently running." + fn = + (function + | _, _, _, [ DInt64 pid ] -> + uply { + try + let proc = System.Diagnostics.Process.GetProcessById(int pid) + let isRunning = not proc.HasExited + return DBool isRunning + with + | :? System.ArgumentException + + | :? System.InvalidOperationException -> + // Process doesn't exist or has exited + return DBool false + } + | _ -> incorrectArgs ()) + sqlSpec = NotYetImplemented + previewable = Impure + deprecated = NotDeprecated } + + + { name = fn "processKill" 0 + typeParams = [] + parameters = [ Param.make "pid" TInt64 "Process ID to kill" ] + returnType = TypeReference.result TUnit TString + description = + "Kills the process with the given PID. Returns unit on success, or an error message on failure." + fn = + (function + | _state, _, _, [ DInt64 pid ] -> + uply { + try + let proc = System.Diagnostics.Process.GetProcessById(int pid) + proc.Kill() + proc.WaitForExit(5000) |> ignore + return Dval.resultOk KTUnit KTString DUnit + with + | :? System.ArgumentException -> + return Dval.resultError KTUnit KTString (DString "Process not found") + | ex -> + return + Dval.resultError + KTUnit + KTString + (DString $"Error killing process: {ex.Message}") + } + | _ -> incorrectArgs ()) + sqlSpec = NotYetImplemented + previewable = Impure + deprecated = NotDeprecated } ] + + +let builtins : Builtins = Builtin.make [] fns diff --git a/backend/src/BuiltinPM/Libs/PackageOps.fs b/backend/src/BuiltinPM/Libs/PackageOps.fs index d9b0de0ba2..0ca545adc9 100644 --- a/backend/src/BuiltinPM/Libs/PackageOps.fs +++ b/backend/src/BuiltinPM/Libs/PackageOps.fs @@ -21,32 +21,38 @@ open Builtin.Shortcuts let packageOpTypeName = FQTypeName.fqPackage PackageIDs.Type.LanguageTools.ProgramTypes.packageOp +let packageOpBatchTypeName = + FQTypeName.fqPackage PackageIDs.Type.LanguageTools.ProgramTypes.packageOpBatch + // TODO: review/reconsider the accessibility of these fns let fns : List = [ { name = fn "scmAddOps" 0 typeParams = [] parameters = - [ Param.make "branchID" (TypeReference.option TUuid) "" + [ Param.make "instanceID" (TypeReference.option TUuid) "" + Param.make "branchID" (TypeReference.option TUuid) "" Param.make "ops" (TList(TCustomType(Ok packageOpTypeName, []))) "" ] returnType = TypeReference.result TInt64 TString description = "Add package ops to the database and apply them to projections. - Returns Ok(insertedCount) on success (duplicates are skipped), or Error with message on failure." + Pass None for instanceID for local ops, or Some(uuid) for ops from remote instances. + Returns the number of inserted ops on success (duplicates are skipped), or an error message on failure." fn = let resultOk = Dval.resultOk KTInt64 KTString let resultError = Dval.resultError KTInt64 KTString (function - | _, _, _, [ branchID; DList(_vtTODO, ops) ] -> + | _, _, _, [ instanceID; branchID; DList(_vtTODO, ops) ] -> uply { try // Deserialize dvals let branchID = C2DT.Option.fromDT D.uuid branchID + let instanceID = C2DT.Option.fromDT D.uuid instanceID let ops = ops |> List.choose PT2DT.PackageOp.fromDT // Insert ops with deduplication, get count of actually inserted ops let! insertedCount = - LibPackageManager.Inserts.insertAndApplyOps branchID ops + LibPackageManager.Inserts.insertAndApplyOps instanceID branchID ops return resultOk (DInt64 insertedCount) with ex -> @@ -110,23 +116,49 @@ let fns : List = { name = fn "scmGetOpsSince" 0 typeParams = [] parameters = - [ Param.make "branchID" (TypeReference.option TUuid) "" + [ Param.make "targetInstanceID" (TypeReference.option TUuid) "" Param.make "since" TDateTime "" ] - returnType = TList(TCustomType(Ok packageOpTypeName, [])) - description = "Get package ops created since the given timestamp." + returnType = TList(TCustomType(Ok packageOpBatchTypeName, [])) + description = + "Get all package ops (from ALL branches) created since the given timestamp, grouped by branch and instance. + Optionally filters for a target instance (pass None to get all ops, or Some(uuid) to exclude ops from that target instance). + Returns a list of PackageOpBatch, where each batch contains ops from one branch with the same instanceID." fn = function - | _, _, _, [ branchID; DDateTime since ] -> + | _, _, _, [ targetInstanceID; DDateTime since ] -> uply { - let branchID = C2DT.Option.fromDT D.uuid branchID - - let! ops = LibPackageManager.Queries.getOpsSince branchID since - - return - DList( - VT.customType PT2DT.PackageOp.typeName [], - ops |> List.map PT2DT.PackageOp.toDT - ) + let targetID = C2DT.Option.fromDT D.uuid targetInstanceID + + let! opsWithMetadata = + LibPackageManager.Queries.getAllOpsSince targetID since + + // Group by (branchID, instanceID) + let grouped = + opsWithMetadata + |> List.groupBy (fun (_, branchID, instanceID) -> + (branchID, instanceID)) + |> Map.toList + + // Convert each group to a PackageOpBatch record + let batches = + grouped + |> List.map (fun ((branchID, instanceID), ops) -> + let opsList = + ops + |> List.map (fun (op, _, _) -> PT2DT.PackageOp.toDT op) + |> fun opDvals -> + DList(VT.customType packageOpTypeName [], opDvals) + + let fields = + [ ("branchID", branchID |> Option.map DUuid |> Dval.option KTUuid) + ("instanceID", + instanceID |> Option.map DUuid |> Dval.option KTUuid) + ("ops", opsList) ] + |> Map + + DRecord(packageOpBatchTypeName, packageOpBatchTypeName, [], fields)) + + return DList(VT.customType packageOpBatchTypeName [], batches) } | _ -> incorrectArgs () sqlSpec = NotQueryable diff --git a/backend/src/LibExecution/PackageIDs.fs b/backend/src/LibExecution/PackageIDs.fs index 9f4da26056..6f1c654b84 100644 --- a/backend/src/LibExecution/PackageIDs.fs +++ b/backend/src/LibExecution/PackageIDs.fs @@ -313,6 +313,8 @@ module Type = p [] "SearchResults" "0660f9dc-a816-4185-9e5c-f936325f83d5" let packageOp = p [] "PackageOp" "7d8e9f0a-1b2c-3d4e-5f6a-7b8c9d0e1f2a" + let packageOpBatch = + p [] "PackageOpBatch" "9f1a2b3c-4d5e-6f7a-8b9c-0d1e2f3a4b5c" let secret = p [] "Secret" "37427120-d71d-41f2-b094-68757570bc41" let db = p [] "DB" "7f219668-f8ac-4b17-a404-1171985dadf9" diff --git a/backend/src/LibExecution/ProgramTypes.fs b/backend/src/LibExecution/ProgramTypes.fs index 38109cc71b..c4fd2b6c70 100644 --- a/backend/src/LibExecution/ProgramTypes.fs +++ b/backend/src/LibExecution/ProgramTypes.fs @@ -699,6 +699,7 @@ Short answer: while Ops are played out type BranchID = uuid +type InstanceID = uuid /// A package entity paired with its location type LocatedItem<'T> = { entity : 'T; location : PackageLocation } diff --git a/backend/src/LibPackageManager/Inserts.fs b/backend/src/LibPackageManager/Inserts.fs index 95cea52c74..c1bce63ea7 100644 --- a/backend/src/LibPackageManager/Inserts.fs +++ b/backend/src/LibPackageManager/Inserts.fs @@ -38,7 +38,10 @@ let computeOpHash (op : PT.PackageOp) : System.Guid = /// Insert PackageOps into the package_ops table and apply them to projection tables /// Returns the count of ops actually inserted (duplicates are skipped via INSERT OR IGNORE) + +// CLEANUP: The 'applied' flag is currently always set to true and all ops are applied immediately let insertAndApplyOps + (instanceID : Option) (branchID : Option) (ops : List) : Task = @@ -60,12 +63,16 @@ let insertAndApplyOps let sql = """ - INSERT OR IGNORE INTO package_ops (id, branch_id, op_blob, applied) - VALUES (@id, @branch_id, @op_blob, @applied) + INSERT OR IGNORE INTO package_ops (id, instance_id, branch_id, op_blob, applied) + VALUES (@id, @instance_id, @branch_id, @op_blob, @applied) """ let parameters = [ "id", Sql.uuid opId + "instance_id", + (match instanceID with + | Some id -> Sql.uuid id + | None -> Sql.dbnull) "branch_id", (match branchID with | Some id -> Sql.uuid id diff --git a/backend/src/LibPackageManager/Queries.fs b/backend/src/LibPackageManager/Queries.fs index fbb51f9b4b..1b69fda0f8 100644 --- a/backend/src/LibPackageManager/Queries.fs +++ b/backend/src/LibPackageManager/Queries.fs @@ -73,45 +73,59 @@ let getRecentOpsAllBranches (limit : int64) : Task> = } -/// Get package ops created since the specified datetime -let getOpsSince - (branchID : Option) +/// Get all package ops (from ALL branches) created since the specified datetime +/// Returns ops with their branch IDs for multi-branch sync +/// +/// targetInstanceID: Optional target instance ID to filter results for: +/// - None: Return all ops +/// - Some(uuid): Exclude ops where instance_id = uuid (used when pushing to target to avoid sending ops back to their source) +let getAllOpsSince + (targetInstanceID : Option) (since : LibExecution.DarkDateTime.T) - : Task> = + : Task * Option>> = task { let sinceStr = LibExecution.DarkDateTime.toIsoString since - match branchID with - | Some id -> - // Query specific branch only + match targetInstanceID with + | Some targetID -> return! Sql.query """ - SELECT id, op_blob + SELECT id, op_blob, branch_id, instance_id FROM package_ops - WHERE branch_id = @branch_id - AND datetime(created_at) > datetime(@since) - ORDER BY created_at ASC + WHERE datetime(created_at) > datetime(@since) + AND (instance_id IS NULL OR instance_id != @target_instance_id) + ORDER BY + CASE WHEN branch_id IS NULL THEN 0 ELSE 1 END, + created_at ASC """ - |> Sql.parameters [ "branch_id", Sql.uuid id; "since", Sql.string sinceStr ] + |> Sql.parameters + [ "since", Sql.string sinceStr; "target_instance_id", Sql.uuid targetID ] |> Sql.executeAsync (fun read -> let opId = read.uuid "id" let opBlob = read.bytes "op_blob" - BinarySerialization.PT.PackageOp.deserialize opId opBlob) + let branchID = read.uuidOrNone "branch_id" + let instanceID = read.uuidOrNone "instance_id" + let op = BinarySerialization.PT.PackageOp.deserialize opId opBlob + (op, branchID, instanceID)) | None -> - // Query main branch only (branch_id IS NULL) return! Sql.query """ - SELECT id, op_blob + SELECT id, op_blob, branch_id, instance_id FROM package_ops - WHERE branch_id IS NULL - AND datetime(created_at) > datetime(@since) - ORDER BY created_at ASC + WHERE datetime(created_at) > datetime(@since) + ORDER BY + CASE WHEN branch_id IS NULL THEN 0 ELSE 1 END, + created_at ASC """ |> Sql.parameters [ "since", Sql.string sinceStr ] |> Sql.executeAsync (fun read -> let opId = read.uuid "id" let opBlob = read.bytes "op_blob" - BinarySerialization.PT.PackageOp.deserialize opId opBlob) + let branchID = read.uuidOrNone "branch_id" + let instanceID = read.uuidOrNone "instance_id" + let op = BinarySerialization.PT.PackageOp.deserialize opId opBlob + (op, branchID, instanceID)) + } diff --git a/backend/src/LibPackageManager/Sync.fs b/backend/src/LibPackageManager/Sync.fs index d6e1ee3eb1..b9682f81db 100644 --- a/backend/src/LibPackageManager/Sync.fs +++ b/backend/src/LibPackageManager/Sync.fs @@ -12,11 +12,12 @@ open Fumble open LibDB.Db module DarkDateTime = LibExecution.DarkDateTime +module PT = LibExecution.ProgramTypes /// Get the most recent sync time with a specific instance /// Returns None if no sync has occurred with this instance -let getLastSyncDate (instanceID : System.Guid) : Ply> = +let getLastSyncDate (instanceID : PT.InstanceID) : Ply> = uply { let! result = Sql.query @@ -41,7 +42,7 @@ let getLastSyncDate (instanceID : System.Guid) : Ply> = /// Record a sync event in the database let recordSync - (instanceID : System.Guid) + (instanceID : PT.InstanceID) (opsPushed : int64) (opsFetched : int64) : Task = diff --git a/backend/src/LocalExec/Canvas.fs b/backend/src/LocalExec/Canvas.fs index e965b4e3c0..f0ae33b30b 100644 --- a/backend/src/LocalExec/Canvas.fs +++ b/backend/src/LocalExec/Canvas.fs @@ -120,7 +120,7 @@ let loadFromDisk let dbs = canvas.dbs |> List.map PT.Toplevel.TLDB // Insert+apply canvas types, values, and fns as PackageOps - let! _ = LibPackageManager.Inserts.insertAndApplyOps None canvas.ops + let! _ = LibPackageManager.Inserts.insertAndApplyOps None None canvas.ops return (dbs @ handlers) |> List.map (fun tl -> tl, LibCloud.Serialize.NotDeleted) diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index 26f68db63a..4676971f23 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -46,7 +46,7 @@ module HandleCommand = do! LibPackageManager.Purge.purge () print "Filling ..." - let! _ = LibPackageManager.Inserts.insertAndApplyOps None ops + let! _ = LibPackageManager.Inserts.insertAndApplyOps None None ops // Get stats after ops are inserted/applied let! stats = LibPackageManager.Stats.get () diff --git a/canvases/dark-packages/main.dark b/canvases/dark-packages/main.dark index 137625a3dc..259cbf9426 100644 --- a/canvases/dark-packages/main.dark +++ b/canvases/dark-packages/main.dark @@ -59,11 +59,25 @@ let parseUuidParam (value: String) : Stdlib.Result.Result] let _handler _req = let bodyStr = Stdlib.String.fromBytesWithReplacement request.body - let branchID = getBranchIDFromQuery request - match Builtin.jsonParse> bodyStr with - | Ok ops -> - match Builtin.scmAddOps branchID ops with + match Builtin.jsonParse> bodyStr with + | Ok batches -> + // Process each batch + let results = + Stdlib.List.fold + batches + (Stdlib.Result.Result.Ok 0L) + (fun resultSoFar batch -> + match resultSoFar with + | Ok insertedSoFar -> + match SCM.PackageOps.add batch.instanceID batch.branchID batch.ops with + | Ok batchInserted -> + Stdlib.Result.Result.Ok(insertedSoFar + batchInserted) + | Error errMsg -> Stdlib.Result.Result.Error errMsg + + | Error msg -> Stdlib.Result.Result.Error msg) + + match results with | Ok insertedCount -> let countStr = Builtin.int64ToString insertedCount let responseMsg = $"Successfully received and applied {countStr} ops" @@ -73,15 +87,13 @@ let _handler _req = Stdlib.Http.response (Stdlib.String.toBytes errorMsg) 500L | Error _err -> - let errorMsg = "Invalid request body. Expected JSON list of PackageOps." + let errorMsg = "Invalid request body. Expected JSON list of PackageOpBatch." Stdlib.Http.response (Stdlib.String.toBytes errorMsg) 400L [] let _handler _req = - let branchID = getBranchIDFromQuery request - match getQueryParam request "since" with | None -> let errorMsg = "Missing required query parameter: 'since'" @@ -90,8 +102,14 @@ let _handler _req = | Some sinceStr -> match Stdlib.DateTime.parse sinceStr with | Ok sinceDate -> - let ops = Darklang.SCM.PackageOps.getSince branchID sinceDate - let json = ops |> Builtin.jsonSerialize> + // Get ops from ALL branches, already grouped by (branchID, instanceID) + // Pass None for targetInstanceID since we're not filtering on pull side + let batches = Builtin.scmGetOpsSince Stdlib.Option.Option.None sinceDate + + let json = + batches + |> Builtin.jsonSerialize> + Stdlib.Http.response (Stdlib.String.toBytes json) 200L | Error _err -> diff --git a/packages/darklang/cli/config.dark b/packages/darklang/cli/config.dark new file mode 100644 index 0000000000..729aa5f210 --- /dev/null +++ b/packages/darklang/cli/config.dark @@ -0,0 +1,117 @@ +module Darklang.Cli.Config + +// Config file path - uses rundir/ in portable mode, ~/.darklang/ when installed +let configFilePath () : String = + match Installation.System.getInstallationMode () with + | Portable -> "rundir/cli-config.json" + | Installed -> + let host = (Stdlib.Cli.Host.getRuntimeHost ()) |> Builtin.unwrap + let darklangHomeDir = Installation.Config.getDarklangHomeDir host + $"{darklangHomeDir}/cli-config.json" + +// Read config from disk +let readConfig () : Dict = + match Builtin.fileExists (configFilePath ()) with + | true -> + match Builtin.fileRead (configFilePath ()) with + | Ok bytes -> + let contents = Stdlib.String.fromBytesWithReplacement bytes + + match Builtin.jsonParse> contents with + | Ok config -> config + | Error _ -> Dict {} + + | Error _ -> Dict {} + + | false -> Dict {} + + +// Write config to disk +let writeConfig (config: Dict) : Unit = + let json = Builtin.jsonSerialize> config + let jsonBytes = Stdlib.String.toBytes json + let _ = Builtin.fileWrite jsonBytes (configFilePath ()) + () + + +// Helper to get a single config value +let get (key: String) : Stdlib.Option.Option = + let config = readConfig () + Stdlib.Dict.get config key + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + match args with + | ["get"; key] -> + let config = readConfig () + match Stdlib.Dict.get config key with + | Some value -> + Stdlib.printLine $"{key} = {value}" + state + + | None -> + Stdlib.printLine (Colors.error $"Configuration key not found: {key}") + state + + | ["set"; key; value] -> + let config = readConfig () + let updatedConfig = Stdlib.Dict.set config key value + writeConfig updatedConfig + Stdlib.printLine (Colors.success $"Set {key} = {value}") + state + + | ["list"] -> + // List all common config keys + Stdlib.printLine "Common configuration keys:" + Stdlib.printLine "" + Stdlib.printLine " sync.default_instance - Default instance for sync service" + Stdlib.printLine " sync.interval_seconds - Sync check interval (default: 20)" + Stdlib.printLine " sync.auto_start - Auto-start sync service (default: true)" + Stdlib.printLine "" + Stdlib.printLine "Use 'config get ' to read a value" + Stdlib.printLine "Use 'config set ' to set a value" + state + + | _ -> + Stdlib.printLine (Colors.error "Invalid arguments") + Stdlib.printLine "" + help state + state + + +let complete (_state: Cli.AppState) (args: List) : List = + match args with + | [] -> ["get"; "set"; "list"] + + | ["get"] -> + [ "sync.default_instance" + "sync.interval_seconds" + "sync.auto_start" ] + + | ["set"] -> + [ "sync.default_instance" + "sync.interval_seconds" + "sync.auto_start" ] + + | _ -> [] + + +let help (_state: Cli.AppState) : Cli.AppState = + [ "Usage:" + " config get - Get a configuration value" + " config set - Set a configuration value" + " config list - List common configuration keys" + "" + "Examples:" + " config set sync.default_instance local" + " config set sync.interval_seconds 60" + " config get sync.default_instance" + " config set sync.auto_start false" + "" + "Common keys:" + " sync.default_instance - Which instance to sync with" + " sync.interval_seconds - How often to check for changes (default: 20)" + " sync.auto_start - Start sync service automatically (default: true)" ] + |> Stdlib.printLines + + _state diff --git a/packages/darklang/cli/core.dark b/packages/darklang/cli/core.dark index 23c0026a92..65b8bc8a5e 100644 --- a/packages/darklang/cli/core.dark +++ b/packages/darklang/cli/core.dark @@ -74,7 +74,7 @@ module Registry = ("help", "Show help for commands", ["commands"; "?"], Help.execute, Help.help, Help.complete) ("branch", "Manage development branches", [], Packages.Branch.execute, Packages.Branch.help, Packages.Branch.complete) ("instance", "Manage remote instances for syncing", [], Instances.execute, Instances.help, Instances.complete) - ("sync", "Sync package ops with remote instance", [], Sync.execute, Sync.help, Sync.complete) + ("config", "Manage CLI configuration", [], Config.execute, Config.help, Config.complete) ("install", "Install CLI globally", [], Installation.Install.execute, Installation.Install.help, Installation.Install.complete) ("update", "Update CLI to latest version", ["upgrade"], Installation.Update.execute, Installation.Update.help, Installation.Update.complete) ("uninstall", "Remove CLI installation", [], Installation.Uninstall.execute, Installation.Uninstall.help, Installation.Uninstall.complete) @@ -166,10 +166,10 @@ module Registry = // Command groups organized by category (shared between compact and detailed views) let commandGroups () : List<(String * List)> = [ ("Packages", ["nav"; "ls"; "view"; "tree"; "back"; "search"; "val"; "let"; "fn"; "type"; "rm"]) - ("Source Control", ["branch"; "instance"; "sync"]) + ("Source Control", ["branch"; "instance"]) ("Execution", ["run"; "eval"; "scripts"]) ("Installation", ["install"; "update"; "uninstall"; "status"; "version"]) - ("Utilities", ["clear"; "help"; "quit"; "experiments"]) ] + ("Utilities", ["clear"; "help"; "quit"; "config"; "experiments"]) ] // Helper function to format a group of commands with details let formatCommandGroup (groupName: String) (commandNames: List) (commands: List) : String = @@ -259,6 +259,7 @@ module Registry = let allNames = Stdlib.List.map commands (fun cmd -> cmd.name) let allAliases = Stdlib.List.fold commands [] (fun acc cmd -> Stdlib.List.append acc cmd.aliases) let allOptions = Stdlib.List.append allNames allAliases + if Stdlib.String.isEmpty partial then allOptions else @@ -411,6 +412,10 @@ module Update = let runInteractiveLoop (state: AppState) : Int64 = if state.isExiting then + // stop sync service if running + match SyncService.stop () with + | Ok () -> () + | Error _ -> () // Silently ignore if not running 0L else // Display current page @@ -462,6 +467,9 @@ let executeCliCommand (args: List) : Int64 = match args with // If someone runs `dark` without args, start the interactive loop | [] -> + // Auto-start sync service if not already running + SyncService.autoStart () + Stdlib.printLine (View.formatWelcome ()) runInteractiveLoop initialState // Otherwise, just execute command, print result, and exit diff --git a/packages/darklang/cli/packages/fn.dark b/packages/darklang/cli/packages/fn.dark index 462a46f9cc..7e78508e43 100644 --- a/packages/darklang/cli/packages/fn.dark +++ b/packages/darklang/cli/packages/fn.dark @@ -97,7 +97,10 @@ let createFnInline LanguageTools.ProgramTypes.PackageOp.SetFnName (packageFn.id, location) ] // Push the ops up to the server - match SCM.PackageOps.add branchID ops with + // Local CLI ops don't need an instanceID to track their origin + let instanceID = Stdlib.Option.Option.None + + match SCM.PackageOps.add instanceID branchID ops with | Ok _ -> // Show success message let fullPath = PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location diff --git a/packages/darklang/cli/packages/type.dark b/packages/darklang/cli/packages/type.dark index a5cfe89ccb..0b1b888cd9 100644 --- a/packages/darklang/cli/packages/type.dark +++ b/packages/darklang/cli/packages/type.dark @@ -99,7 +99,10 @@ let createTypeInline LanguageTools.ProgramTypes.PackageOp.SetTypeName (packageType.id, location) ] // Push the ops up to the server - match SCM.PackageOps.add branchID ops with + // Local CLI ops don't need an instanceID to track their origin + let instanceID = Stdlib.Option.Option.None + + match SCM.PackageOps.add instanceID branchID ops with | Ok _ -> // Show success message let fullPath = PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location diff --git a/packages/darklang/cli/packages/value.dark b/packages/darklang/cli/packages/value.dark index 7ef5646fb7..fe371f4102 100644 --- a/packages/darklang/cli/packages/value.dark +++ b/packages/darklang/cli/packages/value.dark @@ -98,7 +98,10 @@ let createValueInline LanguageTools.ProgramTypes.PackageOp.SetValueName (packageValue.id, location) ] // Push the ops up to the server - match SCM.PackageOps.add branchID ops with + // Local CLI ops don't need an instanceID to track their origin + let instanceID = Stdlib.Option.Option.None + + match SCM.PackageOps.add instanceID branchID ops with | Ok _ -> // Show success message let fullPath = PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location diff --git a/packages/darklang/cli/sync.dark b/packages/darklang/cli/sync.dark deleted file mode 100644 index 1410f31dfa..0000000000 --- a/packages/darklang/cli/sync.dark +++ /dev/null @@ -1,66 +0,0 @@ -module Darklang.Cli.Sync - - -let execute (state: Cli.AppState) (args: List) : Cli.AppState = - match args with - | [instanceName] -> - // Sync with specified instance by name - match SCM.Instances.getByName instanceName with - | Some instance -> - Stdlib.printLine $"Syncing with {instance.name} ({instance.url})..." - Stdlib.printLine "" - - let result = - SCM.Sync.sync instance.id instance.url state.currentBranchId - - match result with - | Ok message -> - Stdlib.printLine (Colors.success message) - state - - | Error errorMsg -> - Stdlib.printLine (Colors.error errorMsg) - state - - | None -> - Stdlib.printLine (Colors.error $"Instance not found: {instanceName}") - Stdlib.printLine "" - Stdlib.printLine "Use 'instance list' to see configured instances." - Stdlib.printLine "Use 'instance add ' to add a new instance." - state - - | _ -> - Stdlib.printLine (Colors.error "Invalid arguments") - Stdlib.printLine "" - help state - state - - -let complete (_state: Cli.AppState) (args: List) : List = - match args with - | [] -> - let instances = SCM.Instances.list () - Stdlib.List.map instances (fun i -> i.name) - - | [partial] -> - let instances = SCM.Instances.list () - - instances - |> Stdlib.List.map (fun i -> i.name) - |> Stdlib.List.filter (fun name -> Stdlib.String.startsWith name partial) - - | _ -> [] - - -let help (_state: Cli.AppState) : Unit = - [ "Usage:" - " sync - Sync with a configured instance" - "" - "Examples:" - " sync local - Sync with 'local' instance" - " sync production - Sync with 'production' instance" - "" - "Before syncing, configure instances using:" - " instance add local http://dark-packages.dlio.localhost:11001" - " instance list" ] - |> Stdlib.printLines diff --git a/packages/darklang/cli/syncService.dark b/packages/darklang/cli/syncService.dark new file mode 100644 index 0000000000..e0d510682c --- /dev/null +++ b/packages/darklang/cli/syncService.dark @@ -0,0 +1,258 @@ +module Darklang.Cli.SyncService + +/// File paths for sync service state - uses rundir/ in portable mode, ~/.darklang/ when installed +let pidFilePath () : String = + match Installation.System.getInstallationMode () with + | Portable -> "rundir/sync-service.pid" + | Installed -> + let host = (Stdlib.Cli.Host.getRuntimeHost ()) |> Builtin.unwrap + let darklangHomeDir = Installation.Config.getDarklangHomeDir host + $"{darklangHomeDir}/sync-service.pid" + +let shutdownSignalPath () : String = + match Installation.System.getInstallationMode () with + | Portable -> "rundir/sync-service.shutdown" + | Installed -> + let host = (Stdlib.Cli.Host.getRuntimeHost ()) |> Builtin.unwrap + let darklangHomeDir = Installation.Config.getDarklangHomeDir host + $"{darklangHomeDir}/sync-service.shutdown" + +// Logging helper function +let log (message: String) : Unit = + let timestamp = (Stdlib.DateTime.now ()) |> Stdlib.DateTime.toString + let logMessage = $"[{timestamp}] {message}\n" + + let logPath = + match Installation.System.getInstallationMode () with + | Portable -> "rundir/logs/sync-service.log" + | Installed -> + let host = (Stdlib.Cli.Host.getRuntimeHost ()) |> Builtin.unwrap + let darklangHomeDir = Installation.Config.getDarklangHomeDir host + $"{darklangHomeDir}/sync-service.log" + + let _ = Builtin.fileAppendText logPath logMessage + () + + +/// Check if sync service is running +let isRunning () : Bool = + match Builtin.fileExists (pidFilePath ()) with + | true -> + match Builtin.fileRead (pidFilePath ()) with + | Ok bytes -> + let pidStr = Stdlib.String.fromBytesWithReplacement bytes + + match Stdlib.Int64.parse pidStr with + | Ok pid -> + if Builtin.processIsRunning pid then + true + else + // Process not running, clean up stale PID file + let _ = Builtin.fileDelete (pidFilePath ()) + false + + | Error _ -> + // Invalid PID file, clean it up + let _ = Builtin.fileDelete (pidFilePath ()) + false + + | Error _ -> + // Error reading file, clean it up + let _ = Builtin.fileDelete (pidFilePath ()) + false + + | false -> false + + +/// Read PID from file +let readPid () : Stdlib.Option.Option = + match Builtin.fileExists (pidFilePath ()) with + | true -> + match Builtin.fileRead (pidFilePath ()) with + | Ok bytes -> + let pidStr = Stdlib.String.fromBytesWithReplacement bytes + + match Stdlib.Int64.parse pidStr with + | Ok pid -> Stdlib.Option.Option.Some pid + | Error _ -> Stdlib.Option.Option.None + + | Error _ -> Stdlib.Option.Option.None + + | false -> Stdlib.Option.Option.None + + +/// Runs the sync loop forever +let syncLoop (intervalSeconds: Int64) : Unit = + let intervalMs = Stdlib.Int64.multiply intervalSeconds 1000L + + // Get default instance from config, or fall back to "instance2" + let instanceName = + match Config.get "sync.default_instance" with + | Some name -> name + // CLEANUP: don't use a hardcoded default instance name here + | None -> "instance2" + + // Look up instance details + match SCM.Instances.getByName instanceName with + | Some instance -> + log $"Sync check starting (instance: {instance.name}, all branches)" + + // Perform sync (syncs all branches) + let syncResult = SCM.Sync.sync instance.id instance.url + + // Log result + (match syncResult with + | Ok message -> log message + + | Error errorMsg -> log $"Sync failed: {errorMsg}") + + // Check shutdown signal before sleeping + if Builtin.fileExists (shutdownSignalPath ()) then + log "Sync service shutting down" + let _ = Builtin.fileDelete (pidFilePath ()) + let _ = Builtin.fileDelete (shutdownSignalPath ()) + () + else + // Sleep + let intervalFloat = Stdlib.Int64.toFloat intervalMs + Builtin.timeSleep intervalFloat + + // Check shutdown signal after sleeping + if Builtin.fileExists (shutdownSignalPath ()) then + log "Sync service shutting down" + let _ = Builtin.fileDelete (pidFilePath ()) + let _ = Builtin.fileDelete (shutdownSignalPath ()) + () + else + syncLoop intervalSeconds + + | None -> + log $"Instance not found: {instanceName}" + log "Run: dark instance list" + // Cleanup PID file since we're exiting + let _ = Builtin.fileDelete (pidFilePath ()) + () + + +/// Start the sync service in the background +/// Returns unit on success, or an error message if failed to start +let startInBackground () : Stdlib.Result.Result = + // Check if already running + if isRunning () then + Stdlib.Result.Result.Error "Sync service is already running" + else + // Get default instance from config, or fall back to "instance2" + let instanceName = + match Config.get "sync.default_instance" with + | Some name -> name + | None -> "instance2" + + // Check if the instance exists + match SCM.Instances.getByName instanceName with + | Some _instance -> + // Get interval from config or use default (20 seconds) + let intervalSeconds = + match Config.get "sync.interval_seconds" with + | Some intervalStr -> + (match Stdlib.Int64.parse intervalStr with + | Ok interval -> interval + | Error _ -> 20L) + | None -> 20L + + // setup args to run syncLoop in background + let args = [ "run"; "@Darklang.Cli.SyncService.syncLoop"; $"{Stdlib.Int64.toString intervalSeconds}L" ] + + // Spawn background process + match Builtin.processSpawnBackground args with + | Ok pid -> + let pidStr = Stdlib.Int64.toString pid + // Write PID file (overwrite if exists) + let _ = Builtin.fileWrite (Stdlib.String.toBytes pidStr) (pidFilePath ()) + log $"Sync service started (PID: {pidStr})" + Stdlib.Result.Result.Ok () + + | Error errMsg -> Stdlib.Result.Result.Error $"Failed to start sync service: {errMsg}" + + | None -> + Stdlib.Result.Result.Error + $"Instance '{instanceName}' not found. Run: dark instance list" + + +/// Helper function to wait for a process to shutdown +let waitForProcessShutdown (pid: Int64) (checksRemaining: Float) : Bool = + let checkIntervalMs = 200.0 + + if Stdlib.Float.lessThanOrEqualTo checksRemaining 0.0 then + false + else if Builtin.processIsRunning pid then + Builtin.timeSleep checkIntervalMs + waitForProcessShutdown pid (Stdlib.Float.subtract checksRemaining 1.0) + else + true + + +/// Stop the sync service +let stop () : Stdlib.Result.Result = + match readPid () with + | Some pid -> + // Create shutdown signal + let _ = Builtin.fileDelete (shutdownSignalPath ()) + let _ = Builtin.fileAppendText (shutdownSignalPath ()) "" + + log "Waiting for sync service to shutdown..." + + // Wait up to 10 seconds for graceful shutdown + let maxWaitMs = 10000.0 + let checkIntervalMs = 200.0 + let maxChecks = Stdlib.Float.divide maxWaitMs checkIntervalMs + + let shutdownGracefully = waitForProcessShutdown pid maxChecks + + if shutdownGracefully then + // Cleanup files + let _ = Builtin.fileDelete (pidFilePath ()) + let _ = Builtin.fileDelete (shutdownSignalPath ()) + log "Sync service stopped" + Stdlib.Result.Result.Ok () + else + // Force kill if didn't shutdown gracefully + log "Force killing sync service..." + + match Builtin.processKill pid with + | Ok () -> + let _ = Builtin.fileDelete (pidFilePath ()) + let _ = Builtin.fileDelete (shutdownSignalPath ()) + log "Sync service stopped (force killed)" + Stdlib.Result.Result.Ok () + + | Error errMsg -> + let _ = Builtin.fileDelete (pidFilePath ()) + let _ = Builtin.fileDelete (shutdownSignalPath ()) + + Stdlib.Result.Result.Error $"Failed to kill sync service: {errMsg}" + | None -> Stdlib.Result.Result.Error "Sync service is not running" + + +/// Auto-start the sync service if not already running +/// Called during CLI startup +let autoStart () : Unit = + if isRunning () then + () + else + // Check if auto-start is enabled (default: true) + let autoStartEnabled = + match Config.get "sync.auto_start" with + | Some "false" -> false + | _ -> true // Default to enabled + + if autoStartEnabled then + // Auto-start syncs all branches + match startInBackground () with + | Ok () -> + log "Auto-started sync service" + Stdlib.printLine (Colors.success "✓ Sync service started") + () + | Error errMsg -> + // Log error but don't block CLI startup + log $"Failed to auto-start sync service: {errMsg}" + () \ No newline at end of file diff --git a/packages/darklang/languageTools/lsp-server/fileSystemProvider.dark b/packages/darklang/languageTools/lsp-server/fileSystemProvider.dark index fd6eacc076..fa63478265 100644 --- a/packages/darklang/languageTools/lsp-server/fileSystemProvider.dark +++ b/packages/darklang/languageTools/lsp-server/fileSystemProvider.dark @@ -461,9 +461,12 @@ module WriteFile = (Stdlib.List.append fnOps valueOps) // Only add ops if there are new items + // Local editor ops don't need an instanceID to track their origin + let instanceID = Stdlib.Option.Option.None + let addResult = if (Stdlib.List.isEmpty allOps) |> Stdlib.Bool.not then - SCM.PackageOps.add state.branchID allOps + SCM.PackageOps.add instanceID state.branchID allOps else Stdlib.Result.Result.Ok 0L diff --git a/packages/darklang/languageTools/lsp-server/handleIncomingMessage.dark b/packages/darklang/languageTools/lsp-server/handleIncomingMessage.dark index 5ed08368a0..9a6d57550d 100644 --- a/packages/darklang/languageTools/lsp-server/handleIncomingMessage.dark +++ b/packages/darklang/languageTools/lsp-server/handleIncomingMessage.dark @@ -276,9 +276,6 @@ let handleIncomingMessage | ("dark/clearBranch", Some requestId, _) -> Branches.handleClearBranchRequest state requestId - | ("dark/sync", Some requestId, Some(Object [ ("instanceID", String instanceID) ])) -> - Sync.handleSyncRequest state requestId instanceID - | ("dark/listInstances", Some requestId, _) -> Sync.handleListInstancesRequest state requestId diff --git a/packages/darklang/languageTools/lsp-server/pendingOps.dark b/packages/darklang/languageTools/lsp-server/pendingOps.dark index ad9e2fa592..7ff610b93a 100644 --- a/packages/darklang/languageTools/lsp-server/pendingOps.dark +++ b/packages/darklang/languageTools/lsp-server/pendingOps.dark @@ -1,70 +1,23 @@ module Darklang.LanguageTools.LspServer.PendingOps -// Get recent package ops from the database with optional filtering +// Get recent package ops from the database let handleGetPendingOpsRequest (state: LspState) (requestID: JsonRPC.RequestId) (params: Stdlib.Option.Option) : LspState = - // Parse filter parameters from request - let limit = - match params with - | Some(Object fields) -> - (match Stdlib.List.findFirst fields (fun (k, v) -> k == "limit") with - | Some((_key, Number n)) -> Stdlib.Float.round n - | Some((k, _v)) -> 50L - | None -> 50L) - | _ -> - 50L + // Use current branch from state + let queryBranchId = state.branchID - let sinceDate = - match params with - | Some(Object fields) -> - (match Stdlib.List.findFirst fields (fun (k, v) -> k == "sinceDate") with - | Some((_key, String dateStr)) -> Stdlib.Option.Option.Some dateStr - | _ -> Stdlib.Option.Option.None) - | _ -> Stdlib.Option.Option.None - - let branchFilter = - match params with - | Some(Object fields) -> - (match Stdlib.List.findFirst fields (fun (k, v) -> k == "branchFilter") with - | Some((_key, String filterValue)) -> filterValue - | _ -> "current") - | _ -> "current" - - - // Determine which branch ID to query - let queryBranchId = - if branchFilter == "current" then - // Use the current branch from state - state.branchID - else - if branchFilter == "all" then - // No branch filter - will query all branches - Stdlib.Option.Option.None - else - // It's a specific branch ID - try to parse it - match Stdlib.Uuid.parse branchFilter with - | Ok uuid -> Stdlib.Option.Option.Some uuid - | Error _ -> state.branchID - - // Get ops based on branch filter and date filter + // Get ops: if on a branch, get all; if on main, get last 20 let ops = - if branchFilter == "all" then - // Query all branches - no branch filter - match sinceDate with - | Some dateStr -> SCM.PackageOps.getRecentAllBranches limit - | None -> SCM.PackageOps.getRecentAllBranches limit - else - // Query specific branch (or main if no branch is set) - match sinceDate with - | Some dateStr -> - (match Stdlib.DateTime.parse dateStr with - | Ok dt -> SCM.PackageOps.getSince queryBranchId dt - | Error _ -> SCM.PackageOps.getRecent queryBranchId limit) - | None -> - SCM.PackageOps.getRecent queryBranchId limit + match queryBranchId with + | Some branchID -> + // On a branch - show ALL ops from this branch + SCM.PackageOps.getRecent (Stdlib.Option.Option.Some branchID) 999999L + | None -> + // On main - show last 20 from all branches + SCM.PackageOps.getRecent (Stdlib.Option.Option.None) 20L let opsCount = Stdlib.List.length ops diff --git a/packages/darklang/languageTools/lsp-server/sync.dark b/packages/darklang/languageTools/lsp-server/sync.dark index 0d9277cf5b..59b3f08fd7 100644 --- a/packages/darklang/languageTools/lsp-server/sync.dark +++ b/packages/darklang/languageTools/lsp-server/sync.dark @@ -1,38 +1,5 @@ module Darklang.LanguageTools.LspServer.Sync -/// Handles `dark/sync` requests from VS Code -let handleSyncRequest - (state: LspState) - (requestID: JsonRPC.RequestId) - (instanceName: String) - : LspState = - let result = - match SCM.Instances.getByName instanceName with - | Some instance -> - let result = SCM.Sync.sync instance.id instance.url state.branchID - - match result with - | Ok message -> - [ ("success", Json.Bool true); ("message", Json.String message) ] - |> Json.Object - | Error errorMsg -> - [ ("success", Json.Bool false); ("message", Json.String errorMsg) ] - |> Json.Object - | None -> - [ ("success", Json.Bool false) - ("message", Json.String $"Instance '{instanceName}' not found.") ] - |> Json.Object - - let responseJson = - let requestID = Stdlib.Option.Option.Some requestID - (JsonRPC.Response.Ok.make requestID result) - |> Stdlib.AltJson.format - - logAndSendToClient responseJson - - state - - /// Handles `dark/listInstances` requests from VS Code let handleListInstancesRequest (state: LspState) diff --git a/packages/darklang/languageTools/programTypes.dark b/packages/darklang/languageTools/programTypes.dark index 9448f3c97f..896aad4253 100644 --- a/packages/darklang/languageTools/programTypes.dark +++ b/packages/darklang/languageTools/programTypes.dark @@ -276,6 +276,8 @@ module Expr = | ERecordFieldAccess(id, _, _) | EVariable(id, _) | EArg(id, _) + | EValue(id, _) + | EFnName(id, _) | EApply(id, _, _, _) | EList(id, _) | EDict(id, _) @@ -285,7 +287,8 @@ module Expr = | EEnum(id, _, _, _, _) | EMatch(id, _, _) | ERecordUpdate(id, _, _) - | EStatement(id, _, _) -> id + | EStatement(id, _, _) + | ESelf id -> id // Used to mark whether a function/type has been deprecated, and if so, @@ -414,6 +417,13 @@ type PackageOp = | SetFnName of FQFnName.Package * PackageLocation +/// A batch of package ops that all share the same branch and instance +type PackageOpBatch = + { branchID: Stdlib.Option.Option + instanceID: Stdlib.Option.Option + ops: List } + + module Search = /// The type of entity to search for type EntityType = diff --git a/packages/darklang/scm/packageOps.dark b/packages/darklang/scm/packageOps.dark index 5aa3896d3f..66e60c39f9 100644 --- a/packages/darklang/scm/packageOps.dark +++ b/packages/darklang/scm/packageOps.dark @@ -2,13 +2,14 @@ module Darklang.SCM.PackageOps /// Add a list of package ops to the database, applying them immediately /// -/// Returns Ok(insertedCount) on success (duplicates are skipped), -/// or Error with message on failure +/// Returns the number of inserted ops on success (duplicates are skipped), +/// or an error message on failure let add + (instanceID: Stdlib.Option.Option) (branchID: Stdlib.Option.Option) (ops: List) : Stdlib.Result.Result = - Builtin.scmAddOps branchID ops + Builtin.scmAddOps instanceID branchID ops /// Get recent package ops from the database @@ -24,11 +25,3 @@ let getRecentAllBranches (limit: Int64) : List = Builtin.scmGetRecentOpsAllBranches limit - - -/// Get package ops created since the specified datetime -let getSince - (branchID: Stdlib.Option.Option) - (since: DateTime) - : List = - Builtin.scmGetOpsSince branchID since diff --git a/packages/darklang/scm/sync.dark b/packages/darklang/scm/sync.dark index 4702d87e52..d873f6d7b7 100644 --- a/packages/darklang/scm/sync.dark +++ b/packages/darklang/scm/sync.dark @@ -5,7 +5,6 @@ module Darklang.SCM.Sync let sync (instanceID: Uuid) (remoteURL: String) - (branchID: Stdlib.Option.Option) : Stdlib.Result.Result = // Get last sync date with this instance let lastSyncOpt = Builtin.scmGetLastSyncDate instanceID @@ -16,74 +15,51 @@ let sync // If never synced, use Unix epoch (will sync everything) | None -> Stdlib.DateTime.fromSeconds 0L - // Get ops to push (created since last sync) - let opsToPush = PackageOps.getSince branchID sinceDate + // Get ops to push from ALL branches (created since last sync), already grouped by branch + // Exclude ops from the target instance to avoid sending ops back to their source + let batches = + Builtin.scmGetOpsSince (Stdlib.Option.Option.Some instanceID) sinceDate - let opsPushedCount = Stdlib.List.length opsToPush + let batchCount = Stdlib.List.length batches - // Push ops to remote in batches + // Count total ops across all batches + let opsPushedCount = + Stdlib.List.fold batches 0L (fun acc batch -> acc + Stdlib.List.length batch.ops) + + // Push ops to remote let pushResult = - if opsPushedCount == 0L then + if batchCount == 0L then Stdlib.Result.Result.Ok 0L else - let batchSize = 100L - - let batches = - match Stdlib.List.chunkBySize opsToPush batchSize with - | Ok chunks -> chunks - | Error _ -> [ opsToPush ] // Fallback to single batch if chunking fails - - let batchCount = Stdlib.List.length batches - let pushUrl = $"{remoteURL}/ops" - // Push each batch sequentially using fold with manual index tracking - let (_, pushAllBatches) = - Stdlib.List.fold + // Push all batches in one request + let opsJson = + Builtin.jsonSerialize> batches - (0L, Stdlib.Result.Result.Ok 0L) - (fun acc batch -> - let (index, batchResult) = acc - - match batchResult with - | Error msg -> (index + 1L, Stdlib.Result.Result.Error msg) - | Ok totalSoFar -> - let batchNum = index + 1L - let batchNumStr = Stdlib.Int64.toString batchNum - let batchOpsCount = Stdlib.List.length batch - - let opsJson = - Builtin.jsonSerialize> batch - - let pushRequest = - Stdlib.HttpClient.post - pushUrl - [ ("Content-Type", "application/json") ] - (Stdlib.String.toBytes opsJson) - - let newResult = - match pushRequest with - | Ok response -> - if response.statusCode >= 200L && response.statusCode < 300L then - Stdlib.Result.Result.Ok(totalSoFar + batchOpsCount) - else - let errorBody = - Stdlib.String.fromBytesWithReplacement response.body - let statusCode = Stdlib.Int64.toString response.statusCode + let pushRequest = + Stdlib.HttpClient.post + pushUrl + [ ("Content-Type", "application/json") ] + (Stdlib.String.toBytes opsJson) - Stdlib.Result.Result.Error - $"Push failed on batch {batchNumStr} with status {statusCode}: {errorBody}" + match pushRequest with + | Ok response -> + if response.statusCode >= 200L && response.statusCode < 300L then + Stdlib.Result.Result.Ok opsPushedCount + else + let errorBody = Stdlib.String.fromBytesWithReplacement response.body - | Error err -> - let errorMsg = Stdlib.HttpClient.toString err + let statusCode = Stdlib.Int64.toString response.statusCode - Stdlib.Result.Result.Error - $"Push request failed on batch {batchNumStr}: {errorMsg}" + Stdlib.Result.Result.Error + $"Push request failed with status {statusCode}: {errorBody}" - (index + 1L, newResult)) + | Error err -> + let errorMsg = Stdlib.HttpClient.toString err - pushAllBatches + Stdlib.Result.Result.Error $"Push request failed: {errorMsg}" // If push succeeded, pull ops from remote match pushResult with @@ -103,49 +79,41 @@ let sync if response.statusCode >= 200L && response.statusCode < 300L then let responseBody = Stdlib.String.fromBytesWithReplacement response.body - match Builtin.jsonParse> responseBody with + match + (Builtin.jsonParse> + responseBody) + with | Error _err -> Stdlib.Result.Result.Error "Failed to parse pulled ops JSON" - | Ok pulledOps -> - let fetchedCount = Stdlib.List.length pulledOps + | Ok pulledBatches -> + let batchCount = Stdlib.List.length pulledBatches - // Add pulled ops to local database in batches, tracking how many were actually inserted + // Process each batch let applyResult = - if fetchedCount > 0L then - let batchSize = 100L - - let batches = - match Stdlib.List.chunkBySize pulledOps batchSize with - | Ok chunks -> chunks - | Error _ -> [ pulledOps ] // Fallback to single batch if chunking fails - - - let (_, batchResults) = - Stdlib.List.fold - batches - (0L, Stdlib.Result.Result.Ok 0L) - (fun acc batch -> - let (index, resultSoFar) = acc - - match resultSoFar with - | Error msg -> (index + 1L, Stdlib.Result.Result.Error msg) - | Ok appliedSoFar -> - let addResult = Builtin.scmAddOps branchID batch - - match addResult with - | Ok batchApplied -> - (index + 1L, Stdlib.Result.Result.Ok(appliedSoFar + batchApplied)) - | Error errMsg -> - let batchNum = Stdlib.Int64.toString(index + 1L) - - ( - index + 1L, - Stdlib.Result.Result.Error - $"Failed to apply batch {batchNum}: {errMsg}" - )) - - batchResults + if batchCount > 0L then + Stdlib.List.fold + pulledBatches + (Stdlib.Result.Result.Ok 0L) + (fun resultSoFar batch -> + match resultSoFar with + | Error msg -> Stdlib.Result.Result.Error msg + | Ok appliedSoFar -> + let addResult = + SCM.PackageOps.add batch.instanceID batch.branchID batch.ops + + match addResult with + | Ok batchApplied -> + Stdlib.Result.Result.Ok(appliedSoFar + batchApplied) + + | Error errMsg -> + let branchName = + match batch.branchID with + | Some id -> Stdlib.Uuid.toString id + | None -> "main" + + Stdlib.Result.Result.Error + $"Failed to apply ops for branch {branchName}: {errMsg}") else Stdlib.Result.Result.Ok 0L diff --git a/scripts/run-second-instance b/scripts/run-second-instance index 470addcbab..d4e3706ce7 100755 --- a/scripts/run-second-instance +++ b/scripts/run-second-instance @@ -27,8 +27,9 @@ MAIN_DB_PATH="${DARK_CONFIG_RUNDIR}/data.db" echo -e "${green}Creating new database for ${INSTANCE_NAME}${reset}" -rm -f $DB_PATH -cp "${MAIN_DB_PATH}" "${DB_PATH}" +rm -f $DB_PATH $DB_PATH-wal $DB_PATH-shm +# Use sqlite3 backup to create consistent copy +sqlite3 "${MAIN_DB_PATH}" ".backup '${DB_PATH}'" echo -e "${green}Database created: ${DB_PATH}${reset}" # Find the BwdServer executable @@ -59,24 +60,23 @@ echo "Stopping any existing ${INSTANCE_NAME} processes..." pkill -f "DARK_CONFIG_DB_NAME=${INSTANCE_DB}" || true sleep 1 -# Reload packages and canvases for the second instance -LOCALEXEC_EXE="backend/Build/out/LocalExec/Debug/net8.0/LocalExec" -echo "Reloading packages for ${INSTANCE_DB}..." -DARK_CONFIG_DB_NAME="${INSTANCE_DB}" "${LOCALEXEC_EXE}" reload-canvases > /dev/null 2>&1 -echo "Packages reloaded" # Record initial sync to mark instances as in-sync at this point # This prevents trying to re-sync all historical ops echo "Recording initial sync state..." -MAIN_INSTANCE_ID="11111111-1111-1111-1111-111111111111" -SECOND_INSTANCE_ID="fcdacd03-8324-4efd-8cc9-3ae1a4157a2a" -# Record sync in main instance's database (synced with second) -# Use SQLite's random UUID generation -sqlite3 "${MAIN_DB_PATH}" "INSERT OR REPLACE INTO syncs (id, instance_id, synced_at, ops_pushed, ops_fetched) VALUES (lower(hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-4' || substr(hex(randomblob(2)),2) || '-' || substr('89ab',abs(random()) % 4 + 1, 1) || substr(hex(randomblob(2)),2) || '-' || hex(randomblob(6))), '${SECOND_INSTANCE_ID}', strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), 0, 0);" +# Get actual instance2 UUID from main database +SECOND_INSTANCE_ID=$(sqlite3 "${MAIN_DB_PATH}" "SELECT id FROM instances WHERE name = 'instance2'") -# Record sync in second instance's database (synced with main) -sqlite3 "${DB_PATH}" "INSERT OR REPLACE INTO syncs (id, instance_id, synced_at, ops_pushed, ops_fetched) VALUES (lower(hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-4' || substr(hex(randomblob(2)),2) || '-' || substr('89ab',abs(random()) % 4 + 1, 1) || substr(hex(randomblob(2)),2) || '-' || hex(randomblob(6))), '${MAIN_INSTANCE_ID}', strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), 0, 0);" +# Clean up instance2's instances table and add instance1 with a known UUID +LOCAL_INSTANCE_ID="00000000-0000-0000-0000-000000000001" +sqlite3 "${DB_PATH}" "DELETE FROM instances WHERE name = 'instance2';" +sqlite3 "${DB_PATH}" "INSERT INTO instances (id, name, url) VALUES ('${LOCAL_INSTANCE_ID}', 'instance1', 'http://dark-packages.dlio.localhost:11001');" + +# Record bootstrap sync records to prevent re-syncing all historical ops. +# Since we just copied the database, both instances are already in sync at this moment. +sqlite3 "${MAIN_DB_PATH}" "INSERT OR REPLACE INTO syncs (id, instance_id, synced_at, ops_pushed, ops_fetched) VALUES ('00000000-0000-0000-0000-000000000000', '${SECOND_INSTANCE_ID}', strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), 0, 0);" +sqlite3 "${DB_PATH}" "INSERT OR REPLACE INTO syncs (id, instance_id, synced_at, ops_pushed, ops_fetched) VALUES ('00000000-0000-0000-0000-000000000000', '${LOCAL_INSTANCE_ID}', strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), 0, 0);" echo "Initial sync recorded" diff --git a/vscode-extension/client/src/commands/instanceCommands.ts b/vscode-extension/client/src/commands/instanceCommands.ts deleted file mode 100644 index e043ea0a18..0000000000 --- a/vscode-extension/client/src/commands/instanceCommands.ts +++ /dev/null @@ -1,73 +0,0 @@ -import * as vscode from "vscode"; -import { LanguageClient } from "vscode-languageclient/node"; -import { StatusBarManager } from "../ui/statusbar/statusBarManager"; -import { WorkspaceTreeDataProvider } from "../providers/treeviews/workspaceTreeDataProvider"; -import { PackagesTreeDataProvider } from "../providers/treeviews/packagesTreeDataProvider"; - -export class InstanceCommands { - private packagesProvider: PackagesTreeDataProvider | null = null; - - constructor( - private client: LanguageClient, - private statusBarManager: StatusBarManager, - private workspaceProvider: WorkspaceTreeDataProvider - ) {} - - setPackagesProvider(provider: PackagesTreeDataProvider): void { - this.packagesProvider = provider; - } - - register(): vscode.Disposable[] { - return [ - // Sync with instance - vscode.commands.registerCommand("darklang.instance.sync", async (treeItem) => { - - // Extract instance info from tree item - const instanceName = treeItem.label || treeItem.instanceData?.name || "unknown"; - const instanceUrl = treeItem.instanceData?.url; - - if (!instanceUrl) { - vscode.window.showErrorMessage("Instance URL not available"); - return; - } - - try { - await vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: `Syncing with ${instanceName}...`, - cancellable: false - }, - async () => { - const syncPromise = this.client.sendRequest<{ - success: boolean; - message: string; - }>("dark/sync", { instanceID: instanceName }); - - const timeoutPromise = new Promise((_, reject) => - setTimeout(() => reject(new Error("Sync request timed out")), 120000) - ); - - const response = await Promise.race([syncPromise, timeoutPromise]); - - if (response.success) { - // Show success message - vscode.window.showInformationMessage(`✓ ${response.message}`); - // Refresh both trees to show synced changes - this.workspaceProvider.refresh(); - if (this.packagesProvider) { - this.packagesProvider.refresh(); - } - } else { - vscode.window.showErrorMessage(`Sync failed: ${response.message}`); - } - } - ); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Sync error: ${errorMessage}`); - } - }), - ]; - } -} \ No newline at end of file diff --git a/vscode-extension/client/src/commands/syncCommands.ts b/vscode-extension/client/src/commands/syncCommands.ts deleted file mode 100644 index c147a28672..0000000000 --- a/vscode-extension/client/src/commands/syncCommands.ts +++ /dev/null @@ -1,63 +0,0 @@ -import * as vscode from "vscode"; -import { LanguageClient } from "vscode-languageclient/node"; - -export class SyncCommands { - constructor(private client: LanguageClient) {} - - register(): vscode.Disposable[] { - return [ - vscode.commands.registerCommand("darklang.sync.execute", async () => { - - // Ask for instance name - const instanceName = await vscode.window.showInputBox({ - prompt: "Enter instance name to sync with", - placeHolder: "e.g., local, origin, production", - value: "local" - }); - - if (!instanceName) return; - - try { - vscode.window.showInformationMessage(`Syncing with ${instanceName}...`); - - const response = await this.client.sendRequest<{ - success: boolean; - message: string; - }>("dark/sync", { instanceID: instanceName }); - - if (response.success) { - vscode.window.showInformationMessage(response.message); - } else { - vscode.window.showErrorMessage(`Sync failed: ${response.message}`); - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Sync error: ${errorMessage}`); - } - }), - - vscode.commands.registerCommand("darklang.sync.quickSync", async () => { - // Quick sync with default local instance - const instanceName = "local"; - - try { - vscode.window.showInformationMessage(`Syncing with ${instanceName}...`); - - const response = await this.client.sendRequest<{ - success: boolean; - message: string; - }>("dark/sync", { instanceID: instanceName }); - - if (response.success) { - vscode.window.showInformationMessage(response.message); - } else { - vscode.window.showErrorMessage(`Sync failed: ${response.message}`); - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - vscode.window.showErrorMessage(`Sync error: ${errorMessage}`); - } - }) - ]; - } -} diff --git a/vscode-extension/client/src/extension.ts b/vscode-extension/client/src/extension.ts index d2ec1bd049..e8379835a2 100644 --- a/vscode-extension/client/src/extension.ts +++ b/vscode-extension/client/src/extension.ts @@ -1,4 +1,5 @@ import * as vscode from "vscode"; +import { spawn } from "child_process"; import { SemanticTokensFeature } from "vscode-languageclient/lib/common/semanticTokens"; import { LanguageClient, @@ -14,8 +15,6 @@ import { BranchesManagerPanel } from "./panels/branchManagerPanel"; import { BranchCommands } from "./commands/branchCommands"; import { PackageCommands } from "./commands/packageCommands"; -import { InstanceCommands } from "./commands/instanceCommands"; -import { SyncCommands } from "./commands/syncCommands"; import { ScriptCommands } from "./commands/scriptCommands"; import { StatusBarManager } from "./ui/statusbar/statusBarManager"; @@ -28,10 +27,30 @@ import { PackageContentProvider } from "./providers/content/packageContentProvid let client: LanguageClient; +const isDebug = process.env.VSCODE_DEBUG_MODE === "true"; +const cwd = "/home/dark/app"; +const cli = isDebug ? "./scripts/run-cli" : "darklang"; + +function startSyncService(): void { + const child = spawn("bash", [cli, "run", "@Darklang.Cli.SyncService.startInBackground", "()"], { + cwd, + detached: true, + stdio: "ignore", + }); + child.unref(); + console.log("Sync service start requested via CLI"); +} + +function stopSyncService(): void { + const child = spawn("bash", [cli, "run", "@Darklang.Cli.SyncService.stop", "()"], { + cwd, + stdio: "ignore", + }); + child.unref(); + console.log("Sync service stop requested via CLI"); +} + function createLSPClient(): LanguageClient { - const isDebug = process.env.VSCODE_DEBUG_MODE === "true"; - const cwd = "/home/dark/app"; - const cli = isDebug ? "./scripts/run-cli" : "darklang"; const args = [cli, "run", "@Darklang.LanguageTools.LspServer.runServerCli", "()" ]; const baseRun = { @@ -68,6 +87,9 @@ export async function activate(context: vscode.ExtensionContext) { BranchStateManager.initialize(client); + // Auto-start sync service via CLI + startSyncService(); + const statusBar = new StatusBarManager(); const fsProvider = new DarkFileSystemProvider(client); const contentProvider = new DarkContentProvider(client); @@ -105,22 +127,10 @@ export async function activate(context: vscode.ExtensionContext) { const packageCommands = new PackageCommands(); const branchCommands = new BranchCommands(statusBar, workspaceProvider); - const instanceCommands = new InstanceCommands(client, statusBar, workspaceProvider); - const syncCommands = new SyncCommands(client); const scriptCommands = new ScriptCommands(); - instanceCommands.setPackagesProvider(packagesProvider); - const reg = (d: vscode.Disposable) => context.subscriptions.push(d); - const ops = [ - ["darklang.ops.setLimit", () => workspaceProvider.configureLimitFilter()], - ["darklang.ops.setDateRange", () => workspaceProvider.configureDateFilter()], - ["darklang.ops.setBranch", () => workspaceProvider.configureBranchFilter()], - ["darklang.ops.setLocation", () => workspaceProvider.configureLocationFilter()], - ["darklang.ops.clearFilters", () => workspaceProvider.clearAllFilters()], - ] as const; - // Core registrations [ statusBar, @@ -130,18 +140,20 @@ export async function activate(context: vscode.ExtensionContext) { vscode.commands.registerCommand("darklang.branches.manageAll", () => { BranchesManagerPanel.createOrShow(context.extensionUri); }), - ...ops.map(([cmd, fn]) => vscode.commands.registerCommand(cmd, fn)), packagesView, packagesProvider, workspaceView, ...packageCommands.register(), ...branchCommands.register(), - ...instanceCommands.register(), - ...syncCommands.register(), ...scriptCommands.register(), ].forEach(reg); } -export function deactivate(): Thenable | undefined { - return client ? client.stop() : undefined; +export async function deactivate(): Promise { + // Stop sync service via CLI (not LSP) + stopSyncService(); + + if (client) { + await client.stop(); + } } diff --git a/vscode-extension/client/src/panels/branchManagerPanel.ts b/vscode-extension/client/src/panels/branchManagerPanel.ts index ba369af070..e275daeef4 100644 --- a/vscode-extension/client/src/panels/branchManagerPanel.ts +++ b/vscode-extension/client/src/panels/branchManagerPanel.ts @@ -9,7 +9,6 @@ interface BranchDisplayModel { active: boolean; isCurrent: boolean; createdAt: string; - conflicts?: number; } /** Branches Manager Panel - Manage all branches (active and inactive) */ @@ -464,8 +463,7 @@ export class BranchesManagerPanel {
${branch.name}${statusBadge}
- ${branch.description ? branch.description + " · " : ""} - ${branch.conflicts ? " · " + branch.conflicts + " conflicts" : ""} + ${branch.description ? branch.description : ""}
diff --git a/vscode-extension/client/src/providers/treeviews/workspaceTreeDataProvider.ts b/vscode-extension/client/src/providers/treeviews/workspaceTreeDataProvider.ts index f4801239c9..1d232ec904 100644 --- a/vscode-extension/client/src/providers/treeviews/workspaceTreeDataProvider.ts +++ b/vscode-extension/client/src/providers/treeviews/workspaceTreeDataProvider.ts @@ -1,16 +1,8 @@ import * as vscode from "vscode"; import { LanguageClient } from "vscode-languageclient/node"; -import { BranchNode } from "../../types"; +import { BranchNode, BranchNodeType } from "../../types"; import { BranchStateManager } from "../../data/branchStateManager"; -interface OpsFilterConfig { - limit: number | null; // null means no limit - branch: 'current' | 'all' | string; // 'current', 'all', or a specific branch ID - dateRange: 'all' | 'today' | 'week' | 'month' | 'custom'; - customStartDate?: Date; - locationFilter?: string; -} - // Interface for pending op response from LSP interface PendingOpResponse { op: string; @@ -26,15 +18,16 @@ interface OpLocation { // Extended BranchNode for pending ops with additional properties interface PendingOpNode extends BranchNode { - type: "pending-op"; + type: BranchNodeType.PendingOp; location?: OpLocation; opData?: string; } -// Interface for branch quick pick items with custom properties -interface BranchQuickPickItem extends vscode.QuickPickItem { - value?: string; - branchID?: string; +// Group node for owner grouping +interface OwnerGroupNode extends BranchNode { + type: BranchNodeType.OwnerGroup; + owner: string; + children: PendingOpNode[]; } // Type for package ops (from file system provider) @@ -43,21 +36,18 @@ interface PackageOp { [key: string]: unknown; } -export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider { - private _onDidChangeTreeData: vscode.EventEmitter = - new vscode.EventEmitter(); - readonly onDidChangeTreeData: vscode.Event = - this._onDidChangeTreeData.event; +export class WorkspaceTreeDataProvider + implements vscode.TreeDataProvider +{ + private _onDidChangeTreeData: vscode.EventEmitter< + BranchNode | undefined | null | void + > = new vscode.EventEmitter(); + readonly onDidChangeTreeData: vscode.Event< + BranchNode | undefined | null | void + > = this._onDidChangeTreeData.event; private branchStateManager = BranchStateManager.getInstance(); - // Filter configuration for ops display - private opsFilter: OpsFilterConfig = { - limit: 50, - branch: 'current', - dateRange: 'all', - }; - constructor(private client: LanguageClient) { // Listen for branch changes and refresh the tree this.branchStateManager.onBranchChanged(() => { @@ -93,9 +83,11 @@ export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider 0) { collapsibleState = vscode.TreeItemCollapsibleState.Collapsed; } else { @@ -108,28 +100,72 @@ export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider { - try { - // Calculate date filter if applicable - let sinceDate: string | undefined; - if (this.opsFilter.dateRange !== 'all') { - const now = new Date(); - const since = new Date(); - - switch (this.opsFilter.dateRange) { - case 'today': - since.setHours(0, 0, 0, 0); - break; - case 'week': - since.setDate(now.getDate() - 7); - break; - case 'month': - since.setMonth(now.getMonth() - 1); - break; - case 'custom': - if (this.opsFilter.customStartDate) { - since.setTime(this.opsFilter.customStartDate.getTime()); - } - break; - } - - sinceDate = since.toISOString(); + private async getBranchChildren(): Promise { + const currentBranchId = this.branchStateManager.getCurrentBranchId(); + const allBranches = this.branchStateManager.getBranches(); + + const children: BranchNode[] = []; + + // 1. Current branch (if any) + if (currentBranchId) { + const currentBranch = allBranches.find(b => b.id === currentBranchId); + if (currentBranch) { + children.push({ + id: `branch-${currentBranch.id}`, + label: currentBranch.name, + type: BranchNodeType.BranchItem, + contextValue: "branch-item", + branchData: { + branchId: currentBranch.id, + branchName: currentBranch.name, + isCurrent: true, + status: undefined, + opsCount: undefined, + lastSynced: undefined, + }, + }); } + } + + // 2. Main/Root branch - show as "main (no branch)" when on a branch + // Only add if we're currently on a specific branch + if (currentBranchId && currentBranchId !== "") { + const mainBranchNode: BranchNode = { + id: "branch-main-clear", + label: "main (no branch)", + type: BranchNodeType.BranchItem, + contextValue: "branch-item-main", + branchData: { + branchId: "", + branchName: "main", + isMain: true, + status: undefined, + opsCount: undefined, + }, + }; + children.push(mainBranchNode); + } + + // 3. All other branches (directly, no grouping) + const otherBranches = allBranches.filter(b => b.id !== currentBranchId); + + otherBranches.forEach(branch => { + children.push({ + id: `branch-${branch.id}`, + label: branch.name, + type: BranchNodeType.BranchItem, + contextValue: "branch-item", + branchData: { + branchId: branch.id, + branchName: branch.name, + isCurrent: false, + status: undefined, + opsCount: undefined, + lastSynced: undefined, + }, + }); + }); + + // 4. Add "Manage All" action node at the bottom + children.push({ + id: "branch-manage-all", + label: "Manage All Branches", + type: BranchNodeType.BranchAction, + contextValue: "branch-manage-all", + }); + + return children; + } + private async getPendingChanges(): Promise { + try { const requestParams = { - limit: this.opsFilter.limit ?? 999999, // Send large number if null - branchFilter: this.opsFilter.branch, - sinceDate: sinceDate, + limit: 20, + branchFilter: "current", + sinceDate: undefined, }; const ops = await this.client.sendRequest( - 'dark/getPendingOps', - requestParams + "dark/getPendingOps", + requestParams, ); - let nodes: PendingOpNode[] = ops.map((opWithLabel, index) => { - const location = this.extractLocationFromOp(opWithLabel.op); - return { - id: `op-${index}`, - label: opWithLabel.label, // Use the formatted label from backend - type: "pending-op", - contextValue: "pending-op", - location: location, // Store location for the command - opData: opWithLabel.op // Store full op for diff view - }; - }); - - // Apply client-side location filter if specified - if (this.opsFilter.locationFilter && this.opsFilter.locationFilter.trim() !== '') { - const filterLower = this.opsFilter.locationFilter.toLowerCase(); - nodes = nodes.filter(node => { - // Filter by label - if (node.label.toLowerCase().includes(filterLower)) { - return true; - } - // Filter by location if available - if (node.location) { - const locStr = `${node.location.owner}.${node.location.modules.join('.')}.${node.location.name}`.toLowerCase(); - return locStr.includes(filterLower); + let nodes: PendingOpNode[] = ops + .filter(opWithLabel => { + // Filter out Add* ops completely (AddFn, AddType, AddValue, etc.) + try { + const parsed = JSON.parse(opWithLabel.op); + if (parsed.AddFn || parsed.AddType || parsed.AddValue) { + return false; // Skip Add* ops + } + } catch (e) { + // If parsing fails, keep the op } - // If no location available, don't match (only label matching works) - return false; + return true; + }) + .map(opWithLabel => { + const location = this.extractLocationFromOp(opWithLabel.op); + // Use the op data itself as a stable ID (not index-based) + // This ensures IDs don't change when ops are reordered + const opId = opWithLabel.op; + + // Clean up label - remove "Set*Name →" prefix text, keep only the fqname + let cleanLabel = opWithLabel.label; + cleanLabel = cleanLabel.replace(/^SetFnName\s+→\s+/, ""); + cleanLabel = cleanLabel.replace(/^SetTypeName\s+→\s+/, ""); + cleanLabel = cleanLabel.replace(/^SetValueName\s+→\s+/, ""); + + return { + id: opId, + label: cleanLabel, + type: BranchNodeType.PendingOp, + contextValue: "pending-op", + location: location, // Store location for the command + opData: opWithLabel.op, // Store full op for diff view + }; }); + + // Add "see more" node if we hit the limit + const hasMore = ops.length === 20; + + // Only group by owner when on a non-main branch + const currentBranchId = this.branchStateManager.getCurrentBranchId(); + const shouldGroupByOwner = + currentBranchId !== null && currentBranchId !== ""; + + let result: BranchNode[]; + if (shouldGroupByOwner) { + result = this.groupByOwner(nodes); + } else { + result = nodes; + } + + // Add "see more" node at the end if there are more items + if (hasMore) { + const seeMoreNode: BranchNode = { + id: "see-more-changes", + label: "See more...", + type: BranchNodeType.SeeMore, + contextValue: "see-more", + }; + result.push(seeMoreNode); } - return nodes; + return result; } catch (error) { - console.error('Failed to get pending ops:', error); + console.error("Failed to get pending ops:", error); return []; } } + private groupByOwner(nodes: PendingOpNode[]): BranchNode[] { + // Group nodes by owner + const groupMap = new Map(); + + for (const node of nodes) { + const owner = node.location?.owner || "Unknown"; + if (!groupMap.has(owner)) { + groupMap.set(owner, []); + } + groupMap.get(owner)!.push(node); + } + + // Convert to owner group nodes + const ownerGroups: OwnerGroupNode[] = []; + for (const [owner, opsInGroup] of groupMap.entries()) { + ownerGroups.push({ + id: `owner-group-${owner}`, + label: owner, + type: BranchNodeType.OwnerGroup, + contextValue: "owner-group", + owner: owner, + children: opsInGroup, + }); + } + + // Sort by owner name + return ownerGroups.sort((a, b) => a.owner.localeCompare(b.owner)); + } + private extractLocationFromOp(op: string): OpLocation | null { try { - if (typeof op === 'string') { + if (typeof op === "string") { const parsed = JSON.parse(op); // SetTypeName has location if (parsed.SetTypeName && parsed.SetTypeName[1]) { const location = parsed.SetTypeName[1]; return { - owner: location.owner || '', + owner: location.owner || "", modules: location.modules || [], - name: location.name || '' + name: location.name || "", }; } @@ -295,9 +459,9 @@ export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider { const branchName = this.branchStateManager.getCurrentBranchName(); const branchID = this.branchStateManager.getCurrentBranchId(); @@ -325,21 +488,20 @@ export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider>( - "dark/listInstances", - {} - ); + const instances = await this.client.sendRequest< + Array<{ id: string; name: string; url: string }> + >("dark/listInstances", {}); instanceChildren = instances.map(inst => ({ id: inst.id, label: inst.name, - type: "instance-item" as const, + type: BranchNodeType.InstanceItem, contextValue: "remote-instance", instanceData: { url: inst.url, - status: "connected" as const + status: "connected" as const, }, - children: [] + children: [], })); } catch (error) { console.error("Failed to fetch instances:", error); @@ -349,246 +511,35 @@ export class WorkspaceTreeDataProvider implements vscode.TreeDataProvider 0) { - return `Pending Changes (${filters.join(', ')})`; - } - return 'Pending Changes'; + return "Changes"; } private getOpsFilterTooltip(): string { - const parts: string[] = []; - - const limitText = this.opsFilter.limit === null ? 'all' : `up to ${this.opsFilter.limit}`; - parts.push(`Showing ${limitText} ops`); - - if (this.opsFilter.dateRange !== 'all') { - const dateLabel = this.opsFilter.dateRange === 'custom' - ? `since ${this.opsFilter.customStartDate?.toLocaleDateString()}` - : `from ${this.opsFilter.dateRange}`; - parts.push(dateLabel); - } - - // Show branch filter information - if (this.opsFilter.branch === 'current') { - parts.push('current branch only'); - } else if (this.opsFilter.branch === 'all') { - parts.push('all branches'); - } else { - // It's a specific branch - const branchLabel = this.getBranchFilterLabel(); - parts.push(`branch: ${branchLabel}`); - } - - if (this.opsFilter.locationFilter) { - parts.push(`filtered by "${this.opsFilter.locationFilter}"`); - } - - return parts.join(', '); - } - - async configureLimitFilter(): Promise { - const currentLimit = this.opsFilter.limit === null ? 'all' : this.opsFilter.limit.toString(); - const choice = await vscode.window.showQuickPick([ - { label: '10', value: 10 }, - { label: '25', value: 25 }, - { label: '50', value: 50 }, - { label: '100', value: 100 }, - { label: '500', value: 500 }, - { label: '1000', value: 1000 }, - { label: 'All (no limit)', value: null }, - ], { - placeHolder: `Current limit: ${currentLimit}` - }); - - if (choice) { - this.opsFilter.limit = choice.value; - vscode.window.showInformationMessage(`Ops limit set to ${choice.label}`); - this.refresh(); - } - } - - async configureDateFilter(): Promise { - type DateRangeOption = { - label: string; - value: OpsFilterConfig['dateRange']; - }; - - const choice = await vscode.window.showQuickPick([ - { label: 'All time', value: 'all' }, - { label: 'Today', value: 'today' }, - { label: 'Last 7 days', value: 'week' }, - { label: 'Last 30 days', value: 'month' }, - { label: 'Custom date...', value: 'custom' }, - ], { - placeHolder: `Current: ${this.opsFilter.dateRange}` - }); - - if (!choice) { - return; - } - - this.opsFilter.dateRange = choice.value; - - if (choice.value === 'custom') { - const dateStr = await vscode.window.showInputBox({ - prompt: 'Enter start date (YYYY-MM-DD)', - placeHolder: '2024-01-01' - }); - - if (dateStr) { - const date = new Date(dateStr); - if (!isNaN(date.getTime())) { - this.opsFilter.customStartDate = date; - vscode.window.showInformationMessage(`Filtering ops since ${dateStr}`); - } else { - vscode.window.showErrorMessage('Invalid date format'); - return; - } - } - } else { - vscode.window.showInformationMessage(`Date filter set to: ${choice.label}`); - } - - this.refresh(); - } - - private getBranchFilterLabel(): string { - if (this.opsFilter.branch === 'all') { - return 'All branches'; - } else if (this.opsFilter.branch === 'current') { - return 'Current branch'; - } else { - // It's a specific branch ID - find the branch name - const branch = this.branchStateManager.getBranches().find(b => b.id === this.opsFilter.branch); - return branch ? branch.name : 'Unknown branch'; - } - } - - async configureBranchFilter(): Promise { - const branches = this.branchStateManager.getBranches(); - const currentBranchId = this.branchStateManager.getCurrentBranchId(); - - // Build the list of branch options - const branchItems: BranchQuickPickItem[] = [ - { - label: '$(git-branch) Current branch only', - value: 'current', - description: currentBranchId ? this.branchStateManager.getCurrentBranchName() : 'No branch selected' - }, - { label: '$(layers) All branches', value: 'all', description: 'Show ops from all branches' }, - { label: '', kind: vscode.QuickPickItemKind.Separator } - ]; - - // Add individual branches (only show non-merged branches) - branches - .filter(b => !b.mergedAt) - .forEach(b => { - const isCurrent = b.id === currentBranchId; - branchItems.push({ - label: b.name, - description: isCurrent ? '● Current' : undefined, - detail: `Branch ID: ${b.id}`, - branchID: b.id - }); - }); - - const choice = await vscode.window.showQuickPick(branchItems, { - placeHolder: `Current filter: ${this.getBranchFilterLabel()}` - }); - - if (choice) { - if (choice.branchID) { - this.opsFilter.branch = choice.branchID; - vscode.window.showInformationMessage(`Branch filter set to: ${choice.label}`); - } else if (choice.value) { - this.opsFilter.branch = choice.value; - vscode.window.showInformationMessage(`Branch filter set to: ${choice.label}`); - } - this.refresh(); - } - } - - async configureLocationFilter(): Promise { - const input = await vscode.window.showInputBox({ - prompt: 'Filter by module/function name (case-insensitive)', - placeHolder: 'e.g., Stdlib.List or MyModule', - value: this.opsFilter.locationFilter || '' - }); - - if (input !== undefined) { - this.opsFilter.locationFilter = input.trim() || undefined; - if (this.opsFilter.locationFilter) { - vscode.window.showInformationMessage(`Filtering by location: ${this.opsFilter.locationFilter}`); - } else { - vscode.window.showInformationMessage('Location filter cleared'); - } - this.refresh(); - } - } - - async clearAllFilters(): Promise { - this.opsFilter = { - limit: 50, - branch: 'current', - dateRange: 'all', - }; - vscode.window.showInformationMessage('All filters cleared'); - this.refresh(); + return "Pending changes from the current branch"; } } diff --git a/vscode-extension/client/src/types/index.ts b/vscode-extension/client/src/types/index.ts index 6ad038ea59..f53919c0e3 100644 --- a/vscode-extension/client/src/types/index.ts +++ b/vscode-extension/client/src/types/index.ts @@ -9,25 +9,22 @@ export interface PackageNode { packagePath?: string; } +export enum BranchNodeType { + PendingOp = "pending-op", + OwnerGroup = "owner-group", + BranchItem = "branch-item", + BranchAction = "branch-action", + InstanceItem = "instance-item", + InstanceRoot = "instance-root", + BranchRoot = "branch-root", + ChangesRoot = "changes-root", + SeeMore = "see-more" +} + export interface BranchNode { id: string; label: string; - type: - "current" - | "recent" - | "shared" - | "actions" - | "operation" - | "conflict" - | "section" - | "instance-root" - | "branch-root" - | "changes-root" - | "instance-item" - | "packages" - | "branches" - | "category" - | "pending-op"; + type: BranchNodeType; contextValue: string; children?: BranchNode[]; instanceData?: { @@ -38,6 +35,15 @@ export interface BranchNode { packageCount?: number; branchCount?: number; }; + branchData?: { + branchId: string; + branchName: string; + isCurrent?: boolean; + isMain?: boolean; + status?: "up-to-date" | "ahead" | "behind"; + opsCount?: number; + lastSynced?: Date; + }; } diff --git a/vscode-extension/package.json b/vscode-extension/package.json index 42b410fcbf..ebf2242e37 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -128,53 +128,44 @@ "command": "darklang.branch.clear", "title": "Clear Branch" }, + { + "command": "darklang.branch.showMenu", + "title": "Branch Menu", + "icon": "$(ellipsis)" + }, { "command": "darklang.branches.manageAll", "title": "Manage All Branches", "icon": "$(list-flat)" }, { - "command": "darklang.sync.execute", - "title": "Darklang: Sync with Remote Instance", - "icon": "$(sync)" - }, - { - "command": "darklang.sync.quickSync", - "title": "Darklang: Quick Sync (instance2)", - "icon": "$(sync)" - }, - { - "command": "darklang.instance.sync", - "title": "Sync with Instance", - "icon": "$(sync)" - }, - { - "command": "darklang.ops.setLimit", - "title": "Set Ops Limit", - "icon": "$(symbol-number)" - }, - { - "command": "darklang.ops.setDateRange", - "title": "Filter by Date Range", - "icon": "$(calendar)" + "command": "darklang.openHomepage", + "title": "Darklang: Open Homepage", + "icon": "./static/logo-dark-transparent.svg" }, { - "command": "darklang.ops.setBranch", - "title": "Filter by Branch", - "icon": "$(git-branch)" + "command": "darklang.packages.search", + "title": "Search Packages", + "icon": "$(search)" }, { - "command": "darklang.ops.setLocation", - "title": "Filter by Location", - "icon": "$(location)" + "command": "darklang.packages.clearSearch", + "title": "Clear Search", + "icon": "$(clear-all)" }, { - "command": "darklang.ops.clearFilters", - "title": "Clear All Filters", - "icon": "$(clear-all)" + "command": "darklang.changes.review", + "title": "Review Changes", + "icon": "$(preview)" } ], "menus": { + "editor/title": [ + { + "command": "darklang.openHomepage", + "group": "navigation" + } + ], "view/item/context": [ { "command": "darklang.openFullModule", @@ -202,34 +193,21 @@ "group": "basic@1" }, { - "command": "darklang.instance.sync", - "when": "view == darklangWorkspace && viewItem == remote-instance", - "group": "inline@1" - }, - { - "command": "darklang.ops.setLimit", - "when": "view == darklangWorkspace && viewItem == workspace-changes-root", - "group": "inline@1" - }, - { - "command": "darklang.ops.setDateRange", - "when": "view == darklangWorkspace && viewItem == workspace-changes-root", - "group": "inline@2" - }, - { - "command": "darklang.ops.setBranch", - "when": "view == darklangWorkspace && viewItem == workspace-changes-root", - "group": "inline@3" - }, + "command": "darklang.branch.rename", + "when": "view == darklangWorkspace && viewItem == branch-item", + "group": "context@1" + } + ], + "view/title": [ { - "command": "darklang.ops.setLocation", - "when": "view == darklangWorkspace && viewItem == workspace-changes-root", - "group": "inline@4" + "command": "darklang.packages.search", + "when": "view == darklangPackages", + "group": "navigation@1" }, { - "command": "darklang.ops.clearFilters", - "when": "view == darklangWorkspace && viewItem == workspace-changes-root", - "group": "inline@5" + "command": "darklang.packages.clearSearch", + "when": "view == darklangPackages", + "group": "navigation@2" } ] },