Feature : Add-PnPFile(s)ToProvisioningTemplate extract web part

Author: stevebeauge

Commands/Base/BaseFileProvisioningCmdlet.cs

@@ -5,11 +5,13 @@
 using OfficeDevPnP.Core.Framework.Provisioning.Providers;
 using OfficeDevPnP.Core.Framework.Provisioning.Providers.Xml;
 using SharePointPnP.PowerShell.Commands.Provisioning;
+using SharePointPnP.PowerShell.Commands.Utilities;
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Management.Automation;
-using System.Net;
+using System.Text.RegularExpressions;
 using PnPFileLevel = OfficeDevPnP.Core.Framework.Provisioning.Model.FileLevel;
 using SPFile = Microsoft.SharePoint.Client.File;
 
@@ -35,9 +37,29 @@ public class BaseFileProvisioningCmdlet : PnPWebCmdlet
         [Parameter(Mandatory = false, Position = 5, HelpMessage = "Set to overwrite in site, Defaults to true")]
         public SwitchParameter FileOverwrite = true;
 
-        [Parameter(Mandatory = false, Position = 6, HelpMessage = "Allows you to specify ITemplateProviderExtension to execute while loading the template.")]
+        [Parameter(Mandatory = false, Position = 6, ParameterSetName = PSNAME_REMOTE_SOURCE, HelpMessage = "Include webparts when the file is a page")]
+        public SwitchParameter ExtractWebParts = true;
+
+        [Parameter(Mandatory = false, Position = 7, HelpMessage = "Allows you to specify ITemplateProviderExtension to execute while loading the template.")]
         public ITemplateProviderExtension[] TemplateProviderExtensions;
 
+        protected readonly ProgressRecord _progressEnumeration = new ProgressRecord(0, "Activity", "Status") { Activity = "Enumerating folder" };
+        protected readonly ProgressRecord _progressFilesEnumeration = new ProgressRecord(1, "Activity", "Status") { Activity = "Extracting files" };
+        protected readonly ProgressRecord _progressFileProcessing = new ProgressRecord(2, "Activity", "Status") { Activity = "Extracting file" };
+
+        protected override void ProcessRecord()
+        {
+            base.ProcessRecord();
+            var ctx = (ClientContext)SelectedWeb.Context;
+            ctx.Load(SelectedWeb, web => web.Id, web => web.ServerRelativeUrl, web => web.Url);
+            if (ExtractWebParts)
+            {
+                ctx.Load(ctx.Site, site => site.Id, site => site.ServerRelativeUrl, site => site.Url);
+                ctx.Load(SelectedWeb.Lists, lists => lists.Include(l => l.Title, l => l.RootFolder.ServerRelativeUrl, l => l.Id));
+            }
+            ctx.ExecuteQueryRetry();
+        }
+
         protected ProvisioningTemplate LoadTemplate()
         {
             if (!System.IO.Path.IsPathRooted(Path))
@@ -65,8 +87,20 @@ protected ProvisioningTemplate LoadTemplate()
         /// <param name="folder">target folder in the provisioning template</param>
         /// <param name="fileName">Name of the file</param>
         /// <param name="container">Container path within the template (pnp file) or related to the xml templage</param>
-        protected void AddFileToTemplate(ProvisioningTemplate template, Stream fs, string folder, string fileName, string container)
+        /// <param name="webParts">WebParts to include</param>
+        protected void AddFileToTemplate(
+            ProvisioningTemplate template,
+            Stream fs,
+            string folder,
+            string fileName,
+            string container,
+            IEnumerable<WebPart> webParts = null
+            )
         {
+            if (template == null) throw new ArgumentNullException(nameof(template));
+            if (fs == null) throw new ArgumentNullException(nameof(fs));
+            if (fileName == null) throw new ArgumentNullException(nameof(fileName));
+
             var source = !string.IsNullOrEmpty(container) ? (container + "/" + fileName) : fileName;
 
             template.Connector.SaveFileStream(fileName, container, fs);
@@ -86,9 +120,11 @@ protected void AddFileToTemplate(ProvisioningTemplate template, Stream fs, strin
                 Src = source,
                 Folder = folder,
                 Level = FileLevel,
-                Overwrite = FileOverwrite,
+                Overwrite = FileOverwrite
             };
 
+            if (webParts != null) newFile.WebParts.AddRange(webParts);
+
             template.Files.Add(newFile);
 
             // Determine the output file name and path
@@ -119,25 +155,72 @@ protected void AddFileToTemplate(ProvisioningTemplate template, Stream fs, strin
         /// <param name="file">The SharePoint file to retrieve and add</param>
         protected void AddSPFileToTemplate(ProvisioningTemplate template, SPFile file)
         {
+            if (template == null) throw new ArgumentNullException(nameof(template));
+            if (file == null) throw new ArgumentNullException(nameof(file));
+
             file.EnsureProperties(f => f.Name, f => f.ServerRelativeUrl);
+
+            _progressFileProcessing.StatusDescription = $"Extracting file {file.ServerRelativeUrl}";
             var folderRelativeUrl = file.ServerRelativeUrl.Substring(0, file.ServerRelativeUrl.Length - file.Name.Length - 1);
             var folderWebRelativeUrl = HttpUtility.UrlKeyValueDecode(folderRelativeUrl.Substring(SelectedWeb.ServerRelativeUrl.TrimEnd('/').Length + 1));
             if (ClientContext.HasPendingRequest) ClientContext.ExecuteQuery();
             try
             {
+                IEnumerable<WebPart> webParts = null;
+                if (ExtractWebParts)
+                {
+                    webParts = ExtractSPFileWebParts(file).ToArray();
+                    _progressFileProcessing.PercentComplete = 25;
+                    _progressFileProcessing.StatusDescription = $"Extracting webpart from {file.ServerRelativeUrl} ";
+                    WriteProgress(_progressFileProcessing);
+                }
+
                 using (var fi = SPFile.OpenBinaryDirect(ClientContext, file.ServerRelativeUrl))
                 using (var ms = new MemoryStream())
                 {
+                    _progressFileProcessing.PercentComplete = 50;
+                    _progressFileProcessing.StatusDescription = $"Reading file {file.ServerRelativeUrl}";
+                    WriteProgress(_progressFileProcessing);
                     // We are using a temporary memory stream because the file connector is seeking in the stream
                     // and the stream provided by OpenBinaryDirect does not allow it
                     fi.Stream.CopyTo(ms);
                     ms.Position = 0;
-                    AddFileToTemplate(template, ms, folderWebRelativeUrl, file.Name, folderWebRelativeUrl);
+                    AddFileToTemplate(template, ms, folderWebRelativeUrl, file.Name, folderWebRelativeUrl, webParts);
+                    _progressFileProcessing.PercentComplete = 100;
+                    _progressFileProcessing.StatusDescription = $"Adding file {file.ServerRelativeUrl} to template";
+                    _progressFileProcessing.RecordType = ProgressRecordType.Completed;
+                    WriteProgress(_progressFileProcessing);
                 }
             }
-            catch (WebException exc)
+            catch (Exception exc)
+            {
+                WriteWarning($"Error trying to add file {file.ServerRelativeUrl} : {exc.Message}");
+            }
+        }
+
+        private IEnumerable<WebPart> ExtractSPFileWebParts(SPFile file)
+        {
+            if (file == null) throw new ArgumentNullException(nameof(file));
+
+            if (string.Compare(System.IO.Path.GetExtension(file.Name), ".aspx", true) == 0)
             {
-                WriteWarning($"Can't add file from url {file.ServerRelativeUrl} : {exc}");
+                foreach (var spwp in SelectedWeb.GetWebParts(file.ServerRelativeUrl))
+                {
+                    spwp.EnsureProperties(wp => wp.WebPart
+#if !SP2016 // Missing ZoneId property in SP2016 version of the CSOM Library
+                , wp => wp.ZoneId
+#endif
+                );
+                    yield return new WebPart
+                    {
+                        Contents = Tokenize(SelectedWeb.GetWebPartXml(spwp.Id, file.ServerRelativeUrl)),
+                        Order = (uint)spwp.WebPart.ZoneIndex,
+                        Title = spwp.WebPart.Title,
+#if !SP2016 // Missing ZoneId property in SP2016 version of the CSOM Library
+                        Zone = spwp.ZoneId
+#endif
+                    };
+                }
             }
         }
 
@@ -149,12 +232,49 @@ protected void AddSPFileToTemplate(ProvisioningTemplate template, SPFile file)
         /// <param name="folder">Destination folder of the added file</param>
         protected void AddLocalFileToTemplate(ProvisioningTemplate template, string file, string folder)
         {
-            var fileName = System.IO.Path.GetFileName(file);
-            var container = !string.IsNullOrEmpty(Container) ? Container : folder.Replace("\\", "/");
-            using (var fs = System.IO.File.OpenRead(file))
+            if (template == null) throw new ArgumentNullException(nameof(template));
+            if (file == null) throw new ArgumentNullException(nameof(file));
+            if (folder == null) throw new ArgumentNullException(nameof(folder));
+
+            _progressFileProcessing.Activity = $"Extracting file {file}";
+            _progressFileProcessing.StatusDescription = "Adding file {file}";
+            _progressFileProcessing.PercentComplete = 0;
+            WriteProgress(_progressFileProcessing);
+
+            try
+            {
+                var fileName = System.IO.Path.GetFileName(file);
+                var container = !string.IsNullOrEmpty(Container) ? Container : folder.Replace("\\", "/");
+
+                using (var fs = System.IO.File.OpenRead(file))
+                {
+                    AddFileToTemplate(template, fs, folder.Replace("\\", "/"), fileName, container);
+                }
+            }
+            catch (Exception exc)
+            {
+                WriteWarning($"Error trying to add file {file} : {exc.Message}");
+            }
+            _progressFileProcessing.RecordType = ProgressRecordType.Completed;
+            WriteProgress(_progressFileProcessing);
+        }
+
+        private string Tokenize(string input)
+        {
+            if (string.IsNullOrEmpty(input)) return input;
+
+            foreach (var list in SelectedWeb.Lists)
             {
-                AddFileToTemplate(template, fs, folder.Replace("\\", "/"), fileName, container);
+                input = input
+                    .ReplaceCaseInsensitive(list.Id.ToString("D"), "{listid:" + Regex.Escape(list.Title) + "}")
+                    .ReplaceCaseInsensitive(list.GetWebRelativeUrl(), "{listurl:" + Regex.Escape(list.Title) + "}");
             }
+            return input.ReplaceCaseInsensitive(SelectedWeb.Url, "{site}")
+                .ReplaceCaseInsensitive(SelectedWeb.ServerRelativeUrl, "{site}")
+                .ReplaceCaseInsensitive(SelectedWeb.Id.ToString(), "{siteid}")
+                .ReplaceCaseInsensitive(((ClientContext)SelectedWeb.Context).Site.ServerRelativeUrl, "{sitecollection}")
+                .ReplaceCaseInsensitive(((ClientContext)SelectedWeb.Context).Site.Id.ToString(), "{sitecollectionid}")
+                .ReplaceCaseInsensitive(((ClientContext)SelectedWeb.Context).Site.Url, "{sitecollection}");
         }
     }
 }

Commands/Provisioning/AddFileToProvisioningTemplate.cs

@@ -27,7 +27,11 @@ namespace SharePointPnP.PowerShell.Commands.Provisioning
     [CmdletExample(
         Code = @"PS:> Add-PnPFileToProvisioningTemplate -Path template.pnp -SourceUrl $urlOfFile",
         Remarks = "Adds a file to a PnP Provisioning Template retrieved from the currently connected web. The url can be either full, server relative or Web relative url.",
-        SortOrder = 4)]
+        SortOrder = 5)]
+    [CmdletExample(
+        Code = @"PS:> Add-PnPFileToProvisioningTemplate -Path template.pnp -SourceUrl $urlOfFile -ExtractWebParts:$false",
+        Remarks = "Adds a file to a PnP Provisioning Template retrieved from the currently connected web, disabling WebPart extraction.",
+        SortOrder = 6)]
     public class AddFileToProvisioningTemplate : BaseFileProvisioningCmdlet
     {
         /*
@@ -45,10 +49,10 @@ public class AddFileToProvisioningTemplate : BaseFileProvisioningCmdlet
 
         protected override void ProcessRecord()
         {
+            base.ProcessRecord();
             var template = LoadTemplate();
             if (this.ParameterSetName == PSNAME_REMOTE_SOURCE)
             {
-                SelectedWeb.EnsureProperty(w => w.ServerRelativeUrl);
                 var sourceUri = new Uri(SourceUrl, UriKind.RelativeOrAbsolute);
 
                 // Get the server relative url of the file, whatever the input url is (absolute, server relative or web relative form)
@@ -57,6 +61,10 @@ protected override void ProcessRecord()
                     SourceUrl.StartsWith("/", StringComparison.Ordinal) ? SourceUrl : // The url is server relative. Take it as is (/sites/web/folder/file)
                     SelectedWeb.ServerRelativeUrl.TrimEnd('/') + "/" + SourceUrl; // The url is web relative, prepend by the web url (folder/file)
 
+                _progressFileProcessing.PercentComplete = 0;
+                _progressFileProcessing.RecordType = ProgressRecordType.Processing;
+                _progressFileProcessing.StatusDescription = $"Getting file info {serverRelativeUrl}";
+
                 var file = SelectedWeb.GetFileByServerRelativeUrl(serverRelativeUrl);
 
                 AddSPFileToTemplate(template, file);
@@ -71,6 +79,11 @@ protected override void ProcessRecord()
                 // Load the file and add it to the .PNP file
                 Folder = Folder.Replace("\\", "/");
 
+                _progressFileProcessing.PercentComplete = 0;
+                _progressFileProcessing.RecordType = ProgressRecordType.Processing;
+                _progressFileProcessing.StatusDescription = $"Getting file info {Source}";
+                WriteProgress(_progressFileProcessing);
+
                 AddLocalFileToTemplate(template, Source, Folder);
             }
         }