diff --git a/ScrapySharp.Core/ScrapySharp.Core.fsproj b/ScrapySharp.Core/ScrapySharp.Core.fsproj index f292c84..f64614d 100644 --- a/ScrapySharp.Core/ScrapySharp.Core.fsproj +++ b/ScrapySharp.Core/ScrapySharp.Core.fsproj @@ -3,8 +3,8 @@ netstandard2.0 - - + + diff --git a/ScrapySharp.Tests/ScrapySharp.Tests.csproj b/ScrapySharp.Tests/ScrapySharp.Tests.csproj index c663c50..8c4e020 100644 --- a/ScrapySharp.Tests/ScrapySharp.Tests.csproj +++ b/ScrapySharp.Tests/ScrapySharp.Tests.csproj @@ -3,11 +3,11 @@ netcoreapp2.0 - - - - - + + + + + diff --git a/ScrapySharp.Tests/When_use_browser.cs b/ScrapySharp.Tests/When_use_browser.cs index e4736ba..73a3c3b 100644 --- a/ScrapySharp.Tests/When_use_browser.cs +++ b/ScrapySharp.Tests/When_use_browser.cs @@ -67,6 +67,27 @@ public void When_forcing_anguage() Assert.AreNotEqual(html1, html2); } - + + [Test] + public void When_skipping_certificate_validation() + { + var uri = new Uri("https://expired.badssl.com/"); + + var browser1 = new ScrapingBrowser { + SkipCertificateValidation = false }; + + var hasException = false; + try { browser1.NavigateToPage(uri); } + catch (Exception) { hasException = true; } + + Assert.IsTrue(hasException); + + var browser2 = new ScrapingBrowser() { + SkipCertificateValidation = true }; + + var page2 = browser2.NavigateToPage(uri); + + Assert.AreEqual(200, page2.RawResponse.StatusCode); + } } } \ No newline at end of file diff --git a/ScrapySharp.sln b/ScrapySharp.sln index e2e032d..624e38e 100644 --- a/ScrapySharp.sln +++ b/ScrapySharp.sln @@ -1,112 +1,115 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.23107.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{6755677C-979E-4D67-A21F-FAAF7598A993}" - ProjectSection(SolutionItems) = preProject - build.cmd = build.cmd - build.fsx = build.fsx - ReleaseNotes.txt = ReleaseNotes.txt - ScrapySharp.nuspec = ScrapySharp.nuspec - EndProjectSection - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScrapySharp", "ScrapySharp\ScrapySharp.csproj", "{19858AEA-E842-4633-AF1B-110ED4FE3770}" - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScrapySharp.Tests", "ScrapySharp.Tests\ScrapySharp.Tests.csproj", "{99F509FE-B51C-48A8-8C79-DD32AD2FDF10}" - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "ScrapySharp.Core", "ScrapySharp.Core\ScrapySharp.Core.fsproj", "{CE27016B-2755-4B44-8A0E-574914B9256F}" - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Libraries", "Libraries", "{F0E92760-C697-4CC6-8767-8929C6C9898E}" - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{D5D992B5-D7B1-4132-A8E1-078513363057}" - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|Mixed Platforms = Debug|Mixed Platforms - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|Mixed Platforms = Release|Mixed Platforms - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|x64.ActiveCfg = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|x86.ActiveCfg = Debug|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Any CPU.Build.0 = Release|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|x64.ActiveCfg = Release|Any CPU - {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|x86.ActiveCfg = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Any CPU.Build.0 = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|x64.ActiveCfg = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|x86.ActiveCfg = Debug|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Any CPU.ActiveCfg = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Any CPU.Build.0 = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|x64.ActiveCfg = Release|Any CPU - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|x86.ActiveCfg = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|x64.ActiveCfg = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|x86.ActiveCfg = Debug|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Any CPU.Build.0 = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|x64.ActiveCfg = Release|Any CPU - {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|x86.ActiveCfg = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {19858AEA-E842-4633-AF1B-110ED4FE3770} = {F0E92760-C697-4CC6-8767-8929C6C9898E} - {99F509FE-B51C-48A8-8C79-DD32AD2FDF10} = {D5D992B5-D7B1-4132-A8E1-078513363057} - {CE27016B-2755-4B44-8A0E-574914B9256F} = {F0E92760-C697-4CC6-8767-8929C6C9898E} - EndGlobalSection - GlobalSection(HgVSProperties) = preSolution - SolutionIsControlled = True - SolutionBindings = - EndGlobalSection - GlobalSection(TestCaseManagementSettings) = postSolution - CategoryFile = ScrapySharp.vsmdi - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31205.134 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{6755677C-979E-4D67-A21F-FAAF7598A993}" + ProjectSection(SolutionItems) = preProject + build.cmd = build.cmd + build.fsx = build.fsx + ReleaseNotes.txt = ReleaseNotes.txt + ScrapySharp.nuspec = ScrapySharp.nuspec + EndProjectSection +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScrapySharp", "ScrapySharp\ScrapySharp.csproj", "{19858AEA-E842-4633-AF1B-110ED4FE3770}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScrapySharp.Tests", "ScrapySharp.Tests\ScrapySharp.Tests.csproj", "{99F509FE-B51C-48A8-8C79-DD32AD2FDF10}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "ScrapySharp.Core", "ScrapySharp.Core\ScrapySharp.Core.fsproj", "{CE27016B-2755-4B44-8A0E-574914B9256F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Libraries", "Libraries", "{F0E92760-C697-4CC6-8767-8929C6C9898E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{D5D992B5-D7B1-4132-A8E1-078513363057}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|Mixed Platforms = Debug|Mixed Platforms + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|Mixed Platforms = Release|Mixed Platforms + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|x64.ActiveCfg = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Debug|x86.ActiveCfg = Debug|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Any CPU.Build.0 = Release|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|x64.ActiveCfg = Release|Any CPU + {19858AEA-E842-4633-AF1B-110ED4FE3770}.Release|x86.ActiveCfg = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|x64.ActiveCfg = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Debug|x86.ActiveCfg = Debug|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Any CPU.Build.0 = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|x64.ActiveCfg = Release|Any CPU + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10}.Release|x86.ActiveCfg = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|x64.ActiveCfg = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Debug|x86.ActiveCfg = Debug|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Any CPU.Build.0 = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|x64.ActiveCfg = Release|Any CPU + {CE27016B-2755-4B44-8A0E-574914B9256F}.Release|x86.ActiveCfg = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {19858AEA-E842-4633-AF1B-110ED4FE3770} = {F0E92760-C697-4CC6-8767-8929C6C9898E} + {99F509FE-B51C-48A8-8C79-DD32AD2FDF10} = {D5D992B5-D7B1-4132-A8E1-078513363057} + {CE27016B-2755-4B44-8A0E-574914B9256F} = {F0E92760-C697-4CC6-8767-8929C6C9898E} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {3205FC55-E7BE-4700-A812-9B2E39AE8E1B} + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(HgVSProperties) = preSolution + SolutionIsControlled = True + SolutionBindings = + EndGlobalSection + GlobalSection(TestCaseManagementSettings) = postSolution + CategoryFile = ScrapySharp.vsmdi + EndGlobalSection +EndGlobal diff --git a/ScrapySharp/Network/ScrapingBrowser.cs b/ScrapySharp/Network/ScrapingBrowser.cs index 636d348..5f7737a 100644 --- a/ScrapySharp/Network/ScrapingBrowser.cs +++ b/ScrapySharp/Network/ScrapingBrowser.cs @@ -30,6 +30,7 @@ public ScrapingBrowser() Language = CultureInfo.CreateSpecificCulture("EN-US"); UseDefaultCookiesParser = true; IgnoreCookies = false; + SkipCertificateValidation = false; ProtocolVersion = HttpVersion.Version10; KeepAlive = false; Proxy = WebRequest.DefaultWebProxy; @@ -159,6 +160,9 @@ private HttpWebRequest CreateRequest(Uri url, HttpVerb verb) else request.SendChunked = SendChunked; + if (SkipCertificateValidation) + request.ServerCertificateValidationCallback = delegate { return true; }; + return request; } @@ -175,13 +179,19 @@ private HttpWebRequest CreateRequest(Uri url, HttpVerb verb) private async Task GetResponseAsync(Uri url, HttpWebRequest request, int iteration, byte[] requestBody) { var response = await GetWebResponseAsync(url, request); - var responseStream = response.GetResponseStream(); + var responseStream = response?.GetResponseStream(); var headers = request.Headers.AllKeys.Select(k => new KeyValuePair(k, request.Headers[k])).ToList(); - if (responseStream == null) + if (responseStream == null) + { + var rawResponse = response != null + ? new RawResponse(response.ProtocolVersion, response.StatusCode, response.StatusDescription, response.Headers, new byte[0], Encoding) + : new RawResponse(request.ProtocolVersion, HttpStatusCode.BadRequest, "", new NameValueCollection(), new byte[0], Encoding); + return new WebPage(this, url, AutoDownloadPagesResources, new RawRequest(request.Method, request.RequestUri, request.ProtocolVersion, headers, requestBody, Encoding), - new RawResponse(response.ProtocolVersion, response.StatusCode, response.StatusDescription, response.Headers, new byte[0], Encoding), Encoding, AutoDetectCharsetEncoding); + rawResponse, Encoding, AutoDetectCharsetEncoding); + } var body = new MemoryStream(); responseStream.CopyTo(body); @@ -278,12 +288,10 @@ private async Task GetWebResponseAsync(Uri url, HttpWebRequest { response = (HttpWebResponse)e.Response; } - - var headers = response.Headers; if (!IgnoreCookies) { - var cookiesExpression = headers["Set-Cookie"]; + var cookiesExpression = response?.Headers["Set-Cookie"]; if (!string.IsNullOrEmpty(cookiesExpression)) { var cookieUrl = @@ -449,6 +457,8 @@ public static string GetHttpPostVars(NameValueCollection variables) public bool IgnoreCookies { get; set; } + public bool SkipCertificateValidation { get; set; } + public TimeSpan Timeout { get; set; } public CultureInfo Language { get; set; } diff --git a/ScrapySharp/ScrapySharp.csproj b/ScrapySharp/ScrapySharp.csproj index 0cb6bb0..ed04dc4 100644 --- a/ScrapySharp/ScrapySharp.csproj +++ b/ScrapySharp/ScrapySharp.csproj @@ -22,12 +22,12 @@ https://github.com/rflechner/ScrapySharp.git - - - + + + - +