From 720af1f71f5c35f9b388da5fe9a65f0f2f4a81fa Mon Sep 17 00:00:00 2001 From: ntr Date: Sat, 16 May 2020 16:09:01 +0200 Subject: [PATCH 1/3] Start from CLR code, then find out when perf digresses --- .../PartitionBench.KeysValues.cs | 215 ++++++++++++++++-- .../PartitionBenchConfig.cs | 38 ++++ .../DotNetCross.Sorting.Benchmarks/Program.cs | 4 +- 3 files changed, 234 insertions(+), 23 deletions(-) create mode 100644 tests/DotNetCross.Sorting.Benchmarks/PartitionBenchConfig.cs diff --git a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs index 9eabb85..d7036a4 100644 --- a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs +++ b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs @@ -17,14 +17,16 @@ namespace DotNetCross.Sorting.Benchmarks public class Int32StringPartitionBench : PartitionBench// { public Int32StringPartitionBench() - : base(maxLength: 6*1000*1000, new[] { 1000000 }, //2, 3, 10, 100, 10000, 1000000 }, + : base(maxLength: 10*1000*1000, new[] { 1000000 }, //2, 3, 10, 100, 10000, 1000000 }, SpanFillers.Default, i => i, i => i.ToString("D9")) { } } [Orderer(SummaryOrderPolicy.Method, MethodOrderPolicy.Alphabetical)] - [Config(typeof(SortBenchConfig))] + [Config(typeof(PartitionBenchConfig))] [MemoryDiagnoser] + [DisassemblyDiagnoser(maxDepth: 4, printSource: true, printInstructionAddresses: true, + exportHtml: true, exportCombinedDisassemblyReport: true, exportDiff: true)] public class PartitionBench// //where TKey : IComparable { @@ -123,10 +125,21 @@ public void DNX_() { var keys = new Span(_work, i, Length); var values = new Span(_workValues, i, Length); - DNX.PickPivotAndPartition(ref MemoryMarshal.GetReference(keys), ref MemoryMarshal.GetReference(values), - keys.Length); + DNX.PickPivotAndPartition(keys, values); } } + + [Benchmark] + public void DNX_N() + { + for (int i = 0; i <= _maxLength - Length; i += Length) + { + var keys = new Span(_work, i, Length); + var values = new Span(_workValues, i, Length); + DNX.PickPivotAndPartitionNew(keys, values); + } + } + //[Benchmark] public void DNX_NullComparer() { @@ -162,10 +175,14 @@ public void DNX_Comparison() public static class DNX { - internal static int PickPivotAndPartition( - ref TKey keys, ref TValue values, int length) + //[MethodImpl(MethodImplOptions.AggressiveInlining)] + // Had to reformulate signature since disassembly did not work + public static int PickPivotAndPartition( + Span spanKeys, Span spanValues) { - + ref TKey keys = ref MemoryMarshal.GetReference(spanKeys); + ref TValue values = ref MemoryMarshal.GetReference(spanValues); + var length = spanKeys.Length; Debug.Assert(length > 2); // // Compute median-of-three. But also partition them, since we've done the comparison. @@ -224,23 +241,147 @@ internal static int PickPivotAndPartition( var v = valuesLeft; valuesLeft = valuesRight; valuesRight = v; + } + // Put pivot in the right location. + right = nextToLast; + if (left != right) + { + Swap(ref keys, left, right); + Swap(ref values, left, right); + } + return left; + } + + public static int PickPivotAndPartitionNew( + Span keys, Span values) + { + //Debug.Assert(keys.Length >= Array.IntrosortSizeThreshold); + + int hi = keys.Length - 1; + + // Compute median-of-three. But also partition them, since we've done the comparison. + int middle = hi >> 1; + + // Sort lo, mid and hi appropriately, then pick mid as the pivot. + Sort2(keys, values, 0, middle); // swap the low with the mid point + Sort2(keys, values, 0, hi); // swap the low with the high + Sort2(keys, values, middle, hi); // swap the middle with the high + + TKey pivot = keys[middle]; + Swap(keys, values, middle, hi - 1); + int left = 0, right = hi - 1; // We already partitioned lo and hi and put the pivot in hi - 1. And we pre-increment & decrement below. + + while (left < right) + { + if (pivot == null) + { + while (left < (hi - 1) && keys[++left] == null) ; + while (right > 0 && keys[--right] != null) ; + } + else + { + while (pivot.CompareTo(keys[++left]) > 0) ; + while (pivot.CompareTo(keys[--right]) < 0) ; + } + + if (left >= right) + break; + + Swap(keys, values, left, right); + } + + // Put pivot in the right location. + if (left != hi - 1) + { + Swap(keys, values, left, hi - 1); + } + return left; + +#if OLDPATH + ref TKey keys = ref MemoryMarshal.GetReference(keys); + ref TValue values = ref MemoryMarshal.GetReference(values); + var length = keys.Length; + Debug.Assert(length > 2); + // + // Compute median-of-three. But also partition them, since we've done the comparison. + // + // Sort left, middle and right appropriately, then pick middle as the pivot. + int middle = (length - 1) >> 1; + ref TKey keysAtMiddle = ref Sort3(ref keys, ref values, 0, middle, length - 1); + + //int hi = length - 1; + //int middle = hi >> 1; + //Sort2(ref keys, ref values, 0, middle); // swap the low with the mid point + //Sort2(ref keys, ref values, 0, hi); // swap the low with the high + //Sort2(ref keys, ref values, middle, hi); // swap the middle with the high + //ref var keysAtMiddle = ref Unsafe.Add(ref keys, middle); - // while (left < nextToLast && comparison(Unsafe.Add(ref keys, ++left), pivot) < 0) ; - // // Check if bad comparable/comparison - // if (left == nextToLast && comparison(Unsafe.Add(ref keys, left), pivot) < 0) - // ThrowHelper.ThrowArgumentException_BadComparer(comparison); + TKey pivot = keysAtMiddle; - // while (right > 0 && comparison(pivot, Unsafe.Add(ref keys, --right)) < 0) ; - // // Check if bad comparable/comparison - // if (right == 0 && comparison(pivot, Unsafe.Add(ref keys, right)) < 0) - // ThrowHelper.ThrowArgumentException_BadComparer(comparison); - //} - //if (left >= right) - // break; + int left = 0; + int nextToLast = length - 2; + int right = nextToLast; + ref TKey keysLeft = ref Unsafe.Add(ref keys, left); + ref TKey keysRight = ref Unsafe.Add(ref keys, right); + // We already partitioned lo and hi and put the pivot in hi - 1. + // And we pre-increment & decrement below. + Swap(ref keysAtMiddle, ref keysRight); + Swap(ref values, middle, right); - //Swap(ref keys, left, right); - //Swap(ref values, left, right); + while (left < right) + { + if (pivot == null) + { + do { ++left; keysLeft = ref Unsafe.Add(ref keysLeft, 1); } + while (left < right && keysLeft == null); + do { --right; keysRight = ref Unsafe.Add(ref keysRight, -1); } + while (right > 0 && keysRight != null); + } + else + { + do { ++left; } + //while (left < right && pivot.CompareTo(keysLeft) > 0); + while (pivot.CompareTo(keysLeft = ref Unsafe.Add(ref keysLeft, 1)) > 0); + // Check if bad comparable/comparer + //if (left == right && pivot.CompareTo(keysLeft) > 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + do { --right; ; } + //while (right > 0 && pivot.CompareTo(keysRight) < 0); + while (pivot.CompareTo(keysRight = ref Unsafe.Add(ref keysRight, -1)) < 0); + // Check if bad comparable/comparer + //if (right == 0 && pivot.CompareTo(keysRight) < 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + //do { ++left; keysLeft = ref Unsafe.Add(ref keysLeft, 1); } + ////while (left < right && pivot.CompareTo(keysLeft) > 0); + //while (pivot.CompareTo(keysLeft) > 0) ; + //// Check if bad comparable/comparer + ////if (left == right && pivot.CompareTo(keysLeft) > 0) + //// ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + //do { --right; keysRight = ref Unsafe.Add(ref keysRight, -1); } + ////while (right > 0 && pivot.CompareTo(keysRight) < 0); + //while (pivot.CompareTo(keysRight) < 0); + //// Check if bad comparable/comparer + ////if (right == 0 && pivot.CompareTo(keysRight) < 0) + //// ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + } + + if (left >= right) + break; + + // PERF: Swap manually inlined here for better code-gen + var t = keysLeft; + keysLeft = keysRight; + keysRight = t; + // PERF: Swap manually inlined here for better code-gen + ref var valuesLeft = ref Unsafe.Add(ref values, left); + ref var valuesRight = ref Unsafe.Add(ref values, right); + var v = valuesLeft; + valuesLeft = valuesRight; + valuesRight = v; } // Put pivot in the right location. right = nextToLast; @@ -250,6 +391,7 @@ internal static int PickPivotAndPartition( Swap(ref values, left, right); } return left; +#endif } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -350,6 +492,37 @@ internal static void Sort2( } } + internal static void Sort2(Span keys, Span values, int i, int j) + { + Debug.Assert(i != j); + + if (keys[i] != null && keys[i].CompareTo(keys[j]) > 0) + { + TKey key = keys[i]; + keys[i] = keys[j]; + keys[j] = key; + + TValue value = values[i]; + values[i] = values[j]; + values[j] = value; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Swap(Span keys, Span values, int i, int j) + { + Debug.Assert(i != j); + + TKey k = keys[i]; + keys[i] = keys[j]; + keys[j] = k; + + TValue v = values[i]; + values[i] = values[j]; + values[j] = v; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void Swap(ref T items, int i, int j) { @@ -394,7 +567,7 @@ public static class CLR // } //} - private static void SwapIfGreaterWithValues(Span keys, Span values, int i, int j) + internal static void SwapIfGreaterWithValues(Span keys, Span values, int i, int j) { Debug.Assert(i != j); diff --git a/tests/DotNetCross.Sorting.Benchmarks/PartitionBenchConfig.cs b/tests/DotNetCross.Sorting.Benchmarks/PartitionBenchConfig.cs new file mode 100644 index 0000000..80887f1 --- /dev/null +++ b/tests/DotNetCross.Sorting.Benchmarks/PartitionBenchConfig.cs @@ -0,0 +1,38 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess; + +namespace DotNetCross.Sorting.Benchmarks +{ + public class PartitionBenchConfig : ManualConfig + { + public PartitionBenchConfig() + { + var runMode = new BenchmarkDotNet.Jobs.RunMode() { LaunchCount = 1, WarmupCount = 3, IterationCount = 7, /*TargetCount = 11, */RunStrategy = RunStrategy.Monitoring }; + var envModes = new[] { + // NOTE: None of the other platforms work... + //new EnvironmentMode { Platform = Platform.X86 }, + //new EnvironmentMode { Platform = Platform.X64 }, + new EnvironmentMode { Runtime = CoreRuntime.Core50, Platform = Platform.X64 }, + //new EnvMode { Runtime = Runtime.Clr, Platform = Platform.X86 }, + //new EnvMode { Runtime = Runtime.Clr, Platform = Platform.X64 }, + }; + foreach (var envMode in envModes) + { + AddJob(new Job(envMode, Job.Dry, runMode) + //.WithToolchain(BenchmarkDotNet.Toolchains.InProcess.NoEmit.InProcessNoEmitToolchain.Instance) + ); + } + //Add(new SpeedupColumn()); + //Add(DisassemblyDiagnoser.Create( + // new DisassemblyDiagnoserConfig(printAsm: true, printSource: true, recursiveDepth: 3))); + + //Add(DisassemblyDiagnoser.Create( + // new DisassemblyDiagnoserConfig(printAsm: true, printPrologAndEpilog: true, printSource: true, recursiveDepth: 3))); + + } + } +} diff --git a/tests/DotNetCross.Sorting.Benchmarks/Program.cs b/tests/DotNetCross.Sorting.Benchmarks/Program.cs index c031056..b6cf875 100644 --- a/tests/DotNetCross.Sorting.Benchmarks/Program.cs +++ b/tests/DotNetCross.Sorting.Benchmarks/Program.cs @@ -526,7 +526,7 @@ static void Main(string[] args) } else if (d == Do.KeysValues1) { - var sut = new Int32StringSortBench(); + var sut = new Int32StringPartitionBench(); //var sut = new ComparableClassInt32Int32SortBench(); //var sut = new StringInt32SortBench(); sut.Filler = new RandomSpanFiller(SpanFillers.RandomSeed); @@ -541,7 +541,7 @@ static void Main(string[] args) //Console.WriteLine("Enter key..."); //Console.ReadKey(); - for (int i = 0; i < 20; i++) + for (int i = 0; i < 40; i++) { sut.IterationSetup(); sut.DNX_(); From ee761c231bffb65bb842f55772d1951f0c9a7e25 Mon Sep 17 00:00:00 2001 From: ntr Date: Sat, 16 May 2020 16:14:34 +0200 Subject: [PATCH 2/3] Use ref inner loop WITH checks (faster than WITHOUT checks) --- .../PartitionBench.KeysValues.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs index d7036a4..6f29c26 100644 --- a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs +++ b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs @@ -271,6 +271,8 @@ public static int PickPivotAndPartitionNew( Swap(keys, values, middle, hi - 1); int left = 0, right = hi - 1; // We already partitioned lo and hi and put the pivot in hi - 1. And we pre-increment & decrement below. + ref TKey keysLeft = ref keys[left]; + ref TKey keysRight = ref keys[right]; while (left < right) { if (pivot == null) @@ -280,8 +282,20 @@ public static int PickPivotAndPartitionNew( } else { - while (pivot.CompareTo(keys[++left]) > 0) ; - while (pivot.CompareTo(keys[--right]) < 0) ; + do { ++left; keysLeft = ref Unsafe.Add(ref keysLeft, 1); } + while (left < right && pivot.CompareTo(keysLeft) > 0); + // Check if bad comparable/comparer + if (left == right && pivot.CompareTo(keysLeft) > 0) + ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + do { --right; keysRight = ref Unsafe.Add(ref keysRight, -1); } + while (right > 0 && pivot.CompareTo(keysRight) < 0); + // Check if bad comparable/comparer + if (right == 0 && pivot.CompareTo(keysRight) < 0) + ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + //while (pivot.CompareTo(keys[++left]) > 0) ; + //while (pivot.CompareTo(keys[--right]) < 0) ; } if (left >= right) From 9c3d1cdfea426aa0e654ff0a4af6bada68905450 Mon Sep 17 00:00:00 2001 From: ntr Date: Sat, 16 May 2020 16:31:25 +0200 Subject: [PATCH 3/3] Switching away from from ref to span based is just faster --- .../PartitionBench.KeysValues.cs | 83 ++++++++++++++----- .../DotNetCross.Sorting.Benchmarks/Program.cs | 2 +- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs index 6f29c26..f0c9f3a 100644 --- a/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs +++ b/tests/DotNetCross.Sorting.Benchmarks/PartitionBench.KeysValues.cs @@ -125,18 +125,18 @@ public void DNX_() { var keys = new Span(_work, i, Length); var values = new Span(_workValues, i, Length); - DNX.PickPivotAndPartition(keys, values); + DNX.PickPivotAndPartitionNew(keys, values); } } [Benchmark] - public void DNX_N() + public void DNX_Org() { for (int i = 0; i <= _maxLength - Length; i += Length) { var keys = new Span(_work, i, Length); var values = new Span(_workValues, i, Length); - DNX.PickPivotAndPartitionNew(keys, values); + DNX.PickPivotAndPartition(keys, values); } } @@ -252,7 +252,7 @@ public static int PickPivotAndPartition( return left; } - public static int PickPivotAndPartitionNew( + public unsafe static int PickPivotAndPartitionNew( Span keys, Span values) { //Debug.Assert(keys.Length >= Array.IntrosortSizeThreshold); @@ -273,33 +273,72 @@ public static int PickPivotAndPartitionNew( ref TKey keysLeft = ref keys[left]; ref TKey keysRight = ref keys[right]; - while (left < right) + ref TKey zeroRef = ref keysLeft; + ref TKey nextToLastRef = ref keysRight; + while (Unsafe.IsAddressLessThan(ref keysLeft, ref keysRight)) { if (pivot == null) { - while (left < (hi - 1) && keys[++left] == null) ; - while (right > 0 && keys[--right] != null) ; + while (Unsafe.IsAddressLessThan(ref keysLeft, ref keysRight) && + (keysLeft = ref Unsafe.Add(ref keysLeft, 1)) == null) ; + while (Unsafe.IsAddressGreaterThan(ref keysRight, ref keysLeft) && + (keysRight = ref Unsafe.Add(ref keysRight, -1)) != null) ; } else { - do { ++left; keysLeft = ref Unsafe.Add(ref keysLeft, 1); } - while (left < right && pivot.CompareTo(keysLeft) > 0); - // Check if bad comparable/comparer - if (left == right && pivot.CompareTo(keysLeft) > 0) - ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); - - do { --right; keysRight = ref Unsafe.Add(ref keysRight, -1); } - while (right > 0 && pivot.CompareTo(keysRight) < 0); - // Check if bad comparable/comparer - if (right == 0 && pivot.CompareTo(keysRight) < 0) - ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); - - //while (pivot.CompareTo(keys[++left]) > 0) ; - //while (pivot.CompareTo(keys[--right]) < 0) ; + while (Unsafe.IsAddressLessThan(ref keysLeft, ref nextToLastRef) && + pivot.CompareTo(keysLeft = ref Unsafe.Add(ref keysLeft, 1)) > 0) ; + while (Unsafe.IsAddressGreaterThan(ref keysRight, ref zeroRef) && + pivot.CompareTo(keysRight = ref Unsafe.Add(ref keysRight, -1)) < 0) ; + + //// PERF: For internal direct comparers the range checks are not needed + //// since we know they cannot be bogus i.e. pass the pivot without being false. + //while (pivot.CompareTo(keysLeft = ref Unsafe.Add(ref keysLeft, 1)) > 0) ; + //// Check if bad comparable/comparer + //if (left == right && pivot.CompareTo(keysLeft) > 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + //while (comparer.LessThan(pivot, keysRight = ref Unsafe.Add(ref keysRight, -1))) ; + //// Check if bad comparable/comparer + //if (right == 0 && pivot.CompareTo(keysRight) < 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); } - if (left >= right) + if (!Unsafe.IsAddressLessThan(ref keysLeft, ref keysRight)) + { break; + } + //if (pivot == null) + //{ + // while (left < (hi - 1) && keys[++left] == null) ; + // while (right > 0 && keys[--right] != null) ; + //} + //else + //{ + // do { ++left; keysLeft = ref Unsafe.Add(ref keysLeft, 1); } + // while (left < right && pivot.CompareTo(keysLeft) > 0); + // // Check if bad comparable/comparer + // if (left == right && pivot.CompareTo(keysLeft) > 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + // do { --right; keysRight = ref Unsafe.Add(ref keysRight, -1); } + // while (right > 0 && pivot.CompareTo(keysRight) < 0); + // // Check if bad comparable/comparer + // if (right == 0 && pivot.CompareTo(keysRight) < 0) + // ThrowHelper.ThrowArgumentException_BadComparable(typeof(TKey)); + + // //while (pivot.CompareTo(keys[++left]) > 0) ; + // //while (pivot.CompareTo(keys[--right]) < 0) ; + //} + + //if (left >= right) + // break; + + left = (sizeof(IntPtr) == 4) + ? (int)Unsafe.ByteOffset(ref zeroRef, ref keysLeft) / Unsafe.SizeOf() + : (int)((long)Unsafe.ByteOffset(ref zeroRef, ref keysLeft) / Unsafe.SizeOf()); + right = (sizeof(IntPtr) == 4) + ? (int)Unsafe.ByteOffset(ref zeroRef, ref keysRight) / Unsafe.SizeOf() + : (int)((long)Unsafe.ByteOffset(ref zeroRef, ref keysRight) / Unsafe.SizeOf()); Swap(keys, values, left, right); } diff --git a/tests/DotNetCross.Sorting.Benchmarks/Program.cs b/tests/DotNetCross.Sorting.Benchmarks/Program.cs index b6cf875..99f8c40 100644 --- a/tests/DotNetCross.Sorting.Benchmarks/Program.cs +++ b/tests/DotNetCross.Sorting.Benchmarks/Program.cs @@ -411,7 +411,7 @@ static void Main(string[] args) // TODO: Refactor to switch/case and methods perhaps, less flexible though // TODO: Add argument parsing for this perhaps - var d = Debugger.IsAttached ? Do.Keys1 : Do.Focus; + var d = Debugger.IsAttached ? Do.Keys1 : Do.KeysValues1; if (d == Do.Focus) { BenchmarkRunner.Run();