Merge pull request #95 from contour-terminal/improvement/tablegen

tablegen: Improve performance for names

Author: christianparpart

.gitignore

@@ -1,4 +1,5 @@
 /build/
+/out/
 /_deps/
 /_ucd/
 /.clangd/

CMakeLists.txt

@@ -48,6 +48,7 @@ option(LIBUNICODE_TOOLS "libunicode: Builds CLI tools [default: ${MASTER_PROJECT
 option(LIBUNICODE_BUILD_STATIC "libunicode: provide static library instead of dynamic [default: ${LIBUNICODE_BUILD_STATIC_DEFAULT}]" ${LIBUNICODE_BUILD_STATIC_DEFAULT})
 option(LIBUNICODE_USE_INTRINSICS "libunicode: Use SIMD extenstion during text read [default: ON]" ON)
 option(LIBUNICODE_USE_STD_SIMD "libunicode: Use std::simd as SIMD extenstion during text read (takes precedence over own intrinsics) [default: ON]" ${LIBUNICODE_USE_INTRINSICS})
+option(LIBUNICODE_TABLEGEN_FASTBUILD "libunicode: Use fast table generation (takes more memory in final tables) [default: OFF]" OFF)
 
 set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Enable testing of the benchmark library." FORCE)
 include(ThirdParties)
@@ -104,6 +105,7 @@ message(STATUS "Build mode:                  ${LIBUNICODE_BUILD_MODE}")
 message(STATUS "Build unit tests:            ${LIBUNICODE_TESTING}")
 message(STATUS "Build benchmark:             ${LIBUNICODE_BENCHMARK}")
 message(STATUS "Build tools:                 ${LIBUNICODE_TOOLS}")
+message(STATUS "Enable tablegen fast build:  ${LIBUNICODE_TABLEGEN_FASTBUILD}")
 message(STATUS "Using ccache:                ${USING_CCACHE_STRING}")
 message(STATUS "Using UCD directory:         ${LIBUNICODE_UCD_DIR}")
 message(STATUS "Enable clang-tidy:           ${ENABLE_TIDY} (${CMAKE_CXX_CLANG_TIDY})")

cmake/presets/common.json

@@ -1,7 +1,7 @@
 {
     "version": 6,
     "configurePresets": [
-        { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } },
+        { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "LIBUNICODE_TABLEGEN_FASTBUILD": "ON" } },
         { "name": "release", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
         { "name": "arch-native", "hidden": true, "cacheVariables": { "CMAKE_CXX_FLAGS": "-march=native" } },
         { "name": "clang", "hidden": true, "cacheVariables": { "CMAKE_CXX_COMPILER": "clang++" } },

src/libunicode/CMakeLists.txt

@@ -84,6 +84,9 @@ add_custom_command(
 
 add_library(unicode_loader ${LIBUNICODE_LIB_MODE} codepoint_properties_loader.h codepoint_properties_loader.cpp)
 add_library(unicode::loader ALIAS unicode_loader)
+if(LIBUNICODE_TABLEGEN_FASTBUILD)
+    target_compile_definitions(unicode_loader PRIVATE LIBUNICODE_TABLEGEN_FASTBUILD)
+endif()
 set_target_properties(unicode_loader PROPERTIES
     VERSION "${PROJECT_VERSION}"
     SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
@@ -104,7 +107,7 @@ add_library(unicode ${LIBUNICODE_LIB_MODE}
     utf8.cpp
     width.cpp
 
-    # auto-generated by unicode_tablgen
+    # auto-generated by unicode_tablegen
     codepoint_properties_data.h
     codepoint_properties_data.cpp
     codepoint_properties_names.cpp
@@ -157,6 +160,7 @@ target_link_libraries(unicode PUBLIC unicode::ucd)
 add_executable(unicode_tablegen tablegen.cpp)
 target_link_libraries(unicode_tablegen PRIVATE unicode::loader)
 
+
 # {{{ installation
 set(LIBUNICODE_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/libunicode" CACHE PATH "Installation directory for cmake files, a relative path that will be joined with ${CMAKE_INSTALL_PREFIX} or an absolute path.")
 set(LIBUNICODE_INSTALL_CMAKE_FILES ${MASTER_PROJECT} CACHE BOOL "Decides whether or not to install CMake config and -version files.")

src/libunicode/codepoint_properties_loader.cpp

@@ -610,12 +610,32 @@ namespace
     {
         {
             auto const _ = scoped_timer { _log, "Creating multistage tables (properties)" };
-            support::generate(_codepoints.data(), _codepoints.size(), _output);
+            support::generate(_codepoints.data(),
+                              _codepoints.size(),
+                              _output,
+                              [](auto const& begin, auto const& end, auto value) noexcept {
+                                  return std::find(begin, end, value);
+                              });
         }
 
         {
             auto const _ = scoped_timer { _log, "Creating multistage tables (names)" };
-            support::generate(_names.data(), _names.size(), _outputNames);
+            support::generate(_names.data(),
+                              _names.size(),
+                              _outputNames,
+                              [&](auto const& begin, auto const& end, auto value) noexcept {
+#if defined(LIBUNICODE_TABLEGEN_FASTBUILD)
+                                  if (value.empty())
+                                      // This case is happening for unassigned codepoints (and quite a lot)
+                                      // We want to keep the names table as small as possible, so we don't
+                                      // re-add empty strings to it.
+                                      return std::find(begin, end, value);
+                                  // Non-empty names are mostly unique (~1.6% are duplicates, that's okay)
+                                  return end;
+#else
+                                  return std::find(begin, end, value);
+#endif
+                              });
         }
     }
 } // namespace

src/libunicode/multistage_table_generator.h

@@ -54,6 +54,7 @@ template <typename T,
           typename SourceType,
           typename Stage1ElementType,
           typename Stage2ElementType,
+          typename Stage3Finder,
           SourceType BlockSize,
           SourceType MaxValue = std::numeric_limits<SourceType>::max()>
 class multistage_table_generator
@@ -62,6 +63,7 @@ class multistage_table_generator
     T const* _input;
     size_t _inputSize;
     multistage_table<T, SourceType, Stage1ElementType, Stage2ElementType, BlockSize, MaxValue>& _output;
+    Stage3Finder _stage3Finder;
 
     void generate()
     {
@@ -143,7 +145,8 @@ class multistage_table_generator
     Stage2ElementType get_or_create_stage3_index(SourceType stage1Index)
     {
         auto& properties = _output.stage3;
-        auto const propertyIterator = find(properties.begin(), properties.end(), _input[stage1Index]);
+        auto const propertyIterator =
+            _stage3Finder(properties.begin(), properties.end(), _input[stage1Index]);
         if (propertyIterator != properties.end())
             return static_cast<Stage2ElementType>(distance(properties.begin(), propertyIterator));
 
@@ -158,17 +161,24 @@ template <typename T,
           typename SourceType,
           typename Stage1ElementType,
           typename Stage2ElementType,
+          typename Stage3Finder,
           SourceType BlockSize,
           SourceType MaxValue = std::numeric_limits<SourceType>::max()>
 void generate(
     T const* input,
     size_t inputSize,
-    multistage_table<T, SourceType, Stage1ElementType, Stage2ElementType, BlockSize, MaxValue>& output)
+    multistage_table<T, SourceType, Stage1ElementType, Stage2ElementType, BlockSize, MaxValue>& output,
+    Stage3Finder&& stage3Finder)
 {
-    auto builder =
-        multistage_table_generator<T, SourceType, Stage1ElementType, Stage2ElementType, BlockSize, MaxValue> {
-            input, inputSize, output
-        };
+    auto builder = multistage_table_generator<T,
+                                              SourceType,
+                                              Stage1ElementType,
+                                              Stage2ElementType,
+                                              Stage3Finder,
+                                              BlockSize,
+                                              MaxValue> {
+        input, inputSize, output, std::forward<Stage3Finder>(stage3Finder)
+    };
     builder.generate();
 }
 

src/libunicode/tablegen.cpp

@@ -207,7 +207,7 @@ int main(int argc, char const* argv[])
     auto headerFile = std::ofstream(cxxHeaderFileName);
     auto implementationFile = std::ofstream(cxxImplementationFileName);
     auto namesFile = std::ofstream(cxxNamesFileName);
-    auto const [props, names] = unicode::load_from_directory(ucdDataDirectory, &std::cout);
+    auto const [props, names] = unicode::load_from_directory(ucdDataDirectory, &std::clog);
 
     write_cxx_tables(props, names, headerFile, implementationFile, namesFile, namespaceName);