Add support for transliterated citation keys

CHANGELOG.md

@@ -14,6 +14,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added "IEEE" as another option for parsing plain text citations. [#14233](github.com/JabRef/jabref/pull/14233)
 - We added automatic date-based groups that create year/month/day subgroups from an entry’s date fields. [#10822](https://github.com/JabRef/jabref/issues/10822)
 - We added `doi-to-bibtex` to `JabKit`. [#14244](https://github.com/JabRef/jabref/pull/14244)
+- We added support for transliteration of fields to English and automatic transliteration of generated citation key. [#11377](https://github.com/JabRef/jabref/issues/11377)
 
 ### Changed
 

jabgui/src/main/java/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTab.java

@@ -16,6 +16,7 @@
 
 public class CitationKeyPatternTab extends AbstractPreferenceTabView<CitationKeyPatternTabViewModel> implements PreferencesTab {
 
+    @FXML private CheckBox transliterateFieldsForCitationKey;
     @FXML private CheckBox overwriteAllow;
     @FXML private CheckBox overwriteWarning;
     @FXML private CheckBox generateOnSave;
@@ -42,6 +43,7 @@ public String getTabName() {
     public void initialize() {
         this.viewModel = new CitationKeyPatternTabViewModel(preferences.getCitationKeyPatternPreferences(), preferences.getImporterPreferences());
 
+        transliterateFieldsForCitationKey.selectedProperty().bindBidirectional(viewModel.transliterateFieldsForCitationKeyProperty());
         overwriteAllow.selectedProperty().bindBidirectional(viewModel.overwriteAllowProperty());
         overwriteWarning.selectedProperty().bindBidirectional(viewModel.overwriteWarningProperty());
         generateOnSave.selectedProperty().bindBidirectional(viewModel.generateOnSaveProperty());

jabgui/src/main/java/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTabViewModel.java

@@ -19,6 +19,7 @@
 
 public class CitationKeyPatternTabViewModel implements PreferenceTabViewModel {
 
+    private final BooleanProperty transliterateFieldsForCitationKeyProperty = new SimpleBooleanProperty();
     private final BooleanProperty overwriteAllowProperty = new SimpleBooleanProperty();
     private final BooleanProperty overwriteWarningProperty = new SimpleBooleanProperty();
     private final BooleanProperty generateOnSaveProperty = new SimpleBooleanProperty();
@@ -53,6 +54,7 @@ public CitationKeyPatternTabViewModel(CitationKeyPatternPreferences keyPatternPr
 
     @Override
     public void setValues() {
+        transliterateFieldsForCitationKeyProperty.setValue(keyPatternPreferences.shouldTransliterateFieldsForCitationKey());
         overwriteAllowProperty.setValue(!keyPatternPreferences.shouldAvoidOverwriteCiteKey());
         overwriteWarningProperty.setValue(keyPatternPreferences.shouldWarnBeforeOverwriteCiteKey());
         generateOnSaveProperty.setValue(keyPatternPreferences.shouldGenerateCiteKeysBeforeSaving());
@@ -106,6 +108,7 @@ public void storeSettings() {
             keySuffix = CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_B;
         }
 
+        keyPatternPreferences.setShouldTransliterateFieldsForCitationKey(transliterateFieldsForCitationKeyProperty.getValue());
         keyPatternPreferences.setAvoidOverwriteCiteKey(!overwriteAllowProperty.getValue());
         keyPatternPreferences.setWarnBeforeOverwriteCiteKey(overwriteWarningProperty.getValue());
         keyPatternPreferences.setGenerateCiteKeysBeforeSaving(generateOnSaveProperty.getValue());
@@ -117,6 +120,10 @@ public void storeSettings() {
         keyPatternPreferences.setKeyPatterns(newKeyPattern);
     }
 
+    public BooleanProperty transliterateFieldsForCitationKeyProperty() {
+        return transliterateFieldsForCitationKeyProperty;
+    }
+
     public BooleanProperty overwriteAllowProperty() {
         return overwriteAllowProperty;
     }

jabgui/src/main/resources/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTab.fxml

@@ -74,6 +74,9 @@
                GridPane.columnIndex="0" GridPane.rowIndex="8"/>
         <TextField fx:id="unwantedCharacters"
                 GridPane.columnIndex="1" GridPane.rowIndex="8"/>
+
+        <CheckBox fx:id="transliterateFieldsForCitationKey" text="%Transliterate fields for citation key"
+                GridPane.columnIndex="0" GridPane.rowIndex="9"/>
     </GridPane>
 
     <HBox spacing="8.0">
@@ -90,4 +93,4 @@
         <Button text="%Reset All" onAction="#resetAllKeyPatterns"
                 AnchorPane.rightAnchor="0.0" AnchorPane.topAnchor="0.0"/>
     </AnchorPane>
-</fx:root>
+</fx:root>

jabgui/src/test/java/org/jabref/gui/externalfiles/AutoRenameFileOnEntryChangeTest.java

@@ -44,6 +44,7 @@ void setUp(@TempDir Path tempDir) {
         GuiPreferences guiPreferences = mock(GuiPreferences.class);
         filePreferences = mock(FilePreferences.class);
         CitationKeyPatternPreferences patternPreferences = new CitationKeyPatternPreferences(
+                false,
                 false,
                 true,
                 false,

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyGenerator.java

@@ -8,6 +8,7 @@
 import java.util.regex.PatternSyntaxException;
 
 import org.jabref.logic.util.strings.StringUtil;
+import org.jabref.logic.util.strings.Transliteration;
 import org.jabref.model.FieldChange;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
@@ -37,7 +38,7 @@ public class CitationKeyGenerator extends BracketedPattern {
 
     /// Source of disallowed characters: <https://tex.stackexchange.com/a/408548/9075>
     /// These characters are disallowed in BibTeX keys.
-    private static final List<Character> DISALLOWED_CHARACTERS = Arrays.asList('{', '}', '(', ')', ',', '=', '\\', '"', '#', '%', '~', '\'');
+    public static final List<Character> DISALLOWED_CHARACTERS = Arrays.asList('{', '}', '(', ')', ',', '=', '\\', '"', '#', '%', '~', '\'');
 
     private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyGenerator.class);
 
@@ -109,7 +110,8 @@ public String generateKey(@NonNull BibEntry entry) {
         String newKey = createCitationKeyFromPattern(entry);
         newKey = replaceWithRegex(newKey);
         newKey = appendLettersToKey(newKey, currentKey);
-        return cleanKey(newKey, unwantedCharacters);
+        newKey = cleanKey(newKey, unwantedCharacters);
+        return transliterateIfNeeded(newKey);
     }
 
     /**
@@ -155,6 +157,15 @@ private String appendLettersToKey(String key, String oldKey) {
         return key;
     }
 
+    public String transliterateIfNeeded(String key) {
+        if (!citationKeyPatternPreferences.shouldTransliterateFieldsForCitationKey()) {
+            return key;
+        }
+
+        String result = Transliteration.transliterate(key);
+        return result.replace(" ", "");
+    }
+
     /**
      * Using preferences, replace matches to the provided regex with a string.
      *

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyGeneratorTestUtils.java

@@ -0,0 +1,22 @@
+package org.jabref.logic.citationkeypattern;
+
+import static org.jabref.logic.citationkeypattern.CitationKeyGenerator.DEFAULT_UNWANTED_CHARACTERS;
+
+public class CitationKeyGeneratorTestUtils {
+
+    public static CitationKeyPatternPreferences getInstanceForTesting() {
+        return new CitationKeyPatternPreferences(
+                true,
+                false,
+                false,
+                false,
+                CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_A,
+                "",
+                "",
+                DEFAULT_UNWANTED_CHARACTERS,
+                GlobalCitationKeyPatterns.fromPattern("[auth][year]"),
+                "",
+                ','
+        );
+    }
+}

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyPatternPreferences.java

@@ -18,6 +18,7 @@ public enum KeySuffix {
         SECOND_WITH_B   // CiteKey, CiteKeyB, CiteKeyC ...
     }
 
+    private final BooleanProperty shouldTransliterateFieldsForCitationKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldAvoidOverwriteCiteKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldWarnBeforeOverwriteCiteKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldGenerateCiteKeysBeforeSaving = new SimpleBooleanProperty();
@@ -29,7 +30,8 @@ public enum KeySuffix {
     private final String defaultPattern;
     private final ReadOnlyObjectProperty<Character> keywordDelimiter;
 
-    public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
+    public CitationKeyPatternPreferences(boolean shouldTransliterateFieldsForCitationKey,
+                                         boolean shouldAvoidOverwriteCiteKey,
                                          boolean shouldWarnBeforeOverwriteCiteKey,
                                          boolean shouldGenerateCiteKeysBeforeSaving,
                                          KeySuffix keySuffix,
@@ -40,6 +42,7 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                                          String defaultPattern,
                                          ReadOnlyObjectProperty<Character> keywordDelimiter) {
 
+        this.shouldTransliterateFieldsForCitationKey.set(shouldTransliterateFieldsForCitationKey);
         this.shouldAvoidOverwriteCiteKey.set(shouldAvoidOverwriteCiteKey);
         this.shouldWarnBeforeOverwriteCiteKey.set(shouldWarnBeforeOverwriteCiteKey);
         this.shouldGenerateCiteKeysBeforeSaving.set(shouldGenerateCiteKeysBeforeSaving);
@@ -54,7 +57,8 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
     }
 
     @VisibleForTesting
-    public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
+    public CitationKeyPatternPreferences(boolean shouldTransliterateFieldsForCitationKey,
+                                         boolean shouldAvoidOverwriteCiteKey,
                                          boolean shouldWarnBeforeOverwriteCiteKey,
                                          boolean shouldGenerateCiteKeysBeforeSaving,
                                          KeySuffix keySuffix,
@@ -65,7 +69,8 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                                          String defaultPattern,
                                          Character keywordDelimiter) {
 
-        this(shouldAvoidOverwriteCiteKey,
+        this(shouldTransliterateFieldsForCitationKey,
+                shouldAvoidOverwriteCiteKey,
                 shouldWarnBeforeOverwriteCiteKey,
                 shouldGenerateCiteKeysBeforeSaving,
                 keySuffix,
@@ -77,6 +82,18 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                 new SimpleObjectProperty<>(keywordDelimiter));
     }
 
+    public boolean shouldTransliterateFieldsForCitationKey() {
+        return shouldTransliterateFieldsForCitationKey.get();
+    }
+
+    public BooleanProperty shouldTransliterateFieldsForCitationKeyProperty() {
+        return shouldTransliterateFieldsForCitationKey;
+    }
+
+    public void setShouldTransliterateFieldsForCitationKey(boolean shouldTransliterateFieldsForCitationKey) {
+        this.shouldTransliterateFieldsForCitationKey.set(shouldTransliterateFieldsForCitationKey);
+    }
+
     public boolean shouldAvoidOverwriteCiteKey() {
         return shouldAvoidOverwriteCiteKey.get();
     }

jablib/src/main/java/org/jabref/logic/formatter/Formatters.java

@@ -29,6 +29,7 @@
 import org.jabref.logic.formatter.bibtexfields.RemoveEnclosingBracesFormatter;
 import org.jabref.logic.formatter.bibtexfields.RemoveWordEnclosingAndOuterEnclosingBracesFormatter;
 import org.jabref.logic.formatter.bibtexfields.ShortenDOIFormatter;
+import org.jabref.logic.formatter.bibtexfields.TransliterateFormatter;
 import org.jabref.logic.formatter.bibtexfields.UnicodeToLatexFormatter;
 import org.jabref.logic.formatter.bibtexfields.UnitsToLatexFormatter;
 import org.jabref.logic.formatter.casechanger.CamelFormatter;
@@ -66,7 +67,8 @@ public static List<Formatter> getConverters() {
                 new HtmlToUnicodeFormatter(),
                 new LatexToUnicodeFormatter(),
                 new UnicodeToLatexFormatter(),
-                new ConvertMSCCodesFormatter()
+                new ConvertMSCCodesFormatter(),
+                new TransliterateFormatter()
         );
     }
 

jablib/src/main/java/org/jabref/logic/formatter/bibtexfields/TransliterateFormatter.java

@@ -0,0 +1,32 @@
+package org.jabref.logic.formatter.bibtexfields;
+
+import org.jabref.logic.cleanup.Formatter;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.strings.Transliteration;
+
+public class TransliterateFormatter extends Formatter {
+    @Override
+    public String getName() {
+        return Localization.lang("Transliterate");
+    }
+
+    @Override
+    public String getKey() {
+        return "transliterate";
+    }
+
+    @Override
+    public String format(String value) {
+        return Transliteration.transliterate(value);
+    }
+
+    @Override
+    public String getDescription() {
+        return Localization.lang("Converts non-Latin characters to their Latin equivalents.");
+    }
+
+    @Override
+    public String getExampleInput() {
+        return "Карпенко Надежда";
+    }
+}

jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java

@@ -240,6 +240,7 @@ public class JabRefCliPreferences implements CliPreferences {
     public static final String UNWANTED_CITATION_KEY_CHARACTERS = "defaultUnwantedBibtexKeyCharacters";
     public static final String CONFIRM_LINKED_FILE_DELETE = "confirmLinkedFileDelete";
     public static final String TRASH_INSTEAD_OF_DELETE = "trashInsteadOfDelete";
+    public static final String TRANSLITERATE_FIELDS_FOR_CITATION_KEY = "transliterateFields";
     public static final String WARN_BEFORE_OVERWRITING_KEY = "warnBeforeOverwritingKey";
     public static final String AVOID_OVERWRITING_KEY = "avoidOverwritingKey";
     public static final String AUTOLINK_EXACT_KEY_ONLY = "autolinkExactKeyOnly";
@@ -649,6 +650,7 @@ public JabRefCliPreferences() {
 
         defaults.put(USE_OWNER, Boolean.FALSE);
         defaults.put(OVERWRITE_OWNER, Boolean.FALSE);
+        defaults.put(TRANSLITERATE_FIELDS_FOR_CITATION_KEY, Boolean.FALSE);
         defaults.put(AVOID_OVERWRITING_KEY, Boolean.FALSE);
         defaults.put(WARN_BEFORE_OVERWRITING_KEY, Boolean.TRUE);
         defaults.put(CONFIRM_LINKED_FILE_DELETE, Boolean.TRUE);
@@ -1541,6 +1543,7 @@ public CitationKeyPatternPreferences getCitationKeyPatternPreferences() {
         }
 
         citationKeyPatternPreferences = new CitationKeyPatternPreferences(
+                getBoolean(TRANSLITERATE_FIELDS_FOR_CITATION_KEY),
                 getBoolean(AVOID_OVERWRITING_KEY),
                 getBoolean(WARN_BEFORE_OVERWRITING_KEY),
                 getBoolean(GENERATE_KEYS_BEFORE_SAVING),
@@ -1552,6 +1555,8 @@ public CitationKeyPatternPreferences getCitationKeyPatternPreferences() {
                 (String) defaults.get(DEFAULT_CITATION_KEY_PATTERN),
                 getBibEntryPreferences().keywordSeparatorProperty());
 
+        EasyBind.listen(citationKeyPatternPreferences.shouldTransliterateFieldsForCitationKeyProperty(),
+                (_, _, newValue) -> putBoolean(TRANSLITERATE_FIELDS_FOR_CITATION_KEY, newValue));
         EasyBind.listen(citationKeyPatternPreferences.shouldAvoidOverwriteCiteKeyProperty(),
                 (_, _, newValue) -> putBoolean(AVOID_OVERWRITING_KEY, newValue));
         EasyBind.listen(citationKeyPatternPreferences.shouldWarnBeforeOverwriteCiteKeyProperty(),

jablib/src/main/java/org/jabref/logic/util/strings/Transliteration.java

@@ -0,0 +1,29 @@
+package org.jabref.logic.util.strings;
+
+import org.jabref.logic.citationkeypattern.CitationKeyGenerator;
+
+import com.ibm.icu.text.Transliterator;
+
+public class Transliteration {
+
+    // This transliterator configuration will transliterate from any language to Latin script
+    // that uses only allowed characters for citation keys.
+    private static final String TRANSLITERATOR_CONFIG = buildTransliteratorConfig();
+    private static final Transliterator TRANSLITERATOR = Transliterator.getInstance(TRANSLITERATOR_CONFIG);
+
+    public static String transliterate(String input) {
+        return TRANSLITERATOR.transliterate(input);
+    }
+
+    private static String buildTransliteratorConfig() {
+        StringBuilder pattern = new StringBuilder();
+
+        for (Character c : CitationKeyGenerator.DISALLOWED_CHARACTERS) {
+            // Generally, only characters like `-` or `[` need to be escaped with a backslash,
+            // but for future proofing we escape all characters.
+            pattern.append("\\").append(c);
+        }
+
+        return "Any-Latin; Latin-ASCII; [" + pattern + "] Remove";
+    }
+}

jablib/src/main/resources/l10n/JabRef_en.properties

@@ -1367,6 +1367,10 @@ Convert\ to\ BibTeX\ format\ (e.g.,\ store\ publication\ date\ in\ year\ and\ mo
 Convert\ MSC\ Keyword\ codes\ to\ their\ respective\ descriptions.=Convert MSC Keyword codes to their respective descriptions.
 MSC\ Codes\ to\ Descriptions=MSC Codes to Descriptions
 
+Converts\ non-Latin\ characters\ to\ their\ Latin\ equivalents.=Converts non-Latin characters to their Latin equivalents.
+Transliterate=Transliterate
+Transliterate\ fields\ for\ citation\ key=Transliterate fields for citation key
+
 Deprecated\ fields=Deprecated fields
 
 Shows\ fields\ having\ a\ successor\ in\ biblatex.=Shows fields having a successor in biblatex.