diff --git a/src/core/annotation.js b/src/core/annotation.js
index f2f642b5008d5..53610f7ec34f8 100644
--- a/src/core/annotation.js
+++ b/src/core/annotation.js
@@ -1278,6 +1278,8 @@ class Annotation {
 
     const text = [];
     const buffer = [];
+    let firstPositionX = Infinity;
+    let firstPositionY = Infinity;
     let firstPosition = null;
     const sink = {
       desiredSize: Math.Infinity,
@@ -1288,7 +1290,8 @@ class Annotation {
           if (item.str === undefined) {
             continue;
           }
-          firstPosition ||= item.transform.slice(-2);
+          firstPositionX = Math.min(firstPositionX, item.transform[4]);
+          firstPositionY = Math.min(firstPositionY, item.transform[5]);
           buffer.push(item.str);
           if (item.hasEOL) {
             text.push(buffer.join("").trimEnd());
@@ -1309,6 +1312,10 @@ class Annotation {
     });
     this.reset();
 
+    if (firstPositionX !== Infinity) {
+      firstPosition = [firstPositionX, firstPositionY];
+    }
+
     if (buffer.length) {
       text.push(buffer.join("").trimEnd());
     }
diff --git a/src/core/default_appearance.js b/src/core/default_appearance.js
index ad49f187e85a3..04e0259d98b87 100644
--- a/src/core/default_appearance.js
+++ b/src/core/default_appearance.js
@@ -26,6 +26,7 @@ import {
   LINE_FACTOR,
   OPS,
   shadow,
+  Util,
   warn,
 } from "../shared/util.js";
 import { ColorSpaceUtils } from "./colorspace_utils.js";
@@ -145,7 +146,8 @@ class AppearanceStreamEvaluator extends EvaluatorPreprocessor {
             result = stack.pop() || result;
             break;
           case OPS.setTextMatrix:
-            result.scaleFactor *= Math.hypot(args[0], args[1]);
+            const tm = Util.transform(this.stateManager.state.ctm, args);
+            result.scaleFactor *= Math.hypot(tm[0], tm[1]);
             break;
           case OPS.setFont:
             const [fontName, fontSize] = args;
diff --git a/test/integration/freetext_editor_spec.mjs b/test/integration/freetext_editor_spec.mjs
index b8a5cb4e6ba81..cb1b0e1d2519f 100644
--- a/test/integration/freetext_editor_spec.mjs
+++ b/test/integration/freetext_editor_spec.mjs
@@ -1799,6 +1799,60 @@ describe("FreeText Editor", () => {
     });
   });
 
+  describe("FreeText (open existing generated with Cairo)", () => {
+    let pages;
+
+    beforeEach(async () => {
+      pages = await loadAndWait(
+        "issue20504.pdf",
+        ".annotationEditorLayer",
+        100
+      );
+    });
+
+    afterEach(async () => {
+      await closePages(pages);
+    });
+
+    it("must open some existing annotations", async () => {
+      await Promise.all(
+        pages.map(async ([browserName, page]) => {
+          const boxes = [];
+          for (const num of [48, 49, 50, 51, 52]) {
+            const id = `${num}R`;
+            await page.waitForSelector(getAnnotationSelector(id), {
+              visible: true,
+            });
+            const rect = await getRect(page, getAnnotationSelector(id));
+            boxes.push(rect);
+          }
+
+          await switchToFreeText(page);
+
+          for (let i = 0; i < boxes.length; i++) {
+            const rect = await getRect(
+              page,
+              `#pdfjs_internal_editor_${i}-editor`
+            );
+
+            expect(Math.abs(rect.width - boxes[i].width) <= 20)
+              .withContext(`In ${browserName}`)
+              .toEqual(true);
+            expect(Math.abs(rect.height - boxes[i].height) <= 20)
+              .withContext(`In ${browserName}`)
+              .toEqual(true);
+            expect(Math.abs(rect.x - boxes[i].x) <= 20)
+              .withContext(`In ${browserName}`)
+              .toEqual(true);
+            expect(Math.abs(rect.y - boxes[i].y) <= 20)
+              .withContext(`In ${browserName}`)
+              .toEqual(true);
+          }
+        })
+      );
+    });
+  });
+
   describe("Keyboard shortcuts when the editor layer isn't focused", () => {
     let pages;
 
diff --git a/test/pdfs/.gitignore b/test/pdfs/.gitignore
index acd3a66ad0020..02a0a4cfaab3e 100644
--- a/test/pdfs/.gitignore
+++ b/test/pdfs/.gitignore
@@ -765,3 +765,4 @@
 !two_paragraphs.pdf
 !paragraph_and_link.pdf
 !issue20225.pdf
+!issue20504.pdf
diff --git a/test/pdfs/issue20504.pdf b/test/pdfs/issue20504.pdf
new file mode 100644
index 0000000000000..7edea68000eb0
Binary files /dev/null and b/test/pdfs/issue20504.pdf differ