From 7f70acf00a813b8a6b8be068dbfe2fb5649615c8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 30 Jan 2025 14:16:49 +0000 Subject: [PATCH 1/8] editor: handle @local typedefs --- assets/js/editor/Editor.tsx | 61 ++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index 9f3e117edc..a9975387e7 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -3,10 +3,44 @@ import type { editor } from 'monaco-editor'; import type { EditorProps as MonacoProps } from '@monaco-editor/react'; import { MonacoEditor, type Monaco } from '../monaco'; -import { fetchDTSListing, fetchFile } from '@openfn/describe-package'; +import * as describe from '@openfn/describe-package'; import createCompletionProvider from './magic-completion'; import { initiateSaveAndRun } from '../common'; +// export async function* fetchDTSListing(packageName: string) { +// for (const f of await fetchFileListing(packageName)) { +// if (dtsExtension.test(f)) { +// yield f; +// } +// } +// } +async function* fetchDTSListing(specifier: string) { + if (specifier.endsWith('@local')) { + const lang = specifier + .replace('@local', '') + .replace('@openfn/language-', ''); + const url = `http://localhost:5000/packages/${lang}/types`; + const r = await fetch(url, { headers: { Accept: 'application/json' } }); + const json = await r.json(); + for (const f of json.files) { + yield `/types/${f.base}`; + } + } else { + return describe.fetchDTSListing(specifier); + } +} +const fetchFile = async (path: string) => { + if (path.includes('@local')) { + path = path.replace('@openfn/language-', '').replace('@local', ''); + const url = `http://localhost:5000/packages/${path}`; + return fetch(url, { headers: { Accept: 'text/plain' } }).then(r => + r.text() + ); + } else { + return describe.fetchFile(path); + } +}; + // static imports for core lib import dts_es5 from './lib/es5.min.dts'; @@ -95,6 +129,8 @@ type Lib = { }; async function loadDTS(specifier: string): Promise { + const useLocal = specifier.endsWith('@local'); + // Work out the module name from the specifier // (his gets a bit tricky with @openfn/ module names) const nameParts = specifier.split('@'); @@ -107,16 +143,18 @@ async function loadDTS(specifier: string): Promise { // TODO maybe we need other dependencies too? collections? if (name !== '@openfn/language-common') { const pkg = await fetchFile(`${specifier}/package.json`); - const commonVersion = JSON.parse(pkg || '{}').dependencies?.[ - '@openfn/language-common' - ]; - - // jsDeliver doesn't appear to support semver range syntax (^1.0.0, 1.x, ~1.1.0) - const commonVersionMatch = commonVersion?.match(/^\d+\.\d+\.\d+/); - if (!commonVersionMatch) { - console.warn( - `@openfn/language-common@${commonVersion} contains semver range syntax.` - ); + const commonVersion = useLocal + ? 'local' + : JSON.parse(pkg || '{}').dependencies?.['@openfn/language-common']; + + if (!useLocal) { + // jsDeliver doesn't appear to support semver range syntax (^1.0.0, 1.x, ~1.1.0) + const commonVersionMatch = commonVersion?.match(/^\d+\.\d+\.\d+/); + if (!commonVersionMatch) { + console.warn( + `@openfn/language-common@${commonVersion} contains semver range syntax.` + ); + } } const commonSpecifier = `@openfn/language-common@${commonVersion.replace( @@ -128,6 +166,7 @@ async function loadDTS(specifier: string): Promise { // Load every common typedef into the common module let content = await fetchFile(`${commonSpecifier}${filePath}`); content = content.replace(/\* +@(.+?)\*\//gs, '*/'); + console.log(content); results.push({ content: `declare module '@openfn/language-common' { ${content} }`, }); From 3ae97aae4cbabdea81d712251ef77e75de6c45f4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 30 Jan 2025 14:23:08 +0000 Subject: [PATCH 2/8] tidy --- assets/js/editor/Editor.tsx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index a9975387e7..95705caec2 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -7,13 +7,6 @@ import * as describe from '@openfn/describe-package'; import createCompletionProvider from './magic-completion'; import { initiateSaveAndRun } from '../common'; -// export async function* fetchDTSListing(packageName: string) { -// for (const f of await fetchFileListing(packageName)) { -// if (dtsExtension.test(f)) { -// yield f; -// } -// } -// } async function* fetchDTSListing(specifier: string) { if (specifier.endsWith('@local')) { const lang = specifier From fc441af3a13e3e3a42d7ba033f4e0e5d5f0fa58e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 26 Feb 2025 16:08:08 +0000 Subject: [PATCH 3/8] editor: fix type tracking issue --- assets/js/editor/Editor.tsx | 22 ++++++++++++++-------- assets/js/editor/lib/es5.dts.js | 5 +++++ assets/js/editor/lib/es5.min.dts.js | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index 95705caec2..a837590d1c 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -158,11 +158,12 @@ async function loadDTS(specifier: string): Promise { if (!filePath.startsWith('node_modules')) { // Load every common typedef into the common module let content = await fetchFile(`${commonSpecifier}${filePath}`); - content = content.replace(/\* +@(.+?)\*\//gs, '*/'); - console.log(content); - results.push({ - content: `declare module '@openfn/language-common' { ${content} }`, - }); + if (!content.startsWith('')) { + content = content.replace(/\* +@(.+?)\*\//gs, '*/'); + results.push({ + content: `declare module '@openfn/language-common' { ${content} }`, + }); + } } } } @@ -176,6 +177,13 @@ async function loadDTS(specifier: string): Promise { for await (const filePath of fetchDTSListing(specifier)) { if (!filePath.startsWith('node_modules')) { let content = await fetchFile(`${specifier}${filePath}`); + if (content.match(//i)) { + continue; + } + // Convert relative paths + content = content + .replace(/from '\.\//g, `from '${name}/`) + .replace(/import '\.\//g, `import '${name}/`); // Remove js doc annotations // this regex means: find a * then an @ (with 1+ space in between), then match everything up to a closing comment */ @@ -186,9 +194,6 @@ async function loadDTS(specifier: string): Promise { // Import the index as the global namespace - but take care to convert all paths to absolute if (fileName === 'index' || fileName === 'Adaptor') { - content = content.replace(/from '\.\//g, `from '${name}/`); - content = content.replace(/import '\.\//g, `import '${name}/`); - // It turns out that "export * as " seems to straight up not work in Monaco // So this little hack will refactor import statements in a way that works content = content.replace( @@ -218,6 +223,7 @@ async function loadDTS(specifier: string): Promise { // This is basically a hack to work around https://github.com/OpenFn/lightning/issues/2641 // If we find a types.d.ts, append it to every other file adaptorDefs = adaptorDefs.map(def => def.replace('{{$TYPES}}', types)); + console.log(adaptorDefs); results.push( ...adaptorDefs.map(content => ({ diff --git a/assets/js/editor/lib/es5.dts.js b/assets/js/editor/lib/es5.dts.js index b27a84f7e3..6df92c65bc 100644 --- a/assets/js/editor/lib/es5.dts.js +++ b/assets/js/editor/lib/es5.dts.js @@ -1,5 +1,7 @@ // https://github.com/microsoft/TypeScript/blob/main/lib/lib.es5.d.ts export default ` + + interface Array { /** * Gets or sets the length of the array. This is a number one higher than the highest index in the array. @@ -178,6 +180,9 @@ interface Array { */ reduceRight(callbackfn: (previousValue: U, currentValue: T, currentIndex: number, array: T[]) => U, initialValue: U): U; + // JC copied out of VSC core. Needed to support union types apparently + copyWithin(target: number, start: number, end?: number): this; + // JC note that this is the only thing we actually need to get the types to track // we could ditch the rest of the definition (if no-one wants to code-complete array.length) [n: number]: T; diff --git a/assets/js/editor/lib/es5.min.dts.js b/assets/js/editor/lib/es5.min.dts.js index 1f66891314..cfb74845ec 100644 --- a/assets/js/editor/lib/es5.min.dts.js +++ b/assets/js/editor/lib/es5.min.dts.js @@ -1,6 +1,7 @@ // Super minimal array definition to support typings export default ` interface Array { + copyWithin(target: number, start: number, end?: number): this; [n: number]: T; } From 9393daa99342d4019955975a110df648115b2116 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 26 Feb 2025 16:09:07 +0000 Subject: [PATCH 4/8] remove log line --- assets/js/editor/Editor.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index a837590d1c..709accec8a 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -223,7 +223,6 @@ async function loadDTS(specifier: string): Promise { // This is basically a hack to work around https://github.com/OpenFn/lightning/issues/2641 // If we find a types.d.ts, append it to every other file adaptorDefs = adaptorDefs.map(def => def.replace('{{$TYPES}}', types)); - console.log(adaptorDefs); results.push( ...adaptorDefs.map(content => ({ From 1a04c2bf0fc991f1288217c426259bf4214681e2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 26 Feb 2025 16:11:13 +0000 Subject: [PATCH 5/8] standardise regex --- assets/js/editor/Editor.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index 709accec8a..219ae98c2f 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -158,7 +158,7 @@ async function loadDTS(specifier: string): Promise { if (!filePath.startsWith('node_modules')) { // Load every common typedef into the common module let content = await fetchFile(`${commonSpecifier}${filePath}`); - if (!content.startsWith('')) { + if (!content.match(//i)) { content = content.replace(/\* +@(.+?)\*\//gs, '*/'); results.push({ content: `declare module '@openfn/language-common' { ${content} }`, From 8dc2f5e697bf555bc563d7324be7503e7c9c77ee Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 26 Feb 2025 16:13:02 +0000 Subject: [PATCH 6/8] slightly improve URL handling --- assets/js/editor/Editor.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/assets/js/editor/Editor.tsx b/assets/js/editor/Editor.tsx index 219ae98c2f..2dded44e24 100644 --- a/assets/js/editor/Editor.tsx +++ b/assets/js/editor/Editor.tsx @@ -7,12 +7,14 @@ import * as describe from '@openfn/describe-package'; import createCompletionProvider from './magic-completion'; import { initiateSaveAndRun } from '../common'; +const LOCAL_ADAPTORS_ROOT = 'http://localhost:5000'; + async function* fetchDTSListing(specifier: string) { if (specifier.endsWith('@local')) { const lang = specifier .replace('@local', '') .replace('@openfn/language-', ''); - const url = `http://localhost:5000/packages/${lang}/types`; + const url = `${LOCAL_ADAPTORS_ROOT}/packages/${lang}/types`; const r = await fetch(url, { headers: { Accept: 'application/json' } }); const json = await r.json(); for (const f of json.files) { @@ -25,7 +27,7 @@ async function* fetchDTSListing(specifier: string) { const fetchFile = async (path: string) => { if (path.includes('@local')) { path = path.replace('@openfn/language-', '').replace('@local', ''); - const url = `http://localhost:5000/packages/${path}`; + const url = `${LOCAL_ADAPTORS_ROOT}/packages/${path}`; return fetch(url, { headers: { Accept: 'text/plain' } }).then(r => r.text() ); From 79a66d5746dbbc2652afc61a78fc1b8ed218b11f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 26 Feb 2025 16:25:43 +0000 Subject: [PATCH 7/8] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98acf838ef..8f55c50d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to - Allow workflow and project concurrency progress windows [#2995](https://github.com/OpenFn/lightning/issues/2995) +- Support for loading local type definitions when using the adaptors monorepo. ### Changed From c6478d80435e9653ec6b166d4db8a0232ffa72d5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 21 Mar 2025 10:21:12 +0000 Subject: [PATCH 8/8] revert lib stuff --- assets/js/editor/lib/es5.dts.js | 5 ----- assets/js/editor/lib/es5.min.dts.js | 1 - 2 files changed, 6 deletions(-) diff --git a/assets/js/editor/lib/es5.dts.js b/assets/js/editor/lib/es5.dts.js index 6df92c65bc..b27a84f7e3 100644 --- a/assets/js/editor/lib/es5.dts.js +++ b/assets/js/editor/lib/es5.dts.js @@ -1,7 +1,5 @@ // https://github.com/microsoft/TypeScript/blob/main/lib/lib.es5.d.ts export default ` - - interface Array { /** * Gets or sets the length of the array. This is a number one higher than the highest index in the array. @@ -180,9 +178,6 @@ interface Array { */ reduceRight(callbackfn: (previousValue: U, currentValue: T, currentIndex: number, array: T[]) => U, initialValue: U): U; - // JC copied out of VSC core. Needed to support union types apparently - copyWithin(target: number, start: number, end?: number): this; - // JC note that this is the only thing we actually need to get the types to track // we could ditch the rest of the definition (if no-one wants to code-complete array.length) [n: number]: T; diff --git a/assets/js/editor/lib/es5.min.dts.js b/assets/js/editor/lib/es5.min.dts.js index cfb74845ec..1f66891314 100644 --- a/assets/js/editor/lib/es5.min.dts.js +++ b/assets/js/editor/lib/es5.min.dts.js @@ -1,7 +1,6 @@ // Super minimal array definition to support typings export default ` interface Array { - copyWithin(target: number, start: number, end?: number): this; [n: number]: T; }