Skip to content

Prototype runtime parameters. #2007

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Prototype runtime parameters. #2007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0a61b27
Prototype runtime parameters.
Ceridan Aug 25, 2025
d0fbdcb
changed param to dynamicVar, added dynamicVars[]
kvlaskarolina Sep 2, 2025
45e58e6
updated rest of files to match {dynamicVar} naming convention
kvlaskarolina Sep 2, 2025
cb1a0ef
draft to convert default values from workflow_settings.ts to protos/c…
kvlaskarolina Sep 4, 2025
e4bfdc6
added <<checking if variables are defined in workflow_settings>>
kvlaskarolina Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion core/actions/assertion.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
/**
* @hidden Stores the generated proto for the compiled graph.
*/
private proto = dataform.Assertion.create();
private proto = dataform.Assertion.create({
dynamicVars: [],
});

/** @hidden We delay contextification until the final compile step, so hold these here for now. */
private contextableQuery: AContextable<string>;
Expand Down Expand Up @@ -334,6 +336,11 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
VerifyProtoErrorBehaviour.SHOW_DOCS_LINK
);
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

/**
Expand Down Expand Up @@ -364,6 +371,11 @@ export class AssertionContext implements IActionContext {
return this.resolve(ref);
}

public dynamicVar(varName: string): string {
this.assertion.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public resolve(ref: Resolvable | string[], ...rest: string[]) {
return this.assertion.session.resolve(ref, ...rest);
}
Expand Down
14 changes: 13 additions & 1 deletion core/actions/data_preparation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
// We delay contextification until the final compile step, so hold these here for now.
public contextableQuery: Contextable<ITableContext, string>;

private proto = dataform.DataPreparation.create();
private proto = dataform.DataPreparation.create({
dynamicVars: [],
});

constructor(
session?: Session,
Expand Down Expand Up @@ -424,6 +426,11 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
}
return columnName;
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

export class DataPreparationContext implements ITableContext {
Expand Down Expand Up @@ -454,6 +461,11 @@ export class DataPreparationContext implements ITableContext {
return this.resolve(ref);
}

public dynamicVar(varName: string): string {
this.dataPreparation.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public resolve(ref: Resolvable | string[], ...rest: string[]) {
return this.dataPreparation.session.resolve(ref, ...rest);
}
Expand Down
4 changes: 3 additions & 1 deletion core/actions/declaration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@ export class Declaration extends ActionBuilder<dataform.Declaration> {
/**
* @hidden Stores the generated proto for the compiled graph.
*/
private proto = dataform.Declaration.create();
private proto = dataform.Declaration.create({
dynamicVars: [],
});

/** @hidden */
constructor(session?: Session, unverifiedConfig?: any, filename?: string) {
Expand Down
15 changes: 13 additions & 2 deletions core/actions/incremental_table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ export class IncrementalTable extends ActionBuilder<dataform.Table> {
type: "incremental",
enumType: dataform.TableType.INCREMENTAL,
disabled: false,
tags: []
tags: [],
dynamicVars: [],
});

/** @hidden */
Expand Down Expand Up @@ -490,7 +491,7 @@ export class IncrementalTable extends ActionBuilder<dataform.Table> {

/** @hidden */
public compile() {
const context = new IncrementalTableContext(this);
const context = new IncrementalTableContext(this);f
const incrementalContext = new IncrementalTableContext(this, true);

this.proto.query = context.apply(this.contextableQuery);
Expand Down Expand Up @@ -647,6 +648,11 @@ export class IncrementalTable extends ActionBuilder<dataform.Table> {
throw new Error(`OnSchemaChange value "${onSchemaChange}" is not supported`);
}
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

/**
Expand All @@ -663,6 +669,11 @@ export class IncrementalTableContext implements ITableContext {
return this.incrementalTable.session.finalizeName(this.incrementalTable.getTarget().name);
}

public dynamicVar(varName: string): string {
this.incrementalTable.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public ref(ref: Resolvable | string[], ...rest: string[]): string {
ref = toResolvable(ref, rest);
if (!resolvableAsTarget(ref)) {
Expand Down
4 changes: 3 additions & 1 deletion core/actions/notebook.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
/**
* @hidden Stores the generated proto for the compiled graph.
*/
private proto = dataform.Notebook.create();
private proto = dataform.Notebook.create({
dynamicVars: [],
});

/** @hidden */
constructor(session?: Session, unverifiedConfig?: any, configPath?: string) {
Expand Down
14 changes: 13 additions & 1 deletion core/actions/operation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
/**
* @hidden Stores the generated proto for the compiled graph.
*/
private proto = dataform.Operation.create();
private proto = dataform.Operation.create({
dynamicVars: [],
});

/** @hidden We delay contextification until the final compile step, so hold these here for now. */
private contextableQueries: Contextable<IActionContext, string | string[]>;
Expand Down Expand Up @@ -395,6 +397,11 @@ export class Operation extends ActionBuilder<dataform.Operation> {
VerifyProtoErrorBehaviour.SHOW_DOCS_LINK
);
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

/**
Expand Down Expand Up @@ -425,6 +432,11 @@ export class OperationContext implements IActionContext {
return this.resolve(ref);
}

public dynamicVar(varName: string): string {
this.operation.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public resolve(ref: Resolvable | string[], ...rest: string[]) {
return this.operation.session.resolve(ref, ...rest);
}
Expand Down
13 changes: 12 additions & 1 deletion core/actions/table.ts
View file
Open in desktop
Copy link

@kvlaskarolina kvlaskarolina Sep 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also add 'inputParams: [],' in proto in core/actions/incremental_table.ts?

Copy link
Contributor Author

@Ceridan Ceridan Sep 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it would be better to define a default as empty array on every action.

Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ export class Table extends ActionBuilder<dataform.Table> {
type: "table",
enumType: dataform.TableType.TABLE,
disabled: false,
tags: []
tags: [],
dynamicVars: [],
});

/** @hidden */
Expand Down Expand Up @@ -595,6 +596,11 @@ export class Table extends ActionBuilder<dataform.Table> {

return config;
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

/**
Expand All @@ -611,6 +617,11 @@ export class TableContext implements ITableContext {
return this.table.session.finalizeName(this.table.getTarget().name);
}

public dynamicVar(varName: string): string {
this.table.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public ref(ref: Resolvable | string[], ...rest: string[]): string {
ref = toResolvable(ref, rest);
if (!resolvableAsTarget(ref)) {
Expand Down
16 changes: 14 additions & 2 deletions core/actions/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@ export class Test extends ActionBuilder<dataform.Test> {
/**
* @hidden Stores the generated proto for the compiled graph.
*/
private proto = dataform.Test.create();
private proto = dataform.Test.create({
dynamicVars: [],
});

/** @hidden */
constructor(session?: Session, config?: ITestConfig) {
Expand Down Expand Up @@ -192,6 +194,11 @@ export class Test extends ActionBuilder<dataform.Test> {
VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM
);
}
public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}
}

/** @hidden */
Expand Down Expand Up @@ -221,7 +228,12 @@ class RefReplacingContext implements ITableContext {
public ref(ref: Resolvable | string[], ...rest: string[]) {
return this.resolve(ref, ...rest);
}


public dynamicVar(varName: string): string {
this.testContext.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public resolve(ref: Resolvable | string[], ...rest: string[]) {
const target = resolvableAsTarget(toResolvable(ref, rest));
if (!this.testContext.test.contextableInputs.has(targetStringifier.stringify(target))) {
Expand Down
14 changes: 13 additions & 1 deletion core/actions/view.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ export class View extends ActionBuilder<dataform.Table> {
type: "view",
enumType: dataform.TableType.VIEW,
disabled: false,
tags: []
tags: [],
dynamicVars: [],
});

/** @hidden */
Expand Down Expand Up @@ -486,6 +487,12 @@ export class View extends ActionBuilder<dataform.Table> {
return dataform.Target.create(this.proto.target);
}

public addInputDynamicVar(varName: string) {
if (!this.proto.dynamicVars.includes(varName)) {
this.proto.dynamicVars.push(varName);
}
}

/** @hidden */
public compile() {
const context = new ViewContext(this);
Expand Down Expand Up @@ -650,6 +657,11 @@ export class ViewContext implements ITableContext {
return this.resolve(ref);
}

public dynamicVar(varName: string): string {
this.view.addInputDynamicVar(varName);
return `\{${varName}\}`;
}

public resolve(ref: Resolvable | string[], ...rest: string[]) {
return this.view.session.resolve(ref, ...rest);
}
Expand Down
3 changes: 2 additions & 1 deletion core/compilers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ const CONTEXT_FUNCTIONS = [
"when",
"incremental",
"schema",
"database"
"database",
"dynamicVar",
]
.map(name => `const ${name} = ctx.${name} ? ctx.${name}.bind(ctx) : undefined;`)
.join("\n");
Expand Down
2 changes: 2 additions & 0 deletions core/contextables.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ export interface IActionContext {
* Returns the database of this dataset, if applicable.
*/
database: () => string;

dynamicVar: (varName: string) => string;
}

/**
Expand Down
40 changes: 40 additions & 0 deletions core/main_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,46 @@ suite("@dataform/core", ({ afterEach }) => {
const tmpDirFixture = new TmpDirFixture(afterEach);

suite("session", () => {
test("dynamic variable resolved correctly", () => {
const projectDir = tmpDirFixture.createNewTmpDir();
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
VALID_WORKFLOW_SETTINGS_YAML
);
fs.mkdirSync(path.join(projectDir, "definitions"));
fs.writeFileSync(
path.join(projectDir, "definitions/view.sqlx"),
`
config { type: "view" }
SELECT 1 AS col WHERE \${dynamicVar("myvar")} == 1`
);

const result = runMainInVm(coreExecutionRequestFromPath(projectDir));

expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([
{
canonicalTarget: {
database: "defaultProject",
name: "view",
schema: "defaultDataset"
},
disabled: false,
enumType: "VIEW",
fileName: "definitions/view.sqlx",
hermeticity: "NON_HERMETIC",
query: "\n\nSELECT 1 AS col WHERE {myvar} == 1",
dynamicVars: ["myvar"],
target: {
database: "defaultProject",
name: "view",
schema: "defaultDataset"
},
type: "view",
}
]);
});

suite("resolve succeeds", () => {
[
WorkflowSettingsTemplates.bigquery,
Expand Down
Loading