From 7d4bfac3b0dbcc6f916c90837398aff1e84588bc Mon Sep 17 00:00:00 2001 From: Carol Schulze Date: Mon, 16 Jan 2023 11:33:33 -0300 Subject: [PATCH 1/6] Add binds for affected IDs in compiled rules After every write, pine has to rerun all rules from the model to ensure consistency. These rules run over the entire database, sometimes causing some queries to run for too long against what should be a simple write. This commit adds a mechanism to help with this issue by narrowing the set of rows that each rule should touch to those rows that were actually changed. Implementing this mechanism safely is doable and not necessarily complex code-wise, but requires a deep modifications from the current architecture. This commit adds a restricted form instead where we only narrow the rows of the root table that were changed. If any other table was changed then narrowing is a no op. It can be proved that this is always safe as long as the root table is selected from only once and the rule is positive ("It is necessary that each ..."). The implementation here adds a single binding into the rule's SQL query which can be bound by pine for each rule where an opportunity to use this optimization arises. The implementation itself is simple: count how many times the root table is selected from and if it is selected from exactly one, then add a narrowing constraint in the form of: $1 = '{}' OR .id = ANY(CAST($1 AS INTEGER[])) Where $1 will be bound to either '{}', which disables narrowing, or to a list of IDs that were affected by the write. This initial implementation can be extended in the future. Change-type: major Signed-off-by: Carol Schulze --- package.json | 2 +- src/AbstractSQLCompiler.ts | 50 +-- src/referenced-fields.ts | 291 ++++++++++++++++++ test/abstract-sql/schema-checks.ts | 5 + .../schema-informative-reference.ts | 4 + test/abstract-sql/schema-rule-optimization.ts | 1 + test/abstract-sql/schema-rule-to-check.ts | 5 + test/abstract-sql/schema-views.ts | 1 + test/sbvr/pilots.js | 72 +++++ 9 files changed, 411 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index b22826a7..6146e804 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "lodash": "^4.17.21" }, "devDependencies": { - "@balena/lf-to-abstract-sql": "^4.6.0", + "@balena/lf-to-abstract-sql": "^5.0.0", "@balena/lint": "^6.2.1", "@balena/odata-parser": "^2.4.2", "@balena/odata-to-abstract-sql": "^5.8.0", diff --git a/src/AbstractSQLCompiler.ts b/src/AbstractSQLCompiler.ts index bbb6a5e1..21780ca9 100644 --- a/src/AbstractSQLCompiler.ts +++ b/src/AbstractSQLCompiler.ts @@ -21,6 +21,7 @@ import { ReferencedFields, RuleReferencedFields, ModifiedFields, + insertAffectedIdsBinds, } from './referenced-fields'; export type { ReferencedFields, RuleReferencedFields, ModifiedFields }; @@ -187,24 +188,21 @@ export type TextTypeNodes = | ExtractJSONPathAsTextNode | UnknownTypeNodes; -export type SelectQueryNode = [ - 'SelectQuery', - ...Array< - | SelectNode - | FromNode - | InnerJoinNode - | LeftJoinNode - | RightJoinNode - | FullJoinNode - | CrossJoinNode - | WhereNode - | GroupByNode - | HavingNode - | OrderByNode - | LimitNode - | OffsetNode - >, -]; +export type SelectQueryStatementNode = + | SelectNode + | FromNode + | InnerJoinNode + | LeftJoinNode + | RightJoinNode + | FullJoinNode + | CrossJoinNode + | WhereNode + | GroupByNode + | HavingNode + | OrderByNode + | LimitNode + | OffsetNode; +export type SelectQueryNode = ['SelectQuery', ...SelectQueryStatementNode[]]; export type UnionQueryNode = [ 'UnionQuery', // tslint:disable-next-line:array-type typescript fails on a circular reference when `Array` form @@ -238,7 +236,7 @@ export type FromTypeNodes = | FromTypeNode[keyof FromTypeNode] | AliasNode; -type AliasableFromTypeNodes = FromTypeNodes | AliasNode; +export type AliasableFromTypeNodes = FromTypeNodes | AliasNode; export type SelectNode = ['Select', AbstractSqlType[]]; export type FromNode = ['From', AliasableFromTypeNodes]; @@ -396,6 +394,17 @@ export interface AbstractSqlModel { body: string; language: 'plpgsql'; }>; + lfInfo: { + rules: { + [key: string]: LfRuleInfo; + }; + }; +} +export interface LfRuleInfo { + root: { + table: string; + alias: string; + }; } export interface SqlModel { synonyms: { @@ -503,6 +512,8 @@ export const isSelectQueryNode = (n: AbstractSqlType): n is SelectQueryNode => n[0] === 'SelectQuery'; export const isSelectNode = (n: AbstractSqlType): n is SelectNode => n[0] === 'Select'; +export const isWhereNode = (n: AbstractSqlType): n is WhereNode => + n[0] === 'Where'; /** * @@ -873,6 +884,7 @@ CREATE TABLE ${ifNotExistsStr}"${table.name}" ( if (typeof ruleSE !== 'string') { throw new Error('Invalid structured English'); } + insertAffectedIdsBinds(ruleBody, abstractSqlModel.lfInfo.rules[ruleSE]); const { query: ruleSQL, bindings: ruleBindings } = compileRule( ruleBody, engine, diff --git a/src/referenced-fields.ts b/src/referenced-fields.ts index f0b13cf1..474a20d9 100644 --- a/src/referenced-fields.ts +++ b/src/referenced-fields.ts @@ -2,16 +2,55 @@ import * as _ from 'lodash'; import { AbstractSqlQuery, AbstractSqlType, + AddDateDurationNode, + AddDateNumberNode, + AliasNode, + AndNode, + AnyNode, + AverageNode, + CountNode, + CrossJoinNode, + DateTruncNode, EngineInstance, + EqualsNode, + ExistsNode, FieldsNode, + FromNode, + FromTypeNodes, + FullJoinNode, + GreaterThanNode, + GreaterThanOrEqualNode, + HavingNode, + InnerJoinNode, + InNode, isAliasNode, isFromNode, isSelectNode, isSelectQueryNode, isTableNode, + isWhereNode, + LeftJoinNode, + LessThanNode, + LessThanOrEqualNode, + LfRuleInfo, + NotEqualsNode, + NotExistsNode, + NotNode, + OrNode, + RightJoinNode, SelectNode, + SelectQueryNode, + SubtractDateDateNode, + SubtractDateDurationNode, + SubtractDateNumberNode, + SumNode, + TableNode, + ToJSONNode, + UnionQueryNode, + WhereNode, } from './AbstractSQLCompiler'; import { AbstractSQLOptimiser } from './AbstractSQLOptimiser'; +import { isAbstractSqlQuery } from './AbstractSQLRules2SQL'; export interface ReferencedFields { [alias: string]: string[]; @@ -294,3 +333,255 @@ export const getModifiedFields: EngineInstance['getModifiedFields'] = ( return checkQuery(abstractSqlQuery); } }; + +// TS requires this to be a funtion declaration +function assertAbstractSqlIsNotLegacy( + abstractSql: AbstractSqlType, +): asserts abstractSql is AbstractSqlQuery { + if (!isAbstractSqlQuery(abstractSql)) { + throw new Error( + 'cannot introspect into the string form of AbstractSqlQuery', + ); + } +} + +// Find how many times an abstract sql fragment selects from the given table +// TODO: +// - Not all abstract sql nodes are supported here yet but hopefully nothing +// important is missing atm +// - Create missing node types +const countTableSelects = ( + abstractSql: AbstractSqlQuery, + table: string, +): number => { + assertAbstractSqlIsNotLegacy(abstractSql); + let sum = 0; + switch (abstractSql[0]) { + // Unary nodes + case 'Alias': + case 'Any': + case 'Average': + case 'CharacterLength': + case 'CrossJoin': + case 'Exists': + case 'From': + case 'Having': + case 'Not': + case 'NotExists': + case 'Sum': + case 'ToJSON': + case 'Where': + // TODO: `CharacterLength` has no node type defined + const unaryOperation = abstractSql as + | AliasNode + | AnyNode + | AverageNode + | CrossJoinNode + | ExistsNode + | FromNode + | HavingNode + | NotExistsNode + | NotNode + | SumNode + | ToJSONNode + | WhereNode; + assertAbstractSqlIsNotLegacy(unaryOperation[1]); + + return countTableSelects(unaryOperation[1], table); + + // `COUNT` is an unary function but we only support the `COUNT(*)` form + case 'Count': + const countNode = abstractSql as CountNode; + if (countNode[1] !== '*') { + throw new Error('Only COUNT(*) is supported'); + } + + return 0; + + // Binary nodes + case 'AddDateDuration': + case 'AddDateNumber': + case 'DateTrunc': + case 'Equals': + case 'GreaterThan': + case 'GreaterThanOrEqual': + case 'IsDistinctFrom': + case 'IsNotDistinctFrom': + case 'LessThan': + case 'LessThanOrEqual': + case 'NotEquals': + case 'SubtractDateDate': + case 'SubtractDateDuration': + case 'SubtractDateNumber': + // TODO: `IsDistinctFrom` and `IsNotDistinctFrom` have no node + // types defined + const binaryOperation = abstractSql as + | AddDateDurationNode + | AddDateNumberNode + | DateTruncNode + | EqualsNode + | GreaterThanNode + | GreaterThanOrEqualNode + | LessThanNode + | LessThanOrEqualNode + | NotEqualsNode + | SubtractDateDateNode + | SubtractDateDurationNode + | SubtractDateNumberNode; + const leftOperand = binaryOperation[1]; + assertAbstractSqlIsNotLegacy(leftOperand); + const rightOperand = binaryOperation[2]; + assertAbstractSqlIsNotLegacy(rightOperand); + + return ( + countTableSelects(leftOperand, table) + + countTableSelects(rightOperand, table) + ); + + // Binary nodes with optional `ON` second argument + case 'FullJoin': + case 'Join': + case 'LeftJoin': + case 'RightJoin': + const joinNode = abstractSql as + | FullJoinNode + | InnerJoinNode + | LeftJoinNode + | RightJoinNode; + assertAbstractSqlIsNotLegacy(joinNode[1]); + if (joinNode[2] !== undefined) { + assertAbstractSqlIsNotLegacy(joinNode[2][1]); + sum = countTableSelects(joinNode[2][1], table); + } + + return sum + countTableSelects(joinNode[1], table); + + // n-ary nodes + case 'And': + case 'Or': + case 'SelectQuery': + case 'UnionQuery': + const selectQueryNode = abstractSql as + | AndNode + | OrNode + | SelectQueryNode + | UnionQueryNode; + for (const arg of selectQueryNode.slice(1)) { + assertAbstractSqlIsNotLegacy(arg); + sum += countTableSelects(arg, table); + } + + return sum; + + // n-ary nodes but the slice starts at the third argument + case 'In': + case 'NotIn': + // TODO: `NotIn` has no node type defined + const inNode = abstractSql as InNode; + for (const arg of inNode.slice(2)) { + assertAbstractSqlIsNotLegacy(arg); + sum += countTableSelects(arg, table); + } + + return sum; + + // n-ary-like node + case 'Select': + const selectNode = abstractSql as SelectNode; + for (const arg of selectNode[1]) { + assertAbstractSqlIsNotLegacy(arg); + sum += countTableSelects(arg, table); + } + + return sum; + + // Uninteresting atomic nodes + case 'Boolean': + case 'Date': + case 'Duration': + case 'EmbeddedText': + case 'GroupBy': + case 'Integer': + case 'JSON': + case 'Null': + case 'Number': + case 'ReferencedField': + case 'Text': + return 0; + + // The atomic node we're looking for: a table selection + case 'Table': + const tableNode = abstractSql as TableNode; + + if (tableNode[1] === table) { + return 1; + } else { + return 0; + } + + default: + throw new Error(`unknown abstract sql type: ${abstractSql[0]}`); + } +}; + +// TODO: +// - This function only narrows the root table of the rule. This is always +// safe when the root table isn't selected from more than once and it is not +// negated in the LF. Right now we conservatively check for the former but +// not the second. The negative forms (e.g. it is forbidden that ...) are +// not fully supported anyway. +// - Removing multiple candidates selecting from the same database table to +// avoid visibility issues is too conservative. The correct criteria is to +// just remove any that are present in at least 2 disjoint subqueries. +// Because in this case the problem is that in those cases there is not a +// single place in the query that has visibility inside both disjoint +// subqueries and that is a requirement for adding the corrent binds for +// narrowing. +// - We assume the ID column is named "id". +// - This is a very restricted implementation of narrowing which could be +// expanded to cover more situations. +// +// This function modifies `abstractSql` in place. +export const insertAffectedIdsBinds = ( + abstractSql: AbstractSqlQuery, + lfRuleInfo: LfRuleInfo, +) => { + const rootTableSelectCount = countTableSelects( + abstractSql, + lfRuleInfo.root.table, + ); + if (rootTableSelectCount !== 1) { + return; + } + + const narrowing: OrNode = [ + 'Or', + ['Equals', ['Bind', lfRuleInfo.root.table], ['EmbeddedText', '{}']], + [ + 'Equals', + ['ReferencedField', lfRuleInfo.root.alias, 'id'], + ['Any', ['Bind', lfRuleInfo.root.table], 'Integer'], + ], + ]; + + // Assume (but check) that the query is of the form: + // + // SELECT (SELECT COUNT(*) ...) = 0 + if ( + abstractSql[0] !== 'Equals' || + abstractSql[1][0] !== 'SelectQuery' || + abstractSql[2][0] !== 'Number' + ) { + throw new Error( + 'Query is not of the form: SELECT (SELECT COUNT(*) ...) = 0', + ); + } + + const selectQueryNode = abstractSql[1] as SelectQueryNode; + const whereNode = selectQueryNode.slice(1).find(isWhereNode); + if (whereNode === undefined) { + selectQueryNode.push(['Where', narrowing]); + } else { + whereNode[1] = ['And', whereNode[1], narrowing]; + } +}; diff --git a/test/abstract-sql/schema-checks.ts b/test/abstract-sql/schema-checks.ts index 5980b03e..d7746bb6 100644 --- a/test/abstract-sql/schema-checks.ts +++ b/test/abstract-sql/schema-checks.ts @@ -8,6 +8,7 @@ it('an empty abstractSql model should produce an empty schema', () => { relationships: {}, tables: {}, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -36,6 +37,7 @@ it('a single table abstractSql model should produce an appropriate schema', () = }, }, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -70,6 +72,7 @@ it('an abstractSql model with a check on a field should produce an appropriate s }, }, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -107,6 +110,7 @@ describe('check constraints on table level', () => { }, }, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -148,6 +152,7 @@ CREATE TABLE IF NOT EXISTS "test" ( }, }, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') diff --git a/test/abstract-sql/schema-informative-reference.ts b/test/abstract-sql/schema-informative-reference.ts index 87eac6ba..bba808ef 100644 --- a/test/abstract-sql/schema-informative-reference.ts +++ b/test/abstract-sql/schema-informative-reference.ts @@ -74,6 +74,7 @@ describe('generate informative reference schema', () => { relationships: {}, rules: [], synonyms: {}, + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -202,6 +203,7 @@ CREATE TABLE IF NOT EXISTS "term history" ( relationships: {}, rules: [], synonyms: {}, + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -301,6 +303,7 @@ CREATE TABLE IF NOT EXISTS "term history" ( relationships: {}, rules: [], synonyms: {}, + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -393,6 +396,7 @@ CREATE TABLE IF NOT EXISTS "term history" ( relationships: {}, rules: [], synonyms: {}, + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') diff --git a/test/abstract-sql/schema-rule-optimization.ts b/test/abstract-sql/schema-rule-optimization.ts index a3f03783..f4c607de 100644 --- a/test/abstract-sql/schema-rule-optimization.ts +++ b/test/abstract-sql/schema-rule-optimization.ts @@ -74,6 +74,7 @@ it('should convert a basic rule to a check', () => { ['StructuredEnglish', 'Test rule abstract sql optimization'], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('rules') diff --git a/test/abstract-sql/schema-rule-to-check.ts b/test/abstract-sql/schema-rule-to-check.ts index e30e831a..ea9de5f8 100644 --- a/test/abstract-sql/schema-rule-to-check.ts +++ b/test/abstract-sql/schema-rule-to-check.ts @@ -67,6 +67,7 @@ it('should convert a basic rule to a check using NOT EXISTS', () => { ], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -138,6 +139,7 @@ it('should convert a basic rule to a check using COUNT(*) = 0', () => { ], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -228,6 +230,7 @@ it('should correctly shorten a converted check rule with a long name', () => { ], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -301,6 +304,7 @@ it('should work with differing table/resource names using NOT EXISTS', () => { ], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') @@ -372,6 +376,7 @@ it('should work with differing table/resource names using COUNT(*) = 0', () => { ], ], ], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') diff --git a/test/abstract-sql/schema-views.ts b/test/abstract-sql/schema-views.ts index 85741613..0f00f77f 100644 --- a/test/abstract-sql/schema-views.ts +++ b/test/abstract-sql/schema-views.ts @@ -24,6 +24,7 @@ it('a table with a static definition should produce a view', () => { }, }, rules: [], + lfInfo: { rules: {} }, }), ) .to.have.property('createSchema') diff --git a/test/sbvr/pilots.js b/test/sbvr/pilots.js index 1a063dc2..50ccd0e0 100644 --- a/test/sbvr/pilots.js +++ b/test/sbvr/pilots.js @@ -160,6 +160,8 @@ SELECT ( FROM "pilot-can fly-plane" AS "pilot.0-can fly-plane.1" WHERE "pilot.0-can fly-plane.1"."pilot" = "pilot.0"."id" ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -177,6 +179,8 @@ SELECT ( WHERE "pilot.0-can fly-plane.1"."pilot" = "pilot.0"."id" ) >= 2 ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -192,6 +196,8 @@ SELECT ( FROM "pilot-can fly-plane" AS "pilot.0-can fly-plane.1" WHERE "pilot.0-can fly-plane.1"."pilot" = "pilot.0"."id" ) >= 3 + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -209,6 +215,8 @@ SELECT ( ) >= 3 ) AND "pilot.0"."is experienced" != 0 + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -224,6 +232,8 @@ SELECT ( WHERE "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) >= 3 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -242,6 +252,8 @@ SELECT ( AND "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) >= 3 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -259,6 +271,8 @@ SELECT ( AND "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) >= 3 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`; test.rule( 'It is necessary that each plane that at least 3 pilots that are not experienced can fly, has a name', @@ -285,6 +299,8 @@ SELECT ( AND "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) >= 3 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -303,6 +319,8 @@ SELECT ( AND "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) >= 3 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -316,6 +334,8 @@ SELECT ( 0 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -330,6 +350,8 @@ SELECT ( FROM "pilot-can fly-plane" AS "pilot.1-can fly-plane.0" WHERE "pilot.1-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -350,6 +372,8 @@ SELECT ( OR 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL) ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -369,6 +393,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -393,6 +419,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -415,6 +443,8 @@ SELECT ( WHERE "pilot.0-can fly-plane.2"."pilot" = "pilot.0"."id" ) = 1) ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -435,6 +465,8 @@ SELECT ( WHERE "pilot.0-can fly-plane.2"."pilot" = "pilot.0"."id" ) = 1) AND "pilot.0"."is experienced" != 1 + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -478,6 +510,8 @@ SELECT ( WHERE "pilot.2-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) = 1) AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -498,6 +532,8 @@ SELECT ( AND 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -517,6 +553,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -539,6 +577,8 @@ SELECT ( WHERE "pilot.0-can fly-plane.2"."pilot" = "pilot.0"."id" ) = 1 ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -559,6 +599,8 @@ SELECT ( WHERE "pilot.0-can fly-plane.2"."pilot" = "pilot.0"."id" ) = 1 AND "pilot.0"."is experienced" != 1 + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -602,6 +644,8 @@ SELECT ( WHERE "pilot.2-can fly-plane.0"."can fly-plane" = "plane.0"."id" ) = 1 AND "plane.0"."name" IS NULL + AND ($1 = '{}' + OR "plane.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -627,6 +671,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -651,6 +697,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -678,6 +726,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -702,6 +752,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -731,6 +783,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -758,6 +812,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -785,6 +841,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -812,6 +870,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -841,6 +901,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -872,6 +934,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -901,6 +965,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -928,6 +994,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -959,6 +1027,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); @@ -990,6 +1060,8 @@ SELECT ( 5 < "pilot.0"."years of experience" AND "pilot.0"."years of experience" IS NOT NULL ) + AND ($1 = '{}' + OR "pilot.0"."id" = ANY(CAST($1 AS INTEGER[]))) ) = 0 AS "result";`, ); From 6ae79335205dfec28069334ea13b1dbc63d2dbfd Mon Sep 17 00:00:00 2001 From: Balena CI Date: Mon, 13 Feb 2023 14:04:27 +0000 Subject: [PATCH 2/6] v8.0.0 --- .versionbot/CHANGELOG.yml | 47 +++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 4 ++++ package.json | 4 ++-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.versionbot/CHANGELOG.yml b/.versionbot/CHANGELOG.yml index 34b5f9ea..c21ca013 100644 --- a/.versionbot/CHANGELOG.yml +++ b/.versionbot/CHANGELOG.yml @@ -1,3 +1,50 @@ +- commits: + - subject: Add binds for affected IDs in compiled rules + hash: 7d4bfac3b0dbcc6f916c90837398aff1e84588bc + body: | + After every write, pine has to rerun all rules from the model to ensure + consistency. These rules run over the entire database, sometimes causing + some queries to run for too long against what should be a simple write. + + This commit adds a mechanism to help with this issue by narrowing the + set of rows that each rule should touch to those rows that were actually + changed. + + Implementing this mechanism safely is doable and not necessarily complex + code-wise, but requires a deep modifications from the current + architecture. This commit adds a restricted form instead where we only + narrow the rows of the root table that were changed. If any other table + was changed then narrowing is a no op. + + It can be proved that this is always safe as long as the root table is + selected from only once and the rule is positive ("It is necessary that + each ..."). + + The implementation here adds a single binding into the rule's SQL query + which can be bound by pine for each rule where an opportunity to use + this optimization arises. + + The implementation itself is simple: count how many times the root table + is selected from and if it is selected from exactly one, then add a + narrowing constraint in the form of: + + $1 = '{}' OR + .id = ANY(CAST($1 AS INTEGER[])) + + Where $1 will be bound to either '{}', which disables narrowing, or to a + list of IDs that were affected by the write. + + This initial implementation can be extended in the future. + footer: + Change-type: major + change-type: major + Signed-off-by: Carol Schulze + signed-off-by: Carol Schulze + author: Carol Schulze + nested: [] + version: 8.0.0 + title: "" + date: 2023-02-13T14:04:25.744Z - commits: - subject: Optimize schema during compilation hash: e0b6b48c1429077d3c3742c9ee6224233a532bf7 diff --git a/CHANGELOG.md b/CHANGELOG.md index b2b28f92..b2561c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY! This project adheres to [Semantic Versioning](http://semver.org/). +## 8.0.0 - 2023-02-13 + +* Add binds for affected IDs in compiled rules [Carol Schulze] + ## 7.26.0 - 2023-02-07 * Optimize schema during compilation [Carol Schulze] diff --git a/package.json b/package.json index 6146e804..891e94f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@balena/abstract-sql-compiler", - "version": "7.26.0", + "version": "8.0.0", "description": "A translator for abstract sql into sql.", "main": "out/AbstractSQLCompiler.js", "types": "out/AbstractSQLCompiler.d.ts", @@ -58,6 +58,6 @@ ] }, "versionist": { - "publishedAt": "2023-02-07T12:42:59.778Z" + "publishedAt": "2023-02-13T14:04:26.240Z" } } From ad73becc26e0254ab92d2a8f79f7dfed910b47d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ramiro=20Gonz=C3=A1lez=20Maciel?= Date: Sat, 18 Feb 2023 09:09:19 -0300 Subject: [PATCH 3/6] Integrate beta version of sbvr-types supporting WebResource Change-type: patch Signed-off-by: Ramiro Gonzalez --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 891e94f4..eebc4b04 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "repository": "https://github.com/balena-io-modules/abstract-sql-compiler.git", "author": "", "dependencies": { - "@balena/sbvr-types": "^3.4.18", + "@balena/sbvr-types": "3.5.0-build-web-resource-2-de61772c60f6efa92f3a400343bba6b454e303e0-1", "lodash": "^4.17.21" }, "devDependencies": { From 27dad8e40759d1778f5cf1fc34b0d08e0a2db5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ramiro=20Gonz=C3=A1lez=20Maciel?= Date: Sat, 18 Feb 2023 09:11:17 -0300 Subject: [PATCH 4/6] add title --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e35295c2..ca09b9da 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -Convert abstract SQL into dialect-specific SQL. \ No newline at end of file +# abstract-sql-compiler + +Convert abstract SQL into dialect-specific SQL. From c251ef1eb62c7b7476b131a940dca61b6fba85ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ramiro=20Gonz=C3=A1lez=20Maciel?= Date: Sat, 18 Feb 2023 09:13:55 -0300 Subject: [PATCH 5/6] trigger flowzone on non master --- .github/workflows/flowzone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flowzone.yml b/.github/workflows/flowzone.yml index 11e6a889..df790c48 100644 --- a/.github/workflows/flowzone.yml +++ b/.github/workflows/flowzone.yml @@ -3,11 +3,11 @@ name: Flowzone on: pull_request: types: [opened, synchronize, closed] - branches: [main, master] + # branches: [main, master] # allow external contributions to use secrets within trusted code pull_request_target: types: [opened, synchronize, closed] - branches: [main, master] + # branches: [main, master] jobs: flowzone: From 10f0550a917ea44acb285041a29d28f84cec86d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ramiro=20Gonz=C3=A1lez=20Maciel?= Date: Tue, 21 Feb 2023 13:24:39 -0300 Subject: [PATCH 6/6] Integrate beta version of sbvr-types supporting WebResource Change-type: patch Signed-off-by: Ramiro Gonzalez --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eebc4b04..31ec8bdc 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "repository": "https://github.com/balena-io-modules/abstract-sql-compiler.git", "author": "", "dependencies": { - "@balena/sbvr-types": "3.5.0-build-web-resource-2-de61772c60f6efa92f3a400343bba6b454e303e0-1", + "@balena/sbvr-types": "3.5.0-build-web-resource-2-cee49a331a438e988e109722d82dad2a04e88c0b-1", "lodash": "^4.17.21" }, "devDependencies": {