diff --git a/package-lock.json b/package-lock.json index c4a1bc368d..454f0da5e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -153,7 +153,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.21.3.tgz", "integrity": "sha512-qIJONzoa/qiHghnm0l1n4i/6IIziDpzqc36FBs4pzMhDUraHqponwJLiAKm1hGLP3OSB/TVNz6rMwVGpwxxySw==", "dev": true, - "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.18.6", @@ -1533,7 +1532,6 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -2880,7 +2878,6 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.55.0.tgz", "integrity": "sha512-ppvmeF7hvdhUUZWSd2EEWfzcFkjJzgNQzVST22nzg958CR+sphy8A6K7LXQZd6V75m1VKjp+J4g/PCEfSCmzhw==", "dev": true, - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.55.0", "@typescript-eslint/types": "5.55.0", @@ -3096,7 +3093,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", "dev": true, - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3759,7 +3755,6 @@ "url": "https://tidelift.com/funding/github/npm/browserslist" } ], - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001449", "electron-to-chromium": "^1.4.284", @@ -5311,7 +5306,6 @@ "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==", "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, - "peer": true, "dependencies": { "@babel/code-frame": "7.12.11", "@eslint/eslintrc": "^0.4.3", @@ -5489,7 +5483,6 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz", "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==", "dev": true, - "peer": true, "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", @@ -5564,7 +5557,6 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-8.0.1.tgz", "integrity": "sha512-ZjOjbjEi6jd82rIpFSgagv4CHWzG9xsQAVp1ZPlhRnnYxcTgENUVBvhYmkQ7GvT1QFijUSo69RaiOJKhMu6i8w==", "dev": true, - "peer": true, "dependencies": { "eslint-plugin-es": "^1.3.1", "eslint-utils": "^1.3.1", @@ -5615,7 +5607,6 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.3.1.tgz", "integrity": "sha512-bY2sGqyptzFBDLh/GMbAxfdJC+b0f23ME63FOE4+Jao0oZ3E1LEwFtWJX/1pGMJLiTtrSSern2CRM/g+dfc0eQ==", "dev": true, - "peer": true, "engines": { "node": ">=6" } @@ -5639,7 +5630,6 @@ "url": "https://feross.org/support" } ], - "peer": true, "peerDependencies": { "eslint": ">=5.0.0" } @@ -8999,7 +8989,6 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -11411,7 +11400,6 @@ "dev": true, "inBundle": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12934,7 +12922,6 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-1.32.1.tgz", "integrity": "sha512-/2HA0Ec70TvQnXdzynFffkjA6XN+1e2pEv/uKS5Ulca40g2L7KuOE3riasHoNVHOsFD5KKZgDsMk1CP3Tw9s+A==", "dev": true, - "peer": true, "dependencies": { "@types/estree": "*", "@types/node": "*", @@ -13173,7 +13160,6 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -15152,7 +15138,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -15509,7 +15494,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true, - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/filters/array.ts b/src/filters/array.ts index f22fcce20f..70de8904fa 100644 --- a/src/filters/array.ts +++ b/src/filters/array.ts @@ -125,7 +125,7 @@ function * filter (this: FilterImpl, include: boolean, arr: T[ const values: unknown[] = [] arr = toArray(arr) this.context.memoryLimit.use(arr.length) - const token = new Tokenizer(stringify(property)).readScopeValue() + const token = new Tokenizer(stringify(property), this.liquid).readScopeValue() for (const item of arr) { values.push(yield evalToken(token, this.context.spawn(item))) } @@ -166,7 +166,7 @@ export function * reject_exp (this: FilterImpl, arr: T[], item export function * group_by (this: FilterImpl, arr: T[], property: string): IterableIterator { const map = new Map() arr = toEnumerable(arr) - const token = new Tokenizer(stringify(property)).readScopeValue() + const token = new Tokenizer(stringify(property), this.liquid).readScopeValue() this.context.memoryLimit.use(arr.length) for (const item of arr) { const key = yield evalToken(token, this.context.spawn(item)) @@ -192,7 +192,7 @@ export function * group_by_exp (this: FilterImpl, arr: T[], it } function * search (this: FilterImpl, arr: T[], property: string, expected: string): IterableIterator { - const token = new Tokenizer(stringify(property)).readScopeValue() + const token = new Tokenizer(stringify(property), this.liquid).readScopeValue() const array = toArray(arr) const matcher = expectedMatcher.call(this, expected) for (let index = 0; index < array.length; index++) { diff --git a/src/liquid-options.ts b/src/liquid-options.ts index a1e0620887..a0ec2bd4f3 100644 --- a/src/liquid-options.ts +++ b/src/liquid-options.ts @@ -90,6 +90,8 @@ export interface LiquidOptions { renderLimit?: number; /** For DoS handling, limit new objects creation, including array concat/join/strftime, etc. A typical PC can handle 1e9 (1G) memory without issue. */ memoryLimit?: number; + /** Allow parenthesized expressions as operands in conditions and loops, e.g. `{% if (foo | upcase) == "BAR" %}`. This is a non-standard extension to Liquid. Defaults to `false`. */ + groupedExpressions?: boolean; } export interface RenderOptions { @@ -162,6 +164,7 @@ export interface NormalizedFullOptions extends NormalizedOptions { parseLimit: number; renderLimit: number; memoryLimit: number; + groupedExpressions: boolean; } export const defaultOptions: NormalizedFullOptions = { @@ -197,7 +200,8 @@ export const defaultOptions: NormalizedFullOptions = { operators: defaultOperators, memoryLimit: Infinity, parseLimit: Infinity, - renderLimit: Infinity + renderLimit: Infinity, + groupedExpressions: false } export function normalize (options: LiquidOptions): NormalizedFullOptions { diff --git a/src/parser/parser.ts b/src/parser/parser.ts index f61f2323ee..2f5fe17656 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -33,7 +33,7 @@ export class Parser { public parse (html: string, filepath?: string): Template[] { html = String(html) this.parseLimit.use(html.length) - const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath) + const tokenizer = new Tokenizer(html, this.liquid, this.liquid.options.operators, filepath) const tokens = tokenizer.readTopLevelTokens(this.liquid.options) return this.parseTokens(tokens) } diff --git a/src/parser/tokenizer.spec.ts b/src/parser/tokenizer.spec.ts index c8d4f6311f..5e063c915a 100644 --- a/src/parser/tokenizer.spec.ts +++ b/src/parser/tokenizer.spec.ts @@ -1,42 +1,45 @@ -import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken } from '../tokens' +import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken, FilteredValueToken } from '../tokens' import { Tokenizer } from './tokenizer' import { defaultOperators } from '../render/operator' import { createTrie } from '../util/operator-trie' +import { Liquid } from '../liquid' +import { TokenizationError } from '../util' describe('Tokenizer', function () { + const liquid = new Liquid() it('should read quoted', () => { - expect(new Tokenizer('"foo" ff').readQuoted()!.getText()).toBe('"foo"') - expect(new Tokenizer(' "foo"ff').readQuoted()!.getText()).toBe('"foo"') + expect(new Tokenizer('"foo" ff', liquid).readQuoted()!.getText()).toBe('"foo"') + expect(new Tokenizer(' "foo"ff', liquid).readQuoted()!.getText()).toBe('"foo"') }) it('should read value', () => { - expect(new Tokenizer('a[ b][ "c d" ]').readValueOrThrow().getText()).toBe('a[ b][ "c d" ]') - expect(new Tokenizer('a.b[c[d.e]]').readValueOrThrow().getText()).toBe('a.b[c[d.e]]') + expect(new Tokenizer('a[ b][ "c d" ]', liquid).readValueOrThrow().getText()).toBe('a[ b][ "c d" ]') + expect(new Tokenizer('a.b[c[d.e]]', liquid).readValueOrThrow().getText()).toBe('a.b[c[d.e]]') }) it('should read identifier', () => { - expect(new Tokenizer('foo bar').readIdentifier()).toHaveProperty('content', 'foo') + expect(new Tokenizer('foo bar', liquid).readIdentifier()).toHaveProperty('content', 'foo') // eslint-disable-next-line deprecation/deprecation - expect(new Tokenizer('foo bar').readWord()).toHaveProperty('content', 'foo') + expect(new Tokenizer('foo bar', liquid).readWord()).toHaveProperty('content', 'foo') }) it('should read integer number', () => { - const token: NumberToken = new Tokenizer('123').readValueOrThrow() as any + const token: NumberToken = new Tokenizer('123', liquid).readValueOrThrow() as any expect(token).toBeInstanceOf(NumberToken) expect(token.getText()).toBe('123') expect(token.content).toBe(123) }) it('should read negative number', () => { - const token: NumberToken = new Tokenizer('-123').readValueOrThrow() as any + const token: NumberToken = new Tokenizer('-123', liquid).readValueOrThrow() as any expect(token).toBeInstanceOf(NumberToken) expect(token.getText()).toBe('-123') expect(token.content).toBe(-123) }) it('should read float number', () => { - const token: NumberToken = new Tokenizer('1.23').readValueOrThrow() as any + const token: NumberToken = new Tokenizer('1.23', liquid).readValueOrThrow() as any expect(token).toBeInstanceOf(NumberToken) expect(token.getText()).toBe('1.23') expect(token.content).toBe(1.23) }) it('should treat 1.2.3 as property read', () => { - const token: PropertyAccessToken = new Tokenizer('1.2.3').readValueOrThrow() as any + const token: PropertyAccessToken = new Tokenizer('1.2.3', liquid).readValueOrThrow() as any expect(token).toBeInstanceOf(PropertyAccessToken) expect(token.props).toHaveLength(3) expect(token.props[0].getText()).toBe('1') @@ -44,33 +47,33 @@ describe('Tokenizer', function () { expect(token.props[2].getText()).toBe('3') }) it('should read quoted value', () => { - const value = new Tokenizer('"foo"a').readValue() + const value = new Tokenizer('"foo"a', liquid).readValue() expect(value).toBeInstanceOf(QuotedToken) expect(value!.getText()).toBe('"foo"') }) it('should read property access value', () => { - expect(new Tokenizer('a[b]["c d"]').readValueOrThrow().getText()).toBe('a[b]["c d"]') + expect(new Tokenizer('a[b]["c d"]', liquid).readValueOrThrow().getText()).toBe('a[b]["c d"]') }) it('should read quoted property access value', () => { - const value = new Tokenizer('["a prop"]').readValue() + const value = new Tokenizer('["a prop"]', liquid).readValue() expect(value).toBeInstanceOf(PropertyAccessToken) expect((value as QuotedToken).getText()).toBe('["a prop"]') }) it('should throw for incomplete quoted property access', () => { - const tokenizer = new Tokenizer('["a prop"') + const tokenizer = new Tokenizer('["a prop"', liquid) expect(() => tokenizer.readValueOrThrow()).toThrow() }) it('should read hash', () => { - const hash1 = new Tokenizer('foo: 3').readHash() + const hash1 = new Tokenizer('foo: 3', liquid).readHash() expect(hash1!.name.content).toBe('foo') expect(hash1!.value!.getText()).toBe('3') - const hash2 = new Tokenizer(', foo: a[ "bar"]').readHash() + const hash2 = new Tokenizer(', foo: a[ "bar"]', liquid).readHash() expect(hash2!.name.content).toBe('foo') expect(hash2!.value!.getText()).toBe('a[ "bar"]') }) it('should read multiple hashes', () => { - const hashes = new Tokenizer(', limit: 3 reverse offset:off').readHashes() + const hashes = new Tokenizer(', limit: 3 reverse offset:off', liquid).readHashes() expect(hashes).toHaveLength(3) const [limit, reverse, offset] = hashes expect(limit.name.content).toBe('limit') @@ -83,7 +86,7 @@ describe('Tokenizer', function () { expect(offset.value!.getText()).toBe('off') }) it('should read hash value with property access', () => { - const hashes = new Tokenizer('cols: 2, rows: data["rows"]').readHashes() + const hashes = new Tokenizer('cols: 2, rows: data["rows"]', liquid).readHashes() expect(hashes).toHaveLength(2) const [cols, rols] = hashes @@ -96,7 +99,7 @@ describe('Tokenizer', function () { describe('#readTopLevelTokens()', () => { it('should read HTML token', function () { const html = '

Lorem Ipsum

' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) @@ -105,7 +108,7 @@ describe('Tokenizer', function () { }) it('should read tag token', function () { const html = '

{% for p in a[1]%}

' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(3) @@ -116,7 +119,7 @@ describe('Tokenizer', function () { }) it('should allow unclosed tag inside {% raw %}', function () { const html = '{%raw%} {%if%} {%else {%endraw%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(3) @@ -125,7 +128,7 @@ describe('Tokenizer', function () { }) it('should allow unclosed endraw tag inside {% raw %}', function () { const html = '{%raw%} {%endraw {%raw%} {%endraw%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(3) @@ -134,12 +137,12 @@ describe('Tokenizer', function () { }) it('should throw when {% raw %} not closed', function () { const html = '{%raw%} {%endraw {%raw%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) expect(() => tokenizer.readTopLevelTokens()).toThrow('raw "{%raw%} {%endraw {%raw%}" not closed, line:1, col:8') }) it('should read output token', function () { const html = '

{{foo | date: "%Y-%m-%d"}}

' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(3) @@ -149,7 +152,7 @@ describe('Tokenizer', function () { }) it('should handle consecutive value and tags', function () { const html = '{{foo}}{{bar}}{%foo%}{%bar%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(4) @@ -171,7 +174,7 @@ describe('Tokenizer', function () { }) it('should keep white spaces and newlines', function () { const html = '{%foo%}\n{%bar %} \n {%alice%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(5) expect(tokens[1]).toBeInstanceOf(HTMLToken) @@ -181,7 +184,7 @@ describe('Tokenizer', function () { }) it('should handle multiple lines tag', function () { const html = '{%foo\na:a\nb:1.23\n%}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) expect(tokens[0]).toBeInstanceOf(TagToken) @@ -190,7 +193,7 @@ describe('Tokenizer', function () { }) it('should handle multiple lines value', function () { const html = '{{foo\n|date:\n"%Y-%m-%d"\n}}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) expect(tokens[0]).toBeInstanceOf(OutputToken) @@ -198,7 +201,7 @@ describe('Tokenizer', function () { }) it('should handle complex object property access', function () { const html = '{{ obj["my:property with anything"] }}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) const output = tokens[0] as OutputToken @@ -207,11 +210,11 @@ describe('Tokenizer', function () { }) it('should throw if tag not closed', function () { const html = '{% assign foo = bar {{foo}}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) expect(() => tokenizer.readTopLevelTokens()).toThrow('tag "{% assign foo = bar {{foo}}" not closed, line:1, col:1') }) it('should throw if output not closed', function () { - const tokenizer = new Tokenizer('{{name}') + const tokenizer = new Tokenizer('{{name}', liquid) expect(() => tokenizer.readTopLevelTokens()).toThrow(/output "{{name}" not closed/) }) }) @@ -220,7 +223,7 @@ describe('Tokenizer', function () { describe('#readOutputToken()', () => { it('should skip quoted delimiters', function () { const html = '{{ "%} {%" | append: "}} {{" }}' - const tokenizer = new Tokenizer(html) + const tokenizer = new Tokenizer(html, liquid) const token = tokenizer.readOutputToken() expect(token).toBeInstanceOf(OutputToken) @@ -229,7 +232,7 @@ describe('Tokenizer', function () { }) describe('#readRange()', () => { it('should read `(1..3)`', () => { - const range = new Tokenizer('(1..3)').readRange() + const range = new Tokenizer('(1..3)', liquid).readGroupOrRange() as RangeToken expect(range).toBeInstanceOf(RangeToken) expect(range!.getText()).toEqual('(1..3)') const { lhs, rhs } = range! @@ -239,23 +242,23 @@ describe('Tokenizer', function () { expect(rhs.getText()).toBe('3') }) it('should throw for `(..3)`', () => { - expect(() => new Tokenizer('(..3)').readRange()).toThrow('unexpected token "..3)", value expected') + expect(() => new Tokenizer('(..3)', liquid).readGroupOrRange()).toThrow('unexpected token "..3)", value expected') }) it('should read `(a.b..c["..d"])`', () => { - const range = new Tokenizer('(a.b..c["..d"])').readRange() + const range = new Tokenizer('(a.b..c["..d"])', liquid).readGroupOrRange() expect(range).toBeInstanceOf(RangeToken) expect(range!.getText()).toEqual('(a.b..c["..d"])') }) }) describe('#readFilter()', () => { it('should read a simple filter', function () { - const tokenizer = new Tokenizer('| plus') + const tokenizer = new Tokenizer('| plus', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token).toHaveProperty('args', []) }) it('should read a filter with argument', function () { - const tokenizer = new Tokenizer(' | plus: 1') + const tokenizer = new Tokenizer(' | plus: 1', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token!.args).toHaveLength(1) @@ -265,18 +268,18 @@ describe('Tokenizer', function () { expect(one.getText()).toBe('1') }) it('should read a filter with colon but no argument', function () { - const tokenizer = new Tokenizer('| plus:') + const tokenizer = new Tokenizer('| plus:', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token).toHaveProperty('args', []) }) it('should read null if name not found', function () { - const tokenizer = new Tokenizer('|') + const tokenizer = new Tokenizer('|', liquid) const token = tokenizer.readFilter() expect(token).toBeNull() }) it('should read a filter with k/v argument', function () { - const tokenizer = new Tokenizer(' | plus: a:1') + const tokenizer = new Tokenizer(' | plus: a:1', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token!.args).toHaveLength(1) @@ -287,7 +290,7 @@ describe('Tokenizer', function () { expect(v.getText()).toBe('1') }) it('should read a filter with "arr[0]" argument', function () { - const tokenizer = new Tokenizer('| plus: arr[0]') + const tokenizer = new Tokenizer('| plus: arr[0]', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token!.args).toHaveLength(1) @@ -300,7 +303,7 @@ describe('Tokenizer', function () { expect(pa.props[1].getText()).toBe('0') }) it('should read a filter with obj.foo argument', function () { - const tokenizer = new Tokenizer('| plus: obj.foo') + const tokenizer = new Tokenizer('| plus: obj.foo', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token!.args).toHaveLength(1) @@ -313,7 +316,7 @@ describe('Tokenizer', function () { expect(pa.props[1].getText()).toBe('foo') }) it('should read a filter with obj["foo"] argument', function () { - const tokenizer = new Tokenizer('| plus: obj["good luck"]') + const tokenizer = new Tokenizer('| plus: obj["good luck"]', liquid) const token = tokenizer.readFilter() expect(token).toHaveProperty('name', 'plus') expect(token!.args).toHaveLength(1) @@ -327,7 +330,7 @@ describe('Tokenizer', function () { }) describe('#readFilters()', () => { it('should read simple filters', function () { - const tokenizer = new Tokenizer('| plus: 3 | capitalize') + const tokenizer = new Tokenizer('| plus: 3 | capitalize', liquid) const tokens = tokenizer.readFilters() expect(tokens).toHaveLength(2) @@ -340,7 +343,7 @@ describe('Tokenizer', function () { expect(tokens[1].args).toHaveLength(0) }) it('should read filters', function () { - const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]') + const tokenizer = new Tokenizer('| plus: a:3 | capitalize | append: foo[a.b["c d"]]', liquid) const tokens = tokenizer.readFilters() expect(tokens).toHaveLength(3) @@ -363,14 +366,14 @@ describe('Tokenizer', function () { }) describe('#readExpression()', () => { it('should read expression `a `', () => { - const exp = [...new Tokenizer('a ').readExpressionTokens()] + const exp = [...new Tokenizer('a ', liquid).readExpressionTokens()] expect(exp).toHaveLength(1) expect(exp[0]).toBeInstanceOf(PropertyAccessToken) expect(exp[0].getText()).toEqual('a') }) it('should read expression `a[][b]`', () => { - const exp = [...new Tokenizer('a[][b]').readExpressionTokens()] + const exp = [...new Tokenizer('a[][b]', liquid).readExpressionTokens()] expect(exp).toHaveLength(1) const pa = exp[0] as PropertyAccessToken @@ -385,7 +388,7 @@ describe('Tokenizer', function () { expect(p2.getText()).toBe('b') }) it('should read expression `a.`', () => { - const exp = [...new Tokenizer('a.').readExpressionTokens()] + const exp = [...new Tokenizer('a.', liquid).readExpressionTokens()] expect(exp).toHaveLength(1) const pa = exp[0] as PropertyAccessToken @@ -394,7 +397,7 @@ describe('Tokenizer', function () { expect((pa.props[0] as any).content).toEqual('a') }) it('should read expression `a ==`', () => { - const exp = [...new Tokenizer('a ==').readExpressionTokens()] + const exp = [...new Tokenizer('a ==', liquid).readExpressionTokens()] expect(exp).toHaveLength(2) expect(exp[0]).toBeInstanceOf(PropertyAccessToken) @@ -403,7 +406,7 @@ describe('Tokenizer', function () { expect(exp[1].getText()).toEqual('==') }) it('should read expression `a==b`', () => { - const exp = new Tokenizer('a==b').readExpressionTokens() + const exp = new Tokenizer('a==b', liquid).readExpressionTokens() const [a, equals, b] = exp expect(a).toBeInstanceOf(PropertyAccessToken) @@ -416,11 +419,11 @@ describe('Tokenizer', function () { expect(b.getText()).toEqual('b') }) it('should read expression `^`', () => { - const exp = new Tokenizer('^').readExpressionTokens() + const exp = new Tokenizer('^', liquid).readExpressionTokens() expect([...exp]).toEqual([]) }) it('should read expression `a == b`', () => { - const exp = new Tokenizer('a == b').readExpressionTokens() + const exp = new Tokenizer('a == b', liquid).readExpressionTokens() const [a, equals, b] = exp expect(a).toBeInstanceOf(PropertyAccessToken) @@ -433,7 +436,7 @@ describe('Tokenizer', function () { expect(b.getText()).toEqual('b') }) it('should read expression `(1..3) contains 3`', () => { - const exp = new Tokenizer('(1..3) contains 3').readExpressionTokens() + const exp = new Tokenizer('(1..3) contains 3', liquid).readExpressionTokens() const [range, contains, rhs] = exp expect(range).toBeInstanceOf(RangeToken) @@ -446,7 +449,7 @@ describe('Tokenizer', function () { expect(rhs.getText()).toEqual('3') }) it('should read expression `a[b] == c`', () => { - const exp = new Tokenizer('a[b] == c').readExpressionTokens() + const exp = new Tokenizer('a[b] == c', liquid).readExpressionTokens() const [lhs, contains, rhs] = exp expect(lhs).toBeInstanceOf(PropertyAccessToken) @@ -459,7 +462,7 @@ describe('Tokenizer', function () { expect(rhs.getText()).toEqual('c') }) it('should read expression `c[a["b"]] >= c`', () => { - const exp = new Tokenizer('c[a["b"]] >= c').readExpressionTokens() + const exp = new Tokenizer('c[a["b"]] >= c', liquid).readExpressionTokens() const [lhs, op, rhs] = exp expect(lhs).toBeInstanceOf(PropertyAccessToken) @@ -472,7 +475,7 @@ describe('Tokenizer', function () { expect(rhs.getText()).toEqual('c') }) it('should read expression `"][" == var`', () => { - const exp = new Tokenizer('"][" == var').readExpressionTokens() + const exp = new Tokenizer('"][" == var', liquid).readExpressionTokens() const [lhs, equals, rhs] = exp expect(lhs).toBeInstanceOf(QuotedToken) @@ -485,7 +488,7 @@ describe('Tokenizer', function () { expect(rhs.getText()).toEqual('var') }) it('should read expression `"\\\'" == "\\""`', () => { - const exp = new Tokenizer('"\\\'" == "\\""').readExpressionTokens() + const exp = new Tokenizer('"\\\'" == "\\""', liquid).readExpressionTokens() const [lhs, equals, rhs] = exp expect(lhs).toBeInstanceOf(QuotedToken) @@ -501,30 +504,30 @@ describe('Tokenizer', function () { describe('#matchTrie()', function () { const opTrie = createTrie(defaultOperators) it('should match contains', () => { - expect(new Tokenizer('contains').matchTrie(opTrie)).toBe(8) + expect(new Tokenizer('contains', liquid).matchTrie(opTrie)).toBe(8) }) it('should match comparison', () => { - expect(new Tokenizer('>').matchTrie(opTrie)).toBe(1) - expect(new Tokenizer('>=').matchTrie(opTrie)).toBe(2) - expect(new Tokenizer('<').matchTrie(opTrie)).toBe(1) - expect(new Tokenizer('<=').matchTrie(opTrie)).toBe(2) + expect(new Tokenizer('>', liquid).matchTrie(opTrie)).toBe(1) + expect(new Tokenizer('>=', liquid).matchTrie(opTrie)).toBe(2) + expect(new Tokenizer('<', liquid).matchTrie(opTrie)).toBe(1) + expect(new Tokenizer('<=', liquid).matchTrie(opTrie)).toBe(2) }) it('should match binary logic', () => { - expect(new Tokenizer('and').matchTrie(opTrie)).toBe(3) - expect(new Tokenizer('or').matchTrie(opTrie)).toBe(2) + expect(new Tokenizer('and', liquid).matchTrie(opTrie)).toBe(3) + expect(new Tokenizer('or', liquid).matchTrie(opTrie)).toBe(2) }) it('should not match if word not terminate', () => { - expect(new Tokenizer('true1').matchTrie(opTrie)).toBe(-1) - expect(new Tokenizer('containsa').matchTrie(opTrie)).toBe(-1) + expect(new Tokenizer('true1', liquid).matchTrie(opTrie)).toBe(-1) + expect(new Tokenizer('containsa', liquid).matchTrie(opTrie)).toBe(-1) }) it('should match if word boundary found', () => { - expect(new Tokenizer('>=1').matchTrie(opTrie)).toBe(2) - expect(new Tokenizer('contains b').matchTrie(opTrie)).toBe(8) + expect(new Tokenizer('>=1', liquid).matchTrie(opTrie)).toBe(2) + expect(new Tokenizer('contains b', liquid).matchTrie(opTrie)).toBe(8) }) }) describe('#readLiquidTagTokens', () => { it('should read newline terminated tokens', () => { - const tokenizer = new Tokenizer('echo \'hello\'') + const tokenizer = new Tokenizer('echo \'hello\'', liquid) const tokens = tokenizer.readLiquidTagTokens() expect(tokens.length).toBe(1) const tag = tokens[0] @@ -537,18 +540,18 @@ describe('Tokenizer', function () { echo 'hello' decrement foo - `) + `, liquid) const tokens = tokenizer.readLiquidTagTokens() expect(tokens.length).toBe(2) }) it('should throw if line does not start with an identifier', () => { - const tokenizer = new Tokenizer('!') + const tokenizer = new Tokenizer('!', liquid) expect(() => tokenizer.readLiquidTagTokens()).toThrow(/illegal liquid tag syntax/) }) }) describe('#read inline comment tags', () => { it('should allow hash characters in tag names', () => { - const tokenizer = new Tokenizer('{% # some comment %}') + const tokenizer = new Tokenizer('{% # some comment %}', liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) const tag = tokens[0] as TagToken @@ -557,7 +560,7 @@ describe('Tokenizer', function () { expect(tag.args).toBe('some comment') }) it('should handle leading whitespace', () => { - const tokenizer = new Tokenizer('{%\n # some comment %}') + const tokenizer = new Tokenizer('{%\n # some comment %}', liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) const tag = tokens[0] as TagToken @@ -566,7 +569,7 @@ describe('Tokenizer', function () { expect(tag.args).toBe('some comment') }) it('should handle no trailing whitespace', () => { - const tokenizer = new Tokenizer('{%\n #some comment %}') + const tokenizer = new Tokenizer('{%\n #some comment %}', liquid) const tokens = tokenizer.readTopLevelTokens() expect(tokens.length).toBe(1) const tag = tokens[0] as TagToken @@ -575,4 +578,55 @@ describe('Tokenizer', function () { expect(tag.args).toBe('some comment') }) }) + + describe('#readGroupedExpression()', () => { + function createGrouped (input: string): Tokenizer { + const liquid = new Liquid({ groupedExpressions: true }) + return new Tokenizer(input, liquid, defaultOperators) + } + it('should read `(foo | upcase)` as FilteredValueToken', () => { + const token = createGrouped('(foo | upcase)').readValue() + expect(token).toBeInstanceOf(FilteredValueToken) + const grouped = token as FilteredValueToken + expect(grouped.getText()).toBe('(foo | upcase)') + expect(grouped.initial.postfix).toHaveLength(1) + expect(grouped.filters).toHaveLength(1) + expect(grouped.filters[0].name).toBe('upcase') + }) + it('should read `(foo | append: "!")` with filter argument', () => { + const token = createGrouped('(foo | append: "!")').readValue() + expect(token).toBeInstanceOf(FilteredValueToken) + const grouped = token as FilteredValueToken + expect(grouped.filters).toHaveLength(1) + expect(grouped.filters[0].name).toBe('append') + expect(grouped.filters[0].args).toHaveLength(1) + }) + it('should read nested `((foo | append: "!") | upcase)`', () => { + const token = createGrouped('((foo | append: "!") | upcase)').readValue() + expect(token).toBeInstanceOf(FilteredValueToken) + const grouped = token as FilteredValueToken + expect(grouped.filters).toHaveLength(1) + expect(grouped.filters[0].name).toBe('upcase') + expect(grouped.initial.postfix).toHaveLength(1) + expect(grouped.initial.postfix[0]).toBeInstanceOf(FilteredValueToken) + }) + it('should parse `(a | upcase) == "BAR"` as expression', () => { + const exp = [...createGrouped('(a | upcase) == "BAR"').readExpressionTokens()] + expect(exp).toHaveLength(3) + expect(exp[0]).toBeInstanceOf(FilteredValueToken) + expect(exp[1]).toBeInstanceOf(OperatorToken) + expect(exp[1].getText()).toBe('==') + expect(exp[2]).toBeInstanceOf(QuotedToken) + }) + it('should still parse `(1..3)` as RangeToken', () => { + const token = createGrouped('(1..3)').readValue() + expect(token).toBeInstanceOf(RangeToken) + }) + it('should return undefined for unclosed parens', () => { + expect(() => { createGrouped('(foo | upcase').readValue() }).toThrow(TokenizationError) + }) + it('should fall back to readRange when flag is off', () => { + expect(() => new Tokenizer('(foo | upcase)', new Liquid({ groupedExpressions: false }), defaultOperators).readValue()).toThrow('invalid range syntax') + }) + }) }) diff --git a/src/parser/tokenizer.ts b/src/parser/tokenizer.ts index 0c2d86df76..9576fd258c 100644 --- a/src/parser/tokenizer.ts +++ b/src/parser/tokenizer.ts @@ -1,10 +1,12 @@ import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens' import { OperatorHandler } from '../render/operator' -import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, NUMBER, SIGN, isWord, isString } from '../util' +import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, NUMBER, SIGN, isWord, isString, toIterableIterator } from '../util' import { Operators, Expression } from '../render' import { NormalizedFullOptions, defaultOptions } from '../liquid-options' import { FilterArg } from './filter-arg' import { whiteSpaceCtrl } from './whitespace-ctrl' +import { Liquid } from '../liquid' +import { PropToken } from '../tokens/prop-token' export class Tokenizer { p: number @@ -15,6 +17,7 @@ export class Tokenizer { constructor ( public input: string, + public liquid: Liquid, operators: Operators = defaultOptions.operators, public file?: string, range?: [number, number] @@ -67,7 +70,7 @@ export class Tokenizer { const initial = this.readExpression() this.assert(initial.valid(), `invalid value expression: ${this.snapshot()}`) const filters = this.readFilters() - return new FilteredValueToken(initial, filters, this.input, begin, this.p, this.file) + return new FilteredValueToken(initial, filters, this.liquid, this.input, begin, this.p, this.file) } readFilters (): FilterToken[] { const filters = [] @@ -79,7 +82,7 @@ export class Tokenizer { } readFilter (): FilterToken | null { this.skipBlank() - if (this.end()) return null + if (this.end() || this.peek() === ')') return null this.assert(this.read() === '|', `expected "|" before filter`) const name = this.readIdentifier() if (!name.size()) { @@ -94,9 +97,9 @@ export class Tokenizer { const arg = this.readFilterArg() arg && args.push(arg) this.skipBlank() - this.assert(this.end() || this.peek() === ',' || this.peek() === '|', () => `unexpected character ${this.snapshot()}`) + this.assert(this.end() || this.peek() === ')' || this.peek() === ',' || this.peek() === '|', () => `unexpected character ${this.snapshot()}`) } while (this.peek() === ',') - } else if (this.peek() === '|' || this.end()) { + } else if (this.peek() === '|' || this.peek() === ')' || this.end()) { // do nothing } else { throw this.error('expected ":" after filter name') @@ -147,7 +150,7 @@ export class Tokenizer { if (this.readToDelimiter(options.tagDelimiterRight) === -1) { throw this.error(`tag ${this.snapshot(begin)} not closed`, begin) } - const token = new TagToken(input, begin, this.p, options, file) + const token = new TagToken(input, begin, this.p, options, this.liquid, file) if (token.name === 'raw') this.rawBeginAt = begin return token } @@ -189,7 +192,7 @@ export class Tokenizer { const end = this.p if (begin === leftPos) { this.rawBeginAt = -1 - return new TagToken(this.input, begin, end, options, this.file) + return new TagToken(this.input, begin, end, options, this.liquid, this.file) } else { this.p = leftPos return new HTMLToken(this.input, begin, leftPos, this.file) @@ -218,7 +221,7 @@ export class Tokenizer { const begin = this.p this.readToDelimiter('\n') const end = this.p - return new LiquidTagToken(this.input, begin, end, options, this.file) + return new LiquidTagToken(this.input, begin, end, options, this.liquid, this.file) } error (msg: string, pos: number = this.p) { @@ -310,7 +313,7 @@ export class Tokenizer { readValue (): ValueToken | undefined { this.skipBlank() const begin = this.p - const variable = this.readLiteral() || this.readQuoted() || this.readRange() || this.readNumber() + const variable = this.readLiteral() || this.readQuoted() || this.readGroupOrRange() || this.readNumber() const props = this.readProperties(!variable) if (!props.length) return variable return new PropertyAccessToken(variable, props, this.input, begin, this.p) @@ -324,12 +327,12 @@ export class Tokenizer { return new PropertyAccessToken(undefined, props, this.input, begin, this.p) } - private readProperties (isBegin = true): (ValueToken | IdentifierToken)[] { - const props: (ValueToken | IdentifierToken)[] = [] + private readProperties (isBegin = true): PropToken[] { + const props: PropToken[] = [] while (true) { if (this.peek() === '[') { this.p++ - const prop = this.readValue() || new IdentifierToken(this.input, this.p, this.p, this.file) + const prop = this.readProperty() || new IdentifierToken(this.input, this.p, this.p, this.file) this.assert(this.readTo(']') !== -1, '[ not closed') props.push(prop) continue @@ -353,6 +356,15 @@ export class Tokenizer { return props } + private readProperty (): PropToken | undefined { + this.skipBlank() + const begin = this.p + const variable = this.readLiteral() || this.readQuoted() || this.readNumber() + const props = this.readProperties(!variable) + if (!props.length) return variable + return new PropertyAccessToken(variable, props, this.input, begin, this.p) + } + readNumber (): NumberToken | undefined { this.skipBlank() let decimalFound = false @@ -385,18 +397,33 @@ export class Tokenizer { return literal } - readRange (): RangeToken | undefined { + readGroupOrRange (): RangeToken | FilteredValueToken | undefined { this.skipBlank() const begin = this.p if (this.peek() !== '(') return ++this.p const lhs = this.readValueOrThrow() this.skipBlank() - this.assert(this.read() === '.' && this.read() === '.', 'invalid range syntax') - const rhs = this.readValueOrThrow() - this.skipBlank() - this.assert(this.read() === ')', 'invalid range syntax') - return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file) + + if (this.peek() === '.' && this.peek(1) === '.') { + this.p += 2 + const rhs = this.readValueOrThrow() + this.skipBlank() + this.assert(this.read() === ')', 'invalid range syntax') + return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file) + } + + if (this.liquid.options.groupedExpressions) { + const expression = new Expression(toIterableIterator(lhs)) + this.skipBlank() + const filters = this.readFilters() + this.skipBlank() + this.assert(this.read() === ')', 'unbalanced parentheses') + this.skipBlank() + return new FilteredValueToken(expression, filters, this.liquid, this.input, begin, this.p, this.file) + } + + throw this.error('invalid range syntax') } readValueOrThrow (): ValueToken { diff --git a/src/render/expression.spec.ts b/src/render/expression.spec.ts index 5f4b0866be..0b4765ae7c 100644 --- a/src/render/expression.spec.ts +++ b/src/render/expression.spec.ts @@ -4,10 +4,12 @@ import { QuotedToken } from '../tokens' import { Context } from '../context' import { toPromise, toValueSync } from '../util' import { evalQuotedToken } from './expression' +import { Liquid } from '../liquid' describe('Expression', function () { + const liquid = new Liquid() const ctx = new Context({}) - const create = (str: string) => new Tokenizer(str).readExpression() + const create = (str: string) => new Tokenizer(str, liquid).readExpression() it('should throw when context not defined', done => { toPromise(create('foo').evaluate(undefined!, false)) diff --git a/src/render/expression.ts b/src/render/expression.ts index 62041ae174..9fb9cd95f6 100644 --- a/src/render/expression.ts +++ b/src/render/expression.ts @@ -1,5 +1,5 @@ -import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes } from '../tokens' -import { isRangeToken, isPropertyAccessToken, UndefinedVariableError, range, isOperatorToken, assert } from '../util' +import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes, FilteredValueToken } from '../tokens' +import { isRangeToken, isPropertyAccessToken, UndefinedVariableError, range, isOperatorToken, assert, isFilteredValueToken } from '../util' import type { Context } from '../context' import type { UnaryOperatorHandler } from '../render' import { Drop } from '../drop' @@ -40,6 +40,7 @@ export function * evalToken (token: Token | undefined, ctx: Context, lenient = f if ('content' in token) return token.content if (isPropertyAccessToken(token)) return yield evalPropertyAccessToken(token, ctx, lenient) if (isRangeToken(token)) return yield evalRangeToken(token, ctx) + if (isFilteredValueToken(token)) return yield evalFilteredValueToken(token, ctx, lenient) } function * evalPropertyAccessToken (token: PropertyAccessToken, ctx: Context, lenient: boolean): IterableIterator { @@ -85,3 +86,13 @@ function * toPostfix (tokens: IterableIterator): IterableIterator yield ops.pop()! } } + +function * evalFilteredValueToken (token: FilteredValueToken, ctx: Context, lenient: boolean): IterableIterator { + lenient = lenient || (ctx.opts.lenientIf && token.filters.length > 0 && token.filters[0].name === 'default') + let val = yield token.initial.evaluate(ctx, lenient) + + for (const filter of token.filters) { + val = yield filter.render(val, ctx) + } + return val +} diff --git a/src/tags/for.ts b/src/tags/for.ts index 779eee207b..dc55ac1285 100644 --- a/src/tags/for.ts +++ b/src/tags/for.ts @@ -26,7 +26,7 @@ export default class extends Tag { this.variable = variable.content this.collection = collection - this.hash = new Hash(this.tokenizer, liquid.options.keyValueSeparator) + this.hash = new Hash(this.tokenizer, liquid, liquid.options.keyValueSeparator) this.templates = [] this.elseTemplates = [] diff --git a/src/tags/include.ts b/src/tags/include.ts index 3ad785b907..4e94c621cc 100644 --- a/src/tags/include.ts +++ b/src/tags/include.ts @@ -23,7 +23,7 @@ export default class extends Tag { } else tokenizer.p = begin } else tokenizer.p = begin - this.hash = new Hash(tokenizer, liquid.options.jekyllInclude || liquid.options.keyValueSeparator) + this.hash = new Hash(tokenizer, liquid, liquid.options.jekyllInclude || liquid.options.keyValueSeparator) } * render (ctx: Context, emitter: Emitter): Generator { const { liquid, hash, withVar } = this diff --git a/src/tags/layout.ts b/src/tags/layout.ts index f1da55f780..392a1dc820 100644 --- a/src/tags/layout.ts +++ b/src/tags/layout.ts @@ -14,7 +14,7 @@ export default class extends Tag { super(token, remainTokens, liquid) this.file = parseFilePath(this.tokenizer, this.liquid, parser) this['currentFile'] = token.file - this.args = new Hash(this.tokenizer, liquid.options.keyValueSeparator) + this.args = new Hash(this.tokenizer, liquid, liquid.options.keyValueSeparator) this.templates = parser.parseTokens(remainTokens) } * render (ctx: Context, emitter: Emitter): Generator { diff --git a/src/tags/render.ts b/src/tags/render.ts index 123e73a933..7dec757f4a 100644 --- a/src/tags/render.ts +++ b/src/tags/render.ts @@ -46,7 +46,7 @@ export default class extends Tag { tokenizer.p = begin break } - this.hash = new Hash(tokenizer, liquid.options.keyValueSeparator) + this.hash = new Hash(tokenizer, liquid, liquid.options.keyValueSeparator) } * render (ctx: Context, emitter: Emitter): Generator { const { liquid, hash } = this diff --git a/src/tags/tablerow.ts b/src/tags/tablerow.ts index 91f8404451..51d55aafae 100644 --- a/src/tags/tablerow.ts +++ b/src/tags/tablerow.ts @@ -22,7 +22,7 @@ export default class extends Tag { this.variable = variable.content this.collection = collectionToken - this.args = new Hash(this.tokenizer, liquid.options.keyValueSeparator) + this.args = new Hash(this.tokenizer, liquid, liquid.options.keyValueSeparator) this.templates = [] let p diff --git a/src/template/hash.spec.ts b/src/template/hash.spec.ts index 9179f661d0..62187d1c54 100644 --- a/src/template/hash.spec.ts +++ b/src/template/hash.spec.ts @@ -2,37 +2,40 @@ import { toPromise } from '../util' import { Hash } from './hash' import { Context } from '../context' import { Tokenizer } from '../parser' +import { Liquid } from '../liquid' describe('Hash', function () { + const liquid = new Liquid() + it('should parse "reverse"', async function () { - const hash = await toPromise(new Hash('reverse').render(new Context({ foo: 3 }))) + const hash = await toPromise(new Hash('reverse', liquid).render(new Context({ foo: 3 }))) expect(hash).toHaveProperty('reverse') expect(hash.reverse).toBeTruthy() }) it('should parse "num:foo"', async function () { - const hash = await toPromise(new Hash('num:foo').render(new Context({ foo: 3 }))) + const hash = await toPromise(new Hash('num:foo', liquid).render(new Context({ foo: 3 }))) expect(hash.num).toBe(3) }) it('should parse "num:3"', async function () { - const hash = await toPromise(new Hash('num:3').render(new Context())) + const hash = await toPromise(new Hash('num:3', liquid).render(new Context())) expect(hash.num).toBe(3) }) it('should parse "num: arr[0]"', async function () { - const hash = await toPromise(new Hash('num:3').render(new Context({ arr: [3] }))) + const hash = await toPromise(new Hash('num:3', liquid).render(new Context({ arr: [3] }))) expect(hash.num).toBe(3) }) it('should parse "num: 2.3"', async function () { - const hash = await toPromise(new Hash('num:2.3').render(new Context())) + const hash = await toPromise(new Hash('num:2.3', liquid).render(new Context())) expect(hash.num).toBe(2.3) }) it('should parse "num:bar.coo"', async function () { - const pending = new Hash('num:bar.coo').render(new Context({ bar: { coo: 3 } })) + const pending = new Hash('num:bar.coo', liquid).render(new Context({ bar: { coo: 3 } })) const hash = await toPromise(pending) expect(hash.num).toBe(3) }) it('should parse "num1:2.3 reverse,num2:bar.coo\n num3: arr[0]"', async function () { const ctx = new Context({ bar: { coo: 3 }, arr: [4] }) - const hash = await toPromise(new Hash('num1:2.3 reverse,num2:bar.coo\n num3: arr[0]').render(ctx)) + const hash = await toPromise(new Hash('num1:2.3 reverse,num2:bar.coo\n num3: arr[0]', liquid).render(ctx)) expect(hash).toEqual({ num1: 2.3, reverse: true, @@ -41,12 +44,12 @@ describe('Hash', function () { }) }) it('should support custom separator', async function () { - const hash = await toPromise(new Hash('num=2.3', '=').render(new Context())) + const hash = await toPromise(new Hash('num=2.3', liquid, '=').render(new Context())) expect(hash.num).toBe(2.3) }) it('should accept an existing tokenizer', async function () { - const tokenizer = new Tokenizer('a:1, b:2') - const hash = await toPromise(new Hash(tokenizer).render(new Context())) + const tokenizer = new Tokenizer('a:1, b:2', liquid) + const hash = await toPromise(new Hash(tokenizer, liquid).render(new Context())) expect(hash.a).toBe(1) expect(hash.b).toBe(2) }) diff --git a/src/template/hash.ts b/src/template/hash.ts index 4899a80450..0713f3d079 100644 --- a/src/template/hash.ts +++ b/src/template/hash.ts @@ -2,6 +2,7 @@ import { evalToken } from '../render/expression' import { Context } from '../context/context' import { Tokenizer } from '../parser/tokenizer' import { Token } from '../tokens/token' +import { Liquid } from '../liquid' type HashValueTokens = Record @@ -16,8 +17,8 @@ type HashValueTokens = Record export class Hash { hash: HashValueTokens = {} - constructor (input: string | Tokenizer, jekyllStyle?: boolean | string) { - const tokenizer = input instanceof Tokenizer ? input : new Tokenizer(input, {}) + constructor (input: string | Tokenizer, liquid: Liquid, jekyllStyle?: boolean | string) { + const tokenizer = input instanceof Tokenizer ? input : new Tokenizer(input, liquid, {}) for (const hash of tokenizer.readHashes(jekyllStyle)) { this.hash[hash.name.content] = hash.value } diff --git a/src/template/output.ts b/src/template/output.ts index cd75ea0b7c..0421e7c27b 100644 --- a/src/template/output.ts +++ b/src/template/output.ts @@ -12,7 +12,7 @@ export class Output extends TemplateImpl implements Template { value: Value public constructor (token: OutputToken, liquid: Liquid) { super(token) - const tokenizer = new Tokenizer(token.input, liquid.options.operators, token.file, token.contentRange) + const tokenizer = new Tokenizer(token.input, liquid, liquid.options.operators, token.file, token.contentRange) this.value = new Value(tokenizer.readFilteredValue(), liquid) const filters = this.value.filters const outputEscape = liquid.options.outputEscape diff --git a/src/template/tag-options-adapter.ts b/src/template/tag-options-adapter.ts index a8a4931ae4..e783c97f0a 100644 --- a/src/template/tag-options-adapter.ts +++ b/src/template/tag-options-adapter.ts @@ -21,7 +21,7 @@ export function createTagClass (options: TagImplOptions): TagClass { } } * render (ctx: Context, emitter: Emitter): TagRenderReturn { - const hash = (yield new Hash(this.token.args, ctx.opts.keyValueSeparator).render(ctx)) as Record + const hash = (yield new Hash(this.token.args, this.liquid, ctx.opts.keyValueSeparator).render(ctx)) as Record return yield options.render.call(this, ctx, emitter, hash) } } diff --git a/src/template/value.ts b/src/template/value.ts index 52c8ad7d22..dbc26872dd 100644 --- a/src/template/value.ts +++ b/src/template/value.ts @@ -1,7 +1,6 @@ import { Filter } from './filter' import { Expression } from '../render' import { Tokenizer } from '../parser' -import { assert } from '../util' import type { FilteredValueToken } from '../tokens' import type { Liquid } from '../liquid' import type { Context } from '../context' @@ -11,14 +10,14 @@ export class Value { public readonly initial: Expression /** - * @param str the value to be valuated, eg.: "foobar" | truncate: 3 + * @param input the value to be valuated, eg.: "foobar" | truncate: 3 */ public constructor (input: string | FilteredValueToken, liquid: Liquid) { const token: FilteredValueToken = typeof input === 'string' - ? new Tokenizer(input, liquid.options.operators).readFilteredValue() + ? new Tokenizer(input, liquid, liquid.options.operators).readFilteredValue() : input this.initial = token.initial - this.filters = token.filters.map(token => new Filter(token, this.getFilter(liquid, token.name), liquid)) + this.filters = token.filters } public * value (ctx: Context, lenient?: boolean): Generator { @@ -30,10 +29,4 @@ export class Value { } return val } - - private getFilter (liquid: Liquid, name: string) { - const impl = liquid.filters[name] - assert(impl || !liquid.options.strictFilters, () => `undefined filter: ${name}`) - return impl - } } diff --git a/src/tokens/filtered-value-token.ts b/src/tokens/filtered-value-token.ts index 3653c285e4..e0b4712887 100644 --- a/src/tokens/filtered-value-token.ts +++ b/src/tokens/filtered-value-token.ts @@ -2,6 +2,9 @@ import { Token } from './token' import { FilterToken } from './filter-token' import { TokenKind } from '../parser' import { Expression } from '../render' +import { Liquid } from '../liquid' +import { Filter } from '../template' +import { assert } from '../util' /** * value expression with optional filters @@ -9,14 +12,24 @@ import { Expression } from '../render' * {% assign foo="bar" | append: "coo" %} */ export class FilteredValueToken extends Token { + public filters: Filter[]; + constructor ( public initial: Expression, - public filters: FilterToken[], + filters: FilterToken[], + liquid: Liquid, public input: string, public begin: number, public end: number, public file?: string ) { super(TokenKind.FilteredValue, input, begin, end, file) + this.filters = filters.map(token => new Filter(token, this.getFilter(liquid, token.name), liquid)) + } + + private getFilter (liquid: Liquid, name: string) { + const impl = liquid.filters[name] + assert(impl || !liquid.options.strictFilters, () => `undefined filter: ${name}`) + return impl } } diff --git a/src/tokens/liquid-tag-token.ts b/src/tokens/liquid-tag-token.ts index 91f119f218..188ea27f96 100644 --- a/src/tokens/liquid-tag-token.ts +++ b/src/tokens/liquid-tag-token.ts @@ -1,6 +1,7 @@ import { DelimitedToken } from './delimited-token' import { NormalizedFullOptions } from '../liquid-options' import { Tokenizer, TokenKind } from '../parser' +import { Liquid } from '../liquid' /** * LiquidTagToken is different from TagToken by not having delimiters `{%` or `%}` @@ -13,10 +14,11 @@ export class LiquidTagToken extends DelimitedToken { begin: number, end: number, options: NormalizedFullOptions, + liquid: Liquid, file?: string ) { super(TokenKind.Tag, [begin, end], input, begin, end, false, false, file) - this.tokenizer = new Tokenizer(input, options.operators, file, this.contentRange) + this.tokenizer = new Tokenizer(input, liquid, options.operators, file, this.contentRange) this.name = this.tokenizer.readTagName() this.tokenizer.assert(this.name, 'illegal liquid tag syntax') this.tokenizer.skipBlank() diff --git a/src/tokens/prop-token.ts b/src/tokens/prop-token.ts new file mode 100644 index 0000000000..9b8b66a84a --- /dev/null +++ b/src/tokens/prop-token.ts @@ -0,0 +1,7 @@ +import { IdentifierToken } from './identifier-token' +import { LiteralToken } from './literal-token' +import { NumberToken } from './number-token' +import { PropertyAccessToken } from './property-access-token' +import { QuotedToken } from './quoted-token' + +export type PropToken = LiteralToken | QuotedToken | NumberToken | PropertyAccessToken |IdentifierToken diff --git a/src/tokens/property-access-token.ts b/src/tokens/property-access-token.ts index 8496da24d6..5479249e39 100644 --- a/src/tokens/property-access-token.ts +++ b/src/tokens/property-access-token.ts @@ -1,16 +1,16 @@ import { Token } from './token' import { LiteralToken } from './literal-token' -import { ValueToken } from './value-token' -import { IdentifierToken } from './identifier-token' import { NumberToken } from './number-token' import { RangeToken } from './range-token' import { QuotedToken } from './quoted-token' import { TokenKind } from '../parser' +import { FilteredValueToken } from './filtered-value-token' +import { PropToken } from './prop-token' export class PropertyAccessToken extends Token { constructor ( - public variable: QuotedToken | RangeToken | LiteralToken | NumberToken | undefined, - public props: (ValueToken | IdentifierToken)[], + public variable: QuotedToken | RangeToken | LiteralToken | NumberToken | FilteredValueToken |undefined, + public props: PropToken[], input: string, begin: number, end: number, diff --git a/src/tokens/tag-token.ts b/src/tokens/tag-token.ts index e42268e754..667e085721 100644 --- a/src/tokens/tag-token.ts +++ b/src/tokens/tag-token.ts @@ -1,6 +1,7 @@ import { DelimitedToken } from './delimited-token' import { Tokenizer, TokenKind } from '../parser' import type { NormalizedFullOptions } from '../liquid-options' +import { Liquid } from '../liquid' export class TagToken extends DelimitedToken { public name: string @@ -11,13 +12,14 @@ export class TagToken extends DelimitedToken { begin: number, end: number, options: NormalizedFullOptions, + liquid: Liquid, file?: string ) { const { trimTagLeft, trimTagRight, tagDelimiterLeft, tagDelimiterRight } = options const [valueBegin, valueEnd] = [begin + tagDelimiterLeft.length, end - tagDelimiterRight.length] super(TokenKind.Tag, [valueBegin, valueEnd], input, begin, end, trimTagLeft, trimTagRight, file) - this.tokenizer = new Tokenizer(input, options.operators, file, this.contentRange) + this.tokenizer = new Tokenizer(input, liquid, options.operators, file, this.contentRange) this.name = this.tokenizer.readTagName() this.tokenizer.assert(this.name, `illegal tag syntax, tag name expected`) this.tokenizer.skipBlank() diff --git a/src/tokens/value-token.ts b/src/tokens/value-token.ts index 37e223925b..1fd6751655 100644 --- a/src/tokens/value-token.ts +++ b/src/tokens/value-token.ts @@ -3,5 +3,6 @@ import { LiteralToken } from './literal-token' import { NumberToken } from './number-token' import { QuotedToken } from './quoted-token' import { PropertyAccessToken } from './property-access-token' +import { FilteredValueToken } from './filtered-value-token' -export type ValueToken = RangeToken | LiteralToken | QuotedToken | PropertyAccessToken | NumberToken +export type ValueToken = RangeToken | LiteralToken | QuotedToken | PropertyAccessToken | NumberToken | FilteredValueToken diff --git a/src/util/type-guards.ts b/src/util/type-guards.ts index 04fca1c39c..9c75f49345 100644 --- a/src/util/type-guards.ts +++ b/src/util/type-guards.ts @@ -1,4 +1,4 @@ -import { RangeToken, NumberToken, QuotedToken, LiteralToken, PropertyAccessToken, OutputToken, HTMLToken, TagToken, IdentifierToken, DelimitedToken, OperatorToken, ValueToken } from '../tokens' +import { RangeToken, NumberToken, QuotedToken, LiteralToken, PropertyAccessToken, OutputToken, HTMLToken, TagToken, IdentifierToken, DelimitedToken, OperatorToken, ValueToken, FilteredValueToken } from '../tokens' import { TokenKind } from '../parser' export function isDelimitedToken (val: any): val is DelimitedToken { @@ -50,6 +50,10 @@ export function isValueToken (val: any): val is ValueToken { return (getKind(val) & 1667) > 0 } +export function isFilteredValueToken (val: any): val is FilteredValueToken { + return getKind(val) === TokenKind.FilteredValue +} + function getKind (val: any) { return val ? val.kind : -1 } diff --git a/src/util/underscore.ts b/src/util/underscore.ts index f3558da1ad..b7758ae1f8 100644 --- a/src/util/underscore.ts +++ b/src/util/underscore.ts @@ -66,6 +66,10 @@ export function toNumber (value: any): number { return +toValue(value) || 0 } +export function * toIterableIterator (item: T): IterableIterator { + yield item +} + export function isNumber (value: any): value is number { return typeof value === 'number' } diff --git a/test/e2e/issues.spec.ts b/test/e2e/issues.spec.ts index c5243fe662..79a491d4ee 100644 --- a/test/e2e/issues.spec.ts +++ b/test/e2e/issues.spec.ts @@ -277,7 +277,7 @@ describe('Issues', function () { expect(() => engine.parse('{% assign headshot = https://testurl.com/not_enclosed_in_quotes.jpg %}')).toThrow(/expected "|" before filter, line:1, col:27/) }) it('export Liquid Expression #527', () => { - const tokenizer = new Tokenizer('a > b') + const tokenizer = new Tokenizer('a > b', new Liquid()) const expression = tokenizer.readExpression() const result = toValueSync(expression.evaluate(new Context({ a: 1, b: 2 }))) expect(result).toBe(false) diff --git a/test/e2e/parse-and-analyze.spec.ts b/test/e2e/parse-and-analyze.spec.ts index f137ea05cc..24edd68e1c 100644 --- a/test/e2e/parse-and-analyze.spec.ts +++ b/test/e2e/parse-and-analyze.spec.ts @@ -6,7 +6,7 @@ class MockTag extends Tag { constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) - this.args = new Hash(token.tokenizer) + this.args = new Hash(token.tokenizer, liquid) this.templates = [] const stream: ParseStream = parser.parseStream(remainTokens) diff --git a/test/integration/tags/if.spec.ts b/test/integration/tags/if.spec.ts index 2573861e07..3fab1e0704 100644 --- a/test/integration/tags/if.spec.ts +++ b/test/integration/tags/if.spec.ts @@ -169,4 +169,43 @@ describe('tags/if', function () { expect(() => liquid.parseAndRenderSync('{% if false %}{% else %}{% elsif true %}{% endif %}')) .toThrow(`unexpected elsif after else`) }) + describe('parenthesized filter chains', function () { + const ge = new Liquid({ groupedExpressions: true }) + it('should support (foo | upcase) == "BAR"', async function () { + const src = '{% if (foo | upcase) == "BAR" %}yes{% else %}no{% endif %}' + const html = await ge.parseAndRender(src, { foo: 'bar' }) + return expect(html).toBe('yes') + }) + it('should support both sides parenthesized', async function () { + const src = '{% if (a | upcase) == (b | upcase) %}yes{% else %}no{% endif %}' + const html = await ge.parseAndRender(src, { a: 'hi', b: 'hi' }) + return expect(html).toBe('yes') + }) + it('should support with logical operators', async function () { + const src = '{% if (a | upcase) == "FOO" and (b | downcase) == "bar" %}yes{% else %}no{% endif %}' + const html = await ge.parseAndRender(src, { a: 'foo', b: 'BAR' }) + return expect(html).toBe('yes') + }) + it('should support standalone parenthesized filter via evalValueSync', function () { + const result = ge.evalValueSync('(foo | upcase)', { foo: 'bar' }) + return expect(result).toBe('BAR') + }) + it('should support comparison via evalValueSync', function () { + const result = ge.evalValueSync('(foo | upcase) == "BAR"', { foo: 'bar' }) + return expect(result).toBe(true) + }) + it('should keep range syntax working', function () { + const result = ge.evalValueSync('(1..5)', {}) + return expect(result).toEqual([1, 2, 3, 4, 5]) + }) + it('should support chained filters in condition', async function () { + const src = '{% if (name | downcase | size) > 3 %}long{% else %}short{% endif %}' + const html = await ge.parseAndRender(src, { name: 'Alice' }) + return expect(html).toBe('long') + }) + it('should support nested parenthesized expressions', function () { + const result = ge.evalValueSync('((foo | append: "!") | upcase)', { foo: 'bar' }) + return expect(result).toBe('BAR!') + }) + }) }) diff --git a/tsconfig.json b/tsconfig.json index 258f536b4b..913801f730 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es6", "module":"CommonJS", - "lib": ["es2015", "es2016", "es2017", "dom"], + "lib": ["es2019", "dom"], "sourceMap": true, "outDir": "dist", "declaration": true, @@ -11,8 +11,10 @@ "resolveJsonModule": true, "downlevelIteration": true, "strict": true, - "suppressImplicitAnyIndexErrors": true + "suppressImplicitAnyIndexErrors": true, + "types": ["jest", "node"], }, "all": true, - "exclude": [ "node_modules", "dist", "demo", "test" ] + "exclude": [ "node_modules", "dist", "demo", "test" ], + }