Dialects

How it works

Yuku is written in Zig and compiled to one WebAssembly module. The yuku npm package runs that module and decodes its output, the AST and the semantic model, into JavaScript objects.

A dialect is compiled into Yuku itself. The result is your own WebAssembly module: Yuku with your syntax in its parser. Pass it to the yuku package as the wasm option, and parse, analyze, walk, and generate work as before, now with your syntax.

1. Write the dialect, one Zig file.
2. Build it into a WebAssembly module with Yuku’s build.
3. Pass the module as wasm in JavaScript.

Standard code parses and analyzes exactly as in Yuku. Your nodes appear in the tree next to standard nodes.

Example

This dialect adds a pipeline operator, |>, with % as the value piped in.

// src/pipeline.zig
const std = @import("std");
const yuku = @import("parser");

const ast = yuku.ast;
const dialect = yuku.dialect;

pub const files: []const dialect.File = &.{.{ .extension = ".pipe.js", .lang = .js }};

pub const Node = enum {
    pipeline,
    topic,

    pub fn shape(node: Node) dialect.Shape {
        return switch (node) {
            .pipeline => .{ .type = "PipelineExpression", .fields = &.{ "left", "right" } },
            .topic => .{ .type = "TopicReference" },
        };
    }
};

pub fn operator(
    p: *dialect.Parser,
    left: ast.NodeIndex,
    precedence: u8,
) dialect.Error!dialect.Claim {
    if (!std.mem.startsWith(u8, p.source()[p.token().span.start..], "|>")) return .decline;
    if (precedence > ast.Precedence.Assignment) return .end;

    if (!try p.advance() or !try p.advance()) return .failed;
    const right = try p.parseExpression(ast.Precedence.Assignment + 1) orelse return .failed;
    const span: ast.Span = .{ .start = p.tree().span(left).start, .end = p.tree().span(right).end };
    return .{ .node = try p.addNode(.pipeline, &.{ left, right }, span) };
}

pub fn expression(p: *dialect.Parser) dialect.Error!dialect.Claim {
    const token = p.token();
    if (token.tag != .percent) return .decline;

    if (!try p.advance()) return .failed;
    return .{ .node = try p.addNode(.topic, &.{}, token.span) };
}

pub fn print(w: *dialect.Printer, kind: Node, children: []const ast.NodeIndex) dialect.Error!void {
    switch (kind) {
        .pipeline => {
            try w.node(children[0]);
            try w.write(" |> ");
            try w.node(children[1]);
        },
        .topic => try w.write("%"),
    }
}

With Yuku as a dependency, see Install, build it into a WebAssembly module.

// build.zig
const std = @import("std");

pub fn build(b: *std.Build) void {
    const yuku = b.dependency("yuku", .{ .dialect = b.path("src/pipeline.zig") });
    b.installArtifact(@import("yuku").addWasm(b, yuku, "pipeline"));
}

zig build writes zig-out/bin/pipeline.wasm. Use it from JavaScript.

import { readFileSync } from "node:fs";
import { parse } from "yuku";

const wasm = new WebAssembly.Module(readFileSync("zig-out/bin/pipeline.wasm"));
const { program } = parse("const total = prices |> sum(%) |> round(%, 2);", {
  path: "app.pipe.js",
  wasm,
});

The pipelines group left to right, and each decodes as { type: "PipelineExpression", start, end, left, right }. The calls inside them are standard nodes, so analyze resolves sum and round, walk visits everything, and generate prints the line back as written.

The dialect file

A dialect is a Zig file with these public declarations. It imports the parser it is compiled into as @import("parser").

Mistakes are compile errors: a missing files, a function that takes a *dialect.Parser but is not named after a position, a wrong signature, a Node without shape, or an invalid shape.

Files

files lists the kinds of file your syntax lives in. Each entry is a dialect.File, an extension and the standard lang it extends: .js, .jsx, .ts, .tsx, or .dts.

pub const files: []const dialect.File = &.{
    .{ .extension = ".pipe.js", .lang = .js },
    .{ .extension = ".pipe.ts", .lang = .ts },
};

• In JavaScript, a source whose path ends in a listed extension parses with the dialect, in that entry’s lang. A source with no path uses the first entry. Any other path, or an explicit lang option, parses as standard syntax. One build reads a whole project.
• In Zig, a parse opts in with the dialect option: parser.parse(allocator, source, .{ .dialect = true }).

Positions

A position is a point in the grammar where the parser asks your dialect first. Define a pub fn with the position’s name, and the parser calls it there. Your function either parses the input as your syntax, or declines and lets standard parsing continue.

pub fn statement(p: *dialect.Parser) dialect.Error!dialect.Claim
pub fn expression(p: *dialect.Parser) dialect.Error!dialect.Claim
pub fn operator(p: *dialect.Parser, left: ast.NodeIndex, precedence: u8) dialect.Error!dialect.Claim
pub fn function_body(p: *dialect.Parser) dialect.Error!dialect.Claim
pub fn jsx_child(p: *dialect.Parser, offset: u32, parent: ast.NodeIndex) dialect.Error!dialect.Claim
pub fn jsx_attribute(p: *dialect.Parser) dialect.Error!dialect.Claim
pub fn jsx_element_name(p: *dialect.Parser) dialect.Error!dialect.Claim
pub fn module_specifier(p: *dialect.Parser) dialect.Error!dialect.Claim

Each returns a dialect.Claim.

• Positions apply everywhere, including inside your own nodes. A parseExpression inside your node asks your expression and operator again, so your syntax and standard syntax nest in each other.
• Decline anything the standard grammar accepts, unless you mean to read it differently.
• operator receives the lowest precedence that continues the current expression. When your operator binds more loosely, answer .end, so an enclosing expression takes it, as the example does.
• function_body allows return and a directive prologue inside, as a block body does.
• A jsx_child ends on its last token, which stays current, since the parser reads the text after it by offset. parent is the opening element or fragment. Use seek(offset) when the child starts with tokens.

The parser

A position function receives a *dialect.Parser, the parser as your dialect drives it.

A parse call returns null after it reports a diagnostic. Answer .failed then.

list() collects children on the parser’s scratch space, for nodes with a variable number of children.

var children = p.list();
defer children.reset();
while (p.token().tag != .right_brace) {
    try children.append(try p.parseStatement() orelse return .failed);
}
const node = try p.addNode(.block, children.items(), span);

Nodes

Your node kinds are a Node enum. Its shape function describes each kind.

pub const Node = enum {
    each,

    pub fn shape(node: Node) dialect.Shape {
        return switch (node) {
            .each => .{
                .type = "EachStatement",
                .fields = &.{ "item", "list", "body[]" },
                .block_scope = true,
                .lexical_bindings = true,
            },
        };
    }
};

addNode(kind, children, span) adds a node. children follow fields in order, with .null for an absent part, and lie inside span. In Zig, the node is an ast.Dialect, with its kind and its children in source order, which tree.extra(node.children) reads.

In JavaScript, a node is a plain object: its type, start and end, each field, and the properties. With the shape above, each (item of list) { ... } decodes as { type: "EachStatement", start, end, item, list, body: [...] }, and item is bound in the statement’s own scope.

Printing

An optional print writes your nodes back to source, so codegen prints a whole tree. Without it, codegen reports a diagnostic for each of your nodes.

pub fn print(w: *dialect.Printer, kind: Node, children: []const ast.NodeIndex) dialect.Error!void

In JavaScript, yuku/codegen prints your nodes through its print option, with the same calls.

Analysis

Semantic analysis walks into your nodes like any other node. Standard code inside them declares, resolves, and reports early errors as usual, and names inside them resolve to the bindings around them. block_scope, lexical_bindings, and catch_bindings decide what scope and bindings your node adds, and iteration makes it a loop.

Walking and lowering

In Zig, every traverser mode walks your nodes, with enter_dialect and exit_dialect hooks that receive the ast.Dialect.

pub fn enter_dialect(self: *Lower, node: ast.Dialect, index: ast.NodeIndex, ctx: *transform.Ctx) traverser.Action
pub fn exit_dialect(self: *Lower, node: ast.Dialect, index: ast.NodeIndex, ctx: *transform.Ctx) void

To compile your syntax to standard JavaScript, replace each of your nodes with standard nodes in a transform pass, with ctx.tree.setData(index, ...) and the tree’s building calls. After that, every tool reads plain JavaScript.

Building

Pass the dialect file to the yuku dependency as dialect.

const yuku = b.dependency("yuku", .{
    .target = target,
    .optimize = optimize,
    .dialect = b.path("src/pipeline.zig"),
});
my_module.addImport("parser", yuku.module("parser"));

addWasm(b, yuku, name) builds the WebAssembly module for JavaScript and installs it as zig-out/bin/<name>.wasm. Inside the Yuku repository, zig build wasm -Ddialect=src/pipeline.zig builds it as zig-out/bin/yuku.wasm.

A build holds one parser, so it holds one dialect.

JavaScript

Pass your module as the wasm option of parse, analyze, or new Analyzer. Files decides which sources parse with the dialect.

const { program } = parse(source, { path: "app.pipe.js", wasm });
const module = analyze(source, { path: "app.pipe.js", wasm });

• yuku/ast walks your nodes by their fields, and visitors can name your types.
• yuku/analyzer resolves the names inside them.
• yuku/codegen prints them through its print option.

For TypeScript, declare your nodes in DialectNodes, which adds them to Node, so visitors and node.type checks know them.

import type { BaseNode, Expression } from "yuku/types";

declare module "yuku/types" {
  interface DialectNodes {
    PipelineExpression: BaseNode & { type: "PipelineExpression"; left: Expression; right: Expression };
    TopicReference: BaseNode & { type: "TopicReference" };
  }
}

Publishing a package

A dialect’s package depends on yuku, ships the WebAssembly module, and passes it for its users.

// src/index.ts
import { parse as parseYuku, type ParseOptions, type ParseResult } from "yuku";
import { analyze as analyzeYuku, type AnalyzeOptions, type Module } from "yuku/analyzer";
import { pipeline } from "#module";

export function parse(source: string, options: Omit<ParseOptions, "wasm"> = {}): ParseResult {
  return parseYuku(source, { ...options, wasm: pipeline });
}

export function analyze(source: string, options: Omit<AnalyzeOptions, "wasm"> = {}): Module {
  return analyzeYuku(source, { ...options, wasm: pipeline });
}

To run in every runtime with no setup, load the module the way yuku does. Inline its bytes, and import the file in Workers, which cannot compile WebAssembly from bytes.

// src/module/inline.ts
import bytes from "../../zig-out/bin/pipeline.wasm";

export const pipeline = new WebAssembly.Module(bytes);

// src/module/workerd.ts
import pipeline from "#wasm";

export { pipeline };

"imports": {
  "#module": {
    "workerd": "./dist/module/workerd.js",
    "default": "./dist/module/inline.js"
  },
  "#wasm": "./dist/pipeline.wasm"
}

Bundle with a bundler that inlines .wasm as bytes, such as tsdown with loader: { ".wasm": "binary" }, keep the # imports external, and copy pipeline.wasm into dist. TypeScript reads the #module, #wasm, and .wasm imports from declarations like yuku’s own imports.d.ts.

Cost

A build without a dialect has no dialect code, since every position sits behind a compile-time check. With one compiled in, a standard file pays one flag check per position, and a dialect file one call.