Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* @cursorless-dev/code-owners
26 changes: 0 additions & 26 deletions .github/workflows/build-and-publish.yml

This file was deleted.

34 changes: 34 additions & 0 deletions .github/workflows/deploy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: Deploy

on:
push:
branches: [main]

jobs:
publish-npm:
name: Publish to NPM
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Install pnpm
uses: pnpm/action-setup@v4

- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version-file: .nvmrc
cache: pnpm

- name: Install dependencies
run: pnpm --color install

- name: Build
run: pnpm build

- name: Publish to NPM
uses: JS-DevTools/npm-publish@v1
with:
token: ${{ secrets.NPM_TOKEN }}
30 changes: 30 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: Run Tests
permissions: read-all

on:
pull_request:
types: [opened, synchronize, reopened]

jobs:
test:
name: Test
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Install pnpm
uses: pnpm/action-setup@v4

- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version-file: .nvmrc
cache: pnpm

- name: Install dependencies
run: pnpm --color install

- name: Build
run: pnpm build
1 change: 1 addition & 0 deletions .nvmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
v20.18.0
3 changes: 3 additions & 0 deletions .prettierrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"tabWidth": 2
}
3 changes: 3 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"editor.defaultFormatter": "esbenp.prettier-vscode"
}
24 changes: 2 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,23 +1,3 @@
# tree-sitter-wasms
Prebuilt WASM binaries for tree-sitter's language parsers. Forked from https://github.com/Menci/tree-sitter-wasm-prebuilt because I wanted to use GitHub Actions to automate publishing.
# Cursorless Tree sitter wasms

## Installation

```bash
pnpm add tree-sitter-wasms
# or
yarn add tree-sitter-wasms
# or
npm install tree-sitter-wasms
```

## Usage

```ts
import treeSitterRust from "tree-sitter-wasms/out/tree-sitter-rust.wasm"
parser.setLanguage(treeSitterCpp);
```

## Supported Languages

Check https://unpkg.com/browse/tree-sitter-wasms@latest/out/ to see all supported languages, and manually download the wasm artifacts directly.
Repository building wasms for use by Cursorless.
105 changes: 70 additions & 35 deletions build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,51 +16,57 @@ const outDir = path.join(__dirname, "out");

let hasErrors = false;

async function gitCloneOverload(name: string, repoUrl: string, useLatest: boolean, {commitHash}: {commitHash?: string}) {
async function gitCloneOverload(
name: string,
repoUrl: string,
useLatest: boolean,
{ commitHash }: { commitHash?: string }
) {
let packagePath;
try {
packagePath = findRoot(require.resolve(name));
} catch (_) {
packagePath = path.join(__dirname, "node_modules", name);
}

let commitInfo = (commitHash === undefined) ? "latest" : commitHash;
let commitInfo = commitHash === undefined ? "latest" : commitHash;

try {
console.log(`🗑️ Deleting cached node dep for ${ name }`);
await exec(`rm -rf ${ packagePath }`)
console.log(`⬇️ Cloning ${ name } from git (${commitInfo})`);
await exec(`git clone ${ repoUrl } ${ packagePath }`)
console.log(`🗑️ Deleting cached node dep for ${name}`);
await exec(`rm -rf ${packagePath}`);
console.log(`⬇️ Cloning ${name} from git (${commitInfo})`);
await exec(`git clone ${repoUrl} ${packagePath}`);
if (!useLatest) {
if (commitHash !== undefined) {
process.chdir(packagePath);
await exec(`git reset --hard ${ commitHash }`);
} else throw new Error("Latest commit is not being used, yet no commit hash was specified");
await exec(`git reset --hard ${commitHash}`);
} else
throw new Error(
"Latest commit is not being used, yet no commit hash was specified"
);
}
} catch (err) {
console.error(`❗Failed to clone git repo for ${ name }:\n`, err);
console.error(`❗Failed to clone git repo for ${name}:\n`, err);
}
}

async function buildParserWASM(
name: string,
{ subPath, generate }:
{ subPath?: string; generate?: boolean } = {}
{ subPath, generate }: { subPath?: string; generate?: boolean } = {}
) {

const label = subPath ? path.join(name, subPath) : name;

let cliPackagePath;
try {
cliPackagePath = findRoot(require.resolve("tree-sitter-cli"));
} catch(_) {
} catch (_) {
cliPackagePath = path.join(__dirname, "node_modules", "tree-sitter-cli");
}

let cliPath = path.join(cliPackagePath, "tree-sitter");
let generateCommand = cliPath.concat(" generate");
let buildCommand = cliPath.concat(" build --wasm");

try {
console.log(`⏳ Building ${label}`);
let packagePath;
Expand All @@ -73,7 +79,7 @@ async function buildParserWASM(
if (generate) {
await exec(generateCommand, { cwd });
}
await exec(buildCommand, { cwd });
await exec(buildCommand, { cwd });
await exec(`mv *.wasm ${outDir}`, { cwd });
console.log(`✅ Finished building ${label}`);
} catch (e) {
Expand All @@ -91,48 +97,77 @@ fs.mkdirSync(outDir);
process.chdir(outDir);

const grammars = Object.keys(packageInfo.devDependencies)
.filter((n) => n.startsWith("tree-sitter-") && n !== "tree-sitter-cli" && n !== "tree-sitter")
.filter(
(n) =>
n.startsWith("tree-sitter-") &&
n !== "tree-sitter-cli" &&
n !== "tree-sitter"
)
.filter((s) => !langArg || s.includes(langArg));

PromisePool.withConcurrency(os.cpus().length)
.for(grammars)
.process(async (name : string) => {
.process(async (name: string) => {
if (name == "tree-sitter-agda") {
await gitCloneOverload(name, "https://github.com/tree-sitter/tree-sitter-agda.git", false, {commitHash: "47802091de0cb8ac2533d67ac37e65692c5902c4"});
await buildParserWASM(name)
await gitCloneOverload(
name,
"https://github.com/tree-sitter/tree-sitter-agda.git",
false,
{ commitHash: "47802091de0cb8ac2533d67ac37e65692c5902c4" }
);
await buildParserWASM(name);
} else if (name == "tree-sitter-perl") {
await gitCloneOverload(name, "https://github.com/tree-sitter-perl/tree-sitter-perl.git", false, {commitHash: "b467c41aed6e52c7f581c18a3b50821995a82c95"});
await buildParserWASM(name, {generate: true});
await gitCloneOverload(
name,
"https://github.com/tree-sitter-perl/tree-sitter-perl.git",
false,
{ commitHash: "b467c41aed6e52c7f581c18a3b50821995a82c95" }
);
await buildParserWASM(name, { generate: true });
} else if (name == "tree-sitter-php") {
await buildParserWASM(name, { subPath: "php" });
} else if (name == "tree-sitter-typescript") {
await buildParserWASM(name, { subPath: "typescript" });
await buildParserWASM(name, { subPath: "tsx" });
} else if (name == "tree-sitter-latex") {
await buildParserWASM(name, {generate: true});
await buildParserWASM(name, { generate: true });
} else if (name == "tree-sitter-xml") {
await buildParserWASM(name, {subPath: "xml"});
await buildParserWASM(name, {subPath: "dtd"});
await buildParserWASM(name, { subPath: "xml" });
await buildParserWASM(name, { subPath: "dtd" });
} else if (name == "tree-sitter-query") {
await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-query.git", false, {commitHash: "a6674e279b14958625d7a530cabe06119c7a1532"});
await gitCloneOverload(
name,
"https://github.com/tree-sitter-grammars/tree-sitter-query.git",
false,
{ commitHash: "a6674e279b14958625d7a530cabe06119c7a1532" }
);
await buildParserWASM(name);
} else if (name == "tree-sitter-elixir") {
await gitCloneOverload(name, "https://github.com/elixir-lang/tree-sitter-elixir.git", false, {commitHash: "02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e"});
await gitCloneOverload(
name,
"https://github.com/elixir-lang/tree-sitter-elixir.git",
false,
{ commitHash: "02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e" }
);
await buildParserWASM(name);
} else if (name == "tree-sitter-markdown") {
await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-markdown", false, {commitHash: "192407ab5a24bfc24f13332979b5e7967518754a"});
await buildParserWASM(name, {subPath: "tree-sitter-markdown"});
await buildParserWASM(name, {subPath: "tree-sitter-markdown-inline"});
}
else {
await gitCloneOverload(
name,
"https://github.com/tree-sitter-grammars/tree-sitter-markdown",
false,
{ commitHash: "192407ab5a24bfc24f13332979b5e7967518754a" }
);
await buildParserWASM(name, { subPath: "tree-sitter-markdown" });
await buildParserWASM(name, { subPath: "tree-sitter-markdown-inline" });
} else {
await buildParserWASM(name);
}
})
.then(async () => {
if (hasErrors) {
//not sure if this failsafe is actually required, but it doesn't hurt to ensure that no wasms can be published from a failed build
//not sure if this failsafe is actually required, but it doesn't hurt to ensure that no wasms can be published from a failed build
fs.rmSync(outDir, { recursive: true, force: true });
fs.mkdirSync(outDir);
process.exit(1);
}
});
});
33 changes: 18 additions & 15 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,23 @@
{
"name": "@cursorless/tree-sitter-wasms",
"version": "0.2.0",
"version": "0.1.0",
"description": "Prebuilt WASM binaries for tree-sitter's language parsers.",
"repository": "https://github.com/cursorless-dev/tree-sitter-wasms",
"author": "Gregor <[email protected]>, Menci <[email protected]>, rinOfTheStars",
"packageManager": "[email protected]",
"main": "bindings/node",
"author": "Cursorless Dev",
"license": "Unlicense",
"files": [
"/out"
],
"repository": {
"type": "git",
"url": "git+https://github.com/cursorless-dev/tree-sitter-wasms.git"
},
"pnpm": {
"onlyBuiltDependencies": [
"tree-sitter-cli"
]
},
"scripts": {
"build": "ts-node build.ts"
},
Expand Down Expand Up @@ -48,26 +61,16 @@
"tree-sitter-rust": "^0.23.2",
"tree-sitter-scala": "^0.23.4",
"tree-sitter-scss": "github:tree-sitter-grammars/tree-sitter-scss#00a4cd98d5fc71aa5a55174e7ed5e34c9cc85a87",
"tree-sitter-sparql": "github:IoannisNezis/tree-sitter-sparql#3c468620e7c2ffa69441de60161c4e4a919969f7",
"tree-sitter-sparql": "github:IoannisNezis/tree-sitter-sparql#3c468620e7c2ffa69441de60161c4e4a919969f7",
"tree-sitter-swift": "^0.6.0",
"tree-sitter-talon": "github:wenkokke/tree-sitter-talon#7119d7f3fefd0b0c7e900120b51f85a23e6c3160",
"tree-sitter-typescript": "^0.23.2",
"tree-sitter-xml": "github:tree-sitter-grammars/tree-sitter-xml#ce150c5be7f617e18ffa2064213e5d9c4cb5d69e",
"ts-node": "^10.9.2",
"typescript": "^5.7.3"
},
"pnpm": {
"onlyBuiltDependencies": [
"tree-sitter-cli"
]
},
"files": [
"/out"
],
"packageManager": "[email protected]",
"main": "bindings/node",
"dependencies": {
"tree-sitter": "^0.22.4",
"tree-sitter-cli": "^0.24.7"
}
}
}
Loading