Compare commits

..

5 Commits

12 changed files with 103 additions and 266 deletions

@ -10,6 +10,8 @@ $prodFlag = If($prod) {"production"} Else {""}
node .\esbuild.config.mjs -- $prodFlag
if($LASTEXITCODE -ne 0) {
$host.SetShouldExit(1);
exit 1;
}
$pluginName = "sqlite3-opfs-test-plugin"

@ -1,11 +1,28 @@
import esbuild from "esbuild";
import process from "process";
import builtins from "builtin-modules";
import { TsconfigPathsPlugin } from "@esbuild-plugins/tsconfig-paths";
import inlineWorkerPlugin from "esbuild-plugin-inline-worker";
import inlineImportPlugin from "esbuild-plugin-inline-import";
import fs from "fs";
const external = [
async function build(prod) {
fs.mkdirSync("build/plugin", { recursive: true });
const result = await esbuild
.build({
plugins: [
inlineWorkerPlugin({
alias: {
"@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-worker1-bundler-friendly":
"node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-worker1-bundler-friendly.mjs",
},
target: "es2020",
format: "cjs",
sourcemap: prod ? false : "inline",
}),
],
entryPoints: ["src/main.ts"],
bundle: true,
external: [
"obsidian",
"electron",
"@codemirror/autocomplete",
@ -20,42 +37,18 @@ const external = [
"@lezer/highlight",
"@lezer/lr",
"pdfjs-dist",
"path",
"url",
"electron",
...builtins,
];
async function build(prod) {
fs.mkdirSync("build/plugin", { recursive: true });
const result = await esbuild
.build({
plugins: [
TsconfigPathsPlugin({}),
inlineImportPlugin(),
inlineWorkerPlugin({
plugins: [TsconfigPathsPlugin({}), inlineImportPlugin()],
external,
target: "es2020",
format: "cjs",
sourcemap: prod ? false : "inline",
banner: {
js: "",
},
define: {
"import.meta.url": "globalThis.__importMetaUrl",
},
}),
],
entryPoints: ["src/main.ts"],
bundle: true,
external,
format: "cjs",
target: "es2020",
logLevel: "info",
sourcemap: prod ? false : "inline",
treeShaking: true,
outfile: "build/plugin/main.js",
alias: {
"@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-worker1-bundler-friendly":
"node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-worker1-bundler-friendly.mjs",
},
})
.catch(() => process.exit(1));
@ -63,7 +56,7 @@ async function build(prod) {
fs.copyFileSync("manifest.json", "build/plugin/manifest.json");
fs.copyFileSync("src/styles.css", "build/plugin/styles.css");
fs.copyFileSync(
"node_modules/@btfash/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm",
"node_modules/@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.wasm",
"build/plugin/sqlite3.wasm"
);
// fs.writeFileSync("build/meta.json", JSON.stringify(result.metafile));

@ -12,21 +12,17 @@
"author": "",
"license": "MIT",
"devDependencies": {
"@esbuild-plugins/tsconfig-paths": "^0.1.2",
"@types/node": "^16.11.6",
"@typescript-eslint/eslint-plugin": "5.29.0",
"@typescript-eslint/parser": "5.29.0",
"builtin-modules": "3.3.0",
"esbuild": "0.17.3",
"esbuild-plugin-inline-import": "^1.1.0",
"obsidian": "latest",
"tslib": "2.4.0",
"typescript": "4.7.4",
"esbuild-plugin-inline-worker": "^0.1.1"
"typescript": "4.7.4"
},
"dependencies": {
"@btfash/sqlite-wasm": "3.46.2",
"esbuild-plugin-inline-worker": "^0.1.1",
"localforage": "^1.10.0"
"@sqlite.org/sqlite-wasm": "^3.45.1-build1",
"esbuild-plugin-inline-worker": "^0.1.1"
}
}

@ -1,33 +1,26 @@
import { Notice, Plugin } from "obsidian";
import * as localforage from "localforage";
import {
Notice,
Plugin,
} from "obsidian";
import TestWorker from "sqlite3.worker";
import { sqlite3Worker1Promiser } from "@btfash/sqlite-wasm/index.mjs";
import { sqlite3Worker1Promiser } from "@sqlite.org/sqlite-wasm";
// Remember to rename these classes and interfaces!
interface Benchmark {
time: number;
name: string;
}
export default class MyPlugin extends Plugin {
public persister: LocalForage;
settings: any;
private worker: Worker;
private promiser: (...args: any[]) => any;
public benches: Benchmark[] = [];
private dbId: string;
async onload() {
await this.loadSettings();
this.benches.push(
await this.time("sqlite-init", async () => await this.initSqlite())
);
await this.sqlite();
this.benches.push(
await this.time("idb-init", async () => await this.initIDB())
);
await this.idb();
console.table(this.benches)
console.log(import.meta);
await this.initSqlite();
await this.start();
(window as any).sqlite3Promiser = this.promiser;
}
onunload() {
@ -48,38 +41,19 @@ export default class MyPlugin extends Plugin {
}
locateFile(path: string, prefix: string): string {
return this.app.vault.adapter.getResourcePath(
`${this.manifest.dir}/${path}`
`${this.manifest.dir}/${path}`,
);
}
async time(benchName: string, func: () => Promise<void>): Promise<Benchmark> {
const start = performance.now();
await func();
const end = performance.now();
return {
name: benchName,
time: end - start,
};
}
async initIDB() {
this.persister = localforage.createInstance({
name: "sqlite-bench",
driver: localforage.INDEXEDDB,
description: "we do a little benchmarking",
});
}
async initSqlite() {
this.worker = new TestWorker();
let w = this.worker;
this.worker.postMessage({
type: "init",
root: this.locateFile("", ""),
path: this.locateFile("sqlite3.wasm", ""),
});
this.promiser = await new Promise<typeof this.promiser>((resolve) => {
const _promiser = (sqlite3Worker1Promiser as (...args: any[]) => any)({
const _promiser = sqlite3Worker1Promiser({
print: this.printNotice,
printErr: console.error,
locateFile: this.locateFile.bind(this),
@ -89,81 +63,44 @@ export default class MyPlugin extends Plugin {
worker: () => w,
});
});
(window as any).sqlite3Promiser = this.promiser;
}
async doInsert(each: (s: number) => Promise<void>) {
for (let i = 0; i < 50; i++) {
await each(i);
}
}
async sqlite() {
this.benches.push(
await this.time("sqlite-open", async () => {
async start() {
let openRes = await this.promiser("open", {
filename: "/test.sqlite3",
vfs: "opfs",
filename: "/managed-sah/test.sqlite3",
vfs: "opfs-sahpool",
});
const { dbId } = openRes;
this.dbId = dbId;
console.log(openRes);
})
);
try {
this.benches.push(await this.time("sqlite-create-table", async () => {
console.log("Creating a table...");
let res = await this.promiser("exec", {
dbId: this.dbId,
dbId,
sql: "DROP TABLE IF EXISTS abcdef; CREATE TABLE IF NOT EXISTS abcdef(a,b,c)",
returnValue: "resultRows",
});
console.log("Result: ", res);
}));
this.benches.push(
await this.time("sqlite-insert-many", async () => {
console.log("Insert some data using exec()...");
await this.doInsert(async (i) => {
for (let i = 20; i <= 25; ++i) {
let innerRes = await this.promiser("exec", {
dbId: this.dbId,
dbId,
sql: "INSERT INTO abcdef(a,b,c) VALUES (?,?,69)",
bind: [i, i * 2],
returnValue: "resultRows",
rowMode: "object",
});
// console.log(innerRes);
});
})
);
this.benches.push(
await this.time("sqlite-query-many", async () => {
console.log(innerRes);
}
console.log("Query data with exec()...");
let rowRes = await this.promiser("exec", {
dbId: this.dbId,
sql: "SELECT * FROM abcdef ORDER BY a LIMIT 50",
dbId,
sql: "SELECT * FROM abcdef ORDER BY a LIMIT 10",
returnValue: "resultRows",
rowMode: "object",
});
console.log(rowRes);
})
);
} finally {
await this.promiser("close", { dbId: this.dbId });
await this.promiser("close", { dbId });
}
}
async idb() {
this.benches.push(await this.time("idb-insert-many", async() => {
await this.doInsert(async (s) => {
await this.persister.setItem("i/" +s, "brian tatler fucked and abused sean harris\n".repeat(Math.floor(s * 1.1)))
})
}))
this.benches.push(await this.time("idb-query-many", async() => {
let q: any[] = [];
await this.doInsert(async (s) => {
q.push(await this.persister.getItem("i/" + s))
})
console.log("qlen", q.length)
}))
}
}

5
src/sqlite.d.ts vendored Normal file

@ -0,0 +1,5 @@
import "@sqlite.org/sqlite-wasm";
declare module "@sqlite.org/sqlite-wasm" {
export const sqlite3Worker1Promiser: any;
}
declare module "@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-worker1-bundler-friendly";

@ -1,22 +1,20 @@
import sqlite3InitModule from "@btfash/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.mjs";
import str from "inline:../node_modules/@btfash/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-opfs-async-proxy.js"
global.__importMetaUrl = ""
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
(async () => {})();
let sqlite3;
globalThis
let inited = false;
onmessage = async (event) => {
console.log(event);
if (event.data.type == "init" && !inited) {
globalThis.__importMetaUrl = event.data.root
if (event.data.type == "init") {
sqlite3 = await sqlite3InitModule({
locateFile: (path: string, prefix: string) => event.data.path,
locateFile: (path, prefix) => event.data.path,
});
// debugger;
console.log(`Running sqlite version: ${sqlite3.version.libVersion}`);
await sqlite3.installOpfsVfs({
proxyUri: URL.createObjectURL(new Blob([str], {type: "application/javascript"}))
await sqlite3.installOpfsSAHPoolVfs({
clearOnInit: false,
initialCapacity: 65536,
directory: "/var/managed-sah"
});
sqlite3.initWorker1API();
console.log(sqlite3.capi.sqlite3_vfs_find("opfs"));
console.log(sqlite3.capi.sqlite3_vfs_find("opfs-sahpool"));
}
};

@ -1,4 +0,0 @@
declare global {
var __importMetaUrl: string
}
export {}

@ -1,8 +0,0 @@
declare module "@btfash/sqlite-wasm/index.mjs" {
import blah from "@btfash/sqlite-wasm";
export const sqlite3Worker1Promiser: any;
export {blah as default};
}
declare module "@btfash/sqlite-wasm/sqlite-wasm/jswasm/sqlite3.mjs";

@ -1,6 +0,0 @@
declare module "sqlite3.worker" {
const WorkerFactory: new () => Worker;
export default WorkerFactory;
}
declare module "inline:../node_modules/@btfash/sqlite-wasm/sqlite-wasm/jswasm/sqlite3-opfs-async-proxy.js"

4
src/workers.d.ts vendored Normal file

@ -0,0 +1,4 @@
declare module "sqlite3.worker" {
const WorkerFactory: new () => Worker;
export default WorkerFactory;
}

@ -11,7 +11,6 @@
"importHelpers": true,
"isolatedModules": true,
"strictNullChecks": true,
"allowSyntheticDefaultImports": true,
"lib": [
"DOM",
"ES5",
@ -19,7 +18,7 @@
"ES7"
]
},
"files": ["src/typings/sqlite.d.ts"],
"include": [
"**/*.ts"
]

@ -2,20 +2,6 @@
# yarn lockfile v1
"@btfash/sqlite-wasm@3.46.2":
version "3.46.2"
resolved "https://registry.yarnpkg.com/@btfash/sqlite-wasm/-/sqlite-wasm-3.46.2.tgz#56446e4e401e3544c34a8660b258918fbd4574ab"
integrity sha512-IPgIlas0XqCQ9/6kfQUd9VmO/diRoZ/ETzpoWAEIk2Bqz7xVCVsswmm/POZBkayGTMnJ331yJDc3iTjgJnYKKw==
"@esbuild-plugins/tsconfig-paths@^0.1.2":
version "0.1.2"
resolved "https://registry.yarnpkg.com/@esbuild-plugins/tsconfig-paths/-/tsconfig-paths-0.1.2.tgz#1955de0a124ecf4364717a2fadbfbea876955232"
integrity sha512-TusFR26Y+Ze+Zm+NdfqZTSG4XyrXKxIaAfYCL3jASEI/gHjSdoCujATjzNWaaXs6Sk6Bv2D7NLr4Jdz1gysy/Q==
dependencies:
debug "^4.3.1"
find-up "^5.0.0"
strip-json-comments "^3.1.1"
"@esbuild/aix-ppc64@0.20.1":
version "0.20.1"
resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.20.1.tgz#eafa8775019b3650a77e8310ba4dbd17ca7af6d5"
@ -262,6 +248,11 @@
"@nodelib/fs.scandir" "2.1.5"
fastq "^1.6.0"
"@sqlite.org/sqlite-wasm@^3.45.1-build1":
version "3.45.1-build1"
resolved "https://registry.yarnpkg.com/@sqlite.org/sqlite-wasm/-/sqlite-wasm-3.45.1-build1.tgz#648f6a0a0a4c3a67aff24b0e1331af655b86fb60"
integrity sha512-1EgshFNhVeBtZ9KtQPm3PzzJ2CtpmXAq2DAPywy7WZ3gOK6p5n8TY+M+mBMpQCF5cLqrdNFb3Kp9uNie9rUAHw==
"@types/codemirror@5.60.8":
version "5.60.8"
resolved "https://registry.yarnpkg.com/@types/codemirror/-/codemirror-5.60.8.tgz#b647d04b470e8e1836dd84b2879988fc55c9de68"
@ -393,13 +384,6 @@ commondir@^1.0.1:
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
integrity sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==
debug@^4.3.1:
version "4.3.5"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e"
integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==
dependencies:
ms "2.1.2"
debug@^4.3.4:
version "4.3.4"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
@ -414,11 +398,6 @@ dir-glob@^3.0.1:
dependencies:
path-type "^4.0.0"
esbuild-plugin-inline-import@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/esbuild-plugin-inline-import/-/esbuild-plugin-inline-import-1.1.0.tgz#9a581a50918021c48962da2309aecf24d3fb9fcd"
integrity sha512-b0xX4tPKBdRjX1CkzpnULpEdeTo9vxD+wf83PKvgUYnOEaJfVxLey4q+sfTUPAdDnRRYasJsQUgQW7/e2Gm5Dw==
esbuild-plugin-inline-worker@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/esbuild-plugin-inline-worker/-/esbuild-plugin-inline-worker-0.1.1.tgz#f21a610ad7410972a408272e0d254654381eac58"
@ -568,14 +547,6 @@ find-up@^4.0.0:
locate-path "^5.0.0"
path-exists "^4.0.0"
find-up@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc"
integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==
dependencies:
locate-path "^6.0.0"
path-exists "^4.0.0"
functional-red-black-tree@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
@ -605,11 +576,6 @@ ignore@^5.2.0:
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef"
integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==
immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==
is-extglob@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
@ -627,20 +593,6 @@ is-number@^7.0.0:
resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
lie@3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
integrity sha512-RiNhHysUjhrDQntfYSfY4MU24coXXdEOgw9WGcKHNeEwffDYbF//u87M1EWaMGzuFoSbqW0C9C6lEEhDOAswfw==
dependencies:
immediate "~3.0.5"
localforage@^1.10.0:
version "1.10.0"
resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.10.0.tgz#5c465dc5f62b2807c3a84c0c6a1b1b3212781dd4"
integrity sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==
dependencies:
lie "3.1.1"
locate-path@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0"
@ -648,13 +600,6 @@ locate-path@^5.0.0:
dependencies:
p-locate "^4.1.0"
locate-path@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286"
integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==
dependencies:
p-locate "^5.0.0"
lru-cache@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
@ -707,13 +652,6 @@ p-limit@^2.2.0:
dependencies:
p-try "^2.0.0"
p-limit@^3.0.2:
version "3.1.0"
resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==
dependencies:
yocto-queue "^0.1.0"
p-locate@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07"
@ -721,13 +659,6 @@ p-locate@^4.1.0:
dependencies:
p-limit "^2.2.0"
p-locate@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834"
integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==
dependencies:
p-limit "^3.0.2"
p-try@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
@ -794,11 +725,6 @@ slash@^3.0.0:
resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
strip-json-comments@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
to-regex-range@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
@ -832,8 +758,3 @@ yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yocto-queue@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==