Codebase
tcmts.cli.codebase.Codebase · #hkewofc7 · class
import { checkerPool } from "#codebase/tcmts/cli/checkers";
import {
augmentationsIn, fileProblem, namedCodebase, registerTransformHook, sessionDirFor, setUpAgents, transformHookSource,
} from "#codebase/tcmts/cli/codebase";
import type { LockEntry, Rehashed, SaltFor } from "#codebase/tcmts/cli/codebase";
import { notFound } from "#codebase/tcmts/cli/help";
import { keepValues, keptValue } from "#codebase/tcmts/cli/kept";
import { librariesIn, libraryGuard, within } from "#codebase/tcmts/cli/libraries";
import { syncMirror } from "#codebase/tcmts/cli/mirror";
import type { Mirror } from "#codebase/tcmts/cli/mirror";
import { providePackages } from "#codebase/tcmts/cli/provided";
import { REACT_CACHE } from "#codebase/tcmts/cli/react_api";
import { codebaseRoles } from "#codebase/tcmts/cli/roles";
import { serverHostFiles } from "#codebase/tcmts/cli/server_api";
import { srcDir, viewOf, writeViews } from "#codebase/tcmts/cli/views";
import { analyzeFile, analyzeProject, saltOf, scratchNamespace } from "#codebase/tcmts/core/analyze";
import type { Analysis, AnalyzeHost, ProjectAnalysis, ProjectDef } from "#codebase/tcmts/core/analyze";
import { hashIR, IR_VERSION, rehashComponent, shortHash } from "#codebase/tcmts/core/hash";
import type { IR } from "#codebase/tcmts/core/hash";
import {
incrementalCompiler, NODE_TYPES_RANGE, projectCompilerOptions, tcmNodeTypes, transpileModule, TS_VERSION,
useNodeTypes,
} from "#codebase/tcmts/core/program";
import type { CompilerOptions } from "#codebase/tcmts/core/program";
import { compareVersions, satisfies as satisfies_ } from "#codebase/tcmts/core/semver";
import { isConflicted } from "#codebase/tcmts/core/types";
import type { DefRecord } from "#codebase/tcmts/core/types";
import {
augmentationRoot, externalsOf, layout, materialize, moduleFile, render, writeIfChanged, writeShim,
} from "#codebase/tcmts/loader";
import type { DefSource, Layout } from "#codebase/tcmts/loader";
import {
ensureExternals, lockAtLeast, npmAlias, parseModule, shippedPins, usePackageLock,
} from "#codebase/tcmts/loader/packages";
import { Store } from "#codebase/tcmts/store";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path_1 from "node:path";
export class Codebase {
readonly lay: Layout;
readonly store: Store;
/** a session's own directory (`TCM_BRANCH`): its scratch files, `#codebase` shim and views */
readonly sessionDir: string | undefined;
/** scratch files are checked over and over (the watcher): unchanged files aren't parsed again */
private compile = incrementalCompiler();
private constructor(root: string) {
const session = process.env.TCM_BRANCH;
this.sessionDir = session ? sessionDirFor(root, session) : undefined;
this.lay = this.sessionDir
? { ...layout(root), shimDir: path_1.join(this.sessionDir, "codebase") }
: layout(root);
this.store = new Store(this.lay.dbFile);
// a library's names change only through `tcm lib`
this.store.guard = libraryGuard(this.store);
// a codebase `tcm init` made before the server library named #tcm/server in its imports: it's #tcm/host now (the
// server API is the server library, mounted with `tcm lib add tcm server`), which sessions copy
const rootPkgFile = path_1.join(root, "package.json");
try {
const pkg = JSON.parse(fs.readFileSync(rootPkgFile, "utf8")) as {
imports?: Record<string, string>;
};
if (pkg.imports?.["#tcm/server"] === "./.tcm/cache/tcm-server.js") {
delete pkg.imports["#tcm/server"];
pkg.imports["#tcm/host"] = "./.tcm/cache/tcm-host.js";
fs.writeFileSync(rootPkgFile, JSON.stringify(pkg, null, 2) + "\n");
}
} catch {
// (none, or one that isn't JSON: someone's own, left as it is)
}
// a session's scratch files resolve `#codebase` to its own shim, and the rest as the codebase's do
if (this.sessionDir) {
fs.mkdirSync(this.sessionDir, { recursive: true });
const rootPkg = path_1.join(root, "package.json");
const imports = fs.existsSync(rootPkg)
? ((JSON.parse(fs.readFileSync(rootPkg, "utf8")) as { imports?: Record<string, string> })
.imports ?? {})
: {};
const up = path_1.relative(this.sessionDir, root).split(path_1.sep).join("/");
// an import map names files of its own package only (`./…`): one of the codebase's (`#tcm/host`) is a module
// here that re-exports it, its types included
const stubs = path_1.join(this.sessionDir, "imports");
const local = Object.fromEntries(
Object.entries(imports).map(([k, v]) => {
if (!v.startsWith("./") || v.includes("*") || !/\.[cm]?js$/.test(v))
return [k, v.startsWith("./") ? `${up}/${v.slice(2)}` : v];
const from = path_1.relative(stubs, path_1.join(root, v)).split(path_1.sep).join("/");
const name = `${k.replace(/[^\w.-]+/g, "_")}${path_1.extname(v)}`;
fs.mkdirSync(stubs, { recursive: true });
writeIfChanged(path_1.join(stubs, name), `export * from ${JSON.stringify(from)};\n`);
writeIfChanged(
path_1.join(stubs, name.replace(/\.([cm]?)js$/, ".d.$1ts")),
`export * from ${JSON.stringify(from)};\n`,
);
return [k, `./imports/${name}`];
}),
);
const pkg = {
private: true,
type: "module",
imports: { ...local, "#codebase": "./codebase/index.ts", "#codebase/*": "./codebase/*.ts" },
};
writeIfChanged(
path_1.join(this.sessionDir, "package.json"),
JSON.stringify(pkg, null, 2) + "\n",
);
writeIfChanged(
path_1.join(this.sessionDir, "tsconfig.json"),
JSON.stringify({ extends: `${up}/tsconfig.json`, include: ["*.ts", "*.tsx"] }, null, 2) +
"\n",
);
}
// the versions generated modules run with
usePackageLock(this.lay, {
get: (name, range) => this.store.lockedVersion(name, range),
set: (name, range, version, integrity) =>
this.store.lockVersion(name, range, version, integrity),
integrity: (name, range) => this.store.lockedIntegrity(name, range),
all: () => this.store.lockedPackages(),
});
// and the ranges it chose for packages tcm ships (`tcm upgrade vitest@<range>`), which are installed, not tcm's
shippedPins.set(this.lay.cacheDir, (name) => this.store.getSetting(`tool.${name}`));
// tcm checks code against its own @types/node; running from a codebase with none installed near it, the
// codebase provides it like any package
if (!tcmNodeTypes()) {
const module = `npm:@types/node@${NODE_TYPES_RANGE}`;
ensureExternals(this.lay, [{ module, subpath: "", export: "*" }]);
useNodeTypes(path_1.join(this.lay.cacheDir, "node_modules", npmAlias(module)));
}
// for what runs from the codebase (tests, watch expressions), which may be far from tcm's own copy
writeIfChanged(
path_1.join(this.lay.cacheDir, "node-types.json"),
JSON.stringify(tcmNodeTypes()) + "\n",
);
// a codebase (or the tcm it was made with) that moved names a @types/node that isn't there any more, where
// `tcm init` wrote it: point that at this tcm's (type roots it doesn't recognize are left alone)
const tsconfig = path_1.join(root, "tsconfig.json");
const types = tcmNodeTypes();
if (types && fs.existsSync(tsconfig)) {
try {
const cfg = JSON.parse(fs.readFileSync(tsconfig, "utf8")) as {
compilerOptions?: { typeRoots?: string[]; types?: string[] };
};
const roots = cfg.compilerOptions?.typeRoots;
if (
cfg.compilerOptions &&
roots?.length &&
roots.every((r) => path_1.isAbsolute(r) && !fs.existsSync(r))
) {
cfg.compilerOptions.typeRoots = types.typeRoots;
cfg.compilerOptions.types = types.types;
fs.writeFileSync(tsconfig, JSON.stringify(cfg, null, 2) + "\n");
}
} catch {
// (one with comments in it is someone's own: left as it is)
}
}
// `tcm init` named tcm's own modules in package.json's imports: only the ones this tcm has (`#tcm/react` is the
// react library now, mounted with `tcm lib add tcm react`)
const pkgFile = path_1.join(root, "package.json");
try {
const pkg = JSON.parse(fs.readFileSync(pkgFile, "utf8")) as {
imports?: Record<string, string>;
};
if (pkg.imports?.["#tcm/react"] === "./.tcm/cache/tcm-react.js") {
delete pkg.imports["#tcm/react"];
fs.writeFileSync(pkgFile, JSON.stringify(pkg, null, 2) + "\n");
}
} catch {
// (none, or one that isn't JSON: someone's own, left as it is)
}
// the #tcm/host runtime every generated module and runner in this codebase shares
const host = serverHostFiles();
fs.mkdirSync(this.lay.cacheDir, { recursive: true });
writeIfChanged(path_1.join(this.lay.cacheDir, "tcm-host.js"), host.js);
writeIfChanged(path_1.join(this.lay.cacheDir, "tcm-host.d.ts"), host.dts);
// and React's cache and cacheSignal, as the server code tcm runs has them (`loader.render` points at it)
writeIfChanged(path_1.join(this.lay.cacheDir, "tcm-cache.js"), REACT_CACHE);
// imported code may use syntax Node can't strip: such modules are compiled on load, here and in runners
// the compilation (with its source maps) is tcm's own, loaded from where tcm runs
writeIfChanged(
path_1.join(this.lay.cacheDir, "tcm-transform.js"),
transformHookSource(transpileModule()),
);
registerTransformHook();
// hashes in another format aren't comparable with this tcm's: refuse rather than mix them
const versions = this.store.irVersions().filter((v) => v !== IR_VERSION);
if (versions.length) {
this.store.close();
throw new Error(
versions.some((v) => v > IR_VERSION)
? `${root} was written with hash format v${versions.join(", v")}, newer than this tcm's (v${IR_VERSION}): use a newer tcm`
: `${root} was written with hash format v${versions.join(", v")}; this tcm uses v${IR_VERSION}: \`tcm migrate\` moves it (history and all, keeping it as it was beside it)`,
);
}
}
static find(cwd: string): Codebase {
return new Codebase(Codebase.findRoot(cwd));
}
/** The codebase in `root`, exactly (not the one `TCM_CODEBASE` names): for code that says which it means. */
static open(root: string): Codebase {
if (!fs.existsSync(path_1.join(root, ".tcm", "codebase.sqlite")))
throw new Error(`no tcm codebase in ${root}`);
return new Codebase(root);
}
/** The nearest directory at or above `cwd` holding a codebase, or the one `TCM_CODEBASE` names. */
static findRoot(cwd: string): string {
if (namedCodebase) return namedCodebase;
for (let dir = cwd; ; dir = path_1.dirname(dir)) {
if (fs.existsSync(path_1.join(dir, ".tcm", "codebase.sqlite"))) return dir;
if (path_1.dirname(dir) === dir)
throw new Error("no tcm codebase here; run `tcm init` first");
}
}
/** The cache module of a component: `.tsx` when it uses JSX. */
modulePath(componentHash: string): string {
return path_1.join(
this.lay.cacheDir,
moduleFile(componentHash, { members: (c) => this.store.componentMembers(c) }),
);
}
static init(root: string): Codebase {
fs.mkdirSync(path_1.join(root, ".tcm", "cache"), { recursive: true });
fs.writeFileSync(
path_1.join(root, ".tcm", ".gitignore"),
"# derived from codebase.sqlite\ncache/\ncodebase/\nsrc/\nweb/\nbuild/\napps/\nexamples/\ntest/\nlint/\nfmt/\npack/\ncompiled/\npackages.json\n# app data and secrets: never commit these\ndata/\nsecret\ntls/\n",
);
const pkgFile = path_1.join(root, "package.json");
const pkg = fs.existsSync(pkgFile)
? JSON.parse(fs.readFileSync(pkgFile, "utf8"))
: {
name: path_1
.basename(root)
.toLowerCase()
.replace(/[^a-z0-9-]/g, "-"),
private: true,
};
pkg.type = "module";
pkg.imports = {
...pkg.imports,
"#codebase": "./.tcm/codebase/index.ts",
"#codebase/*": "./.tcm/codebase/*.ts",
"#tcm/host": "./.tcm/cache/tcm-host.js",
};
fs.writeFileSync(pkgFile, JSON.stringify(pkg, null, 2) + "\n");
// opened first: where @types/node is comes from it (see the constructor)
const cb = new Codebase(root);
const tsconfig = path_1.join(root, "tsconfig.json");
if (!fs.existsSync(tsconfig)) {
const options = {
target: "esnext",
// one codebase holds server and browser code; where each definition runs is inferred (see placement.ts)
lib: ["esnext", "dom"],
module: "nodenext",
moduleResolution: "nodenext",
strict: true,
noEmit: true,
erasableSyntaxOnly: true,
verbatimModuleSyntax: true,
allowImportingTsExtensions: true,
jsx: "react-jsx",
skipLibCheck: true,
...tcmNodeTypes(),
};
fs.writeFileSync(
tsconfig,
JSON.stringify({ compilerOptions: options, include: ["*.ts", "*.tsx"] }, null, 2) + "\n",
);
}
setUpAgents(root);
cb.refreshShim(true);
return cb;
}
/** Definitions from the store, overlaid with in-memory ones (scratch definitions not added yet). */
source(overlay: DefRecord[] = [], nameAt?: (hash: string) => string | undefined): DefSource {
const byHash = new Map(overlay.map((d) => [d.hash, d]));
const byComp = new Map<string, DefRecord[]>();
for (const d of overlay)
byComp.set(d.componentHash, [...(byComp.get(d.componentHash) ?? []), d]);
const views = new Map<string, { file: string; line: number; inserts?: number[] } | undefined>();
const src: DefSource = {
get: (h) => byHash.get(h) ?? this.store.getDef(h),
members: (c) =>
(byComp.get(c) ?? this.store.componentMembers(c)).sort(
(a, b) => a.componentIndex - b.componentIndex,
),
// (names as of another branch or history entry, if asked: a diff reads each side with its own)
nameOf: nameAt ?? ((h) => this.store.pathsOf(h)[0]),
// source maps lead here: the definition's view file, or the scratch file it's still in
viewOf: (h) => viewOf(this, byHash.get(h) ?? this.store.getDef(h), views),
views: srcDir(this),
// pure constants, evaluated once: what they're worth, rather than computed every time
kept: (h) => keptValue(this, byHash.get(h) ?? this.store.getDef(h)),
keep: (hs) => keepValues(this, hs, src),
};
return src;
}
/** whether name changes are written back to the mirror (not while the mirror itself is being imported) */
syncsMirror = true;
/**
* Import the mirror's files (`tcm import <dir> --update`): how a rename of a function or class is finished. Its
* name is part of its hash (code can see it as `.name`), so in a file the rename is a change to the code, and the
* codebase takes the renamed code. Set by the CLI, which owns importing.
*/
adoptMirror?: (m: Mirror) => boolean;
/** Keep `.tcm/codebase` (the `#codebase` import shim) and the mirror, if there is one, in sync with the branch's names. */
refreshShim(force = false) {
// another branch, only read (`--branch`): they're this process's branch's, and stay as they are
if (this.store.reading !== undefined) return;
const stamp = `${this.store.branch().name}:${this.store.logHead()}`;
// (each session has its own shim)
const key = this.sessionDir ? `shim_stamp:${this.store.branch().name}` : "shim_stamp";
const was = this.store.getSetting(key);
const there = fs.existsSync(path_1.join(this.lay.shimDir, "index.ts"));
if (!force && was === stamp && there) return;
// written on this branch before: only the namespaces whose names changed since are written again (what the name
// log says), not every one, with every module checked
const at = was?.lastIndexOf(":") ?? -1;
const since =
!force && there && was && was.slice(0, at) === this.store.branch().name
? Number(was.slice(at + 1))
: NaN;
const changed = Number.isInteger(since)
? new Set(this.store.pathsChangedSince(since))
: undefined;
const only =
changed &&
new Set([...changed].map((p) => (p.includes(".") ? p.slice(0, p.lastIndexOf(".")) : "")));
// code imports files too: they're values
const names = this.store.names(undefined, true);
writeShim(names, this.source(), this.lay, only);
// terms named now that came without a type shape (received: receiving leaves them), worked out, so their
// signatures name types as they're called now (`shownSignature`) and searches by type find them
this.fillShapesQuietly(names.filter((n) => !changed || changed.has(n.path)).map((n) => n.hash));
// and the views (definitions as files, for editors and debuggers): the branch's own, a session's in its directory
writeViews(this, force);
this.store.setSetting(key, stamp);
if (this.syncsMirror) syncMirror(this);
}
private optionsCache = new Map<string, CompilerOptions>();
private warned = new Set<string>();
/**
* Compiler options from the tsconfig.json nearest `dir` (the project root's by default), with the codebase's
* augmentations in `types`, all but those `without` leaves out (a scratch file holds new versions of them).
*/
compilerOptions(
dir = this.lay.root,
mode: "scratch" | "import" = "scratch",
without?: (path: string) => boolean,
): CompilerOptions {
let tsconfig: string | undefined;
for (let d = path_1.resolve(dir); ; d = path_1.dirname(d)) {
if (fs.existsSync(path_1.join(d, "tsconfig.json"))) {
tsconfig = path_1.join(d, "tsconfig.json");
break;
}
if (d === this.lay.root || path_1.dirname(d) === d) break;
}
const key = `${mode} ${tsconfig ?? ""}`;
if (!this.optionsCache.has(key)) {
const { options, warnings } = projectCompilerOptions(tsconfig, mode);
for (const w of warnings)
if (!this.warned.has(w)) {
this.warned.add(w);
console.error(`warning: ${w}`);
}
this.optionsCache.set(key, options);
}
const options = this.optionsCache.get(key)!;
// each augmentation is a type package of its own (see `writeShim`), so a check can leave some out
const root = augmentationRoot(this.lay);
const augmentations = fs.existsSync(root)
? fs.readdirSync(root).filter((p) => !without?.(p))
: [];
if (!augmentations.length) return options;
return {
...options,
typeRoots: [...(options.typeRoots ?? []), root],
// (a project listing no types gets every package in its type roots, these too)
...(options.types ? { types: [...options.types, ...augmentations] } : {}),
};
}
private host(
ignoreDiagnostics: number[],
saltFor: SaltFor | undefined,
compilerOptions = this.compilerOptions(),
wasTable?: (file: string, name: string) => boolean,
): AnalyzeHost {
return {
cacheDir: this.lay.cacheDir,
ignoreDiagnostics,
saltFor,
compilerOptions,
roleOf: (hash) => codebaseRoles(this).get(hash),
...(wasTable ? { wasTable } : {}),
resolveName: (name, meaning) => {
const hs = this.store.hashesAt(name, meaning === "type" ? "type" : "term");
if (hs.length === 0) throw new Error(`'${name}' is not in the codebase`);
if (hs.length > 1)
throw new Error(
`'${name}' is conflicted (${hs.length} definitions); resolve it first (\`tcm todo\`)`,
);
return hs[0]!;
},
};
}
/** `overlay`: unsaved text for files (an editor's buffers), used instead of what's on disk */
analyze(
file: string,
saltFor: SaltFor | undefined,
ignoreDiagnostics: number[] = [],
overlay?: ReadonlyMap<string, string>,
): Analysis {
// packages the file imports that the project doesn't install: tcm provides them
const text = overlay?.get(file) ?? fs.readFileSync(file, "utf8");
const provided = providePackages(this, file, text, this.compilerOptions());
// a provided package with no types anywhere (most ESLint plugins, say) is typed as any, with a warning: there's
// no @types to install, tcm already looked
const untyped = new Set<string>();
const redefined = augmentationsIn(text);
const a = analyzeFile(file, {
...this.host(
ignoreDiagnostics,
saltFor,
this.compilerOptions(this.lay.root, "scratch", (p) => redefined.has(p)),
),
compile: (files, options) => this.compile(files, options, overlay, provided.resolver),
packageDir: (name) => provided.dirs.get(name),
skipDiagnostic: (d) => {
if (d.code !== 7016) return false;
const m = /module '([^']+)'/.exec(d.text);
if (!m || !provided.dirs.has(m[1]!)) return false;
untyped.add(m[1]!);
return true;
},
});
for (const name of untyped)
a.warnings.push(`${name} has no types (neither its own nor @types), so it's typed as any`);
return a;
}
analyzeProject(
files: string[],
saltFor: SaltFor | undefined,
ignoreDiagnostics: number[] = [],
compilerOptions?: CompilerOptions,
typecheck = true,
shapes = typecheck,
wasTable?: (file: string, name: string) => boolean,
): ProjectAnalysis {
return analyzeProject(files, {
...this.host(ignoreDiagnostics, saltFor, compilerOptions, wasTable),
typecheck,
shapes,
});
}
/**
* The lineage salt for a definition about to be bound at `path`: whatever already lives there keeps its
* identity (so updates and re-adds reproduce hashes), and anything new is identified by where it was first added.
*/
lineageSalt(path: string): string {
for (const h of this.store.hashesAt(path)) {
const salt = saltOf(this.store.getDef(h)!.ir);
if (salt !== undefined) return salt;
}
return path;
}
/** Salts for a scratch file whose definitions go into `into` (or the file's namespace pragma). */
scratchSalts(into?: string): SaltFor {
return (file, name, targetPath) => {
if (targetPath) return this.lineageSalt(targetPath);
const ns = into ?? scratchNamespace(fs.readFileSync(file, "utf8")) ?? "";
return this.lineageSalt(ns ? `${ns}.${name}` : name);
};
}
/**
* Options for re-checking definitions: import-mode resolution (rendered modules import by hash, and may use syntax
* only imported code can) plus the type-checking profile the definitions were first checked under.
*/
recheckOptions(profile: Record<string, unknown> | undefined): CompilerOptions {
return {
...this.compilerOptions(this.lay.root, "import"),
...profile,
} as CompilerOptions;
}
/** Salts for re-analyzing rendered definitions: each keeps the salt it was stored with. */
private storedSalts(byFile: Map<string, Map<string, DefRecord>>): SaltFor {
return (file, name) => {
const rec = byFile.get(file)?.get(name);
return rec && saltOf(rec.ir);
};
}
/** For re-analyzing rendered definitions: each that was a table stays one (there are no names to read roles from). */
private storedTables(byFile: Map<string, Map<string, DefRecord>>) {
return (file: string, name: string) => !!byFile.get(file)?.get(name)?.table;
}
/**
* Re-render every live definition from the store and re-hash it. Rendering is supposed to be the inverse of
* hashing, so any mismatch means stored source, splicing, or canonicalization lost information.
*
* A component this same tcm verified already, rendered with the same names, isn't checked again (`fresh` checks
* it anyway): what it renders to and how that hashes follow from its hash, the names it's rendered with, and the
* code doing it, whose own hash covers everything it uses.
*/
verify(fresh = false): {
checked: number;
already: number;
mismatches: { name: string; stored: string; rehashed: string }[];
problems: string[];
} {
const src = this.source();
const live = this.liveHashes();
// a file kept as written hashes from its content, not as a module
const files = [...live].filter((h) => src.get(h)!.kind === "file");
const comps = new Set(
[...live].filter((h) => src.get(h)!.kind !== "file").map((h) => src.get(h)!.componentHash),
);
// (run from a scratch file rather than a build, the code doing this has no hash: everything is checked)
const verifier = (Codebase as unknown as Record<symbol, string | undefined>)[
Symbol.for("tcm.hash")
];
const keyOf = (c: string) => {
const members = src.members(c);
const names = members
.flatMap((m) => [m.hash, ...m.spans.flatMap((s) => (s.hash ? [s.hash] : []))])
.map((h) => `${h}=${src.nameOf(h) ?? ""}`);
return createHash("sha256")
.update(
JSON.stringify([
verifier,
process.version,
c,
members[0]?.checked ?? null,
names,
// (and the text it's rendered from, which can be another version on another branch)
members.map((m) => m.sourceText),
]),
)
.digest("hex")
.slice(0, 32);
};
const keys = new Map(verifier ? [...comps].map((c) => [c, keyOf(c)]) : []);
const done = new Set(
fresh ? [] : (JSON.parse(this.store.getSetting("verified") ?? "[]") as string[]),
);
const todo = new Set([...comps].filter((c) => !done.has(keys.get(c) ?? "")));
const r = todo.size
? this.rehash(src, todo)
: { defs: new Map<string, DefRecord>(), mismatches: [], problems: [] };
const mismatches = r.mismatches.map((m) => ({ name: this.displayRef(`#${m.stored}`), ...m }));
for (const h of files) {
const rehashed = hashIR(JSON.parse(src.get(h)!.ir) as IR);
if (rehashed !== h) mismatches.push({ name: this.displayRef(`#${h}`), stored: h, rehashed });
}
// what passed, for next time (only what's live now, so the list doesn't grow)
const passed =
r.problems.length || mismatches.length ? [...comps].filter((c) => !todo.has(c)) : [...comps];
if (verifier)
this.store.setSetting("verified", JSON.stringify(passed.map((c) => keys.get(c)!)));
const count = (cs: Iterable<string>) => [...cs].reduce((n, c) => n + src.members(c).length, 0);
return {
checked: count(comps) + files.length,
already: count(comps) - count(todo),
mismatches,
problems: r.problems,
};
}
/**
* Render components from `src` the way they will be materialized, then typecheck (unless not `typecheck`) and hash
* the result, working out type shapes if `shapes`. Returns the freshly analyzed definition for each stored hash.
*/
private rehash(
src: DefSource,
comps: Set<string>,
typecheck = true,
shapes = typecheck,
): Rehashed {
const job = this.rehashJob(src, comps, typecheck);
try {
return job.collect(
job.parts(1).map((p) => {
// untyped npm packages are fine: types were checked by whoever added the definition
const a = this.analyzeProject(
p.files,
job.saltFor,
[7016],
p.options,
typecheck,
shapes,
job.wasTable,
);
try {
return {
problems: [...a.errors, ...a.diagnostics.map((d) => d.formatted)],
defs: a.defs,
};
} finally {
// (its compilation goes now, not when the process ends: a sync checks one batch after another, each a
// compilation of thousands of modules)
a.dispose?.();
}
}),
);
} finally {
job.cleanup();
}
}
/**
* `rehash`, its analysis shared among checker processes ({@link checkerPool}) where there are some and enough to
* share: each gets a run of the components (in the order given, so what's used together goes together) to compile
* and analyze, at once. The same definitions come of it either way.
*/
private async rehashInParallel(
src: DefSource,
comps: Set<string>,
typecheck = true,
): Promise<Rehashed> {
const pool = comps.size >= 64 ? checkerPool() : undefined;
if (!pool) return this.rehash(src, comps, typecheck);
const job = this.rehashJob(src, comps, typecheck);
const parts = job.parts(pool.size);
try {
const found = job.collect(
await Promise.all(
parts.map((p) =>
pool.check({
files: p.files,
cacheDir: this.lay.cacheDir,
salts: job.saltsOf(p.files),
tables: job.tablesOf(p.files),
ignore: [7016],
options: p.options,
typecheck,
shapes: typecheck,
}),
),
),
);
return { ...found, checkers: Math.min(pool.size, parts.length) };
} finally {
job.cleanup();
}
}
/**
* What `rehash` analyzes: components rendered as modules (and what they use, materialized), in groups checked
* under the same options (`parts`, each cut into up to `n` runs); `collect` reads what analyzing them found against
* the hashes they're stored under. Not to `typecheck`, a module of what they use that's there already is taken as
* it is, and what it imports isn't visited: deriving their IR takes the modules they import, not what those use.
*/
private rehashJob(src: DefSource, comps: Set<string>, typecheck = true) {
materialize(
[...comps].flatMap((c) => src.members(c).map((m) => m.hash)),
src,
this.lay,
typecheck ? {} : { trustExisting: true },
);
const expected = new Map<string, Map<string, string>>();
const records = new Map<string, Map<string, DefRecord>>();
// definitions are re-checked under the options they were first checked with: group them by that profile
const groups = new Map<string, string[]>();
for (const c of comps) {
const members = src.members(c);
const r = render(members, src, this.lay, { style: "internal" });
// (this process's own: another verifying the same codebase at once, a session's, writes and removes its own)
const file = path_1.join(this.lay.cacheDir, `_verify_${process.pid}_${moduleFile(c, src)}`);
fs.writeFileSync(file, r.text);
expected.set(file, new Map([...r.memberNames].map(([h, n]) => [n, h])));
records.set(file, new Map([...r.memberNames].map(([h, n]) => [n, src.get(h)!])));
const key = JSON.stringify(members[0]?.checked ?? {});
groups.set(key, [...(groups.get(key) ?? []), file]);
}
return {
saltFor: this.storedSalts(records),
wasTable: this.storedTables(records),
tablesOf: (files: string[]) =>
Object.fromEntries(
files.map((f) => [
f,
[...(records.get(f) ?? [])].filter(([, rec]) => rec.table).map(([name]) => name),
]),
),
saltsOf: (files: string[]) =>
Object.fromEntries(
files.map((f) => [
f,
Object.fromEntries(
[...(records.get(f) ?? [])].flatMap(([name, rec]) => {
const salt = saltOf(rec.ir);
return salt === undefined ? [] : [[name, salt]];
}),
),
]),
),
parts: (n: number) =>
[...groups].flatMap(([profile, files]) => {
const options = this.recheckOptions(JSON.parse(profile));
const size = Math.ceil(files.length / Math.max(1, n));
return Array.from({ length: Math.ceil(files.length / size) }, (_, i) => ({
files: files.slice(i * size, i * size + size),
options,
}));
}),
collect: (found: { problems: string[]; defs: ProjectDef[] }[]): Rehashed => {
const problems: string[] = [];
const defs = new Map<string, DefRecord>();
const mismatches: { stored: string; rehashed: string }[] = [];
for (const a of found) {
problems.push(...a.problems);
for (const d of a.defs) {
const stored = expected.get(d.file)?.get(d.nameHint);
if (!stored) continue;
defs.set(stored, d);
if (stored !== d.hash) mismatches.push({ stored, rehashed: d.hash });
}
}
return { defs, mismatches, problems };
},
cleanup: () => {
// TCM_KEEP_VERIFY=1 leaves the rendered files in .tcm/cache, to debug a failing re-check
if (!process.env.TCM_KEEP_VERIFY)
for (const f of expected.keys()) fs.rmSync(f, { force: true });
},
};
}
/**
* Work out the type shapes of these terms where they have none (received ones: receiving leaves them, most of what's
* received being what only history names), and keep them: what search by type matches. A term with no shape to
* have is kept as having none, so it isn't worked out again.
*/
fillShapes(hashes: Iterable<string>) {
const lacking = [...new Set(hashes)].filter((h) => {
const d = this.store.getDef(h);
return d?.kind === "term" && d.shape === undefined;
});
if (!lacking.length) return;
const src = this.source();
const r = this.rehash(src, new Set(lacking.map((h) => src.get(h)!.componentHash)), false, true);
this.store.transaction(() => {
for (const h of lacking)
this.store.setMetadata(h, "shape", JSON.stringify(r.defs.get(h)?.shape ?? null));
});
}
/** `fillShapes`, where a shape not worked out is no matter: a search by type tries again */
fillShapesQuietly(hashes: Iterable<string>) {
try {
this.fillShapes(hashes);
} catch {
// (a shape is for searching and showing signatures: nothing depends on having it now)
}
}
/** Components needed to run or name `hashes`: their own plus everything they reference. */
closureComponents(hashes: Iterable<string>): Set<string> {
return this.store.closureComponents(hashes);
}
/**
* Accept definitions from another codebase. Nothing is taken on the sender's word: each component must be
* complete, every reference must resolve, and rendering and analyzing them locally must reproduce the claimed
* hashes; what's derived from them (IR, dependencies, signatures) is the local analysis's. Their types aren't
* judged here (they're the source's, which is checked to be what it claims): a contribution's are where it's merged.
*/
receive(records: DefRecord[]): { stored: number } {
const { firsts, versions } = this.receivedApart(records);
const r = this.receiving(firsts);
let found: Rehashed | Error | undefined;
try {
found = r.comps.size ? this.rehash(r.src, r.comps, false) : undefined;
} catch (e) {
found = e as Error;
}
const stored = r.store(found);
this.receiveTexts(versions);
return stored;
}
/**
* `receive`, its definitions analyzed by checker processes at once ({@link checkerPool}), where there are some. One
* at a time (a server's clients take turns): two at once would render the same components to the same files.
*/
receiveInParallel(records: DefRecord[]): Promise<{ stored: number; checkers: number }> {
const turn = this.receivingNow.then(() => this.receiveNow(records));
this.receivingNow = turn.catch(() => undefined);
return turn;
}
private receivingNow: Promise<unknown> = Promise.resolve();
private async receiveNow(records: DefRecord[]): Promise<{ stored: number; checkers: number }> {
const { firsts, versions } = this.receivedApart(records);
const r = this.receiving(firsts);
let found: Rehashed | Error | undefined;
try {
found = r.comps.size ? await this.rehashInParallel(r.src, r.comps, false) : undefined;
} catch (e) {
found = e as Error;
}
const stored = r.store(found);
this.receiveTexts(versions);
return { ...stored, checkers: (found instanceof Error ? undefined : found?.checkers) ?? 1 };
}
/** a definition this codebase has (or one that comes twice) can come with another text: its comments, layout or doc */
private receivedApart(records: DefRecord[]) {
const firsts = new Map<string, DefRecord>();
const versions: DefRecord[] = [];
for (const r of records)
if (this.store.hasDef(r.hash) || firsts.has(r.hash)) versions.push(r);
else firsts.set(r.hash, r);
return { firsts: [...firsts.values()], versions };
}
/**
* Accept definitions this codebase doesn't have yet (`receive`): what's checked before analyzing them, the
* components to analyze (`comps`, from `src`), and `store`, which checks what analyzing them found and keeps them.
*/
private receiving(all: DefRecord[]) {
// files kept as written are checked by their content: they aren't modules
const files = all.filter((r) => r.kind === "file");
const fresh = all.filter((r) => r.kind !== "file");
const checkedFiles = files.map((f) => {
const why = fileProblem(f);
if (why) throw new Error(`rejected definitions: ${shortHash(f.hash)} ${why}`);
return {
...f,
deps: [...new Set(f.spans.map((s) => s.hash!))].map((h) => ({
ref: `#${h}`,
pos: "value" as const,
strict: false,
})),
componentHash: f.hash,
componentIndex: 0,
componentSize: 1,
};
});
const byComp = new Map<string, DefRecord[]>();
for (const r of fresh) byComp.set(r.componentHash, [...(byComp.get(r.componentHash) ?? []), r]);
for (const [c, ms] of byComp) {
const indices = ms.map((m) => m.componentIndex).sort((a, b) => a - b);
if (ms.some((m) => m.componentSize !== ms.length) || indices.some((v, i) => v !== i)) {
throw new Error(`component ${shortHash(c)} is incomplete: got ${ms.length} member(s)`);
}
}
// (code is checked with the files it came with: it can import them; and none of it is run, not even a pure
// constant to keep its value: that's for when it's materialized to run)
const src: DefSource = { ...this.source([...fresh, ...checkedFiles]), keep: undefined };
const reject = (why: string): never => {
// cache modules rendered from rejected definitions must not linger under hashes they don't have
for (const c of byComp.keys())
for (const ext of [".ts", ".tsx"])
fs.rmSync(path_1.join(this.lay.cacheDir, `${c}${ext}`), { force: true });
throw new Error(`rejected definitions: ${why}`);
};
const store = (r: Rehashed | Error | undefined): { stored: number } => {
if (fresh.length) {
const found =
r instanceof Error ? reject(r.message) : (r ?? reject("they were not analyzed"));
if (found.problems.length)
reject(`they can't be read as code here:\n${found.problems.join("\n")}`);
const misplaced = fresh.filter((d) => {
const local = found.defs.get(d.hash);
return (
local &&
(local.componentHash !== d.componentHash || local.componentIndex !== d.componentIndex)
);
});
if (found.mismatches.length || misplaced.length || found.defs.size !== fresh.length) {
const bad = [
...found.mismatches.map(
(m) => `${shortHash(m.stored)} re-hashes to ${shortHash(m.rehashed)}`,
),
...misplaced.map((d) => `${shortHash(d.hash)} claims the wrong component`),
];
reject(
`their source does not match their hashes (${bad.join(", ") || "missing members"})`,
);
}
// what's derived from the source comes from the local analysis, never from the sender: the IR (the source is
// what was checked to hash so; IR sent beside it could say anything), dependencies, signatures, schemas, how
// it renders
this.putDefs(
fresh.map((d) => {
const local = found.defs.get(d.hash)!;
return {
...d,
ir: local.ir,
deps: local.deps,
signature: local.signature,
params: local.params,
shape: local.shape,
table: local.table,
globalNames: local.globalNames,
initKind: local.initKind,
nominal: local.nominal,
transform: local.transform,
jsx: local.jsx,
};
}),
);
}
this.putDefs(checkedFiles);
return { stored: fresh.length + checkedFiles.length };
};
return { comps: new Set(byComp.keys()), src, store };
}
/**
* Accept text versions of definitions this codebase has (`receive`): each must be the same code, checked as code is
* (rendered with the rest of its component, analyzed and hashed again), and is kept beside it.
*/
private receiveTexts(versions: DefRecord[]) {
for (const v of versions) {
const stored = this.store.defWithText(v.hash, undefined);
if (!stored || stored.kind === "file") continue;
const same = (t: string | undefined) => {
const d = this.store.defWithText(v.hash, t)!;
return d.sourceText === v.sourceText && d.doc === v.doc;
};
if ([undefined, ...this.store.textVersions(v.hash)].some(same)) continue;
const members = this.store
.componentMembers(stored.componentHash)
.map((m) =>
m.hash === v.hash
? { ...m, sourceText: v.sourceText, spans: v.spans, reservedNames: v.reservedNames }
: m,
);
let r: ReturnType<Codebase["rehash"]> | undefined;
try {
r = this.rehash(this.source(members), new Set([stored.componentHash]), false);
} catch (e) {
throw new Error(
`rejected definitions: a text of ${shortHash(v.hash)}: ${(e as Error).message}`,
);
}
if (r.problems.length || r.mismatches.length || !r.defs.has(v.hash))
throw new Error(
`rejected definitions: a text of ${shortHash(v.hash)} isn't its code (${r.problems[0] ?? "it hashes differently"})`,
);
this.store.addText(v.hash, v);
}
}
/**
* Record the package versions saved code was checked against: the lock moves up to them (within a compatible
* range, newer only adds), never down.
*/
lockPackages(packages: Record<string, string>) {
for (const [module, version] of Object.entries(packages)) {
const { name, version: range } = parseModule(module);
const was = lockAtLeast(this.lay, name, range, version);
if (was)
process.stderr.write(`${name} ${was} → ${version} (code was checked against ${version})\n`);
}
}
/**
* The lock entries for the packages `records` use (`name@range`, their @types included): what another codebase
* needs to run them on exactly these versions.
*/
packagesUsedBy(
records: readonly Pick<DefRecord, "deps">[],
from = this.store.lockedPackages(),
): LockEntry[] {
const used = new Set<string>();
for (const r of records) {
for (const d of r.deps) {
const m = /^npm:((?:@[^/@]+\/)?[^@/]+)@([^/#]+)/.exec(d.ref);
if (m) {
used.add(`${m[1]}@${m[2]}`);
used.add(`@types/${m[1]!.replace(/^@/, "").replace("/", "__")}@${m[2]}`);
}
}
}
return from.filter((l) => used.has(`${l.name}@${l.range}`));
}
/**
* Take lock entries from another codebase, for code received from it. A range not locked here is locked at its
* version; within a range, the newer version wins (semver: newer only adds), unless `keepLocks` (code from a
* library: what this codebase's own code runs on isn't another's to move). An entry whose version isn't in its
* range is no lock of that range, and isn't taken. The same version must be the same tarball on both sides.
*/
adoptPackages(entries: LockEntry[], o: { keepLocks?: boolean } = {}) {
for (const e of entries) {
if (!satisfies_(e.version, e.range)) continue;
const was = this.store.lockedVersion(e.name, e.range);
if (was === e.version) {
const mine = this.store.lockedIntegrity(e.name, e.range);
if (mine && e.integrity && mine !== e.integrity)
throw new Error(
`${e.name}@${e.version} is a different tarball there than here; refusing it`,
);
if (!mine && e.integrity) this.store.lockVersion(e.name, e.range, e.version, e.integrity);
} else if (!was || (!o.keepLocks && compareVersions(e.version, was) > 0))
this.store.lockVersion(e.name, e.range, e.version, e.integrity);
}
}
putDefs(defs: DefRecord[]) {
this.store.putDefs(defs, TS_VERSION, IR_VERSION);
}
/** Resolve `name` or `#hashprefix` to hashes. */
lookup(target: string): string[] {
if (target.startsWith("#")) return this.store.hashesWithPrefix(target.slice(1));
return this.store.hashesAt(target);
}
/** What to say for a name (or #hash) that isn't found: the names meant most likely, or what a namespace holds. */
notFound(target: string): string {
return notFound(
this.store.names(undefined, true).map((r) => r.path),
target,
);
}
/** Resolve to exactly one hash. A name holding a type and a value resolves to the `prefer`red one. */
lookupOne(target: string, prefer?: "type" | "value"): string {
let hs = this.lookup(target);
if (hs.length === 0) throw new Error(this.notFound(target));
if (hs.length > 1 && prefer) {
const kinds = hs.map((h) => this.store.getDef(h)!.kind);
if (!isConflicted(kinds))
hs = hs.filter((_, i) => (kinds[i] === "type") === (prefer === "type"));
}
if (hs.length > 1) {
const which = hs.map((h) => `${shortHash(h)} (${this.store.getDef(h)!.kind})`).join(", ");
throw new Error(`'${target}' names ${hs.length} definitions: ${which}; use a #hash`);
}
return hs[0]!;
}
displayRef(ref: string): string {
if (!ref.startsWith("#")) return ref;
const h = ref.slice(1);
return this.store.pathsOf(h)[0] ?? `#${h.slice(0, 8)}`;
}
/** Every hash reachable from a name on the current branch. */
liveHashes(): Set<string> {
return this.store.liveHashes();
}
/** Typecheck these definitions again, as stored (after the packages they use moved): the problems found. */
recheck(hashes: Iterable<string>): string[] {
const src = this.source();
const comps = new Set(
[...hashes].map((h) => src.get(h)?.componentHash).filter((c): c is string => !!c),
);
return comps.size ? this.rehash(src, comps).problems : [];
}
/**
* After names moved to new hashes (`patches`: old → new), re-derive every live dependent:
* re-render its stored source against the patched references, typecheck, and re-hash.
* Dependents that no longer typecheck go to `todo` and keep their old hash. That includes definitions just
* saved: one saved with a scratch file refers to what it uses as it was, and if that's re-derived too, so is it.
*
* `repin` re-derives some definitions against other package versions too (`tcm upgrade`): those definitions,
* rendered with their imports re-pinned, then everything that depends on them.
*/
propagate(
patches: Map<string, string>,
repin?: { hashes: Set<string>; module: (module: string) => string; command: string },
): { updated: string[]; failed: string[] } {
const { store, lay } = this;
const live = this.liveHashes();
// (a library is as it was added: what its code uses changing elsewhere, the same code named apart from it, leaves it
// as it is)
const libraries = librariesIn(store);
const outside = (p: string) => !libraries.some((l) => within(p, l.path));
const affected = new Set<string>(repin?.hashes ?? []);
const queue = [...patches.keys(), ...(repin?.hashes ?? [])];
// (named ones: an old version a stale definition keeps alive isn't re-derived, which would make a version of
// it nothing names and lose what replaced it; the stale definition catches up instead, when it's re-derived)
const reach = (dep: string) => {
if (
live.has(dep) &&
!patches.has(dep) &&
!affected.has(dep) &&
store.pathsOf(dep).some(outside)
) {
affected.add(dep);
queue.push(dep);
}
};
// a stale definition (using a version nothing names: one that couldn't follow a change, a merge's other side) is
// re-derived with what replaced that version: when it changes now, or is what changes now replace
const staleUsers = new Map<string, string[]>();
for (const u of store.unnamedUses()) {
const now = store.patched(u.uses);
if (now !== u.uses) staleUsers.set(now, [...(staleUsers.get(now) ?? []), u.hash]);
}
for (const h of [...patches.keys(), ...patches.values()])
for (const dep of staleUsers.get(h) ?? []) reach(dep);
while (queue.length) {
const h = queue.pop()!;
for (const dep of store.dependents(h)) reach(dep);
for (const dep of staleUsers.get(h) ?? []) reach(dep);
}
// files that name code by hash (a page's {@link}s) aren't checked as code: they're re-derived last, below
const files = [...affected].filter((h) => store.getDef(h)!.kind === "file");
for (const h of files) affected.delete(h);
const comps = new Map<string, DefRecord[]>();
for (const h of affected) {
const c = store.getDef(h)!.componentHash;
if (!comps.has(c)) comps.set(c, store.componentMembers(c));
}
const compOf = new Map<string, string>();
for (const [c, ms] of comps) for (const m of ms) compOf.set(m.hash, c);
// a dependency is what it was replaced by: one saved and then re-derived here too comes first, and so does one
// replaced before (a definition that couldn't follow a change then catches up now, or is to do again)
// (one replaced before only by what's named now: a reset can leave its replacement unnamed)
const viaThis = (h: string) => {
for (let i = 0; i < 1000 && patches.has(h); i++) h = patches.get(h)!;
return h;
};
// `now` replaces `old`. A version can come back, as the store has it: what replaced it then doesn't now (a merge
// that kept ours of what both sides only re-hashed points theirs at ours, and ours re-derived is theirs)
const replace = (old: string, now: string) => {
patches.delete(now);
patches.set(old, now);
store.addPatch(old, now);
};
const before = new Map<string, string>();
const replacedBefore = (h: string) => {
if (!before.has(h)) {
const later = store.patched(h);
before.set(h, later !== h && store.pathsOf(later).length ? later : h);
}
return before.get(h)!;
};
const resolve = (h: string) => {
const now = viaThis(h);
const later = replacedBefore(now);
return later === now ? now : viaThis(later);
};
const order: string[] = [];
const seen = new Set<string>();
const visit = (c: string) => {
if (seen.has(c)) return;
seen.add(c);
for (const m of comps.get(c)!)
for (const d of m.deps) {
const dc = d.ref.startsWith("#") ? compOf.get(resolve(d.ref.slice(1))) : undefined;
if (dc && dc !== c) visit(dc);
}
order.push(c);
};
[...comps.keys()].forEach(visit);
const updated: string[] = [];
const failed = new Set<string>();
// A dependent of definitions that kept their types is the same code with other hashes in it: it checks as it did,
// and its own type is what it was. So it's re-hashed as it is, not checked again (Unison's type-preserving
// update), and what depends on it can be too: a component all of whose dependencies that moved kept their types
// (a value's or function's type is its signature; a class's is more than that, so a changed class is checked
// below, as is what depends on anything checked there). A cycle is re-hashed as the analyzer would order it
// (`rehashComponent`); one whose members it can't tell apart is checked below.
const typeKept = new Set<string>();
// (a signature names types by name: `Record<Status, string>` reads the same when Status changed. So the types
// it refers to must be the same ones too)
const typesOf = (r: DefRecord) =>
JSON.stringify([...new Set(r.deps.filter((d) => d.pos === "type").map((d) => d.ref))].sort());
if (!repin) {
for (const [o, n] of patches) {
const was = store.getDef(o);
const now = store.getDef(n);
if (
was?.kind === "term" &&
now?.kind === "term" &&
was.signature === now.signature &&
!was.signature.startsWith("typeof ") &&
typesOf(was) === typesOf(now) &&
JSON.stringify(was.checked ?? null) === JSON.stringify(now.checked ?? null)
)
typeKept.add(o);
}
}
const rehashed = new Set<string>();
// (one transaction for all of them: a history entry is made once, not once a component)
if (typeKept.size)
store.transaction(() => {
for (const c of order) {
const members = comps.get(c)!;
const own = new Set(members.map((m) => m.hash));
const refs = members
.flatMap((m) => m.deps.filter((d) => d.ref.startsWith("#")).map((d) => d.ref.slice(1)))
.filter((h) => !own.has(h));
// (one depending on what's still to be checked waits for it, and is checked with it: what it uses, or what
// replaced that, as a merge's other side's code uses this side's replacement, itself re-derived later)
if (
refs.some((h) => {
const d = compOf.get(resolve(h));
return d !== undefined && d !== c && !rehashed.has(d);
})
)
continue;
// (what moved is what this change replaced, or what an earlier one did: that's checked)
const moved = refs.filter((h) => resolve(h) !== h);
if (!moved.length || !moved.every((h) => typeKept.has(h))) continue;
const r = rehashComponent(
members.map((m) => JSON.parse(m.ir) as IR),
resolve,
);
if (!r) continue;
const now = new Map(members.map((m, i) => [m.hash, r.hashes[r.index[i]!]!]));
// (a member's own name is a span too, of its own hash, as are the others' it uses: now their new ones)
const to = (h: string) => now.get(h) ?? resolve(h);
this.putDefs(
members.map((m, i) => ({
...m,
hash: now.get(m.hash)!,
componentHash: r.componentHash,
componentIndex: r.index[i]!,
componentSize: members.length,
ir: JSON.stringify(r.irs[r.index[i]!]),
spans: m.spans.map((s) => (s.hash ? { ...s, hash: to(s.hash) } : s)),
deps: m.deps.map((d) =>
d.ref.startsWith("#") ? { ...d, ref: `#${to(d.ref.slice(1))}` } : d,
),
...(m.docRefs
? { docRefs: m.docRefs.map((x) => (x.hash ? { ...x, hash: to(x.hash) } : x)) }
: {}),
})),
);
store.transaction(() => {
for (const m of members) {
const hash = now.get(m.hash)!;
store.clearTodo(m.hash);
if (hash === m.hash) continue;
replace(m.hash, hash);
for (const p of store.pathsOf(m.hash).filter(outside)) {
store.replaceName(p, m.hash, hash, m.kind, "update (propagated)");
updated.push(p);
}
}
});
for (const m of members) typeKept.add(m.hash);
rehashed.add(c);
}
});
const toCheck = order.filter((c) => !rehashed.has(c));
// components whose dependencies are all re-hashed are re-checked together, in one program: a wave per level
// of the dependency graph, rather than a program per component (each of which checks much of the codebase again)
const level = new Map<string, number>();
for (const c of toCheck) {
let l = 0;
for (const m of comps.get(c)!)
for (const d of m.deps) {
const dc = d.ref.startsWith("#") ? compOf.get(resolve(d.ref.slice(1))) : undefined;
if (dc && dc !== c) l = Math.max(l, level.get(dc)! + 1);
}
level.set(c, l);
}
const waves: string[][] = [];
for (const c of toCheck) (waves[level.get(c)!] ??= []).push(c);
type Item = {
c: string;
members: DefRecord[];
rendered: ReturnType<typeof render>;
tmp: string;
};
for (const wave of waves) {
const src = this.source();
const items: Item[] = [];
// what the wave's components import, written to the cache once for all of them (their closures overlap)
const imported = new Set<string>();
for (const c of wave) {
const members = comps.get(c)!;
const memberSet = new Set(members.map((m) => m.hash));
// (what it uses, or what that was replaced by: a dependency taken from a merge's other side that failed)
const blocker = members
.flatMap((m) => m.deps)
.find(
(d) =>
d.ref.startsWith("#") &&
(failed.has(d.ref.slice(1)) || failed.has(resolve(d.ref.slice(1)))),
);
if (blocker) {
for (const m of members) {
failed.add(m.hash);
store.setTodo(
m.hash,
`depends on ${this.displayRef(`#${resolve(blocker.ref.slice(1))}`)}, which needs fixing`,
);
}
continue;
}
for (const m of members)
for (const s of m.spans)
if (s.hash && !memberSet.has(s.hash)) imported.add(resolve(s.hash));
ensureExternals(lay, externalsOf(members));
const repinned = !!repin && members.some((m) => repin.hashes.has(m.hash));
if (repinned)
ensureExternals(
lay,
externalsOf(members).map((e) => ({ ...e, module: repin.module(e.module) })),
);
const rendered = render(members, src, lay, {
style: "internal",
remap: resolve,
...(repinned ? { remapPackage: repin.module } : {}),
});
const tmp = path_1.join(lay.cacheDir, `_update_${moduleFile(c, src)}`);
fs.writeFileSync(tmp, rendered.text);
items.push({ c, members, rendered, tmp });
}
// the re-check only needs the modules to exist: names and source maps are brought up to date afterwards
materialize([...imported], src, lay, { keepExisting: true });
// checked under the profile each was first checked with; untyped npm packages are fine here (the definitions
// were already checked against their types when added)
const check = (group: Item[]) => {
const byFile = new Map(
group.map((x) => [
x.tmp,
new Map([...x.rendered.memberNames].map(([h, n]) => [n, src.get(h)!])),
]),
);
return this.analyzeProject(
group.map((x) => x.tmp),
this.storedSalts(byFile),
[7016],
this.recheckOptions(group[0]!.members[0]?.checked),
true,
true,
this.storedTables(byFile),
);
};
const outcome = new Map<string, ProjectDef[] | string>();
const byProfile = new Map<string, Item[]>();
for (const x of items) {
const k = JSON.stringify(x.members[0]?.checked ?? null);
byProfile.set(k, [...(byProfile.get(k) ?? []), x]);
}
for (const group of byProfile.values()) {
const a = check(group);
const problem = (r: typeof a) => r.errors[0] ?? r.diagnostics[0]?.message;
if (!problem(a))
for (const x of group)
outcome.set(
x.c,
a.defs.filter((d) => path_1.resolve(d.file) === path_1.resolve(x.tmp)),
);
// a problem somewhere in the group: check each on its own, to say which
else
for (const x of group) {
const b = group.length === 1 ? a : check([x]);
outcome.set(x.c, problem(b) ?? b.defs);
}
}
for (const x of items) fs.rmSync(x.tmp, { force: true });
// (the wave's results in one transaction: a history entry is made once, not once a component)
store.transaction(() => {
for (const { c, members, rendered } of items) {
const result = outcome.get(c)!;
if (typeof result === "string") {
for (const m of members) {
failed.add(m.hash);
store.setTodo(m.hash, result);
}
continue;
}
const oldByName = new Map([...rendered.memberNames].map(([h, n]) => [n, h]));
// a re-derived definition keeps its doc (rendered code has no comments); the doc's references are by hash
// and follow patches when rendered
this.putDefs(
result.map((d) => {
const was = src.get(oldByName.get(d.nameHint) ?? "");
return was?.doc
? { ...d, doc: was.doc, ...(was.docRefs ? { docRefs: was.docRefs } : {}) }
: d;
}),
);
store.transaction(() => {
for (const d of result) {
const old = oldByName.get(d.nameHint);
if (!old) continue;
store.clearTodo(old);
if (d.hash === old) continue;
replace(old, d.hash);
for (const p of store.pathsOf(old).filter(outside)) {
// only this hash moves: a conflicted name keeps its other definitions
store.replaceName(p, old, d.hash, d.kind, repin?.command ?? "update (propagated)");
updated.push(p);
}
}
});
}
});
}
// a file names what replaced what it names; nothing else of it changes
const named = (ir: IR): IR =>
Array.isArray(ir)
? ir[0] === "Ref" && typeof ir[1] === "string"
? ["Ref", resolve(ir[1])]
: ir.map(named)
: ir;
for (const h of files) {
const rec = store.getDef(h)!;
const ir = named(JSON.parse(rec.ir) as IR);
const hash = hashIR(ir);
if (hash === h) continue;
const spans = rec.spans.map((sp) => (sp.hash ? { ...sp, hash: resolve(sp.hash) } : sp));
this.putDefs([
{
...rec,
hash,
componentHash: hash,
ir: JSON.stringify(ir),
spans,
deps: [...new Set(spans.flatMap((sp) => (sp.hash ? [sp.hash] : [])))].map((d) => ({
ref: `#${d}`,
pos: "value" as const,
strict: false,
})),
},
]);
store.transaction(() => {
replace(h, hash);
for (const p of store.pathsOf(h).filter(outside)) {
store.replaceName(p, h, hash, "file", repin?.command ?? "update (propagated)");
updated.push(p);
}
});
}
return { updated, failed: [...failed] };
}
}
uses
used by