Store program to persist in buildinfo and clean it as part of cleanResolutions

Author: sheetalkamat

src/compiler/builder.ts

@@ -1616,7 +1616,7 @@ namespace ts {
                     filesByName.set(newSourceFile.path, newSourceFile);
                 }
             });
-            const oldFilesByNameMap = oldProgram.getFilesByNameMap() as ESMap<Path, SourceFile | Path | false | 0>;
+            const oldFilesByNameMap = oldProgram.getFilesByNameMap() as ESMap<Path, SourceFileOfProgramFromBuildInfo | Path | false | 0>;
             oldFilesByNameMap.forEach((oldFile, path) => {
                 if (!oldFile) {
                     filesByName.set(path, oldFile as false | 0);

src/compiler/program.ts

@@ -25,7 +25,7 @@ namespace ts {
         clear(): void;
     }
 
-    interface ResolutionWithFailedLookupLocations {
+    export interface ResolutionWithFailedLookupLocations {
         readonly failedLookupLocations: string[];
         isInvalidated?: boolean;
         refCount?: number;

src/compiler/resolutionCache.ts

@@ -3827,14 +3827,13 @@ namespace ts {
     /*@internal*/
     export interface IdentifierOfProgramFromBuildInfo {
         kind: SyntaxKind.Identifier;
-        escapedText: string;
+        escapedText: __String;
     }
 
     /*@internal*/
     export interface StringLiteralLikeOfProgramFromBuildInfo {
         kind: SyntaxKind.StringLiteral | SyntaxKind.NoSubstitutionTemplateLiteral;
         text: string;
-        escapedText: string;
     }
 
     /*@internal*/
@@ -3846,14 +3845,16 @@ namespace ts {
         originalFileName: string;
         path: Path;
         resolvedPath: Path;
+        // This currently is set to sourceFile.flags & NodeFlags.PermanentlySetIncrementalFlags but cant be set in type
+        // Change this if it changes in reusing program
         flags: NodeFlags;
         version: string;
 
         typeReferenceDirectives: readonly string[];
         libReferenceDirectives: readonly string[];
         referencedFiles: readonly string[];
         imports: readonly StringLiteralLikeOfProgramFromBuildInfo[];
-        moduleAugmentations: ModuleNameOfProgramFromBuildInfo[];
+        moduleAugmentations: readonly ModuleNameOfProgramFromBuildInfo[];
         ambientModuleNames: readonly string[];
         hasNoDefaultLib: boolean;
 
@@ -3866,22 +3867,21 @@ namespace ts {
 
         getCompilerOptions(): CompilerOptions;
         getRootFileNames(): readonly string[];
-        getSourceFiles(): SourceFileOfProgramFromBuildInfo[];
+        getSourceFiles(): readonly SourceFileOfProgramFromBuildInfo[];
         getSourceFileByPath(path: Path): SourceFileOfProgramFromBuildInfo | undefined;
         getPerFileModuleResolutions(): ESMap<Path, ESMap<string, ResolvedModuleWithFailedLookupLocations>>;
         getPerFileTypeReferenceResolutions(): ESMap<Path, ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>>;
         getProjectReferences(): readonly ProjectReference[] | undefined;
-        getResolvedProjectReferences(): readonly ResolvedProjectReferenceOfProgramFromBuildInfo[] | undefined;
+        getResolvedProjectReferences(): readonly (ResolvedProjectReferenceOfProgramFromBuildInfo | undefined)[] | undefined;
         getMissingFilePaths(): readonly Path[];
         getRefFileMap(): MultiMap<Path, RefFile> | undefined;
         getResolvedTypeReferenceDirectives(): ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
-        getFilesByNameMap(): ESMap<Path, Path | false | 0>;
+        getFilesByNameMap(): ESMap<Path, SourceFileOfProgramFromBuildInfo | Path | false | 0>;
         isSourceFileFromExternalLibraryPath(path: Path): boolean;
         getFileProcessingDiagnostics(): readonly ReusableDiagnostic[];
 
         redirectTargetsMap: MultiMap<Path, string>;
         sourceFileToPackageName: ESMap<Path, string>;
-        structureIsReused?: StructureIsReused;
     }
 
     /* @internal */

src/compiler/types.ts

@@ -28,13 +28,13 @@ namespace ts {
         if (!content) return emitSkippedWithNoDiagnostics;
         const buildInfo = getBuildInfo(content);
         if (buildInfo.version !== version) return emitSkippedWithNoDiagnostics;
-        if (!buildInfo.program) return emitSkippedWithNoDiagnostics;
-        // TODO:: Clean the actual program
-        let newContent = content;
+        if (!buildInfo.program?.peristedProgram) return emitSkippedWithNoDiagnostics;
+        const { program: { peristedProgram, ...program } } = buildInfo;
+        buildInfo.program = program;
 
         // Actual writeFile with new program
         const emitDiagnostics = createDiagnosticCollection();
-        writeFile(host, emitDiagnostics, buildInfoPath, newContent, /*writeByteOrderMark*/ false);
+        writeFile(host, emitDiagnostics, buildInfoPath, getBuildInfoText(buildInfo), /*writeByteOrderMark*/ false);
         return {
             emitSkipped: false,
             diagnostics: emitDiagnostics.getDiagnostics(),

src/compiler/watchPublic.ts

@@ -269,11 +269,12 @@ interface Symbol {
         tick: () => void;
         baseFs: vfs.FileSystem;
         newSys: TscCompileSystem;
+        cleanBuildDescripencies: TscIncremental["cleanBuildDescripencies"];
     }
     function verifyIncrementalCorrectness(input: () => VerifyIncrementalCorrectness, index: number) {
         it(`Verify emit output file text is same when built clean for incremental scenario at:: ${index}`, () => {
             const {
-                scenario, subScenario, commandLineArgs,
+                scenario, subScenario, commandLineArgs, cleanBuildDescripencies,
                 modifyFs, incrementalModifyFs,
                 tick, baseFs, newSys
             } = input();
@@ -288,54 +289,82 @@ interface Symbol {
                     incrementalModifyFs(fs);
                 },
             });
+            const descripencies = cleanBuildDescripencies?.();
             for (const outputFile of arrayFrom(sys.writtenFiles.keys())) {
-                const expectedText = sys.readFile(outputFile);
-                const actualText = newSys.readFile(outputFile);
+                const cleanBuildText = sys.readFile(outputFile);
+                const incrementalBuildText = newSys.readFile(outputFile);
+                const descripencyInClean = descripencies?.get(outputFile);
                 if (!isBuildInfoFile(outputFile)) {
-                    assert.equal(actualText, expectedText, `File: ${outputFile}`);
+                    verifyTextEqual(incrementalBuildText, cleanBuildText, descripencyInClean, `File: ${outputFile}`);
                 }
-                else if (actualText !== expectedText) {
+                else if (incrementalBuildText !== cleanBuildText) {
                     // Verify build info without affectedFilesPendingEmit
-                    const { buildInfo: actualBuildInfo, affectedFilesPendingEmit: actualAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(actualText);
-                    const { buildInfo: expectedBuildInfo, affectedFilesPendingEmit: expectedAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(expectedText);
-                    assert.deepEqual(actualBuildInfo, expectedBuildInfo, `TsBuild info text without affectedFilesPendingEmit: ${outputFile}::\nIncremental buildInfoText:: ${actualText}\nClean buildInfoText:: ${expectedText}`);
+                    const { buildInfo: incrementalBuildInfo, affectedFilesPendingEmit: incrementalBuildAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(incrementalBuildText);
+                    const { buildInfo: cleanBuildInfo, affectedFilesPendingEmit: incrementalAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(cleanBuildText);
+                    verifyTextEqual(incrementalBuildInfo, cleanBuildInfo, descripencyInClean, `TsBuild info text without affectedFilesPendingEmit ${subScenario}:: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
                     // Verify that incrementally pending affected file emit are in clean build since clean build can contain more files compared to incremental depending of noEmitOnError option
-                    if (actualAffectedFilesPendingEmit) {
-                        assert.isDefined(expectedAffectedFilesPendingEmit, `Incremental build contains affectedFilesPendingEmit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${actualText}\nClean buildInfoText:: ${expectedText}`);
+                    if (incrementalBuildAffectedFilesPendingEmit && descripencyInClean === undefined) {
+                        assert.isDefined(incrementalAffectedFilesPendingEmit, `Incremental build contains affectedFilesPendingEmit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
                         let expectedIndex = 0;
-                        actualAffectedFilesPendingEmit.forEach(([actualFile]) => {
-                            expectedIndex = findIndex(expectedAffectedFilesPendingEmit!, ([expectedFile]) => actualFile === expectedFile, expectedIndex);
-                            assert.notEqual(expectedIndex, -1, `Incremental build contains ${actualFile} file as pending emit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${actualText}\nClean buildInfoText:: ${expectedText}`);
+                        incrementalBuildAffectedFilesPendingEmit.forEach(([actualFile]) => {
+                            expectedIndex = findIndex(incrementalAffectedFilesPendingEmit!, ([expectedFile]) => actualFile === expectedFile, expectedIndex);
+                            assert.notEqual(expectedIndex, -1, `Incremental build contains ${actualFile} file as pending emit, clean build should also have it: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
                             expectedIndex++;
                         });
                     }
                 }
             }
+
+            function verifyTextEqual(incrementalText: string | undefined, cleanText: string | undefined, descripencyInClean: CleanBuildDescripency | undefined, message: string) {
+                if (descripencyInClean === undefined) {
+                    assert.equal(incrementalText, cleanText, message);
+                    return;
+                }
+                switch (descripencyInClean) {
+                    case CleanBuildDescripency.CleanFileTextDifferent:
+                        assert.isDefined(incrementalText, `Incremental file should be present:: ${message}`);
+                        assert.isDefined(cleanText, `Clean file should be present present:: ${message}`);
+                        assert.notEqual(incrementalText, cleanText, message);
+                        return;
+                    case CleanBuildDescripency.CleanFilePresent:
+                        assert.isUndefined(incrementalText, `Incremental file should be absent:: ${message}`);
+                        assert.isDefined(cleanText, `Clean file should be present:: ${message}`);
+                        return;
+                    default:
+                        Debug.assertNever(descripencyInClean);
+                }
+            }
         });
     }
 
-    function getBuildInfoForIncrementalCorrectnessCheck(text: string | undefined): { buildInfo: BuildInfo | undefined; affectedFilesPendingEmit?: ProgramBuildInfo["affectedFilesPendingEmit"]; } {
+    function getBuildInfoForIncrementalCorrectnessCheck(text: string | undefined): { buildInfo: string | undefined; affectedFilesPendingEmit?: ProgramBuildInfo["affectedFilesPendingEmit"]; } {
         const buildInfo = text ? getBuildInfo(text) : undefined;
-        if (!buildInfo?.program) return { buildInfo };
+        if (!buildInfo?.program) return { buildInfo: text };
         // Ignore noEmit since that shouldnt be reason to emit the tsbuild info and presence of it in the buildinfo file does not matter
         const { program: { affectedFilesPendingEmit, options: { noEmit, ...optionsRest}, ...programRest }, ...rest } = buildInfo;
         return {
-            buildInfo: {
+            buildInfo: getBuildInfoText({
                 ...rest,
                 program: {
                     options: optionsRest,
                     ...programRest
                 }
-            },
+            }),
             affectedFilesPendingEmit
         };
     }
 
+    export enum CleanBuildDescripency{
+        CleanFileTextDifferent,
+        CleanFilePresent,
+    }
+
     export interface TscIncremental {
         buildKind: BuildKind;
         modifyFs: (fs: vfs.FileSystem) => void;
         subScenario?: string;
         commandLineArgs?: readonly string[];
+        cleanBuildDescripencies?: () => ESMap<string, CleanBuildDescripency>;
     }
 
     export interface VerifyTsBuildInput extends VerifyTsBuildInputWorker {
@@ -395,7 +424,8 @@ interface Symbol {
                 buildKind,
                 modifyFs: incrementalModifyFs,
                 subScenario: incrementalSubScenario,
-                commandLineArgs: incrementalCommandLineArgs
+                commandLineArgs: incrementalCommandLineArgs,
+                cleanBuildDescripencies,
             }, index) => {
                 describe(incrementalSubScenario || buildKind, () => {
                     let newSys: TscCompileSystem;
@@ -424,10 +454,11 @@ interface Symbol {
                     verifyTscBaseline(() => newSys);
                     verifyIncrementalCorrectness(() => ({
                         scenario,
-                        subScenario,
+                        subScenario: incrementalSubScenario || subScenario,
                         baseFs,
                         newSys,
                         commandLineArgs: incrementalCommandLineArgs || commandLineArgs,
+                        cleanBuildDescripencies,
                         incrementalModifyFs,
                         modifyFs,
                         tick
@@ -519,12 +550,13 @@ interface Symbol {
                 }));
             });
             describe("incremental correctness", () => {
-                incrementalScenarios.forEach(({ commandLineArgs: incrementalCommandLineArgs }, index) => verifyIncrementalCorrectness(() => ({
+                incrementalScenarios.forEach(({ commandLineArgs: incrementalCommandLineArgs, subScenario, buildKind, cleanBuildDescripencies }, index) => verifyIncrementalCorrectness(() => ({
                     scenario,
-                    subScenario,
+                    subScenario: subScenario || buildKind,
                     baseFs,
                     newSys: incrementalSys[index],
                     commandLineArgs: incrementalCommandLineArgs || commandLineArgs,
+                    cleanBuildDescripencies,
                     incrementalModifyFs: fs => {
                         for (let i = 0; i <= index; i++) {
                             incrementalScenarios[i].modifyFs(fs);

src/testRunner/unittests/tsbuild/helpers.ts

@@ -43,13 +43,50 @@ namespace ts {
                     subScenario: "Write file that could not be resolved",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: fs => fs.writeFileSync(`/src/project/src/fileNotFound.ts`, "export function something2() { return 20; }"),
+                    cleanBuildDescripencies: () => {
+                        const result = new Map<string, CleanBuildDescripency>();
+                        // when doing clean build, fileNotFound.ts would be resolved so the output order in outFile.js would change
+                        // In build mode the out is generated only when there are no errors
+                        const isBuild = input === "--b";
+                        if (outFile) {
+                            const descripency = isBuild ? CleanBuildDescripency.CleanFilePresent : CleanBuildDescripency.CleanFileTextDifferent;
+                            result.set("/src/project/outFile.tsbuildinfo", CleanBuildDescripency.CleanFileTextDifferent);
+                            result.set("/src/project/outFile.js", descripency);
+                            result.set("/src/project/outFile.d.ts", descripency);
+                            result.set("/src/project/outFile.tsbuildinfo.baseline.txt", descripency);
+                        }
+                        else if (isBuild) {
+                            // Outputs are generated, buildinfo is updated to report no errors
+                            result.set(`/src/project/src/filePresent.js`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/filePresent.d.ts`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/fileNotFound.js`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/fileNotFound.d.ts`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/anotherFileReusingResolution.js`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/anotherFileReusingResolution.d.ts`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/main.js`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/main.d.ts`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/newFile.js`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/src/newFile.d.ts`, CleanBuildDescripency.CleanFilePresent);
+                            result.set(`/src/project/tsconfig.tsbuildinfo`, CleanBuildDescripency.CleanFileTextDifferent);
+                        }
+                        else {
+                            result.set(`/src/project/tsconfig.tsbuildinfo`, CleanBuildDescripency.CleanFileTextDifferent);
+                        }
+                        return result;
+                    },
                 },
                 {
                     subScenario: "Clean resolutions",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: noop,
                     commandLineArgs: [input, "src/project", "--cleanPersistedProgram"]
                 },
+                {
+                    subScenario: "Clean resolutions again",
+                    buildKind: BuildKind.IncrementalDtsChange,
+                    modifyFs: noop,
+                    commandLineArgs: [input, "src/project", "--cleanPersistedProgram"]
+                },
                 noChangeRun,
                 {
                     subScenario: "Modify main file",

src/testRunner/unittests/tsbuild/persistResolutions.ts

@@ -509,13 +509,19 @@ class someClass2 { }`),
                 subScenario: "persistResolutions",
                 baselinePrograms: true,
                 fs: () => projFs,
-                modifyFs: fs => fs.writeFileSync("/src/core/tsconfig.json", JSON.stringify({
-                    compilerOptions: {
-                        composite: true,
-                        skipDefaultLibCheck: true,
-                        persistResolutions: true,
+                modifyFs: fs => {
+                    persistResolutions("/src/core/tsconfig.json");
+                    persistResolutions("/src/logic/tsconfig.json");
+                    persistResolutions("/src/tests/tsconfig.json");
+                    function persistResolutions(file: string) {
+                        const content = JSON.parse(fs.readFileSync(file, "utf-8"));
+                        content.compilerOptions = {
+                            ...content.compilerOptions || {},
+                            persistResolutions: true
+                        };
+                        fs.writeFileSync(file, JSON.stringify(content, /*replacer*/ undefined, 4));
                     }
-                })),
+                },
                 commandLineArgs: ["--b", "/src/tests"],
                 incrementalScenarios: [
                     ...coreChanges,