Store program to persist in buildinfo and clean it as part of cleanResolutions

Author: sheetalkamat

src/compiler/builder.ts

@@ -597,7 +597,11 @@ namespace ts {
     }
 
     /*@internal*/
-    export function isReferencedFile(reason: FileIncludeReason | undefined): reason is ReferencedFile {
+    export function isReferencedFile(reason: FileIncludeReason | undefined): reason is ReferencedFile;
+    /*@internal*/
+    export function isReferencedFile(reason: FileIncludeReason | PersistedProgramFileIncludeReason | undefined): reason is PersistedProgramReferencedFile;
+    /*@internal*/
+    export function isReferencedFile(reason: FileIncludeReason | PersistedProgramFileIncludeReason | undefined): reason is ReferencedFile | PersistedProgramReferencedFile {
         switch (reason?.kind) {
             case FileIncludeKind.Import:
             case FileIncludeKind.ReferenceFile:
@@ -832,7 +836,7 @@ namespace ts {
         const cachedDeclarationDiagnosticsForFile: DiagnosticCache<DiagnosticWithLocation> = {};
 
         let resolvedTypeReferenceDirectives = new Map<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>();
-        let fileProcessingDiagnostics: FilePreprocessingDiagnostics[] | undefined;
+        let fileProcessingDiagnostics: FilePreprocessingDiagnostic[] | undefined;
 
         // The below settings are to track if a .js file should be add to the program if loaded via searching under node_modules.
         // This works as imported modules are discovered recursively in a depth first manner, specifically:
@@ -1677,7 +1681,7 @@ namespace ts {
                     newSourceFile.resolvedTypeReferenceDirectiveNames = oldSourceFile.resolvedTypeReferenceDirectiveNames;
                 }
             }
-            const oldFilesByNameMap = oldProgram.getFilesByNameMap() as ESMap<Path, SourceFile | Path | typeof missingSourceOfProjectReferenceRedirect | typeof missingFile>;
+            const oldFilesByNameMap = oldProgram.getFilesByNameMap() as ESMap<Path, SourceFileOfProgramFromBuildInfo | Path | typeof missingSourceOfProjectReferenceRedirect | typeof missingFile>;
             oldFilesByNameMap.forEach((oldFile, path) => {
                 if (!oldFile) {
                     filesByName.set(path, oldFile as false | 0);
@@ -1696,25 +1700,7 @@ namespace ts {
 
             files = newSourceFiles;
             fileReasons = oldProgram.getFileIncludeReasons();
-            if (!isProgramFromBuildInfo(oldProgram)) {
-                fileProcessingDiagnostics = oldProgram.getFileProcessingDiagnostics();
-            }
-            else {
-                const reusableDiagnostics = oldProgram.getFileProcessingDiagnostics();
-                if (reusableDiagnostics) {
-                    const toPath = getToPathForBuildInfoFilePath(options, currentDirectory, getCanonicalFileName);
-                    fileProcessingDiagnostics = map(reusableDiagnostics, reusable => ({
-                        ...reusable,
-                        diagnostic: Diagnostics[reusable.diagnostic],
-                        file: (reusable as ReusableFilePreprocessingFileExplainingDiagnostic).file !== undefined ?
-                            toPath((reusable as ReusableFilePreprocessingFileExplainingDiagnostic).file!) :
-                            undefined
-                    }));
-                }
-                else {
-                    fileProcessingDiagnostics = undefined;
-                }
-            }
+            fileProcessingDiagnostics = oldProgram.getFileProcessingDiagnostics();
             resolvedTypeReferenceDirectives = oldProgram.getResolvedTypeReferenceDirectives();
 
             sourceFileToPackageName = oldProgram.sourceFileToPackageName;
@@ -2882,7 +2868,8 @@ namespace ts {
         function processTypeReferenceDirectives(file: SourceFile) {
             // We lower-case all type references because npm automatically lowercases all packages. See GH#9824.
             const typeDirectives = map(file.typeReferenceDirectives, ref => toFileNameLowerCase(ref.fileName));
-            if (!typeDirectives) {
+            if (!typeDirectives?.length) {
+                file.resolvedTypeReferenceDirectiveNames = undefined;
                 return;
             }
 
@@ -2902,7 +2889,7 @@ namespace ts {
             resolved: ResolvedTypeReferenceDirectiveWithFailedLookupLocations,
             reason: FileIncludeReason
         ): void {
-            tracing?.push(tracing.Phase.Program, "processTypeReferenceDirective", { directive: typeReferenceDirective, hasResolved: !!resolveModuleNamesReusingOldState, refKind: reason.kind, refPath: isReferencedFile(reason) ? reason.file : undefined });
+            tracing?.push(tracing.Phase.Program, "processTypeReferenceDirective", { directive: typeReferenceDirective, hasResolved: !!resolved.resolvedTypeReferenceDirective, refKind: reason.kind, refPath: isReferencedFile(reason) ? reason.file : undefined });
             processTypeReferenceDirectiveWorker(typeReferenceDirective, resolved, reason);
             tracing?.pop();
         }

src/compiler/program.ts

@@ -27,7 +27,7 @@ namespace ts {
         clear(): void;
     }
 
-    interface ResolutionWithFailedLookupLocations {
+    export interface ResolutionWithFailedLookupLocations {
         readonly failedLookupLocations: string[];
         isInvalidated?: boolean;
         refCount?: number;

src/compiler/resolutionCache.ts

@@ -3485,7 +3485,7 @@ namespace ts {
         // It is used to resolve module names in the checker.
         // Content of this field should never be used directly - use getResolvedModuleFileName/setResolvedModuleFileName functions instead
         /* @internal */ resolvedModules?: ESMap<string, ResolvedModuleWithFailedLookupLocations>;
-        /* @internal */ resolvedTypeReferenceDirectiveNames: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
+        /* @internal */ resolvedTypeReferenceDirectiveNames?: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
         /* @internal */ imports: readonly StringLiteralLike[];
         // Identifier only if `declare global`
         /* @internal */ moduleAugmentations: readonly (StringLiteral | Identifier)[];
@@ -3769,7 +3769,7 @@ namespace ts {
     }
 
     /*@internal*/
-    export type FilePreprocessingDiagnostics = FilePreprocessingReferencedDiagnostic | FilePreprocessingFileExplainingDiagnostic;
+    export type FilePreprocessingDiagnostic = FilePreprocessingReferencedDiagnostic | FilePreprocessingFileExplainingDiagnostic;
 
     /*@internal*/
     export const missingSourceOfProjectReferenceRedirect = false;
@@ -3847,7 +3847,7 @@ namespace ts {
         getInstantiationCount(): number;
         getRelationCacheSizes(): { assignable: number, identity: number, subtype: number, strictSubtype: number };
 
-        /* @internal */ getFileProcessingDiagnostics(): FilePreprocessingDiagnostics[] | undefined;
+        /* @internal */ getFileProcessingDiagnostics(): FilePreprocessingDiagnostic[] | undefined;
         /* @internal */ getResolvedTypeReferenceDirectives(): ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
         isSourceFileFromExternalLibrary(file: SourceFile): boolean;
         /* @internal */ isSourceFileFromExternalLibraryPath(path: Path): boolean;
@@ -3903,14 +3903,13 @@ namespace ts {
     /*@internal*/
     export interface IdentifierOfProgramFromBuildInfo {
         kind: SyntaxKind.Identifier;
-        escapedText: string;
+        escapedText: __String;
     }
 
     /*@internal*/
     export interface StringLiteralLikeOfProgramFromBuildInfo {
         kind: SyntaxKind.StringLiteral | SyntaxKind.NoSubstitutionTemplateLiteral;
         text: string;
-        escapedText: string;
     }
 
     /*@internal*/
@@ -3922,19 +3921,21 @@ namespace ts {
         originalFileName: string;
         path: Path;
         resolvedPath: Path;
+        // This currently is set to sourceFile.flags & NodeFlags.PermanentlySetIncrementalFlags but cant be set in type
+        // Change this if it changes in reusing program
         flags: NodeFlags;
         version: string;
 
         typeReferenceDirectives: readonly string[];
         libReferenceDirectives: readonly string[];
         referencedFiles: readonly string[];
         imports: readonly StringLiteralLikeOfProgramFromBuildInfo[];
-        moduleAugmentations: ModuleNameOfProgramFromBuildInfo[];
+        moduleAugmentations: readonly ModuleNameOfProgramFromBuildInfo[];
         ambientModuleNames: readonly string[];
         hasNoDefaultLib: boolean;
 
         resolvedModules?: ESMap<string, ResolvedModuleWithFailedLookupLocations>;
-        resolvedTypeReferenceDirectiveNames: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
+        resolvedTypeReferenceDirectiveNames?: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
         redirectInfo?: RedirectInfoOfProgramFromBuildInfo;
     }
 
@@ -3944,20 +3945,19 @@ namespace ts {
 
         getCompilerOptions(): CompilerOptions;
         getRootFileNames(): readonly string[];
-        getSourceFiles(): SourceFileOfProgramFromBuildInfo[];
+        getSourceFiles(): readonly SourceFileOfProgramFromBuildInfo[];
         getSourceFileByPath(path: Path): SourceFileOfProgramFromBuildInfo | undefined;
         getProjectReferences(): readonly ProjectReference[] | undefined;
-        getResolvedProjectReferences(): readonly ResolvedProjectReferenceOfProgramFromBuildInfo[] | undefined;
+        getResolvedProjectReferences(): readonly (ResolvedProjectReferenceOfProgramFromBuildInfo | undefined)[] | undefined;
         getMissingFilePaths(): readonly Path[];
         getFileIncludeReasons(): MultiMap<Path, FileIncludeReason>;
         getResolvedTypeReferenceDirectives(): ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
-        getFilesByNameMap(): ESMap<Path, Path | false | 0>;
+        getFilesByNameMap(): ESMap<Path, SourceFileOfProgramFromBuildInfo | Path | typeof missingSourceOfProjectReferenceRedirect | typeof missingFile>;
         isSourceFileFromExternalLibraryPath(path: Path): boolean;
-        getFileProcessingDiagnostics(): readonly ReusableFilePreprocessingDiagnostics[] | undefined;
+        getFileProcessingDiagnostics(): FilePreprocessingDiagnostic[] | undefined;
 
         redirectTargetsMap: MultiMap<Path, string>;
         sourceFileToPackageName: ESMap<Path, string>;
-        structureIsReused?: StructureIsReused;
     }
 
     /*@internal*/
@@ -3978,7 +3978,7 @@ namespace ts {
     }
 
     /*@internal*/
-    export type ReusableFilePreprocessingDiagnostics = ReusableFilePreprocessingReferencedDiagnostic | ReusableFilePreprocessingFileExplainingDiagnostic;
+    export type ReusableFilePreprocessingDiagnostic = ReusableFilePreprocessingReferencedDiagnostic | ReusableFilePreprocessingFileExplainingDiagnostic;
 
     /* @internal */
     export type RedirectTargetsMap = ReadonlyESMap<Path, readonly string[]>;

src/compiler/types.ts

@@ -28,13 +28,13 @@ namespace ts {
         if (!content) return emitSkippedWithNoDiagnostics;
         const buildInfo = getBuildInfo(content);
         if (buildInfo.version !== version) return emitSkippedWithNoDiagnostics;
-        if (!buildInfo.program) return emitSkippedWithNoDiagnostics;
-        // TODO:: Clean the actual program
-        let newContent = content;
+        if (!buildInfo.program?.peristedProgram) return emitSkippedWithNoDiagnostics;
+        const { program: { peristedProgram, ...program } } = buildInfo;
+        buildInfo.program = program;
 
         // Actual writeFile with new program
         const emitDiagnostics = createDiagnosticCollection();
-        writeFile(host, emitDiagnostics, buildInfoPath, newContent, /*writeByteOrderMark*/ false);
+        writeFile(host, emitDiagnostics, buildInfoPath, getBuildInfoText(buildInfo), /*writeByteOrderMark*/ false);
         return {
             emitSkipped: false,
             diagnostics: emitDiagnostics.getDiagnostics(),

src/compiler/watchPublic.ts

@@ -639,7 +639,7 @@ namespace ts {
         public identifiers!: ESMap<string, string>;
         public nameTable: UnderscoreEscapedMap<number> | undefined;
         public resolvedModules: ESMap<string, ResolvedModuleWithFailedLookupLocations> | undefined;
-        public resolvedTypeReferenceDirectiveNames!: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations>;
+        public resolvedTypeReferenceDirectiveNames: ESMap<string, ResolvedTypeReferenceDirectiveWithFailedLookupLocations> | undefined;
         public imports!: readonly StringLiteralLike[];
         public moduleAugmentations!: StringLiteral[];
         private namedDeclarations: ESMap<string, Declaration[]> | undefined;

src/services/services.ts

@@ -46,13 +46,35 @@ namespace ts {
                     subScenario: "Write file that could not be resolved",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: fs => fs.writeFileSync(`/src/project/src/fileNotFound.ts`, "export function something2() { return 20; }"),
+                    // when doing clean build, fileNotFound.ts would be resolved so the output order in outFile.js would change
+                    // In build mode the out is generated only when there are no errors
+                    // Outputs are generated, buildinfo is updated to report no errors
+                    cleanBuildDiscrepancies: () => new Map([
+                        [`/src/project/src/filePresent.js`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/filePresent.d.ts`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/fileNotFound.js`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/fileNotFound.d.ts`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/anotherFileReusingResolution.js`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/anotherFileReusingResolution.d.ts`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/main.js`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/main.d.ts`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/newFile.js`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/src/newFile.d.ts`, CleanBuildDescrepancy.CleanFilePresent],
+                        [`/src/project/tsconfig.tsbuildinfo`, CleanBuildDescrepancy.CleanFileTextDifferent],
+                    ]),
                 },
                 {
                     subScenario: "Clean resolutions",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: noop,
                     commandLineArgs: ["--b", "src/project", "--cleanPersistedProgram"]
                 },
+                {
+                    subScenario: "Clean resolutions again",
+                    buildKind: BuildKind.IncrementalDtsChange,
+                    modifyFs: noop,
+                    commandLineArgs: ["--b", "src/project", "--cleanPersistedProgram"]
+                },
                 noChangeRun,
                 {
                     subScenario: "Modify main file",
@@ -87,13 +109,27 @@ namespace ts {
                     subScenario: "Write file that could not be resolved",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: fs => fs.writeFileSync(`/src/project/src/fileNotFound.ts`, "export function something2() { return 20; }"),
+                    // when doing clean build, fileNotFound.ts would be resolved so the output order in outFile.js would change
+                    // In build mode the out is generated only when there are no errors
+                    cleanBuildDiscrepancies: () => new Map([
+                        ["/src/project/outFile.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent],
+                        ["/src/project/outFile.js", CleanBuildDescrepancy.CleanFilePresent],
+                        ["/src/project/outFile.d.ts", CleanBuildDescrepancy.CleanFilePresent],
+                        ["/src/project/outFile.tsbuildinfo.baseline.txt", CleanBuildDescrepancy.CleanFilePresent],
+                    ]),
                 },
                 {
                     subScenario: "Clean resolutions",
                     buildKind: BuildKind.IncrementalDtsChange,
                     modifyFs: noop,
                     commandLineArgs: ["--b", "src/project", "--cleanPersistedProgram"]
                 },
+                {
+                    subScenario: "Clean resolutions again",
+                    buildKind: BuildKind.IncrementalDtsChange,
+                    modifyFs: noop,
+                    commandLineArgs: ["--b", "src/project", "--cleanPersistedProgram"]
+                },
                 noChangeRun,
                 {
                     subScenario: "Modify main file",

src/testRunner/unittests/tsbuild/persistResolutions.ts

@@ -516,13 +516,19 @@ class someClass2 { }`),
                 subScenario: "persistResolutions",
                 baselinePrograms: true,
                 fs: () => projFs,
-                modifyFs: fs => fs.writeFileSync("/src/core/tsconfig.json", JSON.stringify({
-                    compilerOptions: {
-                        composite: true,
-                        skipDefaultLibCheck: true,
-                        persistResolutions: true,
+                modifyFs: fs => {
+                    persistResolutions("/src/core/tsconfig.json");
+                    persistResolutions("/src/logic/tsconfig.json");
+                    persistResolutions("/src/tests/tsconfig.json");
+                    function persistResolutions(file: string) {
+                        const content = JSON.parse(fs.readFileSync(file, "utf-8"));
+                        content.compilerOptions = {
+                            ...content.compilerOptions || {},
+                            persistResolutions: true
+                        };
+                        fs.writeFileSync(file, JSON.stringify(content, /*replacer*/ undefined, 4));
                     }
-                })),
+                },
                 commandLineArgs: ["--b", "/src/tests"],
                 incrementalScenarios: [
                     ...coreChanges,