Revert "[clang] Don't add documentation comments to the AST if not requested" (#219807)

Reverts llvm/llvm-project#206363 due to huge compile-time regression.

Author: aengelke

clang-tools-extra/clang-doc/tool/ClangDocMain.cpp

@@ -285,8 +285,7 @@ Example usage for a project using a compile commands database:
     llvm::outs() << "Emiting docs in " << Format << " format.\n";
     auto G = ExitOnErr(doc::findGeneratorByName(Format));
 
-    ArgumentsAdjuster ArgAdjuster = getInsertArgumentAdjuster(
-        "-fretain-comments", tooling::ArgumentInsertPosition::END);
+    ArgumentsAdjuster ArgAdjuster;
     if (!DoxygenOnly)
       ArgAdjuster = combineAdjusters(
           getInsertArgumentAdjuster("-fparse-all-comments",

clang-tools-extra/clangd/Compiler.cpp

@@ -121,8 +121,7 @@ buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
   // createInvocationFromCommandLine sets DisableFree.
   CI->getFrontendOpts().DisableFree = false;
   CI->getLangOpts().CommentOpts.ParseAllComments = true;
-  CI->getLangOpts().CommentOpts.RetainComments = true;
-  CI->getLangOpts().CommentOpts.RetainCommentsFromSystemHeaders = true;
+  CI->getLangOpts().RetainCommentsFromSystemHeaders = true;
 
   disableUnsupportedOptions(*CI);
   return CI;

clang-tools-extra/clangd/index/IndexAction.cpp

@@ -167,8 +167,7 @@ class IndexAction : public ASTFrontendAction {
   bool BeginInvocation(CompilerInstance &CI) override {
     // We want all comments, not just the doxygen ones.
     CI.getLangOpts().CommentOpts.ParseAllComments = true;
-    CI.getLangOpts().CommentOpts.RetainComments = true;
-    CI.getLangOpts().CommentOpts.RetainCommentsFromSystemHeaders = true;
+    CI.getLangOpts().RetainCommentsFromSystemHeaders = true;
     // Index the whole file even if there are warnings and -Werror is set.
     // Avoids some analyses too. Set in two places as we're late to the party.
     CI.getDiagnosticOpts().IgnoreWarnings = true;

clang/docs/ReleaseNotes.md

@@ -405,12 +405,6 @@ features cannot lower the translation-unit ABI level;
 
 - Improved how Unicode characters are displayed in diagnostic messages.
 
-- Clang no longer retains source comments in the AST when nothing will read them
-  back. Comments are now collected only when they may be consumed (e.g. with
-  ``-fparse-all-comments``, when ``-Wdocumentation`` is enabled, when emitting a
-  PCH/module, or during code completion), reducing memory overhead for typical
-  compilations.
-
 - `-Wtautological-pointer-compare` and `-Wpointer-bool-conversion` now
   diagnose a reference to a function (e.g. of type `void (&)()`) compared
   against or converted to a null pointer, the same as a bare function name.

clang/include/clang/Basic/CommentOptions.h

@@ -30,14 +30,6 @@ struct CommentOptions {
   /// Treat ordinary comments as documentation comments.
   bool ParseAllComments = false;
 
-  /// Force the front end to retain all documentation comments in the AST, even
-  /// when no comment consuming diagnostic or language option is enabled. Tools
-  /// that query comments after parsing set this.
-  bool RetainComments = false;
-
-  /// Retain documentation comments from system headers in the AST.
-  bool RetainCommentsFromSystemHeaders = false;
-
   CommentOptions() = default;
 };
 

clang/include/clang/Basic/Diagnostic.h

@@ -974,16 +974,6 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
            diag::Severity::Ignored;
   }
 
-  bool areAllIgnored(StringRef Group, SourceLocation Loc) const {
-    llvm::SmallVector<diag::kind> diagsInGroup;
-    bool Failed = Diags->getDiagnosticsInGroup(diag::Flavor::WarningOrError,
-                                               Group, diagsInGroup);
-    assert(!Failed && "Incorrect group name?");
-    (void)Failed;
-    return Diags->getDiagnosticListHighestSeverity(diagsInGroup, Loc, *this) ==
-           diag::Severity::Ignored;
-  }
-
   /// Based on the way the client configured the DiagnosticsEngine
   /// object, classify the specified diagnostic ID into a Level, consumable by
   /// the DiagnosticConsumer.

clang/include/clang/Basic/DiagnosticIDs.h

@@ -516,12 +516,6 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
                         const DiagnosticsEngine &Diag) const LLVM_READONLY;
 
-  /// Given a collection of diagnostic IDs, get the 'highest' severity of them
-  /// at the provided location for this DiagnosticsEngine.
-  diag::Severity getDiagnosticListHighestSeverity(
-      llvm::ArrayRef<diag::kind> DiagIDs, SourceLocation Loc,
-      const DiagnosticsEngine &Diag) const LLVM_READONLY;
-
   Class getDiagClass(unsigned DiagID) const;
 
   /// Whether the diagnostic may leave the AST in a state where some

clang/include/clang/Basic/LangOptions.def

@@ -405,6 +405,8 @@ LANGOPT(ApplePragmaPack, 1, 0, NotCompatible, "Apple gcc-compatible #pragma pack
 
 LANGOPT(XLPragmaPack, 1, 0, NotCompatible, "IBM XL #pragma pack handling")
 
+LANGOPT(RetainCommentsFromSystemHeaders, 1, 0, Compatible, "retain documentation comments from system headers in the AST")
+
 LANGOPT(APINotes, 1, 0, NotCompatible, "use external API notes")
 LANGOPT(APINotesModules, 1, 0, NotCompatible, "use module-based external API notes")
 LANGOPT(SwiftVersionIndependentAPINotes, 1, 0, NotCompatible, "use external API notes capturing all versions")

clang/include/clang/Options/Options.td

@@ -2221,12 +2221,6 @@ defm define_target_os_macros : OptInCC1FFlag<"define-target-os-macros",
 def fparse_all_comments : Flag<["-"], "fparse-all-comments">, Group<f_clang_Group>,
   Visibility<[ClangOption, CC1Option]>,
   MarshallingInfoFlag<LangOpts<"CommentOpts.ParseAllComments">>;
-def fretain_comments : Flag<["-"], "fretain-comments">, Group<f_clang_Group>,
-  Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Retain documentation comments in the AST even when no diagnostic or "
-           "language option would otherwise require them (e.g. for tools that "
-           "query comments after parsing)">,
-  MarshallingInfoFlag<LangOpts<"CommentOpts.RetainComments">>;
 def frecord_command_line : Flag<["-"], "frecord-command-line">,
   DocBrief<[{Generate a section named ".GCC.command.line" containing the
 driver command-line. After linking, the section may contain multiple command
@@ -3948,7 +3942,7 @@ defm implicit_modules : BoolFOption<"implicit-modules",
           [NoXarchOption], [ClangOption, CLOption]>>;
 def fretain_comments_from_system_headers : Flag<["-"], "fretain-comments-from-system-headers">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoFlag<LangOpts<"CommentOpts.RetainCommentsFromSystemHeaders">>;
+  MarshallingInfoFlag<LangOpts<"RetainCommentsFromSystemHeaders">>;
 def fmodule_header : Flag <["-"], "fmodule-header">, Group<f_Group>,
   Visibility<[ClangOption, CLOption]>,
   HelpText<"Build a C++20 Header Unit from a header">;

clang/include/clang/Sema/Sema.h

@@ -1126,11 +1126,6 @@ class Sema final : public SemaBase {
 
   void ActOnComment(SourceRange Comment);
 
-  /// Returns true if a comment at \p Loc should be retained in the AST
-  /// (some consumer such as -Wdocumentation, -fparse-all-comments, code
-  /// completion, or AST-file serialization may read it back).
-  bool shouldRetainCommentsInAST(SourceLocation Loc) const;
-
   /// Retrieve the parser's current scope.
   ///
   /// This routine must only be used when it is certain that semantic analysis