Structured comment parsing, first step.

* Retain comments in the AST
* Serialize/deserialize comments
* Find comments attached to a certain Decl
* Expose raw comment text and SourceRange via libclang


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@158771 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang-c/Index.h

@@ -3187,6 +3187,19 @@ CINDEX_LINKAGE CXCursor clang_getCanonicalCursor(CXCursor);
  */
 CINDEX_LINKAGE int clang_Cursor_getObjCSelectorIndex(CXCursor);
 
+/**
+ * \brief Given a cursor that represents a declaration, return the associated
+ * comment's source range.  The range may include multiple consecutive comments
+ * with whitespace in between.
+ */
+CINDEX_LINKAGE CXSourceRange clang_Cursor_getCommentRange(CXCursor C);
+
+/**
+ * \brief Given a cursor that represents a declaration, return the associated
+ * comment text, including comment markers.
+ */
+CINDEX_LINKAGE CXString clang_Cursor_getRawCommentText(CXCursor C);
+
 /**
  * @}
  */

include/clang/AST/ASTContext.h

@@ -27,6 +27,7 @@
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/CanonicalType.h"
+#include "clang/Comments/RawCommentList.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -51,7 +52,6 @@ namespace clang {
   class ASTMutationListener;
   class IdentifierTable;
   class SelectorTable;
-  class SourceManager;
   class TargetInfo;
   class CXXABI;
   // Decls
@@ -418,6 +418,30 @@ public:
     return FullSourceLoc(Loc,SourceMgr);
   }
 
+  /// \brief All comments in this translation unit.
+  RawCommentList Comments;
+
+  /// \brief True if comments are already loaded from ExternalASTSource.
+  mutable bool CommentsLoaded;
+
+  /// \brief Mapping from declarations to their comments (stored within
+  /// Comments list), once we have already looked up the comment associated
+  /// with a given declaration.
+  mutable llvm::DenseMap<const Decl *, const RawComment *> DeclComments;
+
+  /// \brief Return the Doxygen-style comment attached to a given declaration,
+  /// without looking into cache.
+  const RawComment *getRawCommentForDeclNoCache(const Decl *D) const;
+
+public:
+  void addComment(const RawComment &RC) {
+    Comments.addComment(RC, *this);
+  }
+
+  /// \brief Return the Doxygen-style comment attached to a given declaration.
+  /// Returns NULL if no comment is attached.
+  const RawComment *getRawCommentForDecl(const Decl *D) const;
+
   /// \brief Retrieve the attributes for the given declaration.
   AttrVec& getDeclAttrs(const Decl *D);
 

include/clang/AST/ExternalASTSource.h

@@ -179,6 +179,9 @@ public:
   /// \c ObjCInterfaceDecl::setExternallyCompleted().
   virtual void CompleteType(ObjCInterfaceDecl *Class) { }
 
+  /// \brief Loads comment ranges.
+  virtual void ReadComments() { }
+
   /// \brief Notify ExternalASTSource that we started deserialization of
   /// a decl or type so until FinishedDeserializing is called there may be
   /// decls that are initializing. Must be paired with FinishedDeserializing.

include/clang/Basic/SourceManager.h

@@ -1244,19 +1244,6 @@ public:
   /// \returns true if LHS source location comes before RHS, false otherwise.
   bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const;
 
-  /// \brief Comparison function class.
-  class LocBeforeThanCompare : public std::binary_function<SourceLocation,
-                                                         SourceLocation, bool> {
-    SourceManager &SM;
-
-  public:
-    explicit LocBeforeThanCompare(SourceManager &SM) : SM(SM) { }
-
-    bool operator()(SourceLocation LHS, SourceLocation RHS) const {
-      return SM.isBeforeInTranslationUnit(LHS, RHS);
-    }
-  };
-
   /// \brief Determines the order of 2 source locations in the "source location
   /// address space".
   bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const {
@@ -1500,6 +1487,35 @@ private:
   friend class ASTWriter;
 };
 
+/// \brief Comparison function object.
+template<typename T>
+class BeforeThanCompare;
+
+/// \brief Compare two source locations.
+template<>
+class BeforeThanCompare<SourceLocation> {
+  SourceManager &SM;
+
+public:
+  explicit BeforeThanCompare(SourceManager &SM) : SM(SM) { }
+
+  bool operator()(SourceLocation LHS, SourceLocation RHS) const {
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+};
+
+/// \brief Compare two non-overlapping source ranges.
+template<>
+class BeforeThanCompare<SourceRange> {
+  SourceManager &SM;
+
+public:
+  explicit BeforeThanCompare(SourceManager &SM) : SM(SM) { }
+
+  bool operator()(SourceRange LHS, SourceRange RHS) {
+    return SM.isBeforeInTranslationUnit(LHS.getBegin(), RHS.getBegin());
+  }
+};
 
 }  // end namespace clang
 

include/clang/Comments/RawCommentList.h

@@ -0,0 +1,172 @@
+//===--- RawCommentList.h - Classes for processing raw comments -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_COMMENTS_RAW_COMMENT_LIST_H
+#define LLVM_CLANG_COMMENTS_RAW_COMMENT_LIST_H
+
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/ArrayRef.h"
+
+namespace clang {
+
+class ASTContext;
+class ASTReader;
+
+class RawComment {
+public:
+  enum CommentKind {
+    CK_Invalid,      ///< Invalid comment
+    CK_OrdinaryBCPL, ///< Any normal BCPL comments
+    CK_OrdinaryC,    ///< Any normal C comment
+    CK_BCPLSlash,    ///< \code /// stuff \endcode
+    CK_BCPLExcl,     ///< \code //! stuff \endcode
+    CK_JavaDoc,      ///< \code /** stuff */ \endcode
+    CK_Qt,           ///< \code /*! stuff */ \endcode, also used by HeaderDoc
+    CK_Merged        ///< Two or more Doxygen comments merged together
+  };
+
+  RawComment() : Kind(CK_Invalid), IsAlmostTrailingComment(false) { }
+
+  RawComment(const SourceManager &SourceMgr, SourceRange SR,
+             bool Merged = false);
+
+  CommentKind getKind() const LLVM_READONLY {
+    return (CommentKind) Kind;
+  }
+
+  bool isInvalid() const LLVM_READONLY {
+    return Kind == CK_Invalid;
+  }
+
+  bool isMerged() const LLVM_READONLY {
+    return Kind == CK_Merged;
+  }
+
+  /// Returns true if it is a comment that should be put after a member:
+  /// \code ///< stuff \endcode
+  /// \code //!< stuff \endcode
+  /// \code /**< stuff */ \endcode
+  /// \code /*!< stuff */ \endcode
+  bool isTrailingComment() const LLVM_READONLY {
+    assert(isDoxygen());
+    return IsTrailingComment;
+  }
+
+  /// Returns true if it is a probable typo:
+  /// \code //< stuff \endcode
+  /// \code /*< stuff */ \endcode
+  bool isAlmostTrailingComment() const LLVM_READONLY {
+    return IsAlmostTrailingComment;
+  }
+
+  /// Returns true if this comment is not a Doxygen comment.
+  bool isOrdinary() const LLVM_READONLY {
+    return (Kind == CK_OrdinaryBCPL) || (Kind == CK_OrdinaryC);
+  }
+
+  /// Returns true if this comment any kind of a Doxygen comment.
+  bool isDoxygen() const LLVM_READONLY {
+    return !isInvalid() && !isOrdinary();
+  }
+
+  /// Returns raw comment text with comment markers.
+  StringRef getRawText(const SourceManager &SourceMgr) const {
+    if (RawTextValid)
+      return RawText;
+
+    RawText = getRawTextSlow(SourceMgr);
+    RawTextValid = true;
+    return RawText;
+  }
+
+  SourceRange getSourceRange() const LLVM_READONLY {
+    return Range;
+  }
+
+  unsigned getBeginLine(const SourceManager &SM) const;
+  unsigned getEndLine(const SourceManager &SM) const;
+
+private:
+  SourceRange Range;
+
+  mutable StringRef RawText;
+  mutable bool RawTextValid : 1; ///< True if RawText is valid
+
+  unsigned Kind : 3;
+
+  bool IsTrailingComment : 1;
+  bool IsAlmostTrailingComment : 1;
+
+  mutable bool BeginLineValid : 1; ///< True if BeginLine is valid
+  mutable bool EndLineValid : 1;   ///< True if EndLine is valid
+  mutable unsigned BeginLine;      ///< Cached line number
+  mutable unsigned EndLine;        ///< Cached line number
+
+  /// \brief Constructor for AST deserialization.
+  RawComment(SourceRange SR, CommentKind K, bool IsTrailingComment,
+             bool IsAlmostTrailingComment) :
+    Range(SR), RawTextValid(false), Kind(K),
+    IsTrailingComment(IsTrailingComment),
+    IsAlmostTrailingComment(IsAlmostTrailingComment),
+    BeginLineValid(false), EndLineValid(false)
+  { }
+
+  StringRef getRawTextSlow(const SourceManager &SourceMgr) const;
+
+  friend class ASTReader;
+};
+
+/// \brief Compare comments' source locations.
+template<>
+class BeforeThanCompare<RawComment> {
+  const SourceManager &SM;
+
+public:
+  explicit BeforeThanCompare(const SourceManager &SM) : SM(SM) { }
+
+  bool operator()(const RawComment &LHS, const SourceRange &RHS) {
+    return SM.isBeforeInTranslationUnit(LHS.getSourceRange().getBegin(),
+                                        RHS.getBegin());
+  }
+};
+
+/// \brief This class represents all comments included in the translation unit,
+/// sorted in order of appearance in the translation unit.
+class RawCommentList {
+public:
+  RawCommentList(SourceManager &SourceMgr) :
+    SourceMgr(SourceMgr), OnlyWhitespaceSeen(true) { }
+
+  void addComment(const RawComment &RC, ASTContext &Context);
+
+  ArrayRef<RawComment> getComments() const {
+    return Comments;
+  }
+
+private:
+  SourceManager &SourceMgr;
+  std::vector<RawComment> Comments;
+  RawComment LastComment;
+  bool OnlyWhitespaceSeen;
+
+  void addCommentsToFront(const std::vector<RawComment> &C) {
+    size_t OldSize = Comments.size();
+    Comments.resize(C.size() + OldSize);
+    std::copy_backward(Comments.begin(), Comments.begin() + OldSize,
+                       Comments.end());
+    std::copy(C.begin(), C.end(), Comments.begin());
+  }
+
+  friend class ASTReader;
+};
+
+} // end namespace clang
+
+#endif
+

include/clang/Lex/Preprocessor.h

@@ -510,12 +510,12 @@ public:
   }
 
   /// \brief Add the specified comment handler to the preprocessor.
-  void AddCommentHandler(CommentHandler *Handler);
+  void addCommentHandler(CommentHandler *Handler);
 
   /// \brief Remove the specified comment handler.
   ///
   /// It is an error to remove a handler that has not been registered.
-  void RemoveCommentHandler(CommentHandler *Handler);
+  void removeCommentHandler(CommentHandler *Handler);
 
   /// \brief Set the code completion handler to the given object.
   void setCodeCompletionHandler(CodeCompletionHandler &Handler) {

include/clang/Parse/Parser.h

@@ -170,6 +170,7 @@ class Parser : public CodeCompletionHandler {
   OwningPtr<PragmaHandler> RedefineExtnameHandler;
   OwningPtr<PragmaHandler> FPContractHandler;
   OwningPtr<PragmaHandler> OpenCLExtensionHandler;
+  OwningPtr<CommentHandler> CommentHandler;
 
   /// Whether the '>' token acts as an operator or not. This will be
   /// true except when we are parsing an expression within a C++

include/clang/Sema/Sema.h

@@ -849,6 +849,8 @@ public:
   /// WeakTopLevelDeclDecls - access to \#pragma weak-generated Decls
   SmallVector<Decl*,2> &WeakTopLevelDecls() { return WeakTopLevelDecl; }
 
+  void ActOnComment(SourceRange Comment);
+
   //===--------------------------------------------------------------------===//
   // Type Analysis / Processing: SemaType.cpp.
   //

include/clang/Serialization/ASTBitCodes.h

@@ -207,7 +207,10 @@ namespace clang {
       PREPROCESSOR_DETAIL_BLOCK_ID,
       
       /// \brief The block containing the submodule structure.
-      SUBMODULE_BLOCK_ID
+      SUBMODULE_BLOCK_ID,
+
+      /// \brief The block containing comments.
+      COMMENTS_BLOCK_ID
     };
 
     /// \brief Record types that occur within the AST block itself.
@@ -545,7 +548,12 @@ namespace clang {
       /// \brief Specifies a required feature.
       SUBMODULE_REQUIRES = 7
     };
-    
+
+    /// \brief Record types used within a comments block.
+    enum CommentRecordTypes {
+      COMMENTS_RAW_COMMENT = 0
+    };
+
     /// \defgroup ASTAST AST file AST constants
     ///
     /// The constants in this group describe various components of the

include/clang/Serialization/ASTReader.h

@@ -1501,6 +1501,13 @@ public:
   SwitchCase *getSwitchCaseWithID(unsigned ID);
 
   void ClearSwitchCaseIDs();
+
+  /// \brief Cursors for comments blocks.
+  SmallVector<std::pair<llvm::BitstreamCursor,
+                        serialization::ModuleFile *>, 8> CommentsCursors;
+
+  /// \brief Loads comments ranges.
+  void ReadComments();
 };
 
 /// \brief Helper class that saves the current stream position and

include/clang/Serialization/ASTWriter.h

@@ -414,6 +414,7 @@ private:
   uint64_t WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC);
   void WriteTypeDeclOffsets();
   void WriteFileDeclIDsMap();
+  void WriteComments();
   void WriteSelectors(Sema &SemaRef);
   void WriteReferencedSelectorsPool(Sema &SemaRef);
   void WriteIdentifierTable(Preprocessor &PP, IdentifierResolver &IdResolver,

lib/ARCMigrate/TransEmptyStatementsAndDealloc.cpp

@@ -44,7 +44,7 @@ static bool isEmptyARCMTMacroStatement(NullStmt *S,
   SourceManager &SM = Ctx.getSourceManager();
   std::vector<SourceLocation>::iterator
     I = std::upper_bound(MacroLocs.begin(), MacroLocs.end(), SemiLoc,
-                         SourceManager::LocBeforeThanCompare(SM));
+                         BeforeThanCompare<SourceLocation>(SM));
   --I;
   SourceLocation
       AfterMacroLoc = I->getLocWithOffset(getARCMTMacroName().size());

lib/AST/ASTContext.cpp

@@ -53,6 +53,107 @@ enum FloatingRank {
   HalfRank, FloatRank, DoubleRank, LongDoubleRank
 };
 
+const RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
+  if (!CommentsLoaded && ExternalSource) {
+    ExternalSource->ReadComments();
+    CommentsLoaded = true;
+  }
+
+  assert(D);
+
+  // TODO: handle comments for function parameters properly.
+  if (isa<ParmVarDecl>(D))
+    return NULL;
+
+  ArrayRef<RawComment> RawComments = Comments.getComments();
+
+  // If there are no comments anywhere, we won't find anything.
+  if (RawComments.empty())
+    return NULL;
+
+  // If the declaration doesn't map directly to a location in a file, we
+  // can't find the comment.
+  SourceLocation DeclLoc = D->getLocation();
+  if (DeclLoc.isInvalid() || !DeclLoc.isFileID())
+    return NULL;
+
+  // Find the comment that occurs just after this declaration.
+  ArrayRef<RawComment>::iterator Comment
+      = std::lower_bound(RawComments.begin(),
+                         RawComments.end(),
+                         SourceRange(DeclLoc),
+                         BeforeThanCompare<RawComment>(SourceMgr));
+
+  // Decompose the location for the declaration and find the beginning of the
+  // file buffer.
+  std::pair<FileID, unsigned> DeclLocDecomp = SourceMgr.getDecomposedLoc(DeclLoc);
+
+  // First check whether we have a trailing comment.
+  if (Comment != RawComments.end() &&
+      Comment->isDoxygen() && Comment->isTrailingComment() &&
+      !isa<TagDecl>(D) && !isa<NamespaceDecl>(D)) {
+    std::pair<FileID, unsigned> CommentBeginDecomp
+      = SourceMgr.getDecomposedLoc(Comment->getSourceRange().getBegin());
+    // Check that Doxygen trailing comment comes after the declaration, starts
+    // on the same line and in the same file as the declaration.
+    if (DeclLocDecomp.first == CommentBeginDecomp.first &&
+        SourceMgr.getLineNumber(DeclLocDecomp.first, DeclLocDecomp.second)
+          == SourceMgr.getLineNumber(CommentBeginDecomp.first,
+                                     CommentBeginDecomp.second)) {
+      return &*Comment;
+    }
+  }
+
+  // The comment just after the declaration was not a trailing comment.
+  // Let's look at the previous comment.
+  if (Comment == RawComments.begin())
+    return NULL;
+  --Comment;
+
+  // Check that we actually have a non-member Doxygen comment.
+  if (!Comment->isDoxygen() || Comment->isTrailingComment())
+    return NULL;
+
+  // Decompose the end of the comment.
+  std::pair<FileID, unsigned> CommentEndDecomp
+    = SourceMgr.getDecomposedLoc(Comment->getSourceRange().getEnd());
+
+  // If the comment and the declaration aren't in the same file, then they
+  // aren't related.
+  if (DeclLocDecomp.first != CommentEndDecomp.first)
+    return NULL;
+
+  // Get the corresponding buffer.
+  bool Invalid = false;
+  const char *Buffer = SourceMgr.getBufferData(DeclLocDecomp.first,
+                                               &Invalid).data();
+  if (Invalid)
+    return NULL;
+
+  // Extract text between the comment and declaration.
+  StringRef Text(Buffer + CommentEndDecomp.second,
+                 DeclLocDecomp.second - CommentEndDecomp.second);
+
+  // There should be no other declarations between comment and declaration.
+  if (Text.find_first_of(",;{}") != StringRef::npos)
+    return NULL;
+
+  return &*Comment;
+}
+
+const RawComment *ASTContext::getRawCommentForDecl(const Decl *D) const {
+  // Check whether we have cached a comment string for this declaration
+  // already.
+  llvm::DenseMap<const Decl *, const RawComment *>::iterator Pos
+      = DeclComments.find(D);
+  if (Pos != DeclComments.end())
+      return Pos->second;
+
+  const RawComment *RC = getRawCommentForDeclNoCache(D);
+  DeclComments[D] = RC;
+  return RC;
+}
+
 void 
 ASTContext::CanonicalTemplateTemplateParm::Profile(llvm::FoldingSetNodeID &ID, 
                                                TemplateTemplateParmDecl *Parm) {
@@ -244,6 +345,7 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     BuiltinInfo(builtins),
     DeclarationNames(*this),
     ExternalSource(0), Listener(0),
+    Comments(SM), CommentsLoaded(false),
     LastSDM(0, 0),
     UniqueBlockByRefTypeID(0) 
 {

lib/CMakeLists.txt

@@ -15,3 +15,4 @@ add_subdirectory(Frontend)
 add_subdirectory(FrontendTool)
 add_subdirectory(Tooling)
 add_subdirectory(StaticAnalyzer)
+add_subdirectory(Comments)

lib/Comments/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(LLVM_USED_LIBS clangBasic clangAST clangLex)
+
+add_clang_library(clangComments
+  CommentLexer.cpp
+  RawCommentList.cpp
+  )
+

lib/Comments/Makefile

@@ -0,0 +1,14 @@
+##===- clang/lib/Comments/Makefile -------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../..
+LIBRARYNAME := clangComments
+
+include $(CLANG_LEVEL)/Makefile
+

lib/Comments/RawCommentList.cpp

@@ -0,0 +1,207 @@
+//===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Comments/RawCommentList.h"
+#include "clang/AST/ASTContext.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace clang;
+
+namespace {
+/// Get comment kind and bool describing if it is a trailing comment.
+std::pair<RawComment::CommentKind, bool> getCommentKind(StringRef Comment) {
+  if (Comment.size() < 3 || Comment[0] != '/')
+    return std::make_pair(RawComment::CK_Invalid, false);
+
+  RawComment::CommentKind K;
+  if (Comment[1] == '/') {
+    if (Comment.size() < 3)
+      return std::make_pair(RawComment::CK_OrdinaryBCPL, false);
+
+    if (Comment[2] == '/')
+      K = RawComment::CK_BCPLSlash;
+    else if (Comment[2] == '!')
+      K = RawComment::CK_BCPLExcl;
+    else
+      return std::make_pair(RawComment::CK_OrdinaryBCPL, false);
+  } else {
+    assert(Comment.size() >= 4);
+
+    // Comment lexer does not understand escapes in comment markers, so pretend
+    // that this is not a comment.
+    if (Comment[1] != '*' ||
+        Comment[Comment.size() - 2] != '*' ||
+        Comment[Comment.size() - 1] != '/')
+      return std::make_pair(RawComment::CK_Invalid, false);
+
+    if (Comment[2] == '*')
+      K = RawComment::CK_JavaDoc;
+    else if (Comment[2] == '!')
+      K = RawComment::CK_Qt;
+    else
+      return std::make_pair(RawComment::CK_OrdinaryC, false);
+  }
+  const bool TrailingComment = (Comment.size() > 3) && (Comment[3] == '<');
+  return std::make_pair(K, TrailingComment);
+}
+
+bool mergedCommentIsTrailingComment(StringRef Comment) {
+  return (Comment.size() > 3) && (Comment[3] == '<');
+}
+} // unnamed namespace
+
+RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
+                       bool Merged) :
+    Range(SR), RawTextValid(false), IsAlmostTrailingComment(false),
+    BeginLineValid(false), EndLineValid(false) {
+  // Extract raw comment text, if possible.
+  if (getRawText(SourceMgr).empty()) {
+    Kind = CK_Invalid;
+    return;
+  }
+
+  if (!Merged) {
+    // Guess comment kind.
+    std::pair<CommentKind, bool> K = getCommentKind(RawText);
+    Kind = K.first;
+    IsTrailingComment = K.second;
+
+    IsAlmostTrailingComment = RawText.startswith("//<") ||
+                                 RawText.startswith("/*<");
+  } else {
+    Kind = CK_Merged;
+    IsTrailingComment = mergedCommentIsTrailingComment(RawText);
+  }
+}
+
+unsigned RawComment::getBeginLine(const SourceManager &SM) const {
+  if (BeginLineValid)
+    return BeginLine;
+
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Range.getBegin());
+  BeginLine = SM.getLineNumber(LocInfo.first, LocInfo.second);
+  BeginLineValid = true;
+  return BeginLine;
+}
+
+unsigned RawComment::getEndLine(const SourceManager &SM) const {
+  if (EndLineValid)
+    return EndLine;
+
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Range.getEnd());
+  EndLine = SM.getLineNumber(LocInfo.first, LocInfo.second);
+  EndLineValid = true;
+  return EndLine;
+}
+
+StringRef RawComment::getRawTextSlow(const SourceManager &SourceMgr) const {
+  FileID BeginFileID;
+  FileID EndFileID;
+  unsigned BeginOffset;
+  unsigned EndOffset;
+
+  llvm::tie(BeginFileID, BeginOffset) =
+      SourceMgr.getDecomposedLoc(Range.getBegin());
+  llvm::tie(EndFileID, EndOffset) =
+      SourceMgr.getDecomposedLoc(Range.getEnd());
+
+  const unsigned Length = EndOffset - BeginOffset;
+  if (Length < 2)
+    return StringRef();
+
+  // The comment can't begin in one file and end in another.
+  assert(BeginFileID == EndFileID);
+
+  bool Invalid = false;
+  const char *BufferStart = SourceMgr.getBufferData(BeginFileID,
+                                                    &Invalid).data();
+  if (Invalid)
+    return StringRef();
+
+  return StringRef(BufferStart + BeginOffset, Length);
+}
+
+namespace {
+bool containsOnlyWhitespace(StringRef Str) {
+  return Str.find_first_not_of(" \t\f\v\r\n") == StringRef::npos;
+}
+
+bool onlyWhitespaceBetweenComments(SourceManager &SM,
+                                   const RawComment &C1, const RawComment &C2) {
+  std::pair<FileID, unsigned> C1EndLocInfo = SM.getDecomposedLoc(
+                                                C1.getSourceRange().getEnd());
+  std::pair<FileID, unsigned> C2BeginLocInfo = SM.getDecomposedLoc(
+                                              C2.getSourceRange().getBegin());
+
+  // Question does not make sense if comments are located in different files.
+  if (C1EndLocInfo.first != C2BeginLocInfo.first)
+    return false;
+
+  bool Invalid = false;
+  const char *Buffer = SM.getBufferData(C1EndLocInfo.first, &Invalid).data();
+  if (Invalid)
+    return false;
+
+  StringRef TextBetweenComments(Buffer + C1EndLocInfo.second,
+                                C2BeginLocInfo.second - C1EndLocInfo.second);
+
+  return containsOnlyWhitespace(TextBetweenComments);
+}
+} // unnamed namespace
+
+void RawCommentList::addComment(const RawComment &RC, ASTContext &Context) {
+  if (RC.isInvalid())
+    return;
+
+  assert((Comments.empty() ||
+          SourceMgr.isBeforeInTranslationUnit(
+              Comments[0].getSourceRange().getEnd(),
+              RC.getSourceRange().getBegin())) &&
+         "comments are not coming in source order");
+
+  if (OnlyWhitespaceSeen) {
+    if (!onlyWhitespaceBetweenComments(SourceMgr, LastComment, RC))
+      OnlyWhitespaceSeen = false;
+  }
+
+  LastComment = RC;
+
+  // Ordinary comments are not interesting for us.
+  if (RC.isOrdinary())
+    return;
+
+  // If this is the first Doxygen comment, save it (because there isn't
+  // anything to merge it with).
+  if (Comments.empty()) {
+    Comments.push_back(RC);
+    OnlyWhitespaceSeen = true;
+    return;
+  }
+
+  const RawComment &C1 = Comments.back();
+  const RawComment &C2 = RC;
+
+  // Merge comments only if there is only whitespace between them.
+  // Can't merge trailing and non-trailing comments.
+  // Merge trailing comments if they are on same or consecutive lines.
+  if (OnlyWhitespaceSeen &&
+      (C1.isTrailingComment() == C2.isTrailingComment()) &&
+      (!C1.isTrailingComment() ||
+       C1.getEndLine(SourceMgr) + 1 >= C2.getBeginLine(SourceMgr))) {
+    SourceRange MergedRange(C1.getSourceRange().getBegin(),
+                            C2.getSourceRange().getEnd());
+    RawComment Merged(SourceMgr, MergedRange, true);
+    Comments.pop_back();
+    Comments.push_back(Merged);
+  } else
+    Comments.push_back(RC);
+
+  OnlyWhitespaceSeen = true;
+}
+

lib/Lex/Preprocessor.cpp

@@ -623,14 +623,14 @@ void Preprocessor::LexAfterModuleImport(Token &Result) {
                                      /*IsIncludeDirective=*/false);
 }
 
-void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
+void Preprocessor::addCommentHandler(CommentHandler *Handler) {
   assert(Handler && "NULL comment handler");
   assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
          CommentHandlers.end() && "Comment handler already registered");
   CommentHandlers.push_back(Handler);
 }
 
-void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
+void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
   std::vector<CommentHandler *>::iterator Pos
   = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
   assert(Pos != CommentHandlers.end() && "Comment handler not registered");

lib/Makefile

@@ -10,7 +10,7 @@ CLANG_LEVEL := ..
 
 PARALLEL_DIRS = Headers Basic Lex Parse AST Sema CodeGen Analysis \
                 StaticAnalyzer Edit Rewrite ARCMigrate Serialization Frontend \
-                FrontendTool Tooling Driver
+                FrontendTool Tooling Driver Comments
 
 include $(CLANG_LEVEL)/Makefile
 

lib/Parse/Parser.cpp

@@ -23,6 +23,20 @@
 #include "clang/AST/ASTConsumer.h"
 using namespace clang;
 
+/// \brief A comment handler that passes comments found by the preprocessor
+/// to the parser action.
+class ActionCommentHandler : public CommentHandler {
+  Sema &S;
+
+public:
+  explicit ActionCommentHandler(Sema &S) : S(S) { }
+
+  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) {
+    S.ActOnComment(Comment);
+    return false;
+  }
+};
+
 IdentifierInfo *Parser::getSEHExceptKeyword() {
   // __except is accepted as a (contextual) keyword 
   if (!Ident__except && (getLangOpts().MicrosoftExt || getLangOpts().Borland))
@@ -77,7 +91,10 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool SkipFunctionBodies)
 
     PP.AddPragmaHandler("OPENCL", FPContractHandler.get());
   }
-      
+
+  CommentHandler.reset(new ActionCommentHandler(actions));
+  PP.addCommentHandler(CommentHandler.get());
+
   PP.setCodeCompletionHandler(*this);
 }
 
@@ -422,6 +439,9 @@ Parser::~Parser() {
 
   PP.RemovePragmaHandler("STDC", FPContractHandler.get());
   FPContractHandler.reset();
+
+  PP.removeCommentHandler(CommentHandler.get());
+
   PP.clearCodeCompletionHandler();
 
   assert(TemplateIds.empty() && "Still alive TemplateIdAnnotations around?");

lib/Sema/Sema.cpp

@@ -1014,6 +1014,11 @@ LambdaScopeInfo *Sema::getCurLambda() {
   return dyn_cast<LambdaScopeInfo>(FunctionScopes.back());  
 }
 
+void Sema::ActOnComment(SourceRange Comment) {
+  RawComment RC(SourceMgr, Comment);
+  Context.addComment(RC);
+}
+
 // Pin this vtable to this file.
 ExternalSemaSource::~ExternalSemaSource() {}
 

lib/Sema/SemaType.cpp

@@ -2550,7 +2550,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         //  RemovalLocs.push_back(Chunk.Fun.getRestrictQualifierLoc());
         if (!RemovalLocs.empty()) {
           std::sort(RemovalLocs.begin(), RemovalLocs.end(),
-                    SourceManager::LocBeforeThanCompare(S.getSourceManager()));
+                    BeforeThanCompare<SourceLocation>(S.getSourceManager()));
           RemovalRange = SourceRange(RemovalLocs.front(), RemovalLocs.back());
           Loc = RemovalLocs.front();
         }

lib/Serialization/ASTReader.cpp

@@ -1737,6 +1737,17 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         }
         break;
 
+      case COMMENTS_BLOCK_ID: {
+        llvm::BitstreamCursor C = Stream;
+        if (Stream.SkipBlock() ||
+            ReadBlockAbbrevs(C, COMMENTS_BLOCK_ID)) {
+          Error("malformed comments block in AST file");
+          return Failure;
+        }
+        CommentsCursors.push_back(std::make_pair(C, &F));
+        break;
+      }
+
       default:
         if (!Stream.SkipBlock())
           break;
@@ -6258,6 +6269,61 @@ void ASTReader::ClearSwitchCaseIDs() {
   CurrSwitchCaseStmts->clear();
 }
 
+void ASTReader::ReadComments() {
+  std::vector<RawComment> Comments;
+  for (SmallVectorImpl<std::pair<llvm::BitstreamCursor,
+                                 serialization::ModuleFile *> >::iterator
+       I = CommentsCursors.begin(),
+       E = CommentsCursors.end();
+       I != E; ++I) {
+    llvm::BitstreamCursor &Cursor = I->first;
+    serialization::ModuleFile &F = *I->second;
+    SavedStreamPosition SavedPosition(Cursor);
+
+    RecordData Record;
+    while (true) {
+      unsigned Code = Cursor.ReadCode();
+      if (Code == llvm::bitc::END_BLOCK)
+        break;
+
+      if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+        // No known subblocks, always skip them.
+        Cursor.ReadSubBlockID();
+        if (Cursor.SkipBlock()) {
+          Error("malformed block record in AST file");
+          return;
+        }
+        continue;
+      }
+
+      if (Code == llvm::bitc::DEFINE_ABBREV) {
+        Cursor.ReadAbbrevRecord();
+        continue;
+      }
+
+      // Read a record.
+      Record.clear();
+      switch ((CommentRecordTypes) Cursor.ReadRecord(Code, Record)) {
+        default:  // Default behavior: ignore.
+          break;
+
+        case COMMENTS_RAW_COMMENT: {
+          unsigned Idx = 0;
+          SourceRange SR = ReadSourceRange(F, Record, Idx);
+          RawComment::CommentKind Kind =
+              (RawComment::CommentKind) Record[Idx++];
+          bool IsTrailingComment = Record[Idx++];
+          bool IsAlmostTrailingComment = Record[Idx++];
+          Comments.push_back(RawComment(SR, Kind, IsTrailingComment,
+                                        IsAlmostTrailingComment));
+          break;
+      }
+      }
+    }
+  }
+  Context.Comments.addCommentsToFront(Comments);
+}
+
 void ASTReader::finishPendingActions() {
   while (!PendingIdentifierInfos.empty() || !PendingDeclChains.empty()) {
     // If any identifiers with corresponding top-level declarations have

lib/Serialization/ASTWriter.cpp

@@ -2240,6 +2240,23 @@ void ASTWriter::WriteFileDeclIDsMap() {
   Stream.EmitRecordWithBlob(AbbrevCode, Record, data(FileSortedIDs));
 }
 
+void ASTWriter::WriteComments() {
+  Stream.EnterSubblock(COMMENTS_BLOCK_ID, 3);
+  ArrayRef<RawComment> RawComments = Context->Comments.getComments();
+  RecordData Record;
+  for (ArrayRef<RawComment>::iterator I = RawComments.begin(),
+                                      E = RawComments.end();
+       I != E; ++I) {
+    Record.clear();
+    AddSourceRange(I->getSourceRange(), Record);
+    Record.push_back(I->getKind());
+    Record.push_back(I->isTrailingComment());
+    Record.push_back(I->isAlmostTrailingComment());
+    Stream.EmitRecord(COMMENTS_RAW_COMMENT, Record);
+  }
+  Stream.ExitBlock();
+}
+
 //===----------------------------------------------------------------------===//
 // Global Method Pool and Selector Serialization
 //===----------------------------------------------------------------------===//
@@ -3415,6 +3432,7 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   WriteFileDeclIDsMap();
   WriteSourceManagerBlock(Context.getSourceManager(), PP, isysroot);
+  WriteComments();
   
   if (Chain) {
     // Write the mapping information describing our module dependencies and how

test/Index/annotate-comments.cpp

@@ -0,0 +1,229 @@
+// Run lines are sensitive to line numbers and come below the code.
+
+#ifndef HEADER
+#define HEADER
+
+// Not a Doxygen comment.  NOT_DOXYGEN
+void notdoxy1(void);
+
+/* Not a Doxygen comment.  NOT_DOXYGEN */
+void notdoxy2(void);
+
+/*/ Not a Doxygen comment.  NOT_DOXYGEN */
+void notdoxy3(void);
+
+/** Doxygen comment.  isdoxy4 IS_DOXYGEN_SINGLE */
+void isdoxy4(void);
+
+/**
+ * Doxygen comment.  isdoxy5 IS_DOXYGEN_SINGLE */
+void isdoxy5(void);
+
+/**
+ * Doxygen comment.
+ * isdoxy6 IS_DOXYGEN_SINGLE */
+void isdoxy6(void);
+
+/**
+ * Doxygen comment.
+ * isdoxy7 IS_DOXYGEN_SINGLE
+ */
+void isdoxy7(void);
+
+/*! Doxygen comment.  isdoxy8 IS_DOXYGEN_SINGLE */
+void isdoxy8(void);
+
+/// Doxygen comment.  isdoxy9 IS_DOXYGEN_SINGLE
+void isdoxy9(void);
+
+// Not a Doxygen comment.  NOT_DOXYGEN
+/// Doxygen comment.  isdoxy10 IS_DOXYGEN_SINGLE
+void isdoxy10(void);
+
+/// Doxygen comment.  isdoxy11 IS_DOXYGEN_SINGLE
+// Not a Doxygen comment.  NOT_DOXYGEN
+void isdoxy11(void);
+
+/** Doxygen comment.  isdoxy12  IS_DOXYGEN_SINGLE */
+/* Not a Doxygen comment.  NOT_DOXYGEN */
+void isdoxy12(void);
+
+/// Doxygen comment.  isdoxy13 IS_DOXYGEN_START
+/// Doxygen comment.  IS_DOXYGEN_END
+void isdoxy13(void);
+
+/// Doxygen comment.  isdoxy14 IS_DOXYGEN_START
+/// Blah-blah-blah.
+/// Doxygen comment.  IS_DOXYGEN_END
+void isdoxy14(void);
+
+/// Doxygen comment.  isdoxy15 IS_DOXYGEN_START
+/** Blah-blah-blah */
+/// Doxygen comment.  IS_DOXYGEN_END
+void isdoxy15(void);
+
+/** Blah-blah-blah. isdoxy16 IS_DOXYGEN_START *//** Blah */
+/// Doxygen comment.  IS_DOXYGEN_END
+void isdoxy16(void);
+
+/// isdoxy17 IS_DOXYGEN_START
+// Not a Doxygen comment, but still picked up.
+/// IS_DOXYGEN_END
+void isdoxy17(void);
+
+unsigned
+// NOT_DOXYGEN
+/// isdoxy18 IS_DOXYGEN_START
+// Not a Doxygen comment, but still picked up.
+/// IS_DOXYGEN_END
+// NOT_DOXYGEN
+int isdoxy18(void);
+
+//! It all starts here. isdoxy19 IS_DOXYGEN_START
+/*! It's a little odd to continue line this,
+ *
+ * but we need more multi-line comments. */
+/// This comment comes before my other comments
+/** This is a block comment that is associated with the function f. It
+ *  runs for three lines.  IS_DOXYGEN_END
+ */
+void isdoxy19(int, int);
+
+// NOT IN THE COMMENT  NOT_DOXYGEN
+/// This is a BCPL comment.  isdoxy20 IS_DOXYGEN_START
+/// It has only two lines.
+/** But there are other blocks that are part of the comment, too.  IS_DOXYGEN_END */
+void isdoxy20(int);
+
+void isdoxy21(int); ///< This is a member comment.  isdoxy21 IS_DOXYGEN_SINGLE
+
+void isdoxy22(int); /*!< This is a member comment.  isdoxy22 IS_DOXYGEN_SINGLE */
+
+void isdoxy23(int); /**< This is a member comment.  isdoxy23 IS_DOXYGEN_SINGLE */
+
+void notdoxy24(int); // NOT_DOXYGEN
+
+/// IS_DOXYGEN_SINGLE
+struct isdoxy25 {
+};
+
+struct test26 {
+  /// IS_DOXYGEN_SINGLE
+  int isdoxy26;
+};
+
+struct test27 {
+  int isdoxy27; ///< IS_DOXYGEN_SINGLE
+};
+
+struct notdoxy28 {
+}; ///< IS_DOXYGEN_NOT_ATTACHED
+
+/// IS_DOXYGEN_SINGLE
+enum isdoxy29 {
+};
+
+enum notdoxy30 {
+}; ///< IS_DOXYGEN_NOT_ATTACHED
+
+/// IS_DOXYGEN_SINGLE
+namespace isdoxy31 {
+};
+
+namespace notdoxy32 {
+}; ///< IS_DOXYGEN_NOT_ATTACHED
+
+class test33 {
+                ///< IS_DOXYGEN_NOT_ATTACHED
+  int isdoxy33; ///< isdoxy33 IS_DOXYGEN_SINGLE
+  int isdoxy34; ///< isdoxy34 IS_DOXYGEN_SINGLE
+
+                ///< IS_DOXYGEN_NOT_ATTACHED
+  int isdoxy35, ///< isdoxy35 IS_DOXYGEN_SINGLE
+      isdoxy36; ///< isdoxy36 IS_DOXYGEN_SINGLE
+
+                ///< IS_DOXYGEN_NOT_ATTACHED
+  int isdoxy37  ///< isdoxy37 IS_DOXYGEN_SINGLE
+    , isdoxy38  ///< isdoxy38 IS_DOXYGEN_SINGLE
+    , isdoxy39; ///< isdoxy39 IS_DOXYGEN_SINGLE
+};
+
+// Verified that Doxygen attaches these.
+
+/// isdoxy40 IS_DOXYGEN_SINGLE
+// NOT_DOXYGEN
+void isdoxy40(int);
+
+unsigned
+/// isdoxy41 IS_DOXYGEN_SINGLE
+// NOT_DOXYGEN
+int isdoxy41(int);
+
+class test42 {
+  int isdoxy42; /* NOT_DOXYGEN */ ///< isdoxy42 IS_DOXYGEN_SINGLE
+};
+
+#endif
+
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %clang_cc1 -x c++ -emit-pch -o %t/out.pch %s
+// RUN: %clang_cc1 -x c++ -include-pch %t/out.pch -fsyntax-only %s
+
+// RUN: c-index-test -test-load-source all %s > %t/out.c-index-direct
+// RUN: c-index-test -test-load-tu %t/out.pch all > %t/out.c-index-pch
+
+// RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-direct
+// RUN: FileCheck %s -check-prefix=WRONG < %t/out.c-index-pch
+
+// Declarations without Doxygen comments should not pick up some Doxygen comments.
+// WRONG-NOT: notdoxy{{.*}}Comment=
+// WRONG-NOT: test{{.*}}Comment=
+
+// Non-Doxygen comments should not be attached to anything.
+// WRONG-NOT: NOT_DOXYGEN
+
+// Some Doxygen comments are not attached to anything.
+// WRONG-NOT: IS_DOXYGEN_NOT_ATTACHED
+
+// Ensure we don't pick up extra comments.
+// WRONG-NOT: IS_DOXYGEN_START{{.*}}IS_DOXYGEN_START
+// WRONG-NOT: IS_DOXYGEN_END{{.*}}IS_DOXYGEN_END
+
+// RUN: FileCheck %s < %t/out.c-index-direct
+// RUN: FileCheck %s < %t/out.c-index-pch
+
+// CHECK: annotate-comments.cpp:16:6: FunctionDecl=isdoxy4:{{.*}} isdoxy4 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:20:6: FunctionDecl=isdoxy5:{{.*}} isdoxy5 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:25:6: FunctionDecl=isdoxy6:{{.*}} isdoxy6 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:31:6: FunctionDecl=isdoxy7:{{.*}} isdoxy7 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:34:6: FunctionDecl=isdoxy8:{{.*}} isdoxy8 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:37:6: FunctionDecl=isdoxy9:{{.*}} isdoxy9 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:41:6: FunctionDecl=isdoxy10:{{.*}} isdoxy10 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:53:6: FunctionDecl=isdoxy13:{{.*}} isdoxy13 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:58:6: FunctionDecl=isdoxy14:{{.*}} isdoxy14 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:63:6: FunctionDecl=isdoxy15:{{.*}} isdoxy15 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:67:6: FunctionDecl=isdoxy16:{{.*}} isdoxy16 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:72:6: FunctionDecl=isdoxy17:{{.*}} isdoxy17 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:80:5: FunctionDecl=isdoxy18:{{.*}} isdoxy18 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:90:6: FunctionDecl=isdoxy19:{{.*}} isdoxy19 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:96:6: FunctionDecl=isdoxy20:{{.*}} isdoxy20 IS_DOXYGEN_START{{.*}} IS_DOXYGEN_END
+// CHECK: annotate-comments.cpp:98:6: FunctionDecl=isdoxy21:{{.*}} isdoxy21 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:100:6: FunctionDecl=isdoxy22:{{.*}} isdoxy22 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:102:6: FunctionDecl=isdoxy23:{{.*}} isdoxy23 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:107:8: StructDecl=isdoxy25:{{.*}} IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:112:7: FieldDecl=isdoxy26:{{.*}} IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:116:7: FieldDecl=isdoxy27:{{.*}} IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:123:6: EnumDecl=isdoxy29:{{.*}} IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:130:11: Namespace=isdoxy31:{{.*}} IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:138:7: FieldDecl=isdoxy33:{{.*}} isdoxy33 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:139:7: FieldDecl=isdoxy34:{{.*}} isdoxy34 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:142:7: FieldDecl=isdoxy35:{{.*}} isdoxy35 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:143:7: FieldDecl=isdoxy36:{{.*}} isdoxy36 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:146:7: FieldDecl=isdoxy37:{{.*}} isdoxy37 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:147:7: FieldDecl=isdoxy38:{{.*}} isdoxy38 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:148:7: FieldDecl=isdoxy39:{{.*}} isdoxy39 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:155:6: FunctionDecl=isdoxy40:{{.*}} isdoxy40 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:160:5: FunctionDecl=isdoxy41:{{.*}} isdoxy41 IS_DOXYGEN_SINGLE
+// CHECK: annotate-comments.cpp:163:7: FieldDecl=isdoxy42:{{.*}} isdoxy42 IS_DOXYGEN_SINGLE
+

tools/arcmt-test/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(LLVM_USED_LIBS
   clangARCMigrate
+  clangComments
   clangEdit
   clangRewrite
   )

tools/arcmt-test/Makefile

@@ -19,6 +19,7 @@ NO_INSTALL = 1
 LINK_COMPONENTS := support mc
 USEDLIBS = clangARCMigrate.a clangRewrite.a \
 		 clangFrontend.a clangDriver.a clangSerialization.a clangParse.a \
-		 clangSema.a clangEdit.a clangAnalysis.a clangAST.a clangLex.a clangBasic.a
+		 clangSema.a clangEdit.a clangAnalysis.a clangAST.a clangLex.a clangComments.a \
+		 clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile

tools/c-index-test/Makefile

@@ -20,6 +20,7 @@ TOOL_NO_EXPORTS = 1
 LINK_COMPONENTS := support mc
 USEDLIBS = clang.a clangFrontend.a clangDriver.a \
 	   clangSerialization.a clangParse.a clangSema.a \
-	   clangAnalysis.a clangEdit.a clangAST.a clangLex.a clangBasic.a
+	   clangAnalysis.a clangEdit.a clangAST.a clangLex.a clangComments.a \
+	   clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile

tools/c-index-test/c-index-test.c

@@ -218,7 +218,9 @@ static void PrintCursor(CXCursor Cursor) {
     CXPlatformAvailability PlatformAvailability[2];
     int NumPlatformAvailability;
     int I;
-    
+    CXString Comment;
+    const char *CommentCString;
+
     ks = clang_getCursorKindSpelling(Cursor.kind);
     string = want_display_name? clang_getCursorDisplayName(Cursor) 
                               : clang_getCursorSpelling(Cursor);
@@ -398,6 +400,22 @@ static void PrintCursor(CXCursor Cursor) {
       if (!clang_equalRanges(CursorExtent, RefNameRange))
         PrintRange(RefNameRange, "RefName");
     }
+
+    Comment = clang_Cursor_getRawCommentText(Cursor);
+    CommentCString = clang_getCString(Comment);
+    if (CommentCString != NULL && CommentCString[0] != '\0') {
+      printf(" Comment=[");
+      for ( ; *CommentCString; ++CommentCString) {
+        if (*CommentCString != '\n')
+          putchar(*CommentCString);
+        else
+          printf("\\n");
+      }
+      printf("]");
+
+      PrintRange(clang_Cursor_getCommentRange(Cursor), "CommentRange");
+    }
+    clang_disposeString(Comment);
   }
 }
 

tools/clang-check/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(LLVM_USED_LIBS clangTooling clangBasic)
+set(LLVM_USED_LIBS clangTooling clangBasic clangComments)
 
 add_clang_executable(clang-check
   ClangCheck.cpp

tools/clang-check/Makefile

@@ -18,7 +18,7 @@ TOOL_NO_EXPORTS = 1
 LINK_COMPONENTS := support mc
 USEDLIBS = clangFrontend.a clangSerialization.a clangDriver.a \
            clangTooling.a clangParse.a clangSema.a clangAnalysis.a \
-           clangEdit.a clangAST.a clangLex.a clangBasic.a
+           clangEdit.a clangAST.a clangLex.a clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile
 

tools/diagtool/CMakeLists.txt

@@ -4,6 +4,7 @@ set( LLVM_LINK_COMPONENTS
 
 set( LLVM_USED_LIBS
   clangBasic
+  clangComments
   clangLex
   clangSema
   clangFrontend

tools/diagtool/Makefile

@@ -20,7 +20,7 @@ LINK_COMPONENTS := support mc
 
 USEDLIBS = clangFrontend.a clangDriver.a clangSerialization.a clangParse.a \
            clangSema.a clangAnalysis.a clangEdit.a clangAST.a clangLex.a \
-           clangBasic.a
+           clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile
 

tools/driver/CMakeLists.txt

@@ -4,6 +4,7 @@ set( LLVM_USED_LIBS
   clangAnalysis
   clangBasic
   clangCodeGen
+  clangComments
   clangDriver
   clangEdit
   clangFrontend

tools/driver/Makefile

@@ -36,7 +36,7 @@ USEDLIBS = clangFrontendTool.a clangFrontend.a clangDriver.a \
            clangStaticAnalyzerFrontend.a clangStaticAnalyzerCheckers.a \
            clangStaticAnalyzerCore.a \
            clangAnalysis.a clangARCMigrate.a clangRewrite.a \
-           clangEdit.a clangAST.a clangLex.a clangBasic.a
+           clangEdit.a clangAST.a clangLex.a clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile
 

tools/libclang/CIndex.cpp

@@ -5676,7 +5676,35 @@ CXFile clang_getIncludedFile(CXCursor cursor) {
   InclusionDirective *ID = getCursorInclusionDirective(cursor);
   return (void *)ID->getFile();
 }
-  
+
+CXSourceRange clang_Cursor_getCommentRange(CXCursor C) {
+  if (!clang_isDeclaration(C.kind))
+    return clang_getNullRange();
+
+  const Decl *D = getCursorDecl(C);
+  ASTContext &Context = getCursorContext(C);
+  const RawComment *RC = Context.getRawCommentForDecl(D);
+  if (!RC)
+    return clang_getNullRange();
+
+  return cxloc::translateSourceRange(Context, RC->getSourceRange());
+}
+
+CXString clang_Cursor_getRawCommentText(CXCursor C) {
+  if (!clang_isDeclaration(C.kind))
+    return createCXString((const char *) NULL);
+
+  const Decl *D = getCursorDecl(C);
+  ASTContext &Context = getCursorContext(C);
+  const RawComment *RC = Context.getRawCommentForDecl(D);
+  StringRef RawText = RC ? RC->getRawText(Context.getSourceManager()) :
+                           StringRef();
+
+  // Don't duplicate the string because RawText points directly into source
+  // code.
+  return createCXString(RawText, false);
+}
+
 } // end: extern "C"
 
 

tools/libclang/CMakeLists.txt

@@ -6,6 +6,7 @@ set(LLVM_USED_LIBS
   clangSerialization
   clangSema
   clangEdit
+  clangComments
   clangAST
   clangLex
   clangBasic)

tools/libclang/Makefile

@@ -19,7 +19,7 @@ LINK_COMPONENTS := support mc
 USEDLIBS = clangARCMigrate.a clangRewrite.a clangFrontend.a clangDriver.a \
      clangSerialization.a \
 		 clangParse.a clangSema.a clangEdit.a clangAnalysis.a \
-		 clangAST.a clangLex.a clangBasic.a
+		 clangAST.a clangLex.a clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/Makefile
 

tools/libclang/libclang.exports

@@ -5,6 +5,8 @@ clang_CXIndex_setGlobalOptions
 clang_CXXMethod_isStatic
 clang_CXXMethod_isVirtual
 clang_Cursor_getArgument
+clang_Cursor_getCommentRange
+clang_Cursor_getRawCommentText
 clang_Cursor_getNumArguments
 clang_Cursor_getObjCSelectorIndex
 clang_Cursor_getSpellingNameRange

unittests/Frontend/Makefile

@@ -14,6 +14,6 @@ USEDLIBS = clangFrontendTool.a clangFrontend.a clangDriver.a \
            clangSerialization.a clangCodeGen.a clangParse.a clangSema.a \
            clangStaticAnalyzerCheckers.a clangStaticAnalyzerCore.a \
            clangARCMigrate.a clangRewrite.a clangEdit.a \
-           clangAnalysis.a clangAST.a clangLex.a clangBasic.a
+           clangAnalysis.a clangAST.a clangLex.a clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/unittests/Makefile

unittests/Tooling/Makefile

@@ -12,6 +12,6 @@ TESTNAME = Tooling
 LINK_COMPONENTS := support mc
 USEDLIBS = clangTooling.a clangFrontend.a clangSerialization.a clangDriver.a \
            clangParse.a clangRewrite.a clangSema.a clangAnalysis.a clangEdit.a \
-           clangAST.a clangLex.a clangBasic.a
+           clangAST.a clangLex.a clangComments.a clangBasic.a
 
 include $(CLANG_LEVEL)/unittests/Makefile