PR37189 Fix incorrect end source location and spelling for a split '>>' token.

When a '>>' token is split into two '>' tokens (in C++11 onwards), or (as an
extension) when we do the same for other tokens starting with a '>', we can't
just use a location pointing to the first '>' as the location of the split
token, because that would result in our miscomputing the length and spelling
for the token. As a consequence, for example, a refactoring replacing 'A<X>'
with something else would sometimes replace one character too many, and
similarly diagnostics highlighting a template-id source range would highlight
one character too many.

Fix this by creating an expansion range covering the first character of the
'>>' token, whose spelling is '>'. For this to work, we generalize the
expansion range of a macro FileID to be either a token range (the common case)
or a character range (used in this new case).


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@331155 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Basic/SourceLocation.h

@@ -265,6 +265,7 @@ public:
  
   void setBegin(SourceLocation b) { Range.setBegin(b); }
   void setEnd(SourceLocation e) { Range.setEnd(e); }
+  void setTokenRange(bool TR) { IsTokenRange = TR; }
   
   bool isValid() const { return Range.isValid(); }
   bool isInvalid() const { return !isValid(); }
@@ -359,7 +360,6 @@ public:
   FullSourceLoc getExpansionLoc() const;
   FullSourceLoc getSpellingLoc() const;
   FullSourceLoc getFileLoc() const;
-  std::pair<FullSourceLoc, FullSourceLoc> getImmediateExpansionRange() const;
   PresumedLoc getPresumedLoc(bool UseLineDirectives = true) const;
   bool isMacroArgExpansion(FullSourceLoc *StartLoc = nullptr) const;
   FullSourceLoc getImmediateMacroCallerLoc() const;
@@ -377,8 +377,6 @@ public:
   unsigned getLineNumber(bool *Invalid = nullptr) const;
   unsigned getColumnNumber(bool *Invalid = nullptr) const;
 
-  std::pair<FullSourceLoc, FullSourceLoc> getExpansionRange() const;
-
   const FileEntry *getFileEntry() const;
 
   /// \brief Return a StringRef to the source buffer data for the

include/clang/Basic/SourceManager.h

@@ -317,9 +317,13 @@ namespace SrcMgr {
     /// invalid location.
     unsigned ExpansionLocStart, ExpansionLocEnd;
 
+    /// Whether the expansion range is a token range.
+    bool ExpansionIsTokenRange;
+
   public:
     SourceLocation getSpellingLoc() const {
-      return SourceLocation::getFromRawEncoding(SpellingLoc);
+      SourceLocation SpellLoc = SourceLocation::getFromRawEncoding(SpellingLoc);
+      return SpellLoc.isInvalid() ? getExpansionLocStart() : SpellLoc;
     }
 
     SourceLocation getExpansionLocStart() const {
@@ -332,8 +336,14 @@ namespace SrcMgr {
       return EndLoc.isInvalid() ? getExpansionLocStart() : EndLoc;
     }
 
-    std::pair<SourceLocation,SourceLocation> getExpansionLocRange() const {
-      return std::make_pair(getExpansionLocStart(), getExpansionLocEnd());
+    bool isExpansionTokenRange() const {
+      return ExpansionIsTokenRange;
+    }
+
+    CharSourceRange getExpansionLocRange() const {
+      return CharSourceRange(
+          SourceRange(getExpansionLocStart(), getExpansionLocEnd()),
+          isExpansionTokenRange());
     }
 
     bool isMacroArgExpansion() const {
@@ -359,11 +369,13 @@ namespace SrcMgr {
     /// the characters from the token come from). All three can refer to
     /// normal File SLocs or expansion locations.
     static ExpansionInfo create(SourceLocation SpellingLoc,
-                                SourceLocation Start, SourceLocation End) {
+                                SourceLocation Start, SourceLocation End,
+                                bool ExpansionIsTokenRange = true) {
       ExpansionInfo X;
       X.SpellingLoc = SpellingLoc.getRawEncoding();
       X.ExpansionLocStart = Start.getRawEncoding();
       X.ExpansionLocEnd = End.getRawEncoding();
+      X.ExpansionIsTokenRange = ExpansionIsTokenRange;
       return X;
     }
 
@@ -393,6 +405,17 @@ namespace SrcMgr {
       // than a normal one.
       return create(SpellingLoc, ExpansionLoc, SourceLocation());
     }
+
+    /// \brief Return a special ExpansionInfo representing a token that ends
+    /// prematurely. This is used to model a '>>' token that has been split
+    /// into '>' tokens and similar cases. Unlike for the other forms of
+    /// expansion, the expansion range in this case is a character range, not
+    /// a token range.
+    static ExpansionInfo createForTokenSplit(SourceLocation SpellingLoc,
+                                             SourceLocation Start,
+                                             SourceLocation End) {
+      return create(SpellingLoc, Start, End, false);
+    }
   };
 
   /// \brief This is a discriminated union of FileInfo and ExpansionInfo.
@@ -851,9 +874,16 @@ public:
                                     SourceLocation ExpansionLocStart,
                                     SourceLocation ExpansionLocEnd,
                                     unsigned TokLength,
+                                    bool ExpansionIsTokenRange = true,
                                     int LoadedID = 0,
                                     unsigned LoadedOffset = 0);
 
+  /// \brief Return a new SourceLocation that encodes that the token starting
+  /// at \p TokenStart ends prematurely at \p TokenEnd.
+  SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc,
+                                     SourceLocation TokenStart,
+                                     SourceLocation TokenEnd);
+
   /// \brief Retrieve the memory buffer associated with the given file.
   ///
   /// \param Invalid If non-NULL, will be set \c true if an error
@@ -1102,19 +1132,28 @@ public:
   /// expansion location.
   ///
   /// \pre \p Loc is required to be an expansion location.
-  std::pair<SourceLocation,SourceLocation>
-  getImmediateExpansionRange(SourceLocation Loc) const;
+  CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const;
 
   /// \brief Given a SourceLocation object, return the range of
   /// tokens covered by the expansion in the ultimate file.
-  std::pair<SourceLocation,SourceLocation>
-  getExpansionRange(SourceLocation Loc) const;
+  CharSourceRange getExpansionRange(SourceLocation Loc) const;
 
   /// \brief Given a SourceRange object, return the range of
-  /// tokens covered by the expansion in the ultimate file.
-  SourceRange getExpansionRange(SourceRange Range) const {
-    return SourceRange(getExpansionRange(Range.getBegin()).first,
-                       getExpansionRange(Range.getEnd()).second);
+  /// tokens or characters covered by the expansion in the ultimate file.
+  CharSourceRange getExpansionRange(SourceRange Range) const {
+    SourceLocation Begin = getExpansionRange(Range.getBegin()).getBegin();
+    CharSourceRange End = getExpansionRange(Range.getEnd());
+    return CharSourceRange(SourceRange(Begin, End.getEnd()),
+                           End.isTokenRange());
+  }
+
+  /// \brief Given a CharSourceRange object, return the range of
+  /// tokens or characters covered by the expansion in the ultimate file.
+  CharSourceRange getExpansionRange(CharSourceRange Range) const {
+    CharSourceRange Expansion = getExpansionRange(Range.getAsRange());
+    if (Expansion.getEnd() == Range.getEnd())
+      Expansion.setTokenRange(Range.isTokenRange());
+    return Expansion;
   }
 
   /// \brief Given a SourceLocation object, return the spelling
@@ -1643,7 +1682,7 @@ public:
 
     // Otherwise, the caller of the macro is located where this macro is
     // expanded (while the spelling is part of the macro definition).
-    return getImmediateExpansionRange(Loc).first;
+    return getImmediateExpansionRange(Loc).getBegin();
   }
 
   /// \return Location of the top-level macro caller.

include/clang/Lex/Lexer.h

@@ -336,13 +336,23 @@ public:
                                             const SourceManager &SM,
                                             const LangOptions &LangOpts);
 
+  /// Get the physical length (including trigraphs and escaped newlines) of the
+  /// first \p Characters characters of the token starting at TokStart.
+  static unsigned getTokenPrefixLength(SourceLocation TokStart,
+                                       unsigned Characters,
+                                       const SourceManager &SM,
+                                       const LangOptions &LangOpts);
+
   /// AdvanceToTokenCharacter - If the current SourceLocation specifies a
   /// location at the start of a token, return a new location that specifies a
   /// character within the token.  This handles trigraphs and escaped newlines.
   static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
-                                                unsigned Character,
+                                                unsigned Characters,
                                                 const SourceManager &SM,
-                                                const LangOptions &LangOpts);
+                                                const LangOptions &LangOpts) {
+    return TokStart.getLocWithOffset(
+        getTokenPrefixLength(TokStart, Characters, SM, LangOpts));
+  }
 
   /// \brief Computes the source location just past the end of the
   /// token at this source location.

include/clang/Lex/Preprocessor.h

@@ -1616,6 +1616,11 @@ public:
                     SourceLocation ExpansionLocStart = SourceLocation(),
                     SourceLocation ExpansionLocEnd = SourceLocation());
 
+  /// Split the first Length characters out of the token starting at TokLoc
+  /// and return a location pointing to the split token. Re-lexing from the
+  /// split token will return the split token rather than the original.
+  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
+
   /// \brief Computes the source location just past the end of the
   /// token at this source location.
   ///

include/clang/Rewrite/Core/HTMLRewrite.h

@@ -31,7 +31,8 @@ namespace html {
   /// start/end tags are placed at the start/end of each line if the range is
   /// multiline.
   void HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
-                      const char *StartTag, const char *EndTag);
+                      const char *StartTag, const char *EndTag,
+                      bool IsTokenRange = true);
 
   /// HighlightRange - Highlight a range in the source code with the specified
   /// start/end tags.  The Start/end of the range must be in the same file.

lib/ARCMigrate/PlistReporter.cpp

@@ -107,8 +107,7 @@ void arcmt::writeARCDiagsToPlist(const std::string &outPath,
       o << "   <key>ranges</key>\n";
       o << "   <array>\n";
       for (auto &R : D.getRanges()) {
-        CharSourceRange ExpansionRange(SM.getExpansionRange(R.getAsRange()),
-                                       R.isTokenRange());
+        CharSourceRange ExpansionRange = SM.getExpansionRange(R);
         EmitRange(o, SM, Lexer::getAsCharRange(ExpansionRange, SM, LangOpts),
                   FM, 4);
       }

lib/ARCMigrate/TransGCAttrs.cpp

@@ -92,7 +92,7 @@ public:
     ASTContext &Ctx = MigrateCtx.Pass.Ctx;
     SourceManager &SM = Ctx.getSourceManager();
     if (Loc.isMacroID())
-      Loc = SM.getImmediateExpansionRange(Loc).first;
+      Loc = SM.getImmediateExpansionRange(Loc).getBegin();
     SmallString<32> Buf;
     bool Invalid = false;
     StringRef Spell = Lexer::getSpelling(
@@ -287,7 +287,8 @@ static void checkAllAtProps(MigrationContext &MigrateCtx,
     SourceLocation Loc = ATLs[i].first.getAttrNameLoc();
     if (Loc.isMacroID())
       Loc = MigrateCtx.Pass.Ctx.getSourceManager()
-                                         .getImmediateExpansionRange(Loc).first;
+                .getImmediateExpansionRange(Loc)
+                .getBegin();
     TA.remove(Loc);
     TA.clearDiagnostic(diag::err_objc_property_attr_mutually_exclusive, AtLoc);
     TA.clearDiagnostic(diag::err_arc_inconsistent_property_ownership,

lib/ARCMigrate/TransUnbridgedCasts.cpp

@@ -283,13 +283,12 @@ private:
     SourceManager &SM = Pass.Ctx.getSourceManager();
     SourceLocation Loc = E->getExprLoc();
     assert(Loc.isMacroID());
-    SourceLocation MacroBegin, MacroEnd;
-    std::tie(MacroBegin, MacroEnd) = SM.getImmediateExpansionRange(Loc);
+    CharSourceRange MacroRange = SM.getImmediateExpansionRange(Loc);
     SourceRange SubRange = E->getSubExpr()->IgnoreParenImpCasts()->getSourceRange();
     SourceLocation InnerBegin = SM.getImmediateMacroCallerLoc(SubRange.getBegin());
     SourceLocation InnerEnd = SM.getImmediateMacroCallerLoc(SubRange.getEnd());
 
-    Outer = SourceRange(MacroBegin, MacroEnd);
+    Outer = MacroRange.getAsRange();
     Inner = SourceRange(InnerBegin, InnerEnd);
   }
 

lib/ARCMigrate/TransformActions.cpp

@@ -590,8 +590,12 @@ StringRef TransformActionsImpl::getUniqueText(StringRef text) {
 SourceLocation TransformActionsImpl::getLocForEndOfToken(SourceLocation loc,
                                                          SourceManager &SM,
                                                          Preprocessor &PP) {
-  if (loc.isMacroID())
-    loc = SM.getExpansionRange(loc).second;
+  if (loc.isMacroID()) {
+    CharSourceRange Exp = SM.getExpansionRange(loc);
+    if (Exp.isCharRange())
+      return Exp.getEnd();
+    loc = Exp.getEnd();
+  }
   return PP.getLocForEndOfToken(loc);
 }
 

lib/Basic/SourceLocation.cpp

@@ -103,15 +103,6 @@ FullSourceLoc FullSourceLoc::getFileLoc() const {
   return FullSourceLoc(SrcMgr->getFileLoc(*this), *SrcMgr);
 }
 
-std::pair<FullSourceLoc, FullSourceLoc>
-FullSourceLoc::getImmediateExpansionRange() const {
-  assert(isValid());
-  std::pair<SourceLocation, SourceLocation> Range =
-      SrcMgr->getImmediateExpansionRange(*this);
-  return std::make_pair(FullSourceLoc(Range.first, *SrcMgr),
-                        FullSourceLoc(Range.second, *SrcMgr));
-}
-
 PresumedLoc FullSourceLoc::getPresumedLoc(bool UseLineDirectives) const {
   if (!isValid())
     return PresumedLoc();
@@ -154,15 +145,6 @@ unsigned FullSourceLoc::getColumnNumber(bool *Invalid) const {
   return SrcMgr->getColumnNumber(getFileID(), getFileOffset(), Invalid);
 }
 
-std::pair<FullSourceLoc, FullSourceLoc>
-FullSourceLoc::getExpansionRange() const {
-  assert(isValid());
-  std::pair<SourceLocation, SourceLocation> Range =
-      SrcMgr->getExpansionRange(*this);
-  return std::make_pair(FullSourceLoc(Range.first, *SrcMgr),
-                        FullSourceLoc(Range.second, *SrcMgr));
-}
-
 const FileEntry *FullSourceLoc::getFileEntry() const {
   assert(isValid());
   return SrcMgr->getFileEntryForID(getFileID());

lib/Basic/SourceManager.cpp

@@ -579,13 +579,24 @@ SourceManager::createExpansionLoc(SourceLocation SpellingLoc,
                                   SourceLocation ExpansionLocStart,
                                   SourceLocation ExpansionLocEnd,
                                   unsigned TokLength,
+                                  bool ExpansionIsTokenRange,
                                   int LoadedID,
                                   unsigned LoadedOffset) {
-  ExpansionInfo Info = ExpansionInfo::create(SpellingLoc, ExpansionLocStart,
-                                             ExpansionLocEnd);
+  ExpansionInfo Info = ExpansionInfo::create(
+      SpellingLoc, ExpansionLocStart, ExpansionLocEnd, ExpansionIsTokenRange);
   return createExpansionLocImpl(Info, TokLength, LoadedID, LoadedOffset);
 }
 
+SourceLocation SourceManager::createTokenSplitLoc(SourceLocation Spelling,
+                                                  SourceLocation TokenStart,
+                                                  SourceLocation TokenEnd) {
+  assert(getFileID(TokenStart) == getFileID(TokenEnd) &&
+         "token spans multiple files");
+  return createExpansionLocImpl(
+      ExpansionInfo::createForTokenSplit(Spelling, TokenStart, TokenEnd),
+      TokenEnd.getOffset() - TokenStart.getOffset());
+}
+
 SourceLocation
 SourceManager::createExpansionLocImpl(const ExpansionInfo &Info,
                                       unsigned TokLength,
@@ -895,7 +906,7 @@ SourceLocation SourceManager::getFileLocSlowCase(SourceLocation Loc) const {
     if (isMacroArgExpansion(Loc))
       Loc = getImmediateSpellingLoc(Loc);
     else
-      Loc = getImmediateExpansionRange(Loc).first;
+      Loc = getImmediateExpansionRange(Loc).getBegin();
   } while (!Loc.isFileID());
   return Loc;
 }
@@ -950,7 +961,7 @@ SourceLocation SourceManager::getImmediateSpellingLoc(SourceLocation Loc) const{
 
 /// getImmediateExpansionRange - Loc is required to be an expansion location.
 /// Return the start/end of the expansion information.
-std::pair<SourceLocation,SourceLocation>
+CharSourceRange
 SourceManager::getImmediateExpansionRange(SourceLocation Loc) const {
   assert(Loc.isMacroID() && "Not a macro expansion loc!");
   const ExpansionInfo &Expansion = getSLocEntry(getFileID(Loc)).getExpansion();
@@ -965,19 +976,21 @@ SourceLocation SourceManager::getTopMacroCallerLoc(SourceLocation Loc) const {
 
 /// getExpansionRange - Given a SourceLocation object, return the range of
 /// tokens covered by the expansion in the ultimate file.
-std::pair<SourceLocation,SourceLocation>
-SourceManager::getExpansionRange(SourceLocation Loc) const {
-  if (Loc.isFileID()) return std::make_pair(Loc, Loc);
+CharSourceRange SourceManager::getExpansionRange(SourceLocation Loc) const {
+  if (Loc.isFileID())
+    return CharSourceRange(SourceRange(Loc, Loc), true);
 
-  std::pair<SourceLocation,SourceLocation> Res =
-    getImmediateExpansionRange(Loc);
+  CharSourceRange Res = getImmediateExpansionRange(Loc);
 
   // Fully resolve the start and end locations to their ultimate expansion
   // points.
-  while (!Res.first.isFileID())
-    Res.first = getImmediateExpansionRange(Res.first).first;
-  while (!Res.second.isFileID())
-    Res.second = getImmediateExpansionRange(Res.second).second;
+  while (!Res.getBegin().isFileID())
+    Res.setBegin(getImmediateExpansionRange(Res.getBegin()).getBegin());
+  while (!Res.getEnd().isFileID()) {
+    CharSourceRange EndRange = getImmediateExpansionRange(Res.getEnd());
+    Res.setEnd(EndRange.getEnd());
+    Res.setTokenRange(EndRange.isTokenRange());
+  }
   return Res;
 }
 

lib/CodeGen/CoverageMappingGen.cpp

@@ -183,7 +183,7 @@ public:
 
   /// \brief Find out where the current file is included or macro is expanded.
   SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc) {
-    return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).first
+    return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).getBegin()
                            : SM.getIncludeLoc(SM.getFileID(Loc));
   }
 
@@ -206,7 +206,7 @@ public:
   SourceLocation getStart(const Stmt *S) {
     SourceLocation Loc = S->getLocStart();
     while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc))
-      Loc = SM.getImmediateExpansionRange(Loc).first;
+      Loc = SM.getImmediateExpansionRange(Loc).getBegin();
     return Loc;
   }
 
@@ -214,7 +214,7 @@ public:
   SourceLocation getEnd(const Stmt *S) {
     SourceLocation Loc = S->getLocEnd();
     while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc))
-      Loc = SM.getImmediateExpansionRange(Loc).first;
+      Loc = SM.getImmediateExpansionRange(Loc).getBegin();
     return getPreciseTokenLocEnd(Loc);
   }
 

lib/Edit/EditedSource.cpp

@@ -36,12 +36,14 @@ void EditedSource::deconstructMacroArgLoc(SourceLocation Loc,
                                           SourceLocation &ExpansionLoc,
                                           MacroArgUse &ArgUse) {
   assert(SourceMgr.isMacroArgExpansion(Loc));
-  SourceLocation DefArgLoc = SourceMgr.getImmediateExpansionRange(Loc).first;
+  SourceLocation DefArgLoc =
+      SourceMgr.getImmediateExpansionRange(Loc).getBegin();
   SourceLocation ImmediateExpansionLoc =
-      SourceMgr.getImmediateExpansionRange(DefArgLoc).first;
+      SourceMgr.getImmediateExpansionRange(DefArgLoc).getBegin();
   ExpansionLoc = ImmediateExpansionLoc;
   while (SourceMgr.isMacroBodyExpansion(ExpansionLoc))
-    ExpansionLoc = SourceMgr.getImmediateExpansionRange(ExpansionLoc).first;
+    ExpansionLoc =
+        SourceMgr.getImmediateExpansionRange(ExpansionLoc).getBegin();
   SmallString<20> Buf;
   StringRef ArgName = Lexer::getSpelling(SourceMgr.getSpellingLoc(DefArgLoc),
                                          Buf, SourceMgr, LangOpts);

lib/Frontend/DiagnosticRenderer.cpp

@@ -262,41 +262,54 @@ static SourceLocation
 retrieveMacroLocation(SourceLocation Loc, FileID MacroFileID,
                       FileID CaretFileID,
                       const SmallVectorImpl<FileID> &CommonArgExpansions,
-                      bool IsBegin, const SourceManager *SM) {
+                      bool IsBegin, const SourceManager *SM,
+                      bool &IsTokenRange) {
   assert(SM->getFileID(Loc) == MacroFileID);
   if (MacroFileID == CaretFileID)
     return Loc;
   if (!Loc.isMacroID())
     return {};
 
-  SourceLocation MacroLocation, MacroArgLocation;
+  CharSourceRange MacroRange, MacroArgRange;
 
   if (SM->isMacroArgExpansion(Loc)) {
     // Only look at the immediate spelling location of this macro argument if
     // the other location in the source range is also present in that expansion.
     if (std::binary_search(CommonArgExpansions.begin(),
                            CommonArgExpansions.end(), MacroFileID))
-      MacroLocation = SM->getImmediateSpellingLoc(Loc);
-    MacroArgLocation = IsBegin ? SM->getImmediateExpansionRange(Loc).first
-                               : SM->getImmediateExpansionRange(Loc).second;
+      MacroRange =
+          CharSourceRange(SM->getImmediateSpellingLoc(Loc), IsTokenRange);
+    MacroArgRange = SM->getImmediateExpansionRange(Loc);
   } else {
-    MacroLocation = IsBegin ? SM->getImmediateExpansionRange(Loc).first
-                            : SM->getImmediateExpansionRange(Loc).second;
-    MacroArgLocation = SM->getImmediateSpellingLoc(Loc);
+    MacroRange = SM->getImmediateExpansionRange(Loc);
+    MacroArgRange =
+        CharSourceRange(SM->getImmediateSpellingLoc(Loc), IsTokenRange);
   }
 
+  SourceLocation MacroLocation =
+      IsBegin ? MacroRange.getBegin() : MacroRange.getEnd();
   if (MacroLocation.isValid()) {
     MacroFileID = SM->getFileID(MacroLocation);
+    bool TokenRange = IsBegin ? IsTokenRange : MacroRange.isTokenRange();
     MacroLocation =
         retrieveMacroLocation(MacroLocation, MacroFileID, CaretFileID,
-                              CommonArgExpansions, IsBegin, SM);
-    if (MacroLocation.isValid())
+                              CommonArgExpansions, IsBegin, SM, TokenRange);
+    if (MacroLocation.isValid()) {
+      IsTokenRange = TokenRange;
       return MacroLocation;
+    }
   }
 
+  // If we moved the end of the range to an expansion location, we now have
+  // a range of the same kind as the expansion range.
+  if (!IsBegin)
+    IsTokenRange = MacroArgRange.isTokenRange();
+
+  SourceLocation MacroArgLocation =
+      IsBegin ? MacroArgRange.getBegin() : MacroArgRange.getEnd();
   MacroFileID = SM->getFileID(MacroArgLocation);
   return retrieveMacroLocation(MacroArgLocation, MacroFileID, CaretFileID,
-                               CommonArgExpansions, IsBegin, SM);
+                               CommonArgExpansions, IsBegin, SM, IsTokenRange);
 }
 
 /// Walk up the chain of macro expansions and collect the FileIDs identifying the
@@ -310,7 +323,7 @@ static void getMacroArgExpansionFileIDs(SourceLocation Loc,
       Loc = SM->getImmediateSpellingLoc(Loc);
     } else {
       auto ExpRange = SM->getImmediateExpansionRange(Loc);
-      Loc = IsBegin ? ExpRange.first : ExpRange.second;
+      Loc = IsBegin ? ExpRange.getBegin() : ExpRange.getEnd();
     }
   }
 }
@@ -364,14 +377,16 @@ mapDiagnosticRanges(FullSourceLoc CaretLoc, ArrayRef<CharSourceRange> Ranges,
     llvm::SmallDenseMap<FileID, SourceLocation> BeginLocsMap;
     while (Begin.isMacroID() && BeginFileID != EndFileID) {
       BeginLocsMap[BeginFileID] = Begin;
-      Begin = SM->getImmediateExpansionRange(Begin).first;
+      Begin = SM->getImmediateExpansionRange(Begin).getBegin();
       BeginFileID = SM->getFileID(Begin);
     }
 
     // Then, crawl the expansion chain for the end of the range.
     if (BeginFileID != EndFileID) {
       while (End.isMacroID() && !BeginLocsMap.count(EndFileID)) {
-        End = SM->getImmediateExpansionRange(End).second;
+        auto Exp = SM->getImmediateExpansionRange(End);
+        IsTokenRange = Exp.isTokenRange();
+        End = Exp.getEnd();
         EndFileID = SM->getFileID(End);
       }
       if (End.isMacroID()) {
@@ -384,9 +399,11 @@ mapDiagnosticRanges(FullSourceLoc CaretLoc, ArrayRef<CharSourceRange> Ranges,
     SmallVector<FileID, 4> CommonArgExpansions;
     computeCommonMacroArgExpansionFileIDs(Begin, End, SM, CommonArgExpansions);
     Begin = retrieveMacroLocation(Begin, BeginFileID, CaretLocFileID,
-                                  CommonArgExpansions, /*IsBegin=*/true, SM);
+                                  CommonArgExpansions, /*IsBegin=*/true, SM,
+                                  IsTokenRange);
     End = retrieveMacroLocation(End, BeginFileID, CaretLocFileID,
-                                CommonArgExpansions, /*IsBegin=*/false, SM);
+                                CommonArgExpansions, /*IsBegin=*/false, SM,
+                                IsTokenRange);
     if (Begin.isInvalid() || End.isInvalid()) continue;
 
     // Return the spelling location of the beginning and end of the range.
@@ -511,29 +528,31 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc,
                                              ArrayRef<CharSourceRange> Ranges,
                                              ArrayRef<FixItHint> Hints) {
   assert(Loc.isValid() && "must have a valid source location here");
+  const SourceManager &SM = Loc.getManager();
+  SourceLocation L = Loc;
 
   // Produce a stack of macro backtraces.
-  SmallVector<FullSourceLoc, 8> LocationStack;
+  SmallVector<SourceLocation, 8> LocationStack;
   unsigned IgnoredEnd = 0;
-  while (Loc.isMacroID()) {
+  while (L.isMacroID()) {
     // If this is the expansion of a macro argument, point the caret at the
     // use of the argument in the definition of the macro, not the expansion.
-    if (Loc.isMacroArgExpansion())
-      LocationStack.push_back(Loc.getImmediateExpansionRange().first);
+    if (SM.isMacroArgExpansion(L))
+      LocationStack.push_back(SM.getImmediateExpansionRange(L).getBegin());
     else
-      LocationStack.push_back(Loc);
+      LocationStack.push_back(L);
 
-    if (checkRangesForMacroArgExpansion(Loc, Ranges))
+    if (checkRangesForMacroArgExpansion(FullSourceLoc(L, SM), Ranges))
       IgnoredEnd = LocationStack.size();
 
-    Loc = Loc.getImmediateMacroCallerLoc();
+    L = SM.getImmediateMacroCallerLoc(L);
 
     // Once the location no longer points into a macro, try stepping through
     // the last found location.  This sometimes produces additional useful
     // backtraces.
-    if (Loc.isFileID())
-      Loc = LocationStack.back().getImmediateMacroCallerLoc();
-    assert(Loc.isValid() && "must have a valid source location here");
+    if (L.isFileID())
+      L = SM.getImmediateMacroCallerLoc(LocationStack.back());
+    assert(L.isValid() && "must have a valid source location here");
   }
 
   LocationStack.erase(LocationStack.begin(),
@@ -544,7 +563,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc,
   if (MacroDepth <= MacroLimit || MacroLimit == 0) {
     for (auto I = LocationStack.rbegin(), E = LocationStack.rend();
          I != E; ++I)
-      emitSingleMacroExpansion(*I, Level, Ranges);
+      emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges);
     return;
   }
 
@@ -554,7 +573,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc,
   for (auto I = LocationStack.rbegin(),
             E = LocationStack.rbegin() + MacroStartMessages;
        I != E; ++I)
-    emitSingleMacroExpansion(*I, Level, Ranges);
+    emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges);
 
   SmallString<200> MessageStorage;
   llvm::raw_svector_ostream Message(MessageStorage);
@@ -566,7 +585,7 @@ void DiagnosticRenderer::emitMacroExpansions(FullSourceLoc Loc,
   for (auto I = LocationStack.rend() - MacroEndMessages,
             E = LocationStack.rend();
        I != E; ++I)
-    emitSingleMacroExpansion(*I, Level, Ranges);
+    emitSingleMacroExpansion(FullSourceLoc(*I, SM), Level, Ranges);
 }
 
 DiagnosticNoteRenderer::~DiagnosticNoteRenderer() = default;

lib/Frontend/TextDiagnostic.cpp

@@ -852,23 +852,14 @@ void TextDiagnostic::emitDiagnosticLoc(FullSourceLoc Loc, PresumedLoc PLoc,
       // Ignore invalid ranges.
       if (!RI->isValid()) continue;
 
-      FullSourceLoc B =
-          FullSourceLoc(RI->getBegin(), Loc.getManager()).getExpansionLoc();
-      FullSourceLoc E =
-          FullSourceLoc(RI->getEnd(), Loc.getManager()).getExpansionLoc();
-
-      // If the End location and the start location are the same and are a
-      // macro location, then the range was something that came from a
-      // macro expansion or _Pragma.  If this is an object-like macro, the
-      // best we can do is to highlight the range.  If this is a
-      // function-like macro, we'd also like to highlight the arguments.
-      if (B == E && RI->getEnd().isMacroID())
-        E = FullSourceLoc(RI->getEnd(), Loc.getManager())
-                .getExpansionRange()
-                .second;
-
-      std::pair<FileID, unsigned> BInfo = B.getDecomposedLoc();
-      std::pair<FileID, unsigned> EInfo = E.getDecomposedLoc();
+      auto &SM = Loc.getManager();
+      SourceLocation B = SM.getExpansionLoc(RI->getBegin());
+      CharSourceRange ERange = SM.getExpansionRange(RI->getEnd());
+      SourceLocation E = ERange.getEnd();
+      bool IsTokenRange = ERange.isTokenRange();
+
+      std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
+      std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
 
       // If the start or end of the range is in another file, just discard
       // it.
@@ -878,11 +869,14 @@ void TextDiagnostic::emitDiagnosticLoc(FullSourceLoc Loc, PresumedLoc PLoc,
       // Add in the length of the token, so that we cover multi-char
       // tokens.
       unsigned TokSize = 0;
-      if (RI->isTokenRange())
-        TokSize = Lexer::MeasureTokenLength(E, E.getManager(), LangOpts);
-
-      OS << '{' << B.getLineNumber() << ':' << B.getColumnNumber() << '-'
-         << E.getLineNumber() << ':' << (E.getColumnNumber() + TokSize) << '}';
+      if (IsTokenRange)
+        TokSize = Lexer::MeasureTokenLength(E, SM, LangOpts);
+
+      FullSourceLoc BF(B, SM), EF(E, SM);
+      OS << '{'
+         << BF.getLineNumber() << ':' << BF.getColumnNumber() << '-'
+         << EF.getLineNumber() << ':' << (EF.getColumnNumber() + TokSize)
+         << '}';
       PrintedRange = true;
     }
 

lib/Lex/Lexer.cpp

@@ -708,12 +708,9 @@ PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
                         TheTok.isAtStartOfLine());
 }
 
-/// AdvanceToTokenCharacter - Given a location that specifies the start of a
-/// token, return a new location that specifies a character within the token.
-SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
-                                              unsigned CharNo,
-                                              const SourceManager &SM,
-                                              const LangOptions &LangOpts) {
+unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
+                                     const SourceManager &SM,
+                                     const LangOptions &LangOpts) {
   // Figure out how many physical characters away the specified expansion
   // character is.  This needs to take into consideration newlines and
   // trigraphs.
@@ -722,7 +719,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
 
   // If they request the first char of the token, we're trivially done.
   if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
-    return TokStart;
+    return 0;
 
   unsigned PhysOffset = 0;
 
@@ -731,7 +728,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
   // chars, this method is extremely fast.
   while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
     if (CharNo == 0)
-      return TokStart.getLocWithOffset(PhysOffset);
+      return PhysOffset;
     ++TokPtr;
     --CharNo;
     ++PhysOffset;
@@ -753,7 +750,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
   if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
     PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
 
-  return TokStart.getLocWithOffset(PhysOffset);
+  return PhysOffset;
 }
 
 /// \brief Computes the source location just past the end of the
@@ -987,7 +984,7 @@ StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
 
     // Loc points to the argument id of the macro definition, move to the
     // macro expansion.
-    Loc = SM.getImmediateExpansionRange(Loc).first;
+    Loc = SM.getImmediateExpansionRange(Loc).getBegin();
     SourceLocation SpellLoc = Expansion.getSpellingLoc();
     if (SpellLoc.isFileID())
       break; // No inner macro.
@@ -1020,7 +1017,7 @@ StringRef Lexer::getImmediateMacroNameForDiagnostics(
   assert(Loc.isMacroID() && "Only reasonable to call this on macros");
   // Walk past macro argument expanions.
   while (SM.isMacroArgExpansion(Loc))
-    Loc = SM.getImmediateExpansionRange(Loc).first;
+    Loc = SM.getImmediateExpansionRange(Loc).getBegin();
 
   // If the macro's spelling has no FileID, then it's actually a token paste
   // or stringization (or similar) and not a macro at all.
@@ -1030,7 +1027,7 @@ StringRef Lexer::getImmediateMacroNameForDiagnostics(
   // Find the spelling location of the start of the non-argument expansion
   // range. This is where the macro name was spelled in order to begin
   // expanding this macro.
-  Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).first);
+  Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin());
 
   // Dig out the buffer where the macro name was spelled and the extents of the
   // name so that we can render it into the expansion note.
@@ -1112,10 +1109,9 @@ static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
 
   // Figure out the expansion loc range, which is the range covered by the
   // original _Pragma(...) sequence.
-  std::pair<SourceLocation,SourceLocation> II =
-    SM.getImmediateExpansionRange(FileLoc);
+  CharSourceRange II = SM.getImmediateExpansionRange(FileLoc);
 
-  return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen);
+  return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen);
 }
 
 /// getSourceLocation - Return a source location identifier for the specified

lib/Lex/PPDirectives.cpp

@@ -2043,7 +2043,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   // If the filename string was the result of macro expansions, set the include
   // position on the file where it will be included and after the expansions.
   if (IncludePos.isMacroID())
-    IncludePos = SourceMgr.getExpansionRange(IncludePos).second;
+    IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
   FileID FID = SourceMgr.createFileID(File, IncludePos, FileCharacter);
   assert(FID.isValid() && "Expected valid file ID");
 

lib/Lex/PPMacroExpansion.cpp

@@ -1690,7 +1690,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     // can matter for a function-like macro that expands to contain __LINE__.
     // Skip down through expansion points until we find a file loc for the
     // end of the expansion history.
-    Loc = SourceMgr.getExpansionRange(Loc).second;
+    Loc = SourceMgr.getExpansionRange(Loc).getEnd();
     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
 
     // __LINE__ expands to a simple numeric value.

lib/Lex/Preprocessor.cpp

@@ -487,6 +487,22 @@ void Preprocessor::CreateString(StringRef Str, Token &Tok,
     Tok.setLiteralData(DestPtr);
 }
 
+SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
+  auto &SM = getSourceManager();
+  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
+  bool Invalid = false;
+  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  if (Invalid)
+    return SourceLocation();
+
+  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
+  const char *DestPtr;
+  SourceLocation Spelling =
+      ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
+  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
+}
+
 Module *Preprocessor::getCurrentModule() {
   if (!getLangOpts().isCompilingModule())
     return nullptr;

lib/Lex/TokenLexer.cpp

@@ -865,9 +865,9 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
     EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
   FileID MacroFID = SM.getFileID(MacroExpansionStart);
   while (SM.getFileID(StartLoc) != MacroFID)
-    StartLoc = SM.getImmediateExpansionRange(StartLoc).first;
+    StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();
   while (SM.getFileID(EndLoc) != MacroFID)
-    EndLoc = SM.getImmediateExpansionRange(EndLoc).second;
+    EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();
     
   LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc,
                                         LHSTok.getLength()));

lib/Parse/ParseTemplate.cpp

@@ -775,6 +775,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
   // What will be left once we've consumed the '>'.
   tok::TokenKind RemainingToken;
   const char *ReplacementStr = "> >";
+  bool MergeWithNextToken = false;
 
   switch (Tok.getKind()) {
   default:
@@ -800,6 +801,15 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
   case tok::greaterequal:
     RemainingToken = tok::equal;
     ReplacementStr = "> =";
+
+    // Join two adjacent '=' tokens into one, for cases like:
+    //   void (*p)() = f<int>;
+    //   return f<int>==p;
+    if (NextToken().is(tok::equal) &&
+        areTokensAdjacent(Tok, NextToken())) {
+      RemainingToken = tok::equalequal;
+      MergeWithNextToken = true;
+    }
     break;
 
   case tok::greatergreaterequal:
@@ -807,22 +817,35 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
     break;
   }
 
-  // This template-id is terminated by a token which starts with a '>'. Outside
-  // C++11, this is now error recovery, and in C++11, this is error recovery if
-  // the token isn't '>>' or '>>>'.
-  // '>>>' is for CUDA, where this sequence of characters is parsed into
-  // tok::greatergreatergreater, rather than two separate tokens.
+  // This template-id is terminated by a token that starts with a '>'.
+  // Outside C++11 and Objective-C, this is now error recovery.
+  //
+  // C++11 allows this when the token is '>>', and in CUDA + C++11 mode, we
+  // extend that treatment to also apply to the '>>>' token.
   //
-  // We always allow this for Objective-C type parameter and type argument
-  // lists.
-  RAngleLoc = Tok.getLocation();
+  // Objective-C allows this in its type parameter / argument lists.
+
+  SourceLocation TokBeforeGreaterLoc = PrevTokLocation;
+  SourceLocation TokLoc = Tok.getLocation();
   Token Next = NextToken();
+
+  // Whether splitting the current token after the '>' would undesirably result
+  // in the remaining token pasting with the token after it. This excludes the
+  // MergeWithNextToken cases, which we've already handled.
+  bool PreventMergeWithNextToken =
+      (RemainingToken == tok::greater ||
+       RemainingToken == tok::greatergreater) &&
+      (Next.isOneOf(tok::greater, tok::greatergreater,
+                    tok::greatergreatergreater, tok::equal, tok::greaterequal,
+                    tok::greatergreaterequal, tok::equalequal)) &&
+      areTokensAdjacent(Tok, Next);
+
+  // Diagnose this situation as appropriate.
   if (!ObjCGenericList) {
-    // The source range of the '>>' or '>=' at the start of the token.
-    CharSourceRange ReplacementRange =
-        CharSourceRange::getCharRange(RAngleLoc,
-            Lexer::AdvanceToTokenCharacter(RAngleLoc, 2, PP.getSourceManager(),
-                                           getLangOpts()));
+    // The source range of the replaced token(s).
+    CharSourceRange ReplacementRange = CharSourceRange::getCharRange(
+        TokLoc, Lexer::AdvanceToTokenCharacter(TokLoc, 2, PP.getSourceManager(),
+                                               getLangOpts()));
 
     // A hint to put a space between the '>>'s. In order to make the hint as
     // clear as possible, we include the characters either side of the space in
@@ -833,13 +856,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
     // A hint to put another space after the token, if it would otherwise be
     // lexed differently.
     FixItHint Hint2;
-    if ((RemainingToken == tok::greater ||
-         RemainingToken == tok::greatergreater) &&
-        (Next.isOneOf(tok::greater, tok::greatergreater,
-                      tok::greatergreatergreater, tok::equal,
-                      tok::greaterequal, tok::greatergreaterequal,
-                      tok::equalequal)) &&
-        areTokensAdjacent(Tok, Next))
+    if (PreventMergeWithNextToken)
       Hint2 = FixItHint::CreateInsertion(Next.getLocation(), " ");
 
     unsigned DiagId = diag::err_two_right_angle_brackets_need_space;
@@ -848,50 +865,63 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
       DiagId = diag::warn_cxx98_compat_two_right_angle_brackets;
     else if (Tok.is(tok::greaterequal))
       DiagId = diag::err_right_angle_bracket_equal_needs_space;
-    Diag(Tok.getLocation(), DiagId) << Hint1 << Hint2;
+    Diag(TokLoc, DiagId) << Hint1 << Hint2;
   }
 
+  // Find the "length" of the resulting '>' token. This is not always 1, as it
+  // can contain escaped newlines.
+  unsigned GreaterLength = Lexer::getTokenPrefixLength(
+      TokLoc, 1, PP.getSourceManager(), getLangOpts());
+
+  // Annotate the source buffer to indicate that we split the token after the
+  // '>'. This allows us to properly find the end of, and extract the spelling
+  // of, the '>' token later.
+  RAngleLoc = PP.SplitToken(TokLoc, GreaterLength);
+
   // Strip the initial '>' from the token.
-  Token PrevTok = Tok;
-  if (RemainingToken == tok::equal && Next.is(tok::equal) &&
-      areTokensAdjacent(Tok, Next)) {
-    // Join two adjacent '=' tokens into one, for cases like:
-    //   void (*p)() = f<int>;
-    //   return f<int>==p;
+  bool CachingTokens = PP.IsPreviousCachedToken(Tok);
+
+  Token Greater = Tok;
+  Greater.setLocation(RAngleLoc);
+  Greater.setKind(tok::greater);
+  Greater.setLength(GreaterLength);
+
+  unsigned OldLength = Tok.getLength();
+  if (MergeWithNextToken) {
     ConsumeToken();
-    Tok.setKind(tok::equalequal);
-    Tok.setLength(Tok.getLength() + 1);
-  } else {
-    Tok.setKind(RemainingToken);
-    Tok.setLength(Tok.getLength() - 1);
+    OldLength += Tok.getLength();
   }
-  Tok.setLocation(Lexer::AdvanceToTokenCharacter(RAngleLoc, 1,
-                                                 PP.getSourceManager(),
-                                                 getLangOpts()));
-
-  // The advance from '>>' to '>' in a ObjectiveC template argument list needs
-  // to be properly reflected in the token cache to allow correct interaction
-  // between annotation and backtracking.
-  if (ObjCGenericList && PrevTok.getKind() == tok::greatergreater &&
-      RemainingToken == tok::greater && PP.IsPreviousCachedToken(PrevTok)) {
-    PrevTok.setKind(RemainingToken);
-    PrevTok.setLength(1);
-    // Break tok::greatergreater into two tok::greater but only add the second
-    // one in case the client asks to consume the last token.
+
+  Tok.setKind(RemainingToken);
+  Tok.setLength(OldLength - GreaterLength);
+
+  // Split the second token if lexing it normally would lex a different token
+  // (eg, the fifth token in 'A<B>>>' should re-lex as '>', not '>>').
+  SourceLocation AfterGreaterLoc = TokLoc.getLocWithOffset(GreaterLength);
+  if (PreventMergeWithNextToken)
+    AfterGreaterLoc = PP.SplitToken(AfterGreaterLoc, Tok.getLength());
+  Tok.setLocation(AfterGreaterLoc);
+
+  // Update the token cache to match what we just did if necessary.
+  if (CachingTokens) {
+    // If the previous cached token is being merged, delete it.
+    if (MergeWithNextToken)
+      PP.ReplacePreviousCachedToken({});
+
     if (ConsumeLastToken)
-      PP.ReplacePreviousCachedToken({PrevTok, Tok});
+      PP.ReplacePreviousCachedToken({Greater, Tok});
     else
-      PP.ReplacePreviousCachedToken({PrevTok});
+      PP.ReplacePreviousCachedToken({Greater});
   }
 
-  if (!ConsumeLastToken) {
-    // Since we're not supposed to consume the '>' token, we need to push
-    // this token and revert the current token back to the '>'.
+  if (ConsumeLastToken) {
+    PrevTokLocation = RAngleLoc;
+  } else {
+    PrevTokLocation = TokBeforeGreaterLoc;
     PP.EnterToken(Tok);
-    Tok.setKind(tok::greater);
-    Tok.setLength(1);
-    Tok.setLocation(RAngleLoc);
+    Tok = Greater;
   }
+
   return false;
 }
 

lib/Rewrite/HTMLRewrite.cpp

@@ -30,7 +30,8 @@ using namespace clang;
 /// start/end tags are placed at the start/end of each line if the range is
 /// multiline.
 void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
-                          const char *StartTag, const char *EndTag) {
+                          const char *StartTag, const char *EndTag,
+                          bool IsTokenRange) {
   SourceManager &SM = R.getSourceMgr();
   B = SM.getExpansionLoc(B);
   E = SM.getExpansionLoc(E);
@@ -41,7 +42,8 @@ void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
   unsigned EOffset = SM.getFileOffset(E);
 
   // Include the whole end token in the range.
-  EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
+  if (IsTokenRange)
+    EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
 
   bool Invalid = false;
   const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
@@ -588,16 +590,15 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // Okay, we have the first token of a macro expansion: highlight the
     // expansion by inserting a start tag before the macro expansion and
     // end tag after it.
-    std::pair<SourceLocation, SourceLocation> LLoc =
-      SM.getExpansionRange(Tok.getLocation());
+    CharSourceRange LLoc = SM.getExpansionRange(Tok.getLocation());
 
     // Ignore tokens whose instantiation location was not the main file.
-    if (SM.getFileID(LLoc.first) != FID) {
+    if (SM.getFileID(LLoc.getBegin()) != FID) {
       TmpPP.Lex(Tok);
       continue;
     }
 
-    assert(SM.getFileID(LLoc.second) == FID &&
+    assert(SM.getFileID(LLoc.getEnd()) == FID &&
            "Start and end of expansion must be in the same ultimate file!");
 
     std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
@@ -612,7 +613,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // instantiation.  It would be really nice to pop up a window with all the
     // spelling of the tokens or something.
     while (!Tok.is(tok::eof) &&
-           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
+           SM.getExpansionLoc(Tok.getLocation()) == LLoc.getBegin()) {
       // Insert a newline if the macro expansion is getting large.
       if (LineLen > 60) {
         Expansion += "<br>";
@@ -641,8 +642,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // highlighted.
     Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
 
-    HighlightRange(R, LLoc.first, LLoc.second,
-                   "<span class='macro'>", Expansion.c_str());
+    HighlightRange(R, LLoc.getBegin(), LLoc.getEnd(), "<span class='macro'>",
+                   Expansion.c_str(), LLoc.isTokenRange());
   }
 
   // Restore the preprocessor's old state.

lib/Sema/SemaChecking.cpp

@@ -9562,7 +9562,7 @@ static void DiagnoseNullConversion(Sema &S, Expr *E, QualType T,
     StringRef MacroName = Lexer::getImmediateMacroNameForDiagnostics(
         Loc, S.SourceMgr, S.getLangOpts());
     if (MacroName == "NULL")
-      Loc = S.SourceMgr.getImmediateExpansionRange(Loc).first;
+      Loc = S.SourceMgr.getImmediateExpansionRange(Loc).getBegin();
   }
 
   // Only warn if the null and context location are in the same macro expansion.

lib/Sema/SemaDeclAttr.cpp

@@ -7533,7 +7533,7 @@ void DiagnoseUnguardedAvailability::DiagnoseDeclAvailability(
     SourceLocation StmtEndLoc =
         SM.getExpansionRange(
               (LastStmtOfUse ? LastStmtOfUse : StmtOfUse)->getLocEnd())
-            .second;
+            .getEnd();
     if (SM.getFileID(IfInsertionLoc) != SM.getFileID(StmtEndLoc))
       return;
 

lib/Sema/SemaDeclCXX.cpp

@@ -2771,7 +2771,7 @@ void Sema::DiagnoseAbsenceOfOverrideControl(NamedDecl *D) {
   SourceLocation Loc = MD->getLocation();
   SourceLocation SpellingLoc = Loc;
   if (getSourceManager().isMacroArgExpansion(Loc))
-    SpellingLoc = getSourceManager().getImmediateExpansionRange(Loc).first;
+    SpellingLoc = getSourceManager().getImmediateExpansionRange(Loc).getBegin();
   SpellingLoc = getSourceManager().getSpellingLoc(SpellingLoc);
   if (SpellingLoc.isValid() && getSourceManager().isInSystemHeader(SpellingLoc))
       return;

lib/Sema/SemaInit.cpp

@@ -6487,7 +6487,7 @@ static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr,
   // macro only if it is at the beginning of the macro.
   while (ArgLoc.isMacroID() &&
          S.getSourceManager().isAtStartOfImmediateMacroExpansion(ArgLoc)) {
-    ArgLoc = S.getSourceManager().getImmediateExpansionRange(ArgLoc).first;
+    ArgLoc = S.getSourceManager().getImmediateExpansionRange(ArgLoc).getBegin();
   }
 
   if (LParen.isMacroID())

lib/Sema/SemaType.cpp

@@ -4489,7 +4489,8 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
           // the predefined
           // __strong/__weak/__autoreleasing/__unsafe_unretained.
           if (AttrLoc.isMacroID())
-            AttrLoc = S.SourceMgr.getImmediateExpansionRange(AttrLoc).first;
+            AttrLoc =
+                S.SourceMgr.getImmediateExpansionRange(AttrLoc).getBegin();
 
           S.Diag(AttrLoc, diag::warn_arc_lifetime_result_type)
             << T.getQualifiers().getObjCLifetime();
@@ -5893,7 +5894,8 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
   Sema &S = state.getSema();
   SourceLocation AttrLoc = attr.getLoc();
   if (AttrLoc.isMacroID())
-    AttrLoc = S.getSourceManager().getImmediateExpansionRange(AttrLoc).first;
+    AttrLoc =
+        S.getSourceManager().getImmediateExpansionRange(AttrLoc).getBegin();
 
   if (!attr.isArgIdent(0)) {
     S.Diag(AttrLoc, diag::err_attribute_argument_type)

lib/Serialization/ASTReader.cpp

@@ -1482,6 +1482,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
     SourceMgr.createExpansionLoc(SpellingLoc,
                                      ReadSourceLocation(*F, Record[2]),
                                      ReadSourceLocation(*F, Record[3]),
+                                     Record[5],
                                      Record[4],
                                      ID,
                                      BaseOffset + Record[0]);

lib/Serialization/ASTWriter.cpp

@@ -1905,6 +1905,7 @@ static unsigned CreateSLocExpansionAbbrev(llvm::BitstreamWriter &Stream) {
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Spelling location
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Start location
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // End location
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Is token range
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Token length
   return Stream.EmitAbbrev(std::move(Abbrev));
 }
@@ -2321,6 +2322,7 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
                             ? SourceLocation()
                             : Expansion.getExpansionLocEnd(),
                         Record);
+      Record.push_back(Expansion.isExpansionTokenRange());
 
       // Compute the token length for this macro expansion.
       unsigned NextOffset = SourceMgr.getNextLocalOffset();

lib/StaticAnalyzer/Core/BugReporter.cpp

@@ -2178,7 +2178,7 @@ static void simplifySimpleBranches(PathPieces &pieces) {
 static Optional<size_t> getLengthOnSingleLine(SourceManager &SM,
                                               SourceRange Range) {
   SourceRange ExpansionRange(SM.getExpansionLoc(Range.getBegin()),
-                             SM.getExpansionRange(Range.getEnd()).second);
+                             SM.getExpansionRange(Range.getEnd()).getEnd());
 
   FileID FID = SM.getFileID(ExpansionRange.getBegin());
   if (FID != SM.getFileID(ExpansionRange.getEnd()))

lib/StaticAnalyzer/Core/BugReporterVisitors.cpp

@@ -213,7 +213,7 @@ static bool isFunctionMacroExpansion(SourceLocation Loc,
   if (!Loc.isMacroID())
     return false;
   while (SM.isMacroArgExpansion(Loc))
-    Loc = SM.getImmediateExpansionRange(Loc).first;
+    Loc = SM.getImmediateExpansionRange(Loc).getBegin();
   std::pair<FileID, unsigned> TLInfo = SM.getDecomposedLoc(Loc);
   SrcMgr::SLocEntry SE = SM.getSLocEntry(TLInfo.first);
   const SrcMgr::ExpansionInfo &EInfo = SE.getExpansion();

test/Misc/diag-greatergreater.cpp

@@ -0,0 +1,43 @@
+// RUN: not %clang_cc1 %s -fdiagnostics-print-source-range-info 2>&1 | FileCheck %s --strict-whitespace
+
+template<typename T> class C {};
+template<int> class D {};
+
+void g() {
+  // The range ending in the first > character does not extend to the second >
+  // character.
+  // CHECK:      :{[[@LINE+3]]:5-[[@LINE+3]]:11}: error:
+  // CHECK-NEXT:   D<C<int>> a;
+  // CHECK-NEXT:     ^~~~~~{{$}}
+  D<C<int>> a;
+
+  // The range ending in the second > character does not extend to the third >
+  // character.
+  // CHECK:      :{[[@LINE+3]]:5-[[@LINE+3]]:14}: error:
+  // CHECK-NEXT:   D<C<C<int>>> b;
+  // CHECK-NEXT:     ^~~~~~~~~{{$}}
+  D<C<C<int>>> b;
+}
+
+template<int> int V;
+// Here, we split the >>= token into a > followed by a >=.
+// Then we split the >= token into a > followed by an =,
+// which we merge with the other = to form an ==.
+// CHECK:      error: a space is required
+// CHECK-NEXT: int k = V<C<int>>==0;
+// CHECK-NEXT:                ^~{{$}}
+// CHECK-NEXT:                > >{{$}}
+// CHECK:      error: a space is required
+// CHECK-NEXT: int k = V<C<int>>==0;
+// CHECK-NEXT:                 ^~{{$}}
+// CHECK-NEXT:                 > ={{$}}
+// CHECK:      :{[[@LINE+3]]:11-[[@LINE+3]]:17}: error:
+// CHECK-NEXT: int k = V<C<int>>==0;
+// CHECK-NEXT:           ^~~~~~{{$}}
+int k = V<C<int>>==0;
+
+template<typename> int W;
+// CHECK:      :{[[@LINE+3]]:9-[[@LINE+3]]:18}{[[@LINE+3]]:20-[[@LINE+3]]:22}: error: comparison
+// CHECK-NEXT: int l = W<C<int>>==&k;
+// CHECK-NEXT:         ~~~~~~~~~^ ~~{{$}}
+int l = W<C<int>>==&k;

test/Parser/cxx0x-decl.cpp

@@ -130,15 +130,18 @@ namespace AliasDeclEndLocation {
   using B = AliasDeclEndLocation::A<int
     > // expected-error {{expected ';' after alias declaration}}
     +;
-  // FIXME: After splitting this >> into two > tokens, we incorrectly determine
-  // the end of the template-id to be after the *second* '>'.
-  // Perhaps we could synthesize an expansion FileID containing '> >' to fix this?
   using C = AliasDeclEndLocation::A<int
     >\
 > // expected-error {{expected ';' after alias declaration}}
     ;
   using D = AliasDeclEndLocation::A<int
     > // expected-error {{expected ';' after alias declaration}}
+  // FIXME: After splitting this >> into two > tokens, we incorrectly determine
+  // the end of the template-id to be after the *second* '>'.
+  using E = AliasDeclEndLocation::A<int>>;
+#define GGG >>>
+  using F = AliasDeclEndLocation::A<int GGG;
+  // expected-error@-1 {{expected ';' after alias declaration}}
   B something_else;
 }
 

tools/libclang/CIndex.cpp

@@ -146,9 +146,13 @@ CXSourceRange cxloc::translateSourceRange(const SourceManager &SM,
   // We want the last character in this location, so we will adjust the
   // location accordingly.
   SourceLocation EndLoc = R.getEnd();
-  if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc))
-    EndLoc = SM.getExpansionRange(EndLoc).second;
-  if (R.isTokenRange() && EndLoc.isValid()) {
+  bool IsTokenRange = R.isTokenRange();
+  if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) {
+    CharSourceRange Expansion = SM.getExpansionRange(EndLoc);
+    EndLoc = Expansion.getEnd();
+    IsTokenRange = Expansion.isTokenRange();
+  }
+  if (IsTokenRange && EndLoc.isValid()) {
     unsigned Length = Lexer::MeasureTokenLength(SM.getSpellingLoc(EndLoc),
                                                 SM, LangOpts);
     EndLoc = EndLoc.getLocWithOffset(Length);

unittests/Lex/LexerTest.cpp

@@ -286,9 +286,7 @@ TEST_F(LexerTest, LexAPI) {
   SourceLocation lsqrLoc = toks[0].getLocation();
   SourceLocation idLoc = toks[1].getLocation();
   SourceLocation rsqrLoc = toks[2].getLocation();
-  std::pair<SourceLocation,SourceLocation>
-    macroPair = SourceMgr.getExpansionRange(lsqrLoc);
-  SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
+  CharSourceRange macroRange = SourceMgr.getExpansionRange(lsqrLoc);
 
   SourceLocation Loc;
   EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
@@ -297,6 +295,7 @@ TEST_F(LexerTest, LexAPI) {
   EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
   EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
   EXPECT_EQ(Loc, macroRange.getEnd());
+  EXPECT_TRUE(macroRange.isTokenRange());
 
   CharSourceRange range = Lexer::makeFileCharRange(
            CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
@@ -334,11 +333,11 @@ TEST_F(LexerTest, LexAPI) {
   EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
             range.getAsRange());
 
-  macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
+  macroRange = SourceMgr.getExpansionRange(macroLsqrLoc);
   range = Lexer::makeFileCharRange(
                      CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
                      SourceMgr, LangOpts);
-  EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
+  EXPECT_EQ(SourceRange(macroRange.getBegin(), macroRange.getEnd().getLocWithOffset(1)),
             range.getAsRange());
 
   text = Lexer::getSourceText(