Update LLVM to 21

.bazelrc

@@ -51,6 +51,8 @@ build --per_file_copt="external/llvm-project/.*@-Wno-deprecated-anon-enum-enum-c
 build --per_file_copt="external/llvm-project/.*@-Wno-ambiguous-reversed-operator" # C++20 warning
 build --per_file_copt="external/com_google_protobuf/.*@-Wno-deprecated-declarations" # sprintf on macOS
 build --per_file_copt="external/com_google_protobuf/.*@-Wno-unused-function"
+build --per_file_copt="external/com_google_protobuf/.*@-Wno-missing-field-initializers"
+build --per_file_copt="external/com_google_protobuf/.*@-Wno-unnecessary-virtual-specifier"
 
 build:asserts --copt="-DFORCE_DEBUG=1"
 build:asserts --copt="-DLLVM_ENABLE_ASSERTIONS=1"

README.md

@@ -1,6 +1,6 @@
 # scip-clang: SCIP indexer for C and C++ ![(Status: Beta)](https://img.shields.io/badge/status-beta-yellow?style=flat)
 
-scip-clang is a precise code indexer based on Clang 16,
+scip-clang is a precise code indexer based on Clang 21,
 which supports cross-repository code navigation for C, C++ and CUDA
 in Sourcegraph.
 
@@ -201,7 +201,7 @@ scip-clang --compdb-path=build/small_compdb.json --show-compiler-diagnostics
 <summary>Known diagnostics when indexing CUDA</summary>
 
 1. If you see an error related to the `texture` template, that is likely
-   because of the Clang version not being Clang 16 or newer.
+   because of the Clang version not being Clang 21 or newer.
    See https://github.com/llvm/llvm-project/issues/61340
 2. If you see any errors related to GCC headers, that's a
    [known issue](https://github.com/sourcegraph/scip-clang/issues/440).

fetch_deps.bzl

@@ -4,13 +4,13 @@ _BAZEL_SKYLIB_VERSION = "1.3.0"
 _PLATFORMS_VERSION = "1.0.0"
 _BAZEL_TOOLCHAIN_VERSION = "1.6.0"
 _RULES_BOOST_COMMIT = "00b9b9ecb9b43564de44ea0b10e22b29dcf84d79"
-_LLVM_COMMIT = "e0f3110b854a476c16cce7b44472cd7838d344e9"  # Keep in sync with Version.h
-_ABSL_COMMIT = "4ffaea74c1f5408e0757547a1ca0518ad43fa9f1"
+_LLVM_COMMIT = "2078da43e25a4623cab2d0d60decddf709aaea28"  # Keep in sync with Version.h
+_ABSL_VERSION = "20240722.0"
 _CXXOPTS_VERSION = "3.0.0"
 _RAPIDJSON_COMMIT = "a98e99992bd633a2736cc41f96ec85ef0c50e44d"
 _WYHASH_COMMIT = "ea3b25e1aef55d90f707c3a292eeb9162e2615d8"
-_SPDLOG_COMMIT = "edc51df1bdad8667b628999394a1e7c4dc6f3658"
-_PROTOBUF_VERSION = "3.21.12"
+_SPDLOG_COMMIT = "486b55554f11c9cccc913e11a87085b2a91f706f"  # v1.16.0
+_PROTOBUF_VERSION = "25.3"
 _SCIP_COMMIT = "aa0e511dcfefbacc3b96dcc2fe2abd9894416b1e"
 _UTFCPP_VERSION = "4.0.5"
 # ^ When bumping this version, check if any new fields are introduced
@@ -82,14 +82,14 @@ def fetch_direct_dependencies():
     # Keep the name 'zlib' so that Protobuf doesn't pull in another copy.
     #
     # https://sourcegraph.com/github.com/protocolbuffers/protobuf/-/blob/protobuf_deps.bzl?L48-58
+    # Using zlib 1.3.1 to fix macro conflicts with macOS headers in zlib 1.2.11
     http_archive(
         name = "zlib",
         build_file = "@scip_clang//third_party:zlib.BUILD",
-        sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
-        strip_prefix = "zlib-1.2.11",
+        sha256 = "17e88863f3600672ab49182f217281b6fc4d3c762bde361935e436a95214d05c",
+        strip_prefix = "zlib-1.3.1",
         urls = [
-            "https://mirror.bazel.build/zlib.net/zlib-1.2.11.tar.gz",
-            "https://zlib.net/zlib-1.2.11.tar.gz",
+            "https://github.com/madler/zlib/archive/refs/tags/v1.3.1.tar.gz",
         ],
     )
 
@@ -103,17 +103,33 @@ def fetch_direct_dependencies():
         ],
     )
 
+    # LLVM 18+ uses zlib-ng instead of zlib
+    http_archive(
+        name = "llvm_zlib",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:zlib-ng.BUILD",
+        sha256 = "e36bb346c00472a1f9ff2a0a4643e590a254be6379da7cddd9daeb9a7f296731",
+        strip_prefix = "zlib-ng-2.0.7",
+        urls = [
+            "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.0.7.zip",
+        ],
+    )
+
+    # Cherry-pick fix for CUDA assertion failure from
+    # https://github.com/llvm/llvm-project/pull/173762
+    # Can be removed once LLVM merges the fix and we update the commit.
     http_archive(
         name = "llvm-raw",
-        sha256 = "04b76a5be88331f71a4e4fe96bccfebec302ddd0dbd9418fd5c186a7361c54fb",
+        sha256 = "536a4d64ab21bc85bf95ae4dc412b36e8a9c72d487a476839f3c31c3ded69e96",
         strip_prefix = "llvm-project-%s" % _LLVM_COMMIT,
         build_file_content = "# empty",
         urls = ["https://github.com/llvm/llvm-project/archive/%s.tar.gz" % _LLVM_COMMIT],
+        patch_args = ["-p1"],
+        patches = ["//third_party:llvm-cuda-tooling.patch"],
     )
 
     http_archive(
         name = "com_google_protobuf",
-        sha256 = "f7042d540c969b00db92e8e1066a9b8099c8379c33f40f360eb9e1d98a36ca26",
+        sha256 = "5156b22536feaa88cf95503153a6b2cd67cc80f20f1218f154b84a12c288a220",
         urls = ["https://github.com/protocolbuffers/protobuf/archive/v%s.zip" % _PROTOBUF_VERSION],
         strip_prefix = "protobuf-%s" % _PROTOBUF_VERSION,
     )
@@ -132,11 +148,9 @@ def fetch_direct_dependencies():
     # https://sourcegraph.com/github.com/protocolbuffers/protobuf/-/blob/protobuf_deps.bzl?L39-46
     http_archive(
         name = "com_google_absl",
-        sha256 = "fee8ec623d8bbf0ecb9563a8e08ae319d1ca9fdf8c1c84384520a6992f571659",
-        strip_prefix = "abseil-cpp-%s" % _ABSL_COMMIT,
-        urls = ["https://github.com/abseil/abseil-cpp/archive/%s.zip" % _ABSL_COMMIT],
-        patch_args = ["-p1"],
-        patches = ["//third_party:abseil.patch"],
+        sha256 = "95e90be7c3643e658670e0dd3c1b27092349c34b632c6e795686355f67eca89f",
+        strip_prefix = "abseil-cpp-%s" % _ABSL_VERSION,
+        urls = ["https://github.com/abseil/abseil-cpp/archive/%s.zip" % _ABSL_VERSION],
     )
 
     # Abseil also has a flags/argument parsing library, but let's
@@ -168,7 +182,7 @@ def fetch_direct_dependencies():
     # NOTE: fmt also comes through spdlog, we don't have an explicit dep on fmt.
     http_archive(
         name = "spdlog",
-        sha256 = "93a270dd7ec8fa672eb4feaef443dc14a4a9edc7b59aea998ae5da6cbf7b7119",
+        sha256 = "d2fef585c9879dd239dc498e2e8a1e22982b3ed67b2d14e78622b7ef25bdfdfa",
         build_file = "@scip_clang//third_party:spdlog.BUILD",
         strip_prefix = "spdlog-%s" % _SPDLOG_COMMIT,
         urls = ["https://github.com/gabime/spdlog/archive/%s.tar.gz" % _SPDLOG_COMMIT],

indexer/ApproximateNameResolver.cc

@@ -3,6 +3,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Type.h"
+#include "clang/Sema/HeuristicResolver.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -13,7 +14,6 @@
 #include "spdlog/spdlog.h"
 
 namespace scip_clang {
-
 MemberLookupKey::MemberLookupKey(const clang::Type *type,
                                  const clang::DeclarationNameInfo &declNameInfo)
     : canonicalTypePtr(type->getCanonicalTypeInternal().getTypePtrOrNull()),
@@ -65,8 +65,9 @@ ApproximateNameResolver::tryResolveMember(
       continue;
     }
     cxxRecordDecl = cxxRecordDecl->getDefinition();
-    auto lookupResults =
-        cxxRecordDecl->lookupDependentName(declNameInfo.getName(), filter);
+    clang::HeuristicResolver resolver(this->astContext);
+    auto lookupResults = resolver.lookupDependentName(
+        cxxRecordDecl, declNameInfo.getName(), filter);
     for (auto *namedDecl : lookupResults) {
       auto *unresolvedUsingValueDecl =
           llvm::dyn_cast<clang::UnresolvedUsingValueDecl>(namedDecl);

indexer/ApproximateNameResolver.h

@@ -36,29 +36,23 @@ class MemberLookupKey {
   DERIVE_CMP_ALL(MemberLookupKey)
 };
 
-/// Type similar to clangd's HeuristicResolver, used for performing
-/// best-effort name resolution when encountered unresolved names.
-///
-/// We don't directly reuse the code from clangd, since it's not that
-/// much code (<300 lines), and having own code let's us evolve it
-/// independently and add different heuristics.
-///
-/// (Named differently to reduce risk of confusion.)
+/// Wrapper around clang's HeuristicResolver for performing best-effort
+/// name resolution for unresolved names in template code.
+/// Adds caching and handling of UnresolvedUsingValueDecl chains.
 class ApproximateNameResolver {
-  [[maybe_unused]] const clang::ASTContext &astContext; // for debugging
+  clang::ASTContext &astContext;
   using Self = ApproximateNameResolver;
   using ResultVec = llvm::SmallVector<const clang::NamedDecl *, 1>;
   absl::flat_hash_map<MemberLookupKey, ResultVec> dependentNameLookupCache;
 
 public:
-  ApproximateNameResolver(const clang::ASTContext &astContext)
+  ApproximateNameResolver(clang::ASTContext &astContext)
       : astContext(astContext), dependentNameLookupCache() {}
 
   ResultVec tryResolveMember(const clang::Type *,
                              const clang::DeclarationNameInfo &);
 
 private:
-  // Analog to HeuristicResolver.cc's resolveTypeToRecordDecl
   static clang::CXXRecordDecl *tryFindDeclForType(const clang::Type *);
 };
 

indexer/CompilationDatabase.cc

@@ -20,6 +20,7 @@
 #include "rapidjson/rapidjson.h"
 #include "rapidjson/reader.h"
 #include "spdlog/fmt/fmt.h"
+#include "spdlog/fmt/ranges.h"
 
 #include "indexer/CommandLineCleaner.h"
 #include "indexer/CompilationDatabase.h"
@@ -97,7 +98,7 @@ struct ClangToolchainInfo : public ToolchainInfo {
         findResourceDirInvocation(findResourceDirInvocation),
         compilerDriverPath(compilerDriverPath),
         findDriverInvocation(findDriverInvocation),
-        cleaner(std::move(cleaner)){};
+        cleaner(std::move(cleaner)) {};
 
   virtual CompilerKind kind() const override {
     return CompilerKind::Clang;

indexer/DebugHelpers.cc

@@ -175,7 +175,6 @@ std::string formatKind(clang::NestedNameSpecifier::SpecifierKind kind) {
     HANDLE_KIND(Namespace)
     HANDLE_KIND(NamespaceAlias)
     HANDLE_KIND(TypeSpec)
-    HANDLE_KIND(TypeSpecWithTemplate)
     HANDLE_KIND(Global)
     HANDLE_KIND(Super)
   }

indexer/Driver.cc

@@ -31,6 +31,7 @@
 #include "rapidjson/error/en.h"
 #include "rapidjson/filereadstream.h"
 #include "spdlog/fmt/fmt.h"
+#include "spdlog/fmt/ranges.h"
 #include "spdlog/sinks/stdout_sinks.h"
 #include "spdlog/spdlog.h"
 
@@ -231,8 +232,8 @@ struct DriverOptions {
         projectRootPath(AbsolutePath("/"), RootKind::Project), compdbPath(),
         indexOutputPath(), statsFilePath(), packageMapPath(),
         showCompilerDiagnostics(cliOpts.showCompilerDiagnostics),
-        showProgress(cliOpts.showProgress), ipcOptions{cliOpts.ipcSizeHintBytes,
-                                                       cliOpts.receiveTimeout},
+        showProgress(cliOpts.showProgress),
+        ipcOptions{cliOpts.ipcSizeHintBytes, cliOpts.receiveTimeout},
         numWorkers(cliOpts.numWorkers), deterministic(cliOpts.deterministic),
         preprocessorRecordHistoryFilterRegex(
             cliOpts.preprocessorRecordHistoryFilterRegex),
@@ -257,7 +258,7 @@ struct DriverOptions {
                 : (llvm::sys::path::is_absolute(path)
                        ? AbsolutePath(std::string(path))
                        : this->projectRootPath.makeAbsolute(
-                           RootRelativePathRef(path, RootKind::Project)));
+                             RootRelativePathRef(path, RootKind::Project)));
     };
 
     // Strictly speaking, there is a TOCTOU problem here, as scip-clang

indexer/Indexer.cc

@@ -386,7 +386,7 @@ void ForwardDeclMap::emit(bool deterministic, scip::ForwardDeclIndex &index) {
 
 TuIndexer::TuIndexer(const clang::SourceManager &sourceManager,
                      const clang::LangOptions &langOptions,
-                     const clang::ASTContext &astContext,
+                     clang::ASTContext &astContext,
                      const FileIdsToBeIndexedSet &fileIdsToBeIndexed,
                      SymbolFormatter &symbolFormatter,
                      FileMetadataMap &fileMetadataMap)
@@ -530,7 +530,8 @@ void TuIndexer::saveFunctionDecl(const clang::FunctionDecl &functionDecl) {
   }
   auto symbol = optSymbol.value();
 
-  if (functionDecl.isPure() || functionDecl.isThisDeclarationADefinition()) {
+  if (functionDecl.isPureVirtual()
+      || functionDecl.isThisDeclarationADefinition()) {
     scip::SymbolInformation symbolInfo{};
     this->getDocComment(functionDecl).addTo(symbolInfo);
     if (auto *cxxMethodDecl =
@@ -671,33 +672,6 @@ void TuIndexer::saveNestedNameSpecifierLoc(
     case Kind::NamespaceAlias:
     case Kind::Global:
     case Kind::Super:
-    case Kind::TypeSpecWithTemplate:
-      // FIXME(def: template-specialization-support)
-      // Adding support for TypeSpecWithTemplate needs extra care
-      // for (partial) template specializations. Example code:
-      //
-      //   template <typename T0>
-      //   struct X {
-      //     template <typename T1>
-      //     struct Y {};
-      //   };
-      //
-      //   template <>
-      //   struct X {
-      //     template <typename A>
-      //     struct Y { int[42] magic; };
-      //   };
-      //
-      //   template <typename U0> void f() {
-      //     typename X<U0>::template Y<U0> y{};
-      //                   //^^^^^^^^^^^^^^ TypeSpecWithTemplate
-      //     std::cout << sizeof(y) << '\n';
-      //   }
-      //
-      // In 'template Y<U0>', clangd will navigate to 'Y' in the body of 'X',
-      // even when there is partial template specialization of X
-      // (so calling f<int>() will print a different value).
-      // Ideally, we should surface such specializations too.
       break;
     }
     nameSpecLoc = nameSpecLoc.getPrefix();
@@ -799,6 +773,20 @@ void TuIndexer::saveTagDecl(const clang::TagDecl &tagDecl) {
       }
     }
   }
+  // For explicit template specializations (e.g. template<> struct Foo<int>),
+  // emit a reference to the primary template in addition to the definition.
+  // The name in the specialization refers back to the primary template, and
+  // SCIP uses the same symbol for both, so the reference enables
+  // Go-to-Definition navigation from the specialization to the primary
+  // template.
+  if (auto *specDecl =
+          llvm::dyn_cast<clang::ClassTemplateSpecializationDecl>(&tagDecl);
+      specDecl
+      && !llvm::isa<clang::ClassTemplatePartialSpecializationDecl>(specDecl)
+      && specDecl->getSpecializationKind()
+             == clang::TSK_ExplicitSpecialization) {
+    this->saveReference(symbol, tagDecl.getLocation());
+  }
   this->saveDefinition(symbol, tagDecl.getLocation(), std::move(symbolInfo));
 }
 
@@ -834,6 +822,17 @@ void TuIndexer::saveTemplateSpecializationTypeLoc(
     const clang::TemplateSpecializationTypeLoc &templateSpecializationTypeLoc) {
   auto *templateSpecializationType = templateSpecializationTypeLoc.getTypePtr();
   auto templateName = templateSpecializationType->getTemplateName();
+
+  // Unwrap QualifiedTemplateName and DeducedTemplateStorage to get the
+  // underlying template. These wrappers preserve source-level qualifications
+  // but we need the actual template declaration for indexing.
+  if (auto *qualifiedName = templateName.getAsQualifiedTemplateName()) {
+    templateName = qualifiedName->getUnderlyingTemplate();
+  }
+  if (auto *deducedStorage = templateName.getAsDeducedTemplateName()) {
+    templateName = deducedStorage->getUnderlying();
+  }
+
   using Kind = clang::TemplateName::NameKind;
   switch (templateName.getKind()) {
   case Kind::Template: {
@@ -871,6 +870,7 @@ void TuIndexer::saveTemplateSpecializationTypeLoc(
   case Kind::OverloadedTemplate:
   case Kind::AssumedTemplate:
   case Kind::QualifiedTemplate:
+  case Kind::DeducedTemplate:
   case Kind::DependentTemplate:
   case Kind::SubstTemplateTemplateParm:
   case Kind::SubstTemplateTemplateParmPack:

indexer/Indexer.h

@@ -80,7 +80,6 @@ struct FileLocalSourceRange {
                       r.endLine);
   }
   DERIVE_CMP_ALL(FileLocalSourceRange)
-  DERIVE_EQ_ALL(FileLocalSourceRange)
 
   static std::pair<FileLocalSourceRange, clang::FileID>
   fromNonEmpty(const clang::SourceManager &, clang::SourceRange inclusiveRange);
@@ -299,7 +298,7 @@ using FileIdsToBeIndexedSet =
 class TuIndexer final {
   const clang::SourceManager &sourceManager;
   const clang::LangOptions &langOptions;
-  [[maybe_unused]] const clang::ASTContext &astContext;
+  clang::ASTContext &astContext;
   const FileIdsToBeIndexedSet &fileIdsToBeIndexed;
   SymbolFormatter &symbolFormatter;
   ApproximateNameResolver approximateNameResolver;
@@ -314,7 +313,7 @@ class TuIndexer final {
 
 public:
   TuIndexer(const clang::SourceManager &, const clang::LangOptions &,
-            const clang::ASTContext &, const FileIdsToBeIndexedSet &,
+            clang::ASTContext &, const FileIdsToBeIndexedSet &,
             SymbolFormatter &, FileMetadataMap &);
 
   /// Emit a fake 'definition' for a file, which can be used as a target

indexer/JsonIpcQueue.h

@@ -11,7 +11,6 @@
 #include "boost/interprocess/ipc/message_queue.hpp"
 #pragma clang diagnostic pop
 
-#include "llvm/ADT/Optional.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/raw_ostream.h"

indexer/PackageMap.cc

@@ -64,7 +64,7 @@ PackageMap::PackageMap(const RootPath &projectRootPath,
 
 void PackageMap::populate(const StdPath &packageMapPath) {
   std::error_code error;
-  auto path = std::string_view(packageMapPath.c_str());
+  std::string path(packageMapPath.c_str());
   if (!llvm::sys::fs::exists(path)) {
     spdlog::error("package map not found at path: {}", path);
     std::exit(EXIT_FAILURE);

indexer/Path.cc

@@ -59,7 +59,7 @@ bool AbsolutePathRef::isNormalized() const {
   auto start = llvm::sys::path::begin(this->value);
   auto end = llvm::sys::path::end(this->value);
   for (auto it = start; it != end; ++it) {
-    if (it->equals(".") || it->equals("..")) {
+    if (*it == "." || *it == "..") {
       return false;
     }
   }