fix: track proto_repository configuration inputs

docs/DEVELOPMENT.md

@@ -79,3 +79,30 @@ For example `(cd
 /private/var/tmp/_bazel_i868039/7b08591af2b3d71f45f2e4029050db37/bazel_testing/bazel_go_test/main
 && code .)`.  You can then use `bazel build` directly in this space to explore
 what's going on.
+
+## Repository input tracking
+
+`proto_repository` watches its `cfgs` YAML files and `imports` CSV files.
+Gazelle also writes a temporary JSON manifest through
+`-proto_config_inputs_out` listing the Starlark plugin and rule files loaded
+from YAML or `-proto_plugin` / `-proto_rule`. The repository rule watches those
+files and removes the manifest. Sources inside the fetched repository are
+already covered by the repository definition and are not watched separately.
+The repository Gazelle tool also watches its Go sources so source changes
+rebuild the executable.
+
+The following regression test uses real Bazel and Gazelle in a temporary
+workspace. It checks configuration, CSV, and Starlark edits, propagation to a
+consumer repository, and reuse when no inputs change. Archives and dependencies
+are supplied locally; the test does not require network access.
+
+```sh
+go build -mod=vendor -o /tmp/rules-proto-gazelle ./cmd/gazelle
+python3 tools/test_proto_repository_inputs.py \
+  --bazel "$(command -v bazel)" \
+  --gazelle /tmp/rules-proto-gazelle \
+  --gazelle-repo "$(bazel info output_base)/external/gazelle+"
+```
+
+`--gazelle-repo` must point to the fetched bazel-gazelle sources; adjust the
+canonical repository name if your Bazel version uses a different spelling.

pkg/language/protobuf/config.go

@@ -1,9 +1,11 @@
 package protobuf
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
+	"os"
 	"strings"
 
 	"github.com/bazelbuild/bazel-gazelle/config"
@@ -20,6 +22,9 @@ func (pl *protobufLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Co
 	fs.StringVar(&pl.configFiles,
 		"proto_configs", "",
 		"optional config.yaml file(s) that provide preconfiguration")
+	fs.StringVar(&pl.configInputsOutFile,
+		"proto_config_inputs_out", "",
+		"write loaded Starlark source paths as JSON for repository input tracking")
 	fs.StringVar(&pl.importsInFiles,
 		"proto_imports_in", "",
 		"index files to parse and load symbols from")
@@ -74,6 +79,16 @@ func (pl *protobufLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
 		}
 	}
 
+	if pl.configInputsOutFile != "" {
+		data, err := json.Marshal(protoc.StarlarkFiles(c))
+		if err != nil {
+			return err
+		}
+		if err := os.WriteFile(pl.configInputsOutFile, data, 0o644); err != nil {
+			return fmt.Errorf("writing -proto_config_inputs_out: %w", err)
+		}
+	}
+
 	return nil
 }
 

pkg/language/protobuf/config_filemode_test.go

@@ -1,6 +1,12 @@
 package protobuf
 
 import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/bazelbuild/bazel-gazelle/config"
@@ -35,3 +41,92 @@ func TestGetOrCreatePackageConfig_RebindsConfig(t *testing.T) {
 		t.Errorf("expected FileMode via cfg.Config, got %v", gproto.GetProtoConfig(cfg.Config).Mode)
 	}
 }
+
+// The manifest must use the paths Gazelle actually loads, including the source
+// root fallback used when Gazelle runs inside a fetched repository.
+func TestConfigInputsManifest(t *testing.T) {
+	for _, sourceRootFallback := range []bool{false, true} {
+		t.Run(fmt.Sprint(sourceRootFallback), func(t *testing.T) {
+			dir := t.TempDir()
+			workDir := dir
+			if sourceRootFallback {
+				workDir = filepath.Join(dir, "external")
+				if err := os.Mkdir(workDir, 0o755); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(filepath.Join(workDir, "DO_NOT_BUILD_HERE"), []byte(dir), 0o644); err != nil {
+					t.Fatal(err)
+				}
+			}
+			// Unique registry keys allow repeated tests in the same process.
+			name := filepath.Base(filepath.Dir(dir)) + fmt.Sprint(sourceRootFallback) + ".star"
+			filename := filepath.Join(dir, name)
+			code := `
+protoc.Plugin(name = "yaml_plugin", configure = lambda ctx: None)
+protoc.Plugin(name = "flag_plugin", configure = lambda ctx: None)
+protoc.Rule(
+    name = "yaml_rule",
+    load_info = lambda: None,
+    kind_info = lambda: None,
+    provide_rule = lambda rctx, pctx: None,
+)
+`
+			if err := os.WriteFile(filename, []byte(code), 0o644); err != nil {
+				t.Fatal(err)
+			}
+			yaml := fmt.Sprintf("starlarkPlugins:\n  - %s%%yaml_plugin\nstarlarkRules:\n  - %s%%yaml_rule\n", name, name)
+			cfgFile := filepath.Join(dir, "config.yaml")
+			if err := os.WriteFile(cfgFile, []byte(yaml), 0o644); err != nil {
+				t.Fatal(err)
+			}
+			out := filepath.Join(dir, "inputs.json")
+			c := config.New()
+			c.WorkDir = workDir
+			pl := NewProtobufLang("protobuf")
+			fs := flag.NewFlagSet("test", flag.ContinueOnError)
+			pl.RegisterFlags(fs, "update", c)
+			if err := fs.Parse([]string{"-proto_configs", cfgFile, "-proto_plugin", name + "%flag_plugin", "-proto_config_inputs_out", out}); err != nil {
+				t.Fatal(err)
+			}
+			if err := pl.CheckFlags(fs, c); err != nil {
+				t.Fatal(err)
+			}
+			data, err := os.ReadFile(out)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got []string
+			if err := json.Unmarshal(data, &got); err != nil {
+				t.Fatal(err)
+			}
+			if want := []string{filename}; !reflect.DeepEqual(got, want) {
+				t.Fatalf("manifest = %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestConfigInputsManifestEmpty(t *testing.T) {
+	c := config.New()
+	pl := NewProtobufLang("protobuf")
+	out := filepath.Join(t.TempDir(), "inputs.json")
+	fs := flag.NewFlagSet("test", flag.ContinueOnError)
+	pl.RegisterFlags(fs, "update", c)
+	if err := fs.Parse([]string{"-proto_config_inputs_out", out}); err != nil {
+		t.Fatal(err)
+	}
+	if err := pl.CheckFlags(fs, c); err != nil {
+		t.Fatal(err)
+	}
+	data, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != "[]" {
+		t.Fatalf("manifest = %s, want []", data)
+	}
+	pl.configInputsOutFile = filepath.Join(out, "cannot-write.json")
+	if err := pl.CheckFlags(fs, c); err == nil {
+		t.Fatal("expected an error writing the manifest")
+	}
+}

pkg/language/protobuf/lang.go

@@ -24,6 +24,8 @@ type protobufLang struct {
 	packages map[string]*protoc.Package
 	// configFiles contains yconfig yaml files to parse.  May be comma-separated.
 	configFiles string
+	// configInputsOutFile is a JSON manifest of loaded Starlark source paths.
+	configInputsOutFile string
 	// repoName is the name (if this an external repository)
 	repoName string
 	// importsOutFile is the name of the file to create.  If "", skip writing

pkg/protoc/package_config.go

@@ -3,6 +3,7 @@ package protoc
 import (
 	"fmt"
 	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -265,16 +266,24 @@ func RegisterStarlarkPlugin(c *config.Config, starlarkPlugin string) error {
 	if len(parts) != 2 {
 		return fmt.Errorf("invalid starlark plugin name %q", starlarkPlugin)
 	}
-	fileName := parts[0]
+	fileName, err := resolveStarlarkFilename(c.WorkDir, parts[0])
+	if err != nil {
+		return err
+	}
+	fileName, err = filepath.Abs(fileName)
+	if err != nil {
+		return err
+	}
 	ruleName := parts[1]
-	impl, err := LoadStarlarkPluginFromFile(c.WorkDir, fileName, ruleName, func(msg string) {
+	impl, err := LoadStarlarkPluginFromFile("", fileName, ruleName, func(msg string) {
 		log.Printf("%s: %v", starlarkPlugin, msg)
 	}, func(err error) {
 		log.Fatalf("starlark plugin configuration error (plugin %q will not be registered): %v", starlarkPlugin, err)
 	})
 	if err != nil {
 		return err
 	}
+	recordStarlarkFile(c, fileName)
 	Plugins().RegisterPlugin(starlarkPlugin, impl)
 	return nil
 }
@@ -284,16 +293,48 @@ func RegisterStarlarkRule(c *config.Config, starlarkRule string) error {
 	if len(parts) != 2 {
 		return fmt.Errorf("invalid starlark rule name %q", starlarkRule)
 	}
-	fileName := parts[0]
+	fileName, err := resolveStarlarkFilename(c.WorkDir, parts[0])
+	if err != nil {
+		return err
+	}
+	fileName, err = filepath.Abs(fileName)
+	if err != nil {
+		return err
+	}
 	ruleName := parts[1]
 
-	impl, err := LoadStarlarkLanguageRuleFromFile(c.WorkDir, fileName, ruleName, func(msg string) {
+	impl, err := LoadStarlarkLanguageRuleFromFile("", fileName, ruleName, func(msg string) {
 	}, func(err error) {
 		log.Panicf("starlark rule configuration error (rule %q will not be registered): %v", starlarkRule, err)
 	})
 	if err != nil {
 		return err
 	}
+	recordStarlarkFile(c, fileName)
 	Rules().MustRegisterRule(starlarkRule, impl)
 	return nil
 }
+
+const starlarkFilesKey = "rules_proto_starlark_files"
+
+func recordStarlarkFile(c *config.Config, filename string) {
+	files, ok := c.Exts[starlarkFilesKey].(map[string]bool)
+	if !ok {
+		files = make(map[string]bool)
+		c.Exts[starlarkFilesKey] = files
+	}
+	files[filename] = true
+}
+
+// StarlarkFiles returns the sorted, absolute paths of plugin and rule sources
+// loaded by this configuration. Repository rules use them to track inputs read
+// by the Gazelle subprocess.
+func StarlarkFiles(c *config.Config) []string {
+	files, _ := c.Exts[starlarkFilesKey].(map[string]bool)
+	out := make([]string, 0, len(files))
+	for filename := range files {
+		out = append(out, filename)
+	}
+	sort.Strings(out)
+	return out
+}

rules/private/proto_repository_tools.bzl

@@ -13,7 +13,7 @@
 # limitations under the License.
 """"""
 
-load("@bazel_gazelle//internal:common.bzl", "env_execute", "executable_extension")
+load("@bazel_gazelle//internal:common.bzl", "env_execute", "executable_extension", "watch")
 load("@bazel_gazelle//internal:go_repository_cache.bzl", "read_cache_env")
 load("@build_stack_rules_proto//rules/private:proto_repository_tools_srcs.bzl", "PROTO_REPOSITORY_TOOLS_SRCS")
 
@@ -33,6 +33,13 @@ def _proto_repository_tools_impl(ctx):
     go_tool = env["GOROOT"] + "/bin/go" + extension
 
     rules_proto_path = ctx.path(Label("@build_stack_rules_proto//:MODULE.bazel"))
+
+    # The compiler reads sources through the symlink below. Resolve paths from
+    # the source root: the generated list can still name deleted packages.
+    for src in ctx.attr._proto_repository_tools_srcs + [ctx.attr._list_repository_tools_srcs]:
+        relative_path = src.package + "/" + src.name if src.package else src.name
+        watch(ctx, rules_proto_path.dirname.get_child(relative_path))
+
     ctx.symlink(
         rules_proto_path.dirname,
         "src/github.com/stackb/rules_proto/v4",

rules/proto/proto_repository.bzl

@@ -318,6 +318,13 @@ def _proto_repository_impl(ctx):
         if is_module_extension_repo:
             cmd.append("-bzlmod")
 
+        # These files are read by Gazelle, not by repository_ctx.
+        for input in ctx.attr.cfgs + ctx.attr.imports:
+            watch(ctx, ctx.path(input))
+
+        config_inputs = ctx.path(".gazelle_config_inputs.json")
+        cmd.extend(["-proto_config_inputs_out", config_inputs])
+
         # BEGIN protobuf extension flags
         if ctx.attr.languages:
             cmd.extend(["-lang", ",".join(ctx.attr.languages)])
@@ -344,6 +351,19 @@ def _proto_repository_impl(ctx):
                 ctx.attr.importpath,
                 result.stderr,
             ))
+
+        # YAML and command-line configuration can load Starlark files outside
+        # this repository. Gazelle reports the actual paths it resolved.
+        repo_root = str(ctx.path("")) + "/"
+        real_repo_root = str(ctx.path("").realpath) + "/"
+        for filename in json.decode(ctx.read(config_inputs, watch = "no")):
+            input_path = ctx.path(filename)
+            if str(input_path).startswith(repo_root):
+                input_path = input_path.realpath
+            if not str(input_path.realpath).startswith(real_repo_root):
+                watch(ctx, input_path)
+        ctx.delete(config_inputs)
+
         if ctx.attr.debug_mode and result.stderr:
             # buildifier: disable=print
             print("%s gazelle.stdout: %s" % (ctx.name, result.stdout))