Add configurable startup delay for JVM async-profiler (#302)

Author: tinchi

flags/flags.go

@@ -17,6 +17,7 @@ var (
 	DisableGPUMonitoring    = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool()
 	EnableJavaTls           = kingpin.Flag("enable-java-tls", "Enable Java TLS instrumentation via dynamic agent loading").Default("false").Envar("ENABLE_JAVA_TLS").Bool()
 	EnableJavaAsyncProfiler = kingpin.Flag("enable-java-async-profiler", "Enable Java profiling via async-profiler (CPU, memory allocations, lock contention)").Default("false").Envar("ENABLE_JAVA_ASYNC_PROFILER").Bool()
+	JavaAsyncProfilerDelay  = kingpin.Flag("java-async-profiler-delay", "Delay in seconds before starting async-profiler after JVM process is detected").Default("30s").Envar("JAVA_ASYNC_PROFILER_DELAY").Duration()
 	GoHeapProfilerMode      = kingpin.Flag("go-heap-profiler", "Go heap profiling mode: disabled, enabled (collect from apps with profiling on), force (enable profiling in all Go apps)").Default("enabled").Envar("GO_HEAP_PROFILER").String()
 
 	ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings()

profiling/profiling.go

@@ -193,6 +193,7 @@ func collectAsyncProfilerProfiles() {
 		serviceName string
 		containerID string
 		started     bool
+		startedAt   time.Time
 	}
 	var jvms []jvmInfo
 	for pid, pi := range targetFinder.processes {
@@ -202,13 +203,22 @@ func collectAsyncProfilerProfiles() {
 				serviceName: pi.serviceName,
 				containerID: pi.containerId,
 				started:     pi.asyncProfilerStarted,
+				startedAt:   time.Unix(0, pi.startedAt),
 			})
 		}
 	}
 	targetFinder.lock.Unlock()
 
 	for _, j := range jvms {
 		if !j.started {
+			delay := *flags.JavaAsyncProfilerDelay
+			if delay > 0 && !j.startedAt.IsZero() {
+				if time.Since(j.startedAt) < delay {
+					klog.Infof("pid=%d: delaying async-profiler start (waiting for %s since start)", j.pid, delay)
+					continue
+				}
+			}
+
 			if jvm.IsAsyncProfilerAlreadyLoaded(j.pid) {
 				klog.Infof("pid=%d: async-profiler already loaded by another tool, skipping", j.pid)
 				targetFinder.lock.Lock()
@@ -219,13 +229,14 @@ func collectAsyncProfilerProfiles() {
 				continue
 			}
 			if err := jvm.DeployAndStartAsyncProfiler(j.pid); err != nil {
-				klog.Warningf("async-profiler start pid=%d: %v", j.pid, err)
+				klog.Warningf("async-profiler start pid=%d: %s", j.pid, err)
 				targetFinder.lock.Lock()
 				if pi := targetFinder.processes[j.pid]; pi != nil {
 					pi.asyncProfilerErr = true
 				}
 				targetFinder.lock.Unlock()
 			} else {
+				klog.Infof("pid=%d: async-profiler started", j.pid)
 				targetFinder.lock.Lock()
 				if pi := targetFinder.processes[j.pid]; pi != nil {
 					pi.asyncProfilerStarted = true