feat(browser): replace stealth with custom profile functionality

Author: karust

baidu/search.go

@@ -74,6 +74,7 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool {
 // It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
 	ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
+	ctx = core.WithProfileRegion(ctx, query.LangCode)
 	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
 	scoped := *baid
 	scoped.logger = baid.logger.WithRequest(ctx)
@@ -177,6 +178,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
 // results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
 	ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
+	ctx = core.WithProfileRegion(ctx, query.LangCode)
 	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
 	scoped := *baid
 	scoped.logger = baid.logger.WithRequest(ctx)

bing/search.go

@@ -98,6 +98,7 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
 // It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
 	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
+	ctx = core.WithProfileRegion(ctx, query.LangCode)
 	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
 	scoped := *bing
 	scoped.logger = bing.logger.WithRequest(ctx)
@@ -274,6 +275,7 @@ type BingImageData struct {
 // results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
 func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
 	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
+	ctx = core.WithProfileRegion(ctx, query.LangCode)
 	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
 	scoped := *bing
 	scoped.logger = bing.logger.WithRequest(ctx)

cmd/root.go

@@ -7,14 +7,15 @@ import (
 	"strings"
 
 	"github.com/karust/openserp/core"
+	browserprofile "github.com/karust/openserp/core/browser"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 )
 
 const (
-	version               = "0.6.5"
+	version               = "0.6.6"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -53,10 +54,10 @@ type ServerConfig struct {
 type AppConfig struct {
 	Timeout        int    `mapstructure:"timeout"`
 	BrowserPath    string `mapstructure:"browser_path"`
+	ProfilesJSON   string `mapstructure:"profiles"`
 	IsBrowserHead  bool   `mapstructure:"head"`
 	IsLeaveHead    bool   `mapstructure:"leave_head"`
 	IsLeakless     bool   `mapstructure:"leakless"`
-	IsStealth      bool   `mapstructure:"stealth"`
 	DebugEndpoints bool   `mapstructure:"debug_endpoints"`
 	LogFormat      string `mapstructure:"log_format"`
 }
@@ -102,6 +103,7 @@ var flagToConfigKey = map[string]string{
 	"timeout":                 "app.timeout",
 	"config":                  "server.config_path",
 	"browser-path":            "app.browser_path",
+	"profiles-json":           "app.profiles",
 	"verbose":                 "server.verbose",
 	"debug":                   "server.debug",
 	"head":                    "app.head",
@@ -110,7 +112,6 @@ var flagToConfigKey = map[string]string{
 	"leave":                   "app.leave_head",
 	"2captcha_key":            "2captcha.apikey",
 	"proxy":                   "proxies.global",
-	"stealth":                 "app.stealth",
 	"debug-endpoints":         "app.debug_endpoints",
 	"insecure":                "server.insecure",
 	"cache_ttl":               "cache.ttl_seconds",
@@ -134,6 +135,9 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		if err := browserprofile.LoadProfilesFromJSON(config.App.ProfilesJSON); err != nil {
+			return fmt.Errorf("load app.profiles: %w", err)
+		}
 
 		logFormat, err := core.NormalizeLogFormat(config.App.LogFormat)
 		if err != nil {
@@ -316,10 +320,10 @@ func setConfigDefaults(v *viper.Viper) {
 
 	v.SetDefault("app.timeout", 30)
 	v.SetDefault("app.browser_path", "")
+	v.SetDefault("app.profiles", "")
 	v.SetDefault("app.head", false)
 	v.SetDefault("app.leave_head", false)
 	v.SetDefault("app.leakless", false)
-	v.SetDefault("app.stealth", false)
 	v.SetDefault("app.debug_endpoints", false)
 
 	v.SetDefault("proxies.entries", []interface{}{})
@@ -348,6 +352,7 @@ func init() {
 	RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
 	RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
 	RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
+	RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
@@ -356,7 +361,6 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
 	RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
-	RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
 	RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
 	RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")

cmd/search.go

@@ -117,7 +117,6 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
 		BrowserPath:          config.App.BrowserPath,
 		ProxyURL:             browserProxyURL,
 		Insecure:             config.Server.Insecure,
-		UseStealth:           config.App.IsStealth,
 	}
 
 	if config.Server.IsDebug {

cmd/serve.go

@@ -132,7 +132,6 @@ func buildFingerprintBrowserOptions() core.BrowserOpts {
 		Timeout:     time.Second * time.Duration(config.App.Timeout),
 		BrowserPath: config.App.BrowserPath,
 		Insecure:    config.Server.Insecure,
-		UseStealth:  config.App.IsStealth,
 	}
 	if config.Server.IsDebug {
 		opts.IsHeadless = false

config.yaml

@@ -11,10 +11,10 @@ app:
   log_format: "text"
   timeout: 15 # Browser/search timeout in seconds
   browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
+  profiles: "" # Optional JSON file path overriding built-in browser profiles
   head: false # Show browser UI (headful mode)
   leakless: false # Force browser process cleanup after request
   leave_head: false # Keep tabs open after request for debugging
-  stealth: false # Enable stealth browser plugin
 
 proxies:
   # Force a single proxy for all engines.