Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pkg/scraper/mapped.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,14 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.
logger.Debug(`Processing scene:`)
results := sceneMap.process(ctx, q, s.Common, urlsIsMulti)

// Ensure ret is non-nil before calling processSceneRelationships so it can
// safely populate relationship fields even when no direct results were found.
// This preserves the intent of #3953: returning a scene with only relationships.
var ret *models.ScrapedScene
if len(results) > 0 {
ret = results[0].scrapedScene()
} else {
ret = &models.ScrapedScene{}
}
hasRelationships := s.processSceneRelationships(ctx, q, 0, ret)

Expand Down
127 changes: 59 additions & 68 deletions pkg/scraper/xpath_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -683,74 +683,65 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin
}
}

func TestApplySceneXPathConfig(t *testing.T) {
reader := strings.NewReader(sceneHTML)
doc, err := htmlquery.Parse(reader)

if err != nil {
t.Errorf("Error loading document: %s", err.Error())
return
}

scraper := makeSceneXPathConfig()

q := &xpathQuery{
doc: doc,
}
scene, err := scraper.scrapeScene(context.Background(), q)

if err != nil {
t.Errorf("Error scraping scene: %s", err.Error())
return
}

const title = "Test Video"

verifyField(t, title, scene.Title, "Title")

// verify tags
expectedTags := []string{
"Amateur",
"Babe",
"Blowjob",
"Exclusive",
"HD Porn",
"Pornstar",
"Public",
"Pussy Licking",
"Threesome",
"Verified Models",
}
verifyTags(t, expectedTags, scene.Tags)

// verify movies
expectedMovies := []string{
"Video",
"of",
"verified",
"member",
}
verifyMovies(t, expectedMovies, scene.Movies)

expectedPerformerNames := []string{
"Alex D",
"Mia Malkova",
"Riley Reid",
}

expectedPerformerURLs := []string{
"/pornstar/alex-d",
"/pornstar/mia-malkova",
"/pornstar/riley-reid",
}

verifyPerformers(t, expectedPerformerNames, expectedPerformerURLs, scene.Performers)

const expectedStudioName = "Sis Loves Me"
const expectedStudioURL = "/channels/sis-loves-me"

verifyField(t, expectedStudioName, &scene.Studio.Name, "Studio.Name")
verifyField(t, expectedStudioURL, scene.Studio.URL, "Studio.URL")
func TestApplySceneXPathConfigRelationshipsOnly(t *testing.T) {
reader := strings.NewReader(sceneHTML)
doc, err := htmlquery.Parse(reader)
if err != nil {
t.Fatalf("Error loading document: %s", err.Error())
}

scraper := makeSceneXPathConfig()

// Simulate the relationships-only scrape path:
// no direct scene fields, but populated Performers/Tags mappings.
scraper.Scene.mappedConfig = make(mappedConfig)
scraper.Scene.Studio = nil
scraper.Scene.Movies = nil
scraper.Scene.Groups = nil

q := &xpathQuery{
doc: doc,
}

var scene *models.ScrapedScene
assert.NotPanics(t, func() {
scene, err = scraper.scrapeScene(context.Background(), q)
}, "relationships-only scene scrape should not panic")

assert.NoError(t, err)
assert.NotNil(t, scene)

// No direct scene fields should be populated.
assert.Nil(t, scene.Title)
assert.Nil(t, scene.Date)

expectedTags := []string{
"Amateur",
"Babe",
"Blowjob",
"Exclusive",
"HD Porn",
"Pornstar",
"Public",
"Pussy Licking",
"Threesome",
"Verified Models",
}
verifyTags(t, expectedTags, scene.Tags)

expectedPerformerNames := []string{
"Alex D",
"Mia Malkova",
"Riley Reid",
}

expectedPerformerURLs := []string{
"/pornstar/alex-d",
"/pornstar/mia-malkova",
"/pornstar/riley-reid",
}

verifyPerformers(t, expectedPerformerNames, expectedPerformerURLs, scene.Performers)
}

func TestLoadXPathScraperFromYAML(t *testing.T) {
Expand Down