Autotag support for images and galleries (#1345)

* Add compound queries for images and galleries
* Implement image and gallery auto tagging
This commit is contained in:
WithoutPants
2021-05-03 13:09:46 +10:00
committed by GitHub
parent 2c52fd711b
commit a3609079bb
27 changed files with 2910 additions and 521 deletions
+212 -186
View File
@@ -159,7 +159,69 @@ func (qb *galleryQueryBuilder) All() ([]*models.Gallery, error) {
return qb.queryGalleries(selectAll("galleries")+qb.getGallerySort(nil), nil)
}
func (qb *galleryQueryBuilder) makeQuery(galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) queryBuilder {
func (qb *galleryQueryBuilder) validateFilter(galleryFilter *models.GalleryFilterType) error {
const and = "AND"
const or = "OR"
const not = "NOT"
if galleryFilter.And != nil {
if galleryFilter.Or != nil {
return illegalFilterCombination(and, or)
}
if galleryFilter.Not != nil {
return illegalFilterCombination(and, not)
}
return qb.validateFilter(galleryFilter.And)
}
if galleryFilter.Or != nil {
if galleryFilter.Not != nil {
return illegalFilterCombination(or, not)
}
return qb.validateFilter(galleryFilter.Or)
}
if galleryFilter.Not != nil {
return qb.validateFilter(galleryFilter.Not)
}
return nil
}
func (qb *galleryQueryBuilder) makeFilter(galleryFilter *models.GalleryFilterType) *filterBuilder {
query := &filterBuilder{}
if galleryFilter.And != nil {
query.and(qb.makeFilter(galleryFilter.And))
}
if galleryFilter.Or != nil {
query.or(qb.makeFilter(galleryFilter.Or))
}
if galleryFilter.Not != nil {
query.not(qb.makeFilter(galleryFilter.Not))
}
query.handleCriterionFunc(boolCriterionHandler(galleryFilter.IsZip, "galleries.zip"))
query.handleCriterionFunc(stringCriterionHandler(galleryFilter.Path, "galleries.path"))
query.handleCriterionFunc(intCriterionHandler(galleryFilter.Rating, "galleries.rating"))
query.handleCriterionFunc(stringCriterionHandler(galleryFilter.URL, "galleries.url"))
query.handleCriterionFunc(boolCriterionHandler(galleryFilter.Organized, "galleries.organized"))
query.handleCriterionFunc(galleryIsMissingCriterionHandler(qb, galleryFilter.IsMissing))
query.handleCriterionFunc(galleryTagsCriterionHandler(qb, galleryFilter.Tags))
query.handleCriterionFunc(galleryTagCountCriterionHandler(qb, galleryFilter.TagCount))
query.handleCriterionFunc(galleryPerformersCriterionHandler(qb, galleryFilter.Performers))
query.handleCriterionFunc(galleryPerformerCountCriterionHandler(qb, galleryFilter.PerformerCount))
query.handleCriterionFunc(galleryStudioCriterionHandler(qb, galleryFilter.Studios))
query.handleCriterionFunc(galleryPerformerTagsCriterionHandler(qb, galleryFilter.PerformerTags))
query.handleCriterionFunc(galleryAverageResolutionCriterionHandler(qb, galleryFilter.AverageResolution))
query.handleCriterionFunc(galleryImageCountCriterionHandler(qb, galleryFilter.ImageCount))
return query
}
func (qb *galleryQueryBuilder) makeQuery(galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
if galleryFilter == nil {
galleryFilter = &models.GalleryFilterType{}
}
@@ -169,15 +231,7 @@ func (qb *galleryQueryBuilder) makeQuery(galleryFilter *models.GalleryFilterType
query := qb.newQuery()
query.body = selectDistinctIDs("galleries")
query.body += `
left join performers_galleries as performers_join on performers_join.gallery_id = galleries.id
left join scenes_galleries as scenes_join on scenes_join.gallery_id = galleries.id
left join studios as studio on studio.id = galleries.studio_id
left join galleries_tags as tags_join on tags_join.gallery_id = galleries.id
left join galleries_images as images_join on images_join.gallery_id = galleries.id
left join images on images_join.image_id = images.id
`
query.body = selectDistinctIDs(galleryTable)
if q := findFilter.Q; q != nil && *q != "" {
searchColumns := []string{"galleries.title", "galleries.path", "galleries.checksum"}
@@ -186,110 +240,23 @@ func (qb *galleryQueryBuilder) makeQuery(galleryFilter *models.GalleryFilterType
query.addArg(thisArgs...)
}
if zipFilter := galleryFilter.IsZip; zipFilter != nil {
var favStr string
if *zipFilter == true {
favStr = "1"
} else {
favStr = "0"
}
query.addWhere("galleries.zip = " + favStr)
if err := qb.validateFilter(galleryFilter); err != nil {
return nil, err
}
filter := qb.makeFilter(galleryFilter)
query.handleStringCriterionInput(galleryFilter.Path, "galleries.path")
query.handleIntCriterionInput(galleryFilter.Rating, "galleries.rating")
query.handleStringCriterionInput(galleryFilter.URL, "galleries.url")
query.handleCountCriterion(galleryFilter.ImageCount, galleryTable, galleriesImagesTable, galleryIDColumn)
qb.handleAverageResolutionFilter(&query, galleryFilter.AverageResolution)
if Organized := galleryFilter.Organized; Organized != nil {
var organized string
if *Organized == true {
organized = "1"
} else {
organized = "0"
}
query.addWhere("galleries.organized = " + organized)
}
if isMissingFilter := galleryFilter.IsMissing; isMissingFilter != nil && *isMissingFilter != "" {
switch *isMissingFilter {
case "scenes":
query.addWhere("scenes_join.gallery_id IS NULL")
case "studio":
query.addWhere("galleries.studio_id IS NULL")
case "performers":
query.addWhere("performers_join.gallery_id IS NULL")
case "date":
query.addWhere("galleries.date IS \"\" OR galleries.date IS \"0001-01-01\"")
case "tags":
query.addWhere("tags_join.gallery_id IS NULL")
default:
query.addWhere("galleries." + *isMissingFilter + " IS NULL")
}
}
if tagsFilter := galleryFilter.Tags; tagsFilter != nil && len(tagsFilter.Value) > 0 {
for _, tagID := range tagsFilter.Value {
query.addArg(tagID)
}
query.body += " LEFT JOIN tags on tags_join.tag_id = tags.id"
whereClause, havingClause := getMultiCriterionClause("galleries", "tags", "galleries_tags", "gallery_id", "tag_id", tagsFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
if tagCountFilter := galleryFilter.TagCount; tagCountFilter != nil {
clause, count := getCountCriterionClause(galleryTable, galleriesTagsTable, galleryIDColumn, *tagCountFilter)
if count == 1 {
query.addArg(tagCountFilter.Value)
}
query.addWhere(clause)
}
if performersFilter := galleryFilter.Performers; performersFilter != nil && len(performersFilter.Value) > 0 {
for _, performerID := range performersFilter.Value {
query.addArg(performerID)
}
query.body += " LEFT JOIN performers ON performers_join.performer_id = performers.id"
whereClause, havingClause := getMultiCriterionClause("galleries", "performers", "performers_galleries", "gallery_id", "performer_id", performersFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
if performerCountFilter := galleryFilter.PerformerCount; performerCountFilter != nil {
clause, count := getCountCriterionClause(galleryTable, performersGalleriesTable, galleryIDColumn, *performerCountFilter)
if count == 1 {
query.addArg(performerCountFilter.Value)
}
query.addWhere(clause)
}
if studiosFilter := galleryFilter.Studios; studiosFilter != nil && len(studiosFilter.Value) > 0 {
for _, studioID := range studiosFilter.Value {
query.addArg(studioID)
}
whereClause, havingClause := getMultiCriterionClause("galleries", "studio", "", "", "studio_id", studiosFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
handleGalleryPerformerTagsCriterion(&query, galleryFilter.PerformerTags)
query.addFilter(filter)
query.sortAndPagination = qb.getGallerySort(findFilter) + getPagination(findFilter)
return query
return &query, nil
}
func (qb *galleryQueryBuilder) Query(galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) ([]*models.Gallery, int, error) {
query := qb.makeQuery(galleryFilter, findFilter)
query, err := qb.makeQuery(galleryFilter, findFilter)
if err != nil {
return nil, 0, err
}
idsResult, countResult, err := query.executeFind()
if err != nil {
@@ -310,98 +277,155 @@ func (qb *galleryQueryBuilder) Query(galleryFilter *models.GalleryFilterType, fi
}
func (qb *galleryQueryBuilder) QueryCount(galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) (int, error) {
query := qb.makeQuery(galleryFilter, findFilter)
query, err := qb.makeQuery(galleryFilter, findFilter)
if err != nil {
return 0, err
}
return query.executeCount()
}
func (qb *galleryQueryBuilder) handleAverageResolutionFilter(query *queryBuilder, resolutionFilter *models.ResolutionEnum) {
if resolutionFilter == nil {
return
}
if resolution := resolutionFilter.String(); resolutionFilter.IsValid() {
var low int
var high int
switch resolution {
case "VERY_LOW":
high = 240
case "LOW":
low = 240
high = 360
case "R360P":
low = 360
high = 480
case "STANDARD":
low = 480
high = 540
case "WEB_HD":
low = 540
high = 720
case "STANDARD_HD":
low = 720
high = 1080
case "FULL_HD":
low = 1080
high = 1440
case "QUAD_HD":
low = 1440
high = 1920
case "VR_HD":
low = 1920
high = 2160
case "FOUR_K":
low = 2160
high = 2880
case "FIVE_K":
low = 2880
high = 3384
case "SIX_K":
low = 3384
high = 4320
case "EIGHT_K":
low = 4320
}
havingClause := ""
if low != 0 {
havingClause = "avg(MIN(images.width, images.height)) >= " + strconv.Itoa(low)
}
if high != 0 {
if havingClause != "" {
havingClause += " AND "
func galleryIsMissingCriterionHandler(qb *galleryQueryBuilder, isMissing *string) criterionHandlerFunc {
return func(f *filterBuilder) {
if isMissing != nil && *isMissing != "" {
switch *isMissing {
case "scenes":
f.addJoin("scenes_galleries", "scenes_join", "scenes_join.gallery_id = galleries.id")
f.addWhere("scenes_join.gallery_id IS NULL")
case "studio":
f.addWhere("galleries.studio_id IS NULL")
case "performers":
qb.performersRepository().join(f, "performers_join", "galleries.id")
f.addWhere("performers_join.gallery_id IS NULL")
case "date":
f.addWhere("galleries.date IS \"\" OR galleries.date IS \"0001-01-01\"")
case "tags":
qb.tagsRepository().join(f, "tags_join", "galleries.id")
f.addWhere("tags_join.gallery_id IS NULL")
default:
f.addWhere("(galleries." + *isMissing + " IS NULL OR TRIM(galleries." + *isMissing + ") = '')")
}
havingClause += "avg(MIN(images.width, images.height)) < " + strconv.Itoa(high)
}
if havingClause != "" {
query.addHaving(havingClause)
}
}
}
func handleGalleryPerformerTagsCriterion(query *queryBuilder, performerTagsFilter *models.MultiCriterionInput) {
if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 {
for _, tagID := range performerTagsFilter.Value {
query.addArg(tagID)
func (qb *galleryQueryBuilder) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{
primaryTable: galleryTable,
foreignTable: foreignTable,
joinTable: joinTable,
primaryFK: galleryIDColumn,
foreignFK: foreignFK,
addJoinsFunc: addJoinsFunc,
}
}
func galleryTagsCriterionHandler(qb *galleryQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
qb.tagsRepository().join(f, "tags_join", "galleries.id")
f.addJoin(tagTable, "", "tags_join.tag_id = tags.id")
}
h := qb.getMultiCriterionHandlerBuilder(tagTable, galleriesTagsTable, tagIDColumn, addJoinsFunc)
return h.handler(tags)
}
func galleryTagCountCriterionHandler(qb *galleryQueryBuilder, tagCount *models.IntCriterionInput) criterionHandlerFunc {
h := countCriterionHandlerBuilder{
primaryTable: galleryTable,
joinTable: galleriesTagsTable,
primaryFK: galleryIDColumn,
}
return h.handler(tagCount)
}
func galleryPerformersCriterionHandler(qb *galleryQueryBuilder, performers *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
qb.performersRepository().join(f, "performers_join", "galleries.id")
f.addJoin(performerTable, "", "performers_join.performer_id = performers.id")
}
h := qb.getMultiCriterionHandlerBuilder(performerTable, performersGalleriesTable, performerIDColumn, addJoinsFunc)
return h.handler(performers)
}
func galleryPerformerCountCriterionHandler(qb *galleryQueryBuilder, performerCount *models.IntCriterionInput) criterionHandlerFunc {
h := countCriterionHandlerBuilder{
primaryTable: galleryTable,
joinTable: performersGalleriesTable,
primaryFK: galleryIDColumn,
}
return h.handler(performerCount)
}
func galleryImageCountCriterionHandler(qb *galleryQueryBuilder, imageCount *models.IntCriterionInput) criterionHandlerFunc {
h := countCriterionHandlerBuilder{
primaryTable: galleryTable,
joinTable: galleriesImagesTable,
primaryFK: galleryIDColumn,
}
return h.handler(imageCount)
}
func galleryStudioCriterionHandler(qb *galleryQueryBuilder, studios *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
f.addJoin(studioTable, "studio", "studio.id = galleries.studio_id")
}
h := qb.getMultiCriterionHandlerBuilder("studio", "", studioIDColumn, addJoinsFunc)
return h.handler(studios)
}
func galleryPerformerTagsCriterionHandler(qb *galleryQueryBuilder, performerTagsFilter *models.MultiCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 {
qb.performersRepository().join(f, "performers_join", "galleries.id")
f.addJoin("performers_tags", "performer_tags_join", "performers_join.performer_id = performer_tags_join.performer_id")
var args []interface{}
for _, tagID := range performerTagsFilter.Value {
args = append(args, tagID)
}
if performerTagsFilter.Modifier == models.CriterionModifierIncludes {
// includes any of the provided ids
f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...)
} else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll {
// includes all of the provided ids
f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...)
f.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierExcludes {
f.addWhere(fmt.Sprintf(`not exists
(select performers_galleries.performer_id from performers_galleries
left join performers_tags on performers_tags.performer_id = performers_galleries.performer_id where
performers_galleries.gallery_id = galleries.id AND
performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))), args...)
}
}
}
}
query.body += " LEFT JOIN performers_tags AS performer_tags_join on performers_join.performer_id = performer_tags_join.performer_id"
func galleryAverageResolutionCriterionHandler(qb *galleryQueryBuilder, resolution *models.ResolutionEnum) criterionHandlerFunc {
return func(f *filterBuilder) {
if resolution != nil && resolution.IsValid() {
qb.imagesRepository().join(f, "images_join", "galleries.id")
f.addJoin("images", "", "images_join.image_id = images.id")
if performerTagsFilter.Modifier == models.CriterionModifierIncludes {
// includes any of the provided ids
query.addWhere("performer_tags_join.tag_id IN " + getInBinding(len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll {
// includes all of the provided ids
query.addWhere("performer_tags_join.tag_id IN " + getInBinding(len(performerTagsFilter.Value)))
query.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierExcludes {
query.addWhere(fmt.Sprintf(`not exists
(select performers_galleries.performer_id from performers_galleries
left join performers_tags on performers_tags.performer_id = performers_galleries.performer_id where
performers_galleries.gallery_id = galleries.id AND
performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))))
min := resolution.GetMinResolution()
max := resolution.GetMaxResolution()
const widthHeight = "avg(MIN(images.width, images.height))"
if min > 0 {
f.addHaving(widthHeight + " >= " + strconv.Itoa(min))
}
if max > 0 {
f.addHaving(widthHeight + " < " + strconv.Itoa(max))
}
}
}
}
@@ -418,6 +442,8 @@ func (qb *galleryQueryBuilder) getGallerySort(findFilter *models.FindFilterType)
}
switch sort {
case "images_count":
return getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)
case "tag_count":
return getCountSort(galleryTable, galleriesTagsTable, galleryIDColumn, direction)
case "performer_count":
+153
View File
@@ -193,6 +193,143 @@ func verifyGalleriesPath(t *testing.T, sqb models.GalleryReader, pathCriterion m
}
}
func TestGalleryQueryPathOr(t *testing.T) {
const gallery1Idx = 1
const gallery2Idx = 2
gallery1Path := getGalleryStringValue(gallery1Idx, "Path")
gallery2Path := getGalleryStringValue(gallery2Idx, "Path")
galleryFilter := models.GalleryFilterType{
Path: &models.StringCriterionInput{
Value: gallery1Path,
Modifier: models.CriterionModifierEquals,
},
Or: &models.GalleryFilterType{
Path: &models.StringCriterionInput{
Value: gallery2Path,
Modifier: models.CriterionModifierEquals,
},
},
}
withTxn(func(r models.Repository) error {
sqb := r.Gallery()
galleries := queryGallery(t, sqb, &galleryFilter, nil)
assert.Len(t, galleries, 2)
assert.Equal(t, gallery1Path, galleries[0].Path.String)
assert.Equal(t, gallery2Path, galleries[1].Path.String)
return nil
})
}
func TestGalleryQueryPathAndRating(t *testing.T) {
const galleryIdx = 1
galleryPath := getGalleryStringValue(galleryIdx, "Path")
galleryRating := getRating(galleryIdx)
galleryFilter := models.GalleryFilterType{
Path: &models.StringCriterionInput{
Value: galleryPath,
Modifier: models.CriterionModifierEquals,
},
And: &models.GalleryFilterType{
Rating: &models.IntCriterionInput{
Value: int(galleryRating.Int64),
Modifier: models.CriterionModifierEquals,
},
},
}
withTxn(func(r models.Repository) error {
sqb := r.Gallery()
galleries := queryGallery(t, sqb, &galleryFilter, nil)
assert.Len(t, galleries, 1)
assert.Equal(t, galleryPath, galleries[0].Path.String)
assert.Equal(t, galleryRating.Int64, galleries[0].Rating.Int64)
return nil
})
}
func TestGalleryQueryPathNotRating(t *testing.T) {
const galleryIdx = 1
galleryRating := getRating(galleryIdx)
pathCriterion := models.StringCriterionInput{
Value: "gallery_.*1_Path",
Modifier: models.CriterionModifierMatchesRegex,
}
ratingCriterion := models.IntCriterionInput{
Value: int(galleryRating.Int64),
Modifier: models.CriterionModifierEquals,
}
galleryFilter := models.GalleryFilterType{
Path: &pathCriterion,
Not: &models.GalleryFilterType{
Rating: &ratingCriterion,
},
}
withTxn(func(r models.Repository) error {
sqb := r.Gallery()
galleries := queryGallery(t, sqb, &galleryFilter, nil)
for _, gallery := range galleries {
verifyNullString(t, gallery.Path, pathCriterion)
ratingCriterion.Modifier = models.CriterionModifierNotEquals
verifyInt64(t, gallery.Rating, ratingCriterion)
}
return nil
})
}
func TestGalleryIllegalQuery(t *testing.T) {
assert := assert.New(t)
const galleryIdx = 1
subFilter := models.GalleryFilterType{
Path: &models.StringCriterionInput{
Value: getGalleryStringValue(galleryIdx, "Path"),
Modifier: models.CriterionModifierEquals,
},
}
galleryFilter := &models.GalleryFilterType{
And: &subFilter,
Or: &subFilter,
}
withTxn(func(r models.Repository) error {
sqb := r.Gallery()
_, _, err := sqb.Query(galleryFilter, nil)
assert.NotNil(err)
galleryFilter.Or = nil
galleryFilter.Not = &subFilter
_, _, err = sqb.Query(galleryFilter, nil)
assert.NotNil(err)
galleryFilter.And = nil
galleryFilter.Or = &subFilter
_, _, err = sqb.Query(galleryFilter, nil)
assert.NotNil(err)
return nil
})
}
func TestGalleryQueryURL(t *testing.T) {
const sceneIdx = 1
galleryURL := getGalleryStringValue(sceneIdx, urlField)
@@ -712,6 +849,22 @@ func verifyGalleriesPerformerCount(t *testing.T, performerCountCriterion models.
})
}
func TestGalleryQueryAverageResolution(t *testing.T) {
withTxn(func(r models.Repository) error {
qb := r.Gallery()
resolution := models.ResolutionEnumLow
galleryFilter := models.GalleryFilterType{
AverageResolution: &resolution,
}
// not verifying average - just ensure we get at least one
galleries := queryGallery(t, qb, &galleryFilter, nil)
assert.Greater(t, len(galleries), 0)
return nil
})
}
func TestGalleryQueryImageCount(t *testing.T) {
const imageCount = 0
imageCountCriterion := models.IntCriterionInput{
+190 -195
View File
@@ -12,35 +12,6 @@ const imageIDColumn = "image_id"
const performersImagesTable = "performers_images"
const imagesTagsTable = "images_tags"
var imagesForPerformerQuery = selectAll(imageTable) + `
LEFT JOIN performers_images as performers_join on performers_join.image_id = images.id
WHERE performers_join.performer_id = ?
GROUP BY images.id
`
var countImagesForPerformerQuery = `
SELECT performer_id FROM performers_images as performers_join
WHERE performer_id = ?
GROUP BY image_id
`
var imagesForStudioQuery = selectAll(imageTable) + `
JOIN studios ON studios.id = images.studio_id
WHERE studios.id = ?
GROUP BY images.id
`
var imagesForMovieQuery = selectAll(imageTable) + `
LEFT JOIN movies_images as movies_join on movies_join.image_id = images.id
WHERE movies_join.movie_id = ?
GROUP BY images.id
`
var countImagesForTagQuery = `
SELECT tag_id AS id FROM images_tags
WHERE images_tags.tag_id = ?
GROUP BY images_tags.image_id
`
var imagesForGalleryQuery = selectAll(imageTable) + `
LEFT JOIN galleries_images as galleries_join on galleries_join.image_id = images.id
WHERE galleries_join.gallery_id = ?
@@ -216,7 +187,69 @@ func (qb *imageQueryBuilder) All() ([]*models.Image, error) {
return qb.queryImages(selectAll(imageTable)+qb.getImageSort(nil), nil)
}
func (qb *imageQueryBuilder) makeQuery(imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) queryBuilder {
func (qb *imageQueryBuilder) validateFilter(imageFilter *models.ImageFilterType) error {
const and = "AND"
const or = "OR"
const not = "NOT"
if imageFilter.And != nil {
if imageFilter.Or != nil {
return illegalFilterCombination(and, or)
}
if imageFilter.Not != nil {
return illegalFilterCombination(and, not)
}
return qb.validateFilter(imageFilter.And)
}
if imageFilter.Or != nil {
if imageFilter.Not != nil {
return illegalFilterCombination(or, not)
}
return qb.validateFilter(imageFilter.Or)
}
if imageFilter.Not != nil {
return qb.validateFilter(imageFilter.Not)
}
return nil
}
func (qb *imageQueryBuilder) makeFilter(imageFilter *models.ImageFilterType) *filterBuilder {
query := &filterBuilder{}
if imageFilter.And != nil {
query.and(qb.makeFilter(imageFilter.And))
}
if imageFilter.Or != nil {
query.or(qb.makeFilter(imageFilter.Or))
}
if imageFilter.Not != nil {
query.not(qb.makeFilter(imageFilter.Not))
}
query.handleCriterionFunc(stringCriterionHandler(imageFilter.Path, "images.path"))
query.handleCriterionFunc(intCriterionHandler(imageFilter.Rating, "images.rating"))
query.handleCriterionFunc(intCriterionHandler(imageFilter.OCounter, "images.o_counter"))
query.handleCriterionFunc(boolCriterionHandler(imageFilter.Organized, "images.organized"))
query.handleCriterionFunc(resolutionCriterionHandler(imageFilter.Resolution, "images.height", "images.width"))
query.handleCriterionFunc(imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
query.handleCriterionFunc(imageTagsCriterionHandler(qb, imageFilter.Tags))
query.handleCriterionFunc(imageTagCountCriterionHandler(qb, imageFilter.TagCount))
query.handleCriterionFunc(imageGalleriesCriterionHandler(qb, imageFilter.Galleries))
query.handleCriterionFunc(imagePerformersCriterionHandler(qb, imageFilter.Performers))
query.handleCriterionFunc(imagePerformerCountCriterionHandler(qb, imageFilter.PerformerCount))
query.handleCriterionFunc(imageStudioCriterionHandler(qb, imageFilter.Studios))
query.handleCriterionFunc(imagePerformerTagsCriterionHandler(qb, imageFilter.PerformerTags))
return query
}
func (qb *imageQueryBuilder) makeQuery(imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
if imageFilter == nil {
imageFilter = &models.ImageFilterType{}
}
@@ -227,12 +260,6 @@ func (qb *imageQueryBuilder) makeQuery(imageFilter *models.ImageFilterType, find
query := qb.newQuery()
query.body = selectDistinctIDs(imageTable)
query.body += `
left join performers_images as performers_join on performers_join.image_id = images.id
left join studios as studio on studio.id = images.studio_id
left join images_tags as tags_join on tags_join.image_id = images.id
left join galleries_images as galleries_join on galleries_join.image_id = images.id
`
if q := findFilter.Q; q != nil && *q != "" {
searchColumns := []string{"images.title", "images.path", "images.checksum"}
@@ -241,154 +268,23 @@ func (qb *imageQueryBuilder) makeQuery(imageFilter *models.ImageFilterType, find
query.addArg(thisArgs...)
}
query.handleStringCriterionInput(imageFilter.Path, "images.path")
if rating := imageFilter.Rating; rating != nil {
clause, count := getIntCriterionWhereClause("images.rating", *imageFilter.Rating)
query.addWhere(clause)
if count == 1 {
query.addArg(imageFilter.Rating.Value)
}
if err := qb.validateFilter(imageFilter); err != nil {
return nil, err
}
filter := qb.makeFilter(imageFilter)
if oCounter := imageFilter.OCounter; oCounter != nil {
clause, count := getIntCriterionWhereClause("images.o_counter", *imageFilter.OCounter)
query.addWhere(clause)
if count == 1 {
query.addArg(imageFilter.OCounter.Value)
}
}
if Organized := imageFilter.Organized; Organized != nil {
var organized string
if *Organized == true {
organized = "1"
} else {
organized = "0"
}
query.addWhere("images.organized = " + organized)
}
if resolutionFilter := imageFilter.Resolution; resolutionFilter != nil {
if resolution := resolutionFilter.String(); resolutionFilter.IsValid() {
switch resolution {
case "VERY_LOW":
query.addWhere("MIN(images.height, images.width) < 240")
case "LOW":
query.addWhere("(MIN(images.height, images.width) >= 240 AND MIN(images.height, images.width) < 360)")
case "R360P":
query.addWhere("(MIN(images.height, images.width) >= 360 AND MIN(images.height, images.width) < 480)")
case "STANDARD":
query.addWhere("(MIN(images.height, images.width) >= 480 AND MIN(images.height, images.width) < 540)")
case "WEB_HD":
query.addWhere("(MIN(images.height, images.width) >= 540 AND MIN(images.height, images.width) < 720)")
case "STANDARD_HD":
query.addWhere("(MIN(images.height, images.width) >= 720 AND MIN(images.height, images.width) < 1080)")
case "FULL_HD":
query.addWhere("(MIN(images.height, images.width) >= 1080 AND MIN(images.height, images.width) < 1440)")
case "QUAD_HD":
query.addWhere("(MIN(images.height, images.width) >= 1440 AND MIN(images.height, images.width) < 1920)")
case "VR_HD":
query.addWhere("(MIN(images.height, images.width) >= 1920 AND MIN(images.height, images.width) < 2160)")
case "FOUR_K":
query.addWhere("(MIN(images.height, images.width) >= 2160 AND MIN(images.height, images.width) < 2880)")
case "FIVE_K":
query.addWhere("(MIN(images.height, images.width) >= 2880 AND MIN(images.height, images.width) < 3384)")
case "SIX_K":
query.addWhere("(MIN(images.height, images.width) >= 3384 AND MIN(images.height, images.width) < 4320)")
case "EIGHT_K":
query.addWhere("MIN(images.height, images.width) >= 4320")
}
}
}
if isMissingFilter := imageFilter.IsMissing; isMissingFilter != nil && *isMissingFilter != "" {
switch *isMissingFilter {
case "studio":
query.addWhere("images.studio_id IS NULL")
case "performers":
query.addWhere("performers_join.image_id IS NULL")
case "galleries":
query.addWhere("galleries_join.image_id IS NULL")
case "tags":
query.addWhere("tags_join.image_id IS NULL")
default:
query.addWhere("(images." + *isMissingFilter + " IS NULL OR TRIM(images." + *isMissingFilter + ") = '')")
}
}
if tagsFilter := imageFilter.Tags; tagsFilter != nil && len(tagsFilter.Value) > 0 {
for _, tagID := range tagsFilter.Value {
query.addArg(tagID)
}
query.body += " LEFT JOIN tags on tags_join.tag_id = tags.id"
whereClause, havingClause := getMultiCriterionClause("images", "tags", "images_tags", "image_id", "tag_id", tagsFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
if tagCountFilter := imageFilter.TagCount; tagCountFilter != nil {
clause, count := getCountCriterionClause(imageTable, imagesTagsTable, imageIDColumn, *tagCountFilter)
if count == 1 {
query.addArg(tagCountFilter.Value)
}
query.addWhere(clause)
}
if galleriesFilter := imageFilter.Galleries; galleriesFilter != nil && len(galleriesFilter.Value) > 0 {
for _, galleryID := range galleriesFilter.Value {
query.addArg(galleryID)
}
query.body += " LEFT JOIN galleries ON galleries_join.gallery_id = galleries.id"
whereClause, havingClause := getMultiCriterionClause("images", "galleries", "galleries_images", "image_id", "gallery_id", galleriesFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
if performersFilter := imageFilter.Performers; performersFilter != nil && len(performersFilter.Value) > 0 {
for _, performerID := range performersFilter.Value {
query.addArg(performerID)
}
query.body += " LEFT JOIN performers ON performers_join.performer_id = performers.id"
whereClause, havingClause := getMultiCriterionClause("images", "performers", "performers_images", "image_id", "performer_id", performersFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
if performerCountFilter := imageFilter.PerformerCount; performerCountFilter != nil {
clause, count := getCountCriterionClause(imageTable, performersImagesTable, imageIDColumn, *performerCountFilter)
if count == 1 {
query.addArg(performerCountFilter.Value)
}
query.addWhere(clause)
}
if studiosFilter := imageFilter.Studios; studiosFilter != nil && len(studiosFilter.Value) > 0 {
for _, studioID := range studiosFilter.Value {
query.addArg(studioID)
}
whereClause, havingClause := getMultiCriterionClause("images", "studio", "", "", "studio_id", studiosFilter)
query.addWhere(whereClause)
query.addHaving(havingClause)
}
handleImagePerformerTagsCriterion(&query, imageFilter.PerformerTags)
query.addFilter(filter)
query.sortAndPagination = qb.getImageSort(findFilter) + getPagination(findFilter)
return query
return &query, nil
}
func (qb *imageQueryBuilder) Query(imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) ([]*models.Image, int, error) {
query := qb.makeQuery(imageFilter, findFilter)
query, err := qb.makeQuery(imageFilter, findFilter)
if err != nil {
return nil, 0, err
}
idsResult, countResult, err := query.executeFind()
if err != nil {
@@ -409,32 +305,131 @@ func (qb *imageQueryBuilder) Query(imageFilter *models.ImageFilterType, findFilt
}
func (qb *imageQueryBuilder) QueryCount(imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (int, error) {
query := qb.makeQuery(imageFilter, findFilter)
query, err := qb.makeQuery(imageFilter, findFilter)
if err != nil {
return 0, err
}
return query.executeCount()
}
func handleImagePerformerTagsCriterion(query *queryBuilder, performerTagsFilter *models.MultiCriterionInput) {
if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 {
for _, tagID := range performerTagsFilter.Value {
query.addArg(tagID)
func imageIsMissingCriterionHandler(qb *imageQueryBuilder, isMissing *string) criterionHandlerFunc {
return func(f *filterBuilder) {
if isMissing != nil && *isMissing != "" {
switch *isMissing {
case "studio":
f.addWhere("images.studio_id IS NULL")
case "performers":
qb.performersRepository().join(f, "performers_join", "images.id")
f.addWhere("performers_join.image_id IS NULL")
case "galleries":
qb.galleriesRepository().join(f, "galleries_join", "images.id")
f.addWhere("galleries_join.image_id IS NULL")
case "tags":
qb.tagsRepository().join(f, "tags_join", "images.id")
f.addWhere("tags_join.image_id IS NULL")
default:
f.addWhere("(images." + *isMissing + " IS NULL OR TRIM(images." + *isMissing + ") = '')")
}
}
}
}
query.body += " LEFT JOIN performers_tags AS performer_tags_join on performers_join.performer_id = performer_tags_join.performer_id"
func (qb *imageQueryBuilder) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{
primaryTable: imageTable,
foreignTable: foreignTable,
joinTable: joinTable,
primaryFK: imageIDColumn,
foreignFK: foreignFK,
addJoinsFunc: addJoinsFunc,
}
}
if performerTagsFilter.Modifier == models.CriterionModifierIncludes {
// includes any of the provided ids
query.addWhere("performer_tags_join.tag_id IN " + getInBinding(len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll {
// includes all of the provided ids
query.addWhere("performer_tags_join.tag_id IN " + getInBinding(len(performerTagsFilter.Value)))
query.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierExcludes {
query.addWhere(fmt.Sprintf(`not exists
(select performers_images.performer_id from performers_images
left join performers_tags on performers_tags.performer_id = performers_images.performer_id where
performers_images.image_id = images.id AND
performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))))
func imageTagsCriterionHandler(qb *imageQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
qb.tagsRepository().join(f, "tags_join", "images.id")
f.addJoin(tagTable, "", "tags_join.tag_id = tags.id")
}
h := qb.getMultiCriterionHandlerBuilder(tagTable, imagesTagsTable, tagIDColumn, addJoinsFunc)
return h.handler(tags)
}
func imageTagCountCriterionHandler(qb *imageQueryBuilder, tagCount *models.IntCriterionInput) criterionHandlerFunc {
h := countCriterionHandlerBuilder{
primaryTable: imageTable,
joinTable: imagesTagsTable,
primaryFK: imageIDColumn,
}
return h.handler(tagCount)
}
func imageGalleriesCriterionHandler(qb *imageQueryBuilder, galleries *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
qb.galleriesRepository().join(f, "galleries_join", "images.id")
f.addJoin(galleryTable, "", "galleries_join.gallery_id = galleries.id")
}
h := qb.getMultiCriterionHandlerBuilder(galleryTable, galleriesImagesTable, galleryIDColumn, addJoinsFunc)
return h.handler(galleries)
}
func imagePerformersCriterionHandler(qb *imageQueryBuilder, performers *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
qb.performersRepository().join(f, "performers_join", "images.id")
f.addJoin(performerTable, "", "performers_join.performer_id = performers.id")
}
h := qb.getMultiCriterionHandlerBuilder(performerTable, performersImagesTable, performerIDColumn, addJoinsFunc)
return h.handler(performers)
}
func imagePerformerCountCriterionHandler(qb *imageQueryBuilder, performerCount *models.IntCriterionInput) criterionHandlerFunc {
h := countCriterionHandlerBuilder{
primaryTable: imageTable,
joinTable: performersImagesTable,
primaryFK: imageIDColumn,
}
return h.handler(performerCount)
}
func imageStudioCriterionHandler(qb *imageQueryBuilder, studios *models.MultiCriterionInput) criterionHandlerFunc {
addJoinsFunc := func(f *filterBuilder) {
f.addJoin(studioTable, "studio", "studio.id = images.studio_id")
}
h := qb.getMultiCriterionHandlerBuilder("studio", "", studioIDColumn, addJoinsFunc)
return h.handler(studios)
}
func imagePerformerTagsCriterionHandler(qb *imageQueryBuilder, performerTagsFilter *models.MultiCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 {
qb.performersRepository().join(f, "performers_join", "images.id")
f.addJoin("performers_tags", "performer_tags_join", "performers_join.performer_id = performer_tags_join.performer_id")
var args []interface{}
for _, tagID := range performerTagsFilter.Value {
args = append(args, tagID)
}
if performerTagsFilter.Modifier == models.CriterionModifierIncludes {
// includes any of the provided ids
f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...)
} else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll {
// includes all of the provided ids
f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...)
f.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value)))
} else if performerTagsFilter.Modifier == models.CriterionModifierExcludes {
f.addWhere(fmt.Sprintf(`not exists
(select performers_images.performer_id from performers_images
left join performers_tags on performers_tags.performer_id = performers_images.performer_id where
performers_images.image_id = images.id AND
performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))), args...)
}
}
}
}
+201
View File
@@ -155,6 +155,143 @@ func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput, ex
})
}
func TestImageQueryPathOr(t *testing.T) {
const image1Idx = 1
const image2Idx = 2
image1Path := getImageStringValue(image1Idx, "Path")
image2Path := getImageStringValue(image2Idx, "Path")
imageFilter := models.ImageFilterType{
Path: &models.StringCriterionInput{
Value: image1Path,
Modifier: models.CriterionModifierEquals,
},
Or: &models.ImageFilterType{
Path: &models.StringCriterionInput{
Value: image2Path,
Modifier: models.CriterionModifierEquals,
},
},
}
withTxn(func(r models.Repository) error {
sqb := r.Image()
images := queryImages(t, sqb, &imageFilter, nil)
assert.Len(t, images, 2)
assert.Equal(t, image1Path, images[0].Path)
assert.Equal(t, image2Path, images[1].Path)
return nil
})
}
func TestImageQueryPathAndRating(t *testing.T) {
const imageIdx = 1
imagePath := getImageStringValue(imageIdx, "Path")
imageRating := getRating(imageIdx)
imageFilter := models.ImageFilterType{
Path: &models.StringCriterionInput{
Value: imagePath,
Modifier: models.CriterionModifierEquals,
},
And: &models.ImageFilterType{
Rating: &models.IntCriterionInput{
Value: int(imageRating.Int64),
Modifier: models.CriterionModifierEquals,
},
},
}
withTxn(func(r models.Repository) error {
sqb := r.Image()
images := queryImages(t, sqb, &imageFilter, nil)
assert.Len(t, images, 1)
assert.Equal(t, imagePath, images[0].Path)
assert.Equal(t, imageRating.Int64, images[0].Rating.Int64)
return nil
})
}
func TestImageQueryPathNotRating(t *testing.T) {
const imageIdx = 1
imageRating := getRating(imageIdx)
pathCriterion := models.StringCriterionInput{
Value: "image_.*1_Path",
Modifier: models.CriterionModifierMatchesRegex,
}
ratingCriterion := models.IntCriterionInput{
Value: int(imageRating.Int64),
Modifier: models.CriterionModifierEquals,
}
imageFilter := models.ImageFilterType{
Path: &pathCriterion,
Not: &models.ImageFilterType{
Rating: &ratingCriterion,
},
}
withTxn(func(r models.Repository) error {
sqb := r.Image()
images := queryImages(t, sqb, &imageFilter, nil)
for _, image := range images {
verifyString(t, image.Path, pathCriterion)
ratingCriterion.Modifier = models.CriterionModifierNotEquals
verifyInt64(t, image.Rating, ratingCriterion)
}
return nil
})
}
func TestImageIllegalQuery(t *testing.T) {
assert := assert.New(t)
const imageIdx = 1
subFilter := models.ImageFilterType{
Path: &models.StringCriterionInput{
Value: getImageStringValue(imageIdx, "Path"),
Modifier: models.CriterionModifierEquals,
},
}
imageFilter := &models.ImageFilterType{
And: &subFilter,
Or: &subFilter,
}
withTxn(func(r models.Repository) error {
sqb := r.Image()
_, _, err := sqb.Query(imageFilter, nil)
assert.NotNil(err)
imageFilter.Or = nil
imageFilter.Not = &subFilter
_, _, err = sqb.Query(imageFilter, nil)
assert.NotNil(err)
imageFilter.And = nil
imageFilter.Or = &subFilter
_, _, err = sqb.Query(imageFilter, nil)
assert.NotNil(err)
return nil
})
}
func TestImageQueryRating(t *testing.T) {
const rating = 3
ratingCriterion := models.IntCriterionInput{
@@ -449,6 +586,70 @@ func TestImageQueryIsMissingRating(t *testing.T) {
})
}
func TestImageQueryGallery(t *testing.T) {
withTxn(func(r models.Repository) error {
sqb := r.Image()
galleryCriterion := models.MultiCriterionInput{
Value: []string{
strconv.Itoa(galleryIDs[galleryIdxWithImage]),
},
Modifier: models.CriterionModifierIncludes,
}
imageFilter := models.ImageFilterType{
Galleries: &galleryCriterion,
}
images, _, err := sqb.Query(&imageFilter, nil)
if err != nil {
t.Errorf("Error querying image: %s", err.Error())
}
assert.Len(t, images, 1)
// ensure ids are correct
for _, image := range images {
assert.True(t, image.ID == imageIDs[imageIdxWithGallery])
}
galleryCriterion = models.MultiCriterionInput{
Value: []string{
strconv.Itoa(galleryIDs[galleryIdx1WithImage]),
strconv.Itoa(galleryIDs[galleryIdx2WithImage]),
},
Modifier: models.CriterionModifierIncludesAll,
}
images, _, err = sqb.Query(&imageFilter, nil)
if err != nil {
t.Errorf("Error querying image: %s", err.Error())
}
assert.Len(t, images, 1)
assert.Equal(t, imageIDs[imageIdxWithTwoGalleries], images[0].ID)
galleryCriterion = models.MultiCriterionInput{
Value: []string{
strconv.Itoa(performerIDs[galleryIdx1WithImage]),
},
Modifier: models.CriterionModifierExcludes,
}
q := getImageStringValue(imageIdxWithTwoGalleries, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
images, _, err = sqb.Query(&imageFilter, &findFilter)
if err != nil {
t.Errorf("Error querying image: %s", err.Error())
}
assert.Len(t, images, 0)
return nil
})
}
func TestImageQueryPerformers(t *testing.T) {
withTxn(func(r models.Repository) error {
sqb := r.Image()
+20
View File
@@ -48,6 +48,9 @@ const (
const (
imageIdxWithGallery = iota
imageIdx1WithGallery
imageIdx2WithGallery
imageIdxWithTwoGalleries
imageIdxWithPerformer
imageIdx1WithPerformer
imageIdx2WithPerformer
@@ -102,6 +105,9 @@ const (
const (
galleryIdxWithScene = iota
galleryIdxWithImage
galleryIdx1WithImage
galleryIdx2WithImage
galleryIdxWithTwoImages
galleryIdxWithPerformer
galleryIdx1WithPerformer
galleryIdx2WithPerformer
@@ -230,6 +236,10 @@ var (
var (
imageGalleryLinks = [][2]int{
{imageIdxWithGallery, galleryIdxWithImage},
{imageIdx1WithGallery, galleryIdxWithTwoImages},
{imageIdx2WithGallery, galleryIdxWithTwoImages},
{imageIdxWithTwoGalleries, galleryIdx1WithImage},
{imageIdxWithTwoGalleries, galleryIdx2WithImage},
}
imageStudioLinks = [][2]int{
{imageIdxWithStudio, studioIdxWithImage},
@@ -513,6 +523,14 @@ func getHeight(index int) sql.NullInt64 {
}
}
func getWidth(index int) sql.NullInt64 {
height := getHeight(index)
return sql.NullInt64{
Int64: height.Int64 * 2,
Valid: height.Valid,
}
}
func getSceneDate(index int) models.SQLiteDate {
dates := []string{"null", "", "0001-01-01", "2001-02-03"}
date := dates[index%len(dates)]
@@ -571,6 +589,7 @@ func createImages(qb models.ImageReaderWriter, n int) error {
Rating: getRating(i),
OCounter: getOCounter(i),
Height: getHeight(i),
Width: getWidth(i),
}
created, err := qb.Create(image)
@@ -599,6 +618,7 @@ func createGalleries(gqb models.GalleryReaderWriter, n int) error {
Path: models.NullString(getGalleryStringValue(i, pathField)),
URL: getGalleryNullStringValue(i, urlField),
Checksum: getGalleryStringValue(i, checksumField),
Rating: getRating(i),
}
created, err := gqb.Create(gallery)