From c961de1d579da68d2cd078f63aa8e502b3d0c650 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Tue, 3 Aug 2021 18:49:04 +0545 Subject: [PATCH] Remove apiBugDemonstration test suite --- .drone.star | 4 +- docs/ocis/development/testing.md | 25 ++++----- tests/acceptance/config/behat.yml | 17 ------ tests/acceptance/docker/Makefile | 54 ++++-------------- .../apiWebdavPreviews-previews.feature | 55 ------------------- 5 files changed, 21 insertions(+), 134 deletions(-) delete mode 100644 tests/acceptance/features/apiBugDemonstration/apiWebdavPreviews-previews.feature diff --git a/.drone.star b/.drone.star index 6e0291a885..0050ba67ef 100644 --- a/.drone.star +++ b/.drone.star @@ -242,8 +242,6 @@ def testPipelines(ctx): pipelines = [] if "skip" not in config["localApiTests"] or not config["localApiTests"]["skip"]: pipelines = [ - localApiTests(ctx, "owncloud", "apiBugDemonstration"), - localApiTests(ctx, "ocis", "apiBugDemonstration"), localApiTests(ctx, "owncloud", "apiAccountsHashDifficulty", "default"), localApiTests(ctx, "ocis", "apiAccountsHashDifficulty", "default"), ] @@ -435,7 +433,7 @@ def uploadScanResults(ctx): }, } -def localApiTests(ctx, storage = "owncloud", suite = "apiBugDemonstration", accounts_hash_difficulty = 4): +def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): earlyFail = config["localApiTests"]["earlyFail"] if "earlyFail" in config["localApiTests"] else False if ("full-ci" in ctx.build.title.lower()): diff --git a/docs/ocis/development/testing.md b/docs/ocis/development/testing.md index 908155e272..aecf53bdc7 100644 --- a/docs/ocis/development/testing.md +++ b/docs/ocis/development/testing.md @@ -39,7 +39,7 @@ You can invoke two types of test suite runs: The names of the full test suite make targets have the same naming as in the CI pipeline. -For example `make -C tests/acceptance/docker localApiTests-apiBugDemonstration-ocis` runs the same tests as the `localApiTests-apiBugDemonstration-ocis` CI pipeline, which runs the oCIS test suite "apiBugDemonstration" against an oCIS with oCIS storage. +For example `make -C tests/acceptance/docker localApiTests-apiAccountsHashDifficulty-ocis` runs the same tests as the `localApiTests-apiAccountsHashDifficulty-ocis` CI pipeline, which runs the oCIS test suite "apiAccountsHashDifficulty" against an oCIS with oCIS storage. For example `make -C tests/acceptance/docker Core-API-Tests-owncloud-storage-3`runs the same tests as the `Core-API-Tests-owncloud-storage-3` CI pipeline, which runs the third (out of ten) ownCloud test suite against an oCIS with owncloud storage. @@ -48,7 +48,7 @@ For example `make -C tests/acceptance/docker Core-API-Tests-owncloud-storage-3`r The single feature tests can also be run against the different storage backends. Therefore multiple make targets with the schema test--feature- exists. For selecting a single feature test you have to add an additional `BEHAT_FEATURE=...` parameter when invoking the make command: ``` -make -C tests/acceptance/docker test-ocis-feature-ocis BEHAT_FEATURE='tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature' +make -C tests/acceptance/docker test-ocis-feature-ocis BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature' ``` This must be pointing to a valid feature definition. @@ -58,7 +58,7 @@ This must be pointing to a valid feature definition. By default the tests will be run against docker image built from your current working state of the oCIS repository. For some purposes it might also be handy to use a oCIS image from Docker Hub. Therefore you can provide the optional flag `OCIS_IMAGE_TAG=...` which must contain an available docker tag of the [owncloud/ocis registry on Docker Hub](https://hub.docker.com/r/owncloud/ocis) (eg. 'latest'). ``` -make -C tests/acceptance/docker localApiTests-apiBugDemonstration-ocis OCIS_IMAGE_TAG=latest +make -C tests/acceptance/docker localApiTests-apiAccountsHashDifficulty-ocis OCIS_IMAGE_TAG=latest ``` ### Test log output @@ -139,20 +139,17 @@ Every scenario that does not work in oCIS with "ocis" storage, is listed in `tes Those scenarios are run in the ordinary acceptance test pipeline in CI. The scenarios that fail are checked against the expected failures. If there are any differences then the CI pipeline fails. Similarly, scenarios that do not work in oCIS with EOS storage are listed in `tests/acceptance/expected-failures-on-EOS-storage.md`. -Additionally, some issues have scenarios that demonstrate the current buggy behaviour in ocis(reva). -Those scenarios are in this ocis repository in `tests/acceptance/features/apiBugDemonstration`. -Have a look into the [documentation](https://doc.owncloud.com/server/developer_manual/testing/acceptance-tests.html#writing-scenarios-for-bugs) to understand why we are writing those tests. If you want to work on a specific issue -1. adjust the core commit id to the latest commit in core so that CI will run the latest test code and scenarios from core. +1. adjust the core commit id to the latest commit in core so that CI will run the latest test code and scenarios from core. For that change `CORE_COMMITID` in `.drone.env`: # The test runner source for API tests CORE_COMMITID=38c91e5cf5fc4ffdc0536ba1d147a2a618ef83b5 CORE_BRANCH=master -2. locally run each of the tests marked with that issue in the expected failures file. +2. locally run each of the tests marked with that issue in the expected failures file. E.g.: @@ -164,10 +161,8 @@ If you want to work on a specific issue BEHAT_FEATURE='tests/acceptance/features/apiComments/comments.feature:123' ``` -3. the tests will fail, try to understand how and why they are failing -4. fix the code -5. go back to 2. and repeat till the tests are passing. -6. remove those tests from the expected failures file -7. run each of the local tests that were demonstrating the **buggy** behavior. They should fail. -8. delete each of the local tests that were demonstrating the **buggy** behavior. -9. make a PR that has the fixed code, relevant lines removed from the expected failures file and bug demonstration tests deleted. +3. the tests will fail, try to understand how and why they are failing +4. fix the code +5. go back to 2. and repeat till the tests are passing. +6. remove those tests from the expected failures file +7. make a PR that has the fixed code, and the relevant lines removed from the expected failures file. diff --git a/tests/acceptance/config/behat.yml b/tests/acceptance/config/behat.yml index f785972736..131a0d0afd 100644 --- a/tests/acceptance/config/behat.yml +++ b/tests/acceptance/config/behat.yml @@ -29,23 +29,6 @@ default: - TrashbinContext: - WebDavPropertiesContext: - apiBugDemonstration: - paths: - - '%paths.base%/../features/apiBugDemonstration' - context: *common_ldap_suite_context - contexts: - - RevaContext: - - OccContext: - - FeatureContext: *common_feature_context_params - - CapabilitiesContext: - - ChecksumContext: - - FavoritesContext: - - FilesVersionsContext: - - PublicWebDavContext: - - TrashbinContext: - - WebDavPropertiesContext: - - TUSContext: - extensions: jarnaiz\JUnitFormatter\JUnitFormatterExtension: filename: report.xml diff --git a/tests/acceptance/docker/Makefile b/tests/acceptance/docker/Makefile index 3e2d2a41c1..602fd199c4 100644 --- a/tests/acceptance/docker/Makefile +++ b/tests/acceptance/docker/Makefile @@ -52,15 +52,12 @@ help: @echo @echo -e "${GREEN}Run full oCIS test suites against oCIS with oCIS storage:${RESET}\n" @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}" - @echo -e "\tmake localApiTests-apiBugDemonstration-ocis\t\t${BLUE}run apiBugDemonstration test suite${RESET}" @echo @echo -e "${GREEN}Run full oCIS test suites against oCIS with s3ng storage:${RESET}\n" - @echo -e "\tmake localApiTests-apiBasic-s3ng\t\t${BLUE}run apiBasic test suite${RESET}" - @echo -e "\tmake localApiTests-apiOcisSpecific-s3ng\t\t${BLUE}run apiOcisSPecific test suite${RESET}" + @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-s3ng\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}" @echo @echo -e "${GREEN}Run full oCIS test suites against oCIS with ownCloud storage:${RESET}\n" @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-owncloud\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}" - @echo -e "\tmake localApiTests-apiBugDemonstration-owncloud\t${BLUE}run apiBugDemonstration test suite${RESET}" @echo @echo -e "${GREEN}Run full ownCloud test suites against oCIS with oCIS storage:${RESET}\n" @echo -e "\tmake Core-API-Tests-ocis-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @@ -75,19 +72,19 @@ help: @echo -e "\tmake test-ocis-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." - @echo -e "\texample: ${RED}tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature${RESET}" + @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo @echo -e "${GREEN}Run an oCIS feature test against oCIS with s3ng storage:${RESET}\n" @echo -e "\tmake test-ocis-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." - @echo -e "\texample: ${RED}tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature${RESET}" + @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo @echo -e "${GREEN}Run an oCIS feature test against oCIS with owncloud storage:${RESET}\n" @echo -e "\tmake test-ocis-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." - @echo -e "\texample: ${RED}tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature${RESET}" + @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo @echo -e "${GREEN}Run an ownCloud feature test against oCIS with oCIS storage:${RESET}\n" @echo -e "\tmake test-oc10-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @@ -118,14 +115,14 @@ help: @echo -e "\tmake clean-docker-volumes\t\t${BLUE}removes used docker volumes (used for caching)${RESET}" @echo .PHONY: test-ocis-feature-ocis-storage -test-ocis-feature-ocis-storage: ## test a ocis feature with oCIS storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature:7' +test-ocis-feature-ocis-storage: ## test a ocis feature with oCIS storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' @TEST_SOURCE=ocis \ STORAGE_DRIVER=ocis \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite .PHONY: test-ocis-feature-s3ng-storage -test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature:7' +test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' @TEST_SOURCE=ocis \ STORAGE_DRIVER=s3ng \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ @@ -133,21 +130,21 @@ test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, useage $(MAKE) --no-print-directory testSuite .PHONY: test-ocis-feature-owncloud-storage -test-ocis-feature-owncloud-storage: ## test a ocis feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature:7' +test-ocis-feature-owncloud-storage: ## test a ocis feature with oc10 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' @TEST_SOURCE=ocis \ STORAGE_DRIVER=owncloud \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite .PHONY: test-oc10-feature-ocis-storage -test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' +test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' @TEST_SOURCE=oc10 \ STORAGE_DRIVER=ocis \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite .PHONY: test-oc10-feature-s3ng-storage -test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' +test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' @TEST_SOURCE=oc10 \ STORAGE_DRIVER=s3ng \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ @@ -155,19 +152,12 @@ test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, useage $(MAKE) --no-print-directory testSuite .PHONY: test-oc10-feature-owncloud-storage -test-oc10-feature-owncloud-storage: ## test a oC10 feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' +test-oc10-feature-owncloud-storage: ## test a oC10 feature with oc10 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' @TEST_SOURCE=oc10 \ STORAGE_DRIVER=owncloud \ BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite -.PHONY: localApiTests-apiBugDemonstration-owncloud -localApiTests-apiBugDemonstration-owncloud: ## run apiBugDemonstration test suite with owncloud storage - @TEST_SOURCE=oc10 \ - STORAGE_DRIVER=owncloud \ - BEHAT_SUITE=apiBugDemonstration \ - $(MAKE) --no-print-directory testSuite - .PHONY: localApiTests-apiAccountsHashDifficulty-owncloud localApiTests-apiAccountsHashDifficulty-owncloud: ## run apiAccountsHashDifficulty test suite with owncloud storage @TEST_SOURCE=ocis \ @@ -175,13 +165,6 @@ localApiTests-apiAccountsHashDifficulty-owncloud: ## run apiAccountsHashDifficul BEHAT_SUITE=apiAccountsHashDifficulty \ $(MAKE) --no-print-directory testSuite -.PHONY: localApiTests-apiBugDemonstration-ocis -localApiTests-apiBugDemonstration-ocis: ## run apiBugDemonstration test suite with ocis storage - @TEST_SOURCE=ocis \ - STORAGE_DRIVER=ocis \ - BEHAT_SUITE=apiBugDemonstration \ - $(MAKE) --no-print-directory testSuite - .PHONY: localApiTests-apiAccountsHashDifficulty-ocis localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty test suite with ocis storage @TEST_SOURCE=ocis \ @@ -189,23 +172,6 @@ localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty t BEHAT_SUITE=apiAccountsHashDifficulty \ $(MAKE) --no-print-directory testSuite -.PHONY: localApiTests-apiOcisSpecific-s3ng -localApiTests-apiOcisSpecific-s3ng: ## run apiOcisSPecific test suite with s3ng storage - @TEST_SOURCE=ocis \ - STORAGE_DRIVER=s3ng \ - BEHAT_SUITE=apiOcisSpecific \ - START_CEPH=1 \ - $(MAKE) --no-print-directory testSuite - - -.PHONY: localApiTests-apiBasic-s3ng -localApiTests-apiBasic-s3ng: ## run apiBasic test suite with s3ng storage - @TEST_SOURCE=ocis \ - STORAGE_DRIVER=s3ng \ - BEHAT_SUITE=apiBasic \ - START_CEPH=1 \ - $(MAKE) --no-print-directory testSuite - targets = $(addprefix Core-API-Tests-owncloud-storage-,$(PARTS)) .PHONY: $(targets) $(targets): diff --git a/tests/acceptance/features/apiBugDemonstration/apiWebdavPreviews-previews.feature b/tests/acceptance/features/apiBugDemonstration/apiWebdavPreviews-previews.feature deleted file mode 100644 index b66fa9da09..0000000000 --- a/tests/acceptance/features/apiBugDemonstration/apiWebdavPreviews-previews.feature +++ /dev/null @@ -1,55 +0,0 @@ -@api @issue-ocis-187 -Feature: previews of files downloaded through the webdav API - - Background: - Given user "Alice" has been created with default attributes and without skeleton files - - @issue-ocis-2069 - Scenario Outline: download different sizes of previews of file on ocis - Given user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt" - When user "Alice" downloads the preview of "/parent.txt" with width and height using the WebDAV API - Then the HTTP status code should be "200" - And the downloaded image should be pixels wide and pixels high - Examples: - | width | height | expected-width | expected-height | - | 1 | 1 | 16 | 16 | - | 32 | 32 | 32 | 32 | - | 1024 | 1024 | 640 | 480 | - | 1 | 1024 | 16 | 16 | - | 1024 | 1 | 640 | 480 | - - @issue-ocis-2071 - Scenario: download previews of other users files in ocis - Given user "Brian" has been created with default attributes and without skeleton files - And user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt" - When user "Brian" downloads the preview of "/parent.txt" of "Alice" with width "32" and height "32" using the WebDAV API - Then the HTTP status code should be "404" - And the value of the item "/d:error/s:message" in the response about user "Alice" should be "File with name parent.txt could not be located" - And the value of the item "/d:error/s:exception" in the response about user "Alice" should be "Sabre\DAV\Exception\NotFound" - - @issue-ocis-2070 - # after fixing all issues delete this Scenario and use the one from oC10 core - Scenario: Download file previews when it is disabled by the administrator - Given the administrator has updated system config key "enable_previews" with value "false" and type "boolean" - And user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt" - When user "Alice" downloads the preview of "/parent.txt" with width "32" and height "32" using the WebDAV API - Then the HTTP status code should be "200" - - @issue-ocis-2070 - # after fixing all issues delete this Scenario and use the one from oC10 core - Scenario: unset maximum size of previews - Given user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt" - And the administrator has updated system config key "preview_max_x" with value "null" - And the administrator has updated system config key "preview_max_y" with value "null" - When user "Alice" downloads the preview of "/parent.txt" with width "32" and height "32" using the WebDAV API - Then the HTTP status code should be "200" - - @issue-ocis-2070 - # after fixing all issues delete this Scenario and use the one from oC10 core - Scenario: set maximum size of previews - Given user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt" - When the administrator updates system config key "preview_max_x" with value "null" using the occ command - And the administrator updates system config key "preview_max_y" with value "null" using the occ command - Then the HTTP status code should be "201" - When user "Alice" downloads the preview of "/parent.txt" with width "null" and height "null" using the WebDAV API - Then the HTTP status code should be "400"